import configparser
import enum
import hashlib
import json
import math
import os
import sys
from collections import defaultdict
from os.path import join, getsize, realpath, relpath, normpath

class FileSizeUnit(enum.Enum):
    B = 'b'
    KB = 'kb'
    MB = 'mb'
    GB = 'gb'
    TB = 'tb'
    PB = 'pb'

    @staticmethod
    def from_str(text: str):
        if text.lower() == 'b':
            return FileSizeUnit.B
        elif text.lower() == 'kb':
            return FileSizeUnit.KB
        elif text.lower() == 'mb':
            return FileSizeUnit.MB
        elif text.lower() == 'gb':
            return FileSizeUnit.GB
        elif text.lower() == 'tb':
            return FileSizeUnit.TB
        elif text.lower() == 'pb':
            return FileSizeUnit.PB

def convert_to_bytes(size: float, unit: FileSizeUnit) -> int:
    if unit == FileSizeUnit.KB:
        return int(size * 1024)
    elif unit == FileSizeUnit.MB:
        return int(size * 1024 * 1024)
    elif unit == FileSizeUnit.GB:
        return int(size * 1024 * 1024 * 1024)
    elif unit == FileSizeUnit.TB:
        return int(size * 1024 * 1024 * 1024 * 1024)
    elif unit == FileSizeUnit.PB:
        return int(size * 1024 * 1024 * 1024 * 1024 * 1024)
    else:
        return int(size)

def get_chunks(file, chunk_size=1024) -> str:
    while True:
        chunk = file.read(chunk_size)
        if not chunk:
            return
        yield chunk

def get_hash(file_name, hash_alg: str = 'sha512', first_chunk: bool = False, chunk_size: int = 1024) -> bytes:
    hash_obj = hashlib.new(hash_alg.lower())

    with open(file_name, 'rb') as file:
        if first_chunk:
            hash_obj.update(file.read(chunk_size))
        else:
            for chunk in get_chunks(file, chunk_size):
                hash_obj.update(chunk)
    return hash_obj.digest()

def find_duplicates(paths: list[str]) -> None:
    config = configparser.ConfigParser()
    config.read('search_rules.ini')

    recursive = bool(config.get('General', 'Recursive'))
    hash_alg = config.get('General', 'HashAlgorithm')

    file_list = list()
    print('Retrieving all files {}'.format('recursively' if recursive else ''))
    for path in paths:
        excluded_dirs = [normpath(path) for path in json.loads(config.get('Filter', 'ExcludedDirectories'))]
        for dirpath, dirname, filenames in os.walk(path, topdown=True):
            dirname[:] = [d for d in dirname if join(dirpath, d) not in excluded_dirs]

            for name in filenames:
                extensions_tuple = tuple(json.loads(config.get('Filter', 'IncludeFileExtensions')))

                if len(extensions_tuple) > 0:
                    if name.endswith(extensions_tuple):
                        file_list.append(join(dirpath, name))
                else:
                    extensions_tuple = tuple(json.loads(config.get('Filter', 'ExcludeFileExtensions')))
                    if not name.endswith(extensions_tuple):
                        file_list.append(join(dirpath, name))

            if not recursive:
                break

        # Create dict of each file size with a list of files having that size
        size_dict = defaultdict(list)
        print('Getting file sizes')
        for file_path in file_list:
            try:
                file_size = getsize(realpath(file_path))
                file_size_unit = FileSizeUnit.from_str(config.get('Filter', 'FileSizeUnit'))
                file_size_eq = config.get('Filter', 'FileSizeEqualTo')
                file_size_gt = config.get('Filter', 'FileSizeGreaterThan')
                file_size_lt = config.get('Filter', 'FileSizeLessThan')

                if file_size_eq:
                    file_size_eq = convert_to_bytes(float(file_size_eq), file_size_unit)
                    if file_size == file_size_eq:
                        size_dict[file_size].append(file_path)
                else:
                    file_size_gt = convert_to_bytes(float(file_size_gt), file_size_unit) if file_size_gt else -math.inf
                    file_size_lt = convert_to_bytes(float(file_size_lt), file_size_unit) if file_size_lt else math.inf

                    if file_size_gt < file_size < file_size_lt:
                        size_dict[file_size].append(file_path)
            except OSError:
                print('Unable to access file: ' + file_path)
                continue

        small_hash_dict = defaultdict(list)
        try:
            chunk_size = int(config.get('General', 'ChunkSize'))
        except ValueError:
            print('Invalid chunk size, must specify a positive integer')
            return
        print('Checking first {} bytes'.format(chunk_size))
        counter = 0
        for size, file_list in size_dict.items():
            if len(file_list) < 2:
                continue

            for file_path in file_list:
                counter += 1
                try:
                    chunk_hash = get_hash(file_path, hash_alg=hash_alg, first_chunk=True, chunk_size=chunk_size)
                    small_hash_dict[(chunk_hash, size)].append(file_path)
                except OSError:
                    print('Unable to access file: ' + file_path)
                    continue
                except ValueError as e:
                    print(e)
                    print('Invalid hash algorithm specified')
                    return
        print('{} files with identical sizes'.format(counter))

        hash_dict = dict()
        print('Calculating full hashes')
        for (_, _), file_list in small_hash_dict.items():
            if len(file_list) < 2:
                continue

            for file_path in file_list:
                try:
                    full_hash = get_hash(file_path, hash_alg=hash_alg, chunk_size=chunk_size)
                    if full_hash in hash_dict.keys():
                        file_path = relpath(file_path, start=path)
                        duplicate = relpath(hash_dict[full_hash], start=path)
                        print('Duplicate found: {} and {}'.format(file_path, duplicate))
                    else:
                        hash_dict[full_hash] = file_path
                except OSError:
                    print('Unable to access file: ' + file_path)
                    continue
                except ValueError as e:
                    print(e)
                    print('Invalid hash algorithm specified')
                    return

def main():
    if sys.argv[1:]:
        find_duplicates(sys.argv[1:])
    else:
        with open('paths.txt', 'r') as paths_list:
            paths_list = paths_list.read().split('\n')
            find_duplicates(paths_list)

if __name__ == "__main__":
    main()


from django.http import HttpResponse
from django.shortcuts import render,redirect
from app import models,func       #导入blog模块
from django.contrib import auth
# 表单
def test(request):
    if request.user.is_authenticated==False:
        return redirect("/app/login")
    utype=func.get_usertype(request)
    if utype=="server":
        sid=func.get_userid(request)
        return redirect("/app/jiacai/"+str(sid))
    mm=func.get_allserver()
    return render(request,'app/gouwu.html',{
        'username':request.user.username,
        'server':mm,
        'num_server':len(mm),
    })

def test2(request,*args,**kwargs):
    sid=kwargs['page']
    if request.method=="POST":
        action=request.POST.get("action")
        if action=='add':
            dname=request.POST.get("dishname")
            pr=request.POST.get("price")
            img=request.POST.get("img")
            models.Dishes.objects.create(serverid=sid,dishname=dname,price=int(pr),img=img)
        else :
            dname=request.POST.get("dishname")
            models.Dishes.objects.filter(dishname=dname).delete()
    mm=func.get_dish(sid)
    return render(request,'app/jiacai.html',{
        'username':request.user.username,
        'dish':mm,
    })

def test3(request,*args,**kwargs):
    url = request.path_info
    url= url.split('/')
    sid=url[-1]
    if request.method=="POST":
        action=request.POST.get("action")
        servername=func.get_servername(sid)
        dname=request.POST.get("dishname")
        pr=request.POST.get("price")
        if action=='add':
            func.addonedish(servername,dname,pr)
        else :
            func.deleteonedish(servername,dname,pr)
    mm=func.get_dish(sid)
    allorder=func.get_ordered()
    sumprice=0
    for item in allorder:
        sumprice+=item['price']*item['num']
    return render(request,'app/diancan.html',{
        'username':request.user.username,
        'dish':mm,
        'order':allorder,
        'sumprice':sumprice,
    })


# -*- coding: utf-8 -*-
"""
Created on Thu Jan  9 22:40:03 2020

@author: wyue
"""

import copy

class Vertex():
    def __init__(self, tail=0,edgescore=[],explored=False):
        self.explored = explored
        self.edgescore = edgescore
        self.tail = tail

def readtxt(filename):
    with open(filename) as f:
        graph = f.readlines()

    for i in range(0,len(graph)):
        graph[i]=graph[i].strip('\n')
        graph[i]= graph[i].split()
        graph[i] = list(map(int,graph[i]))
    return graph

def initvertices(N,graph):
    vertices = []
    for i in range(0,N):
        vertices.append(Vertex(i,[],False))

    vertices[0].explored = True

    for e in graph:
        v1=e[0]-1
        v2=e[1]-1
        cost=e[2]
        vertices[v1].edgescore.append([v2,cost])
        vertices[v2].edgescore.append([v1,cost])
    return vertices

def prim(vertices):
    #X is the min weightes
    W = []
    #UX is the vertices unexplored
    UX = copy.deepcopy(vertices)
    del UX[0]
    vertices[0].explored = True

    #vindex = 0
    while UX:
        minw = 100000000
        mintail = 0
        vUX = 0
        #minhead = 0
        for v in range(len(UX)):
            tail = UX[v].tail
            edges = UX[v].edgescore
            for e in range(len(edges)):
                head = edges[e][0]
                #weight = edges[e][1]
                if vertices[head].explored == True:#This edge cross frontier
                    weight = edges[e][1]
                    if weight < minw:
                        minw = weight
                        mintail = tail
                        vUX = v
                        #minhead = head
        vertices[mintail].explored = True
        W.append(minw)
        del UX[vUX]
    return W

filename = "Data/edges.txt"
graph = readtxt(filename)
#N is number of nodes
N = graph[0][0]
#M is number of edges
M = graph[0][1]
del graph[0]

vertices = initvertices(N,graph)

W = prim(vertices)
print(sum(W))


class Solution:
    def kClosest(self, points, K):
        array = []
        for point in points:
            array.append(point[0] ** 2 + point[1] ** 2)

        idxs = [i[0] for i in sorted(enumerate(array), key=lambda x: x[1])]

        res = []
        for i in range(K):
            res.append(points[idxs[i]])

        return res

s = Solution()
print(s.kClosest([[1,3],[-2,2]], K = 1))

Interview Questions/Closests K points to Origin.py


import os.path as osp
import os
import numpy as np
import torch
from torch.utils.data import Dataset
from torchvision import transforms
from torch.utils.data import DataLoader
from bindsnet.datasets import MNIST
from bindsnet.encoding import PoissonEncoder
from PIL import Image
import random

random.seed(0)

class DatasetLoader(Dataset):
    """The class to load the dataset"""
    def __init__(self,
                path,
                time,
                dt,
                intensity):
        self.path = path
        data = []
        label = []
        classes = os.listdir(path)
        print(classes)
        for class_folder in classes:
            folder_images_path = os.listdir(osp.join(path, class_folder))
            for image_path in folder_images_path:
                data.append(osp.join(path, class_folder, image_path))
                label.append(int(class_folder))

        self.data = data
        self.label = label
        self.encoding = PoissonEncoder(time=time, dt=dt)
        self.transform = transforms.Compose([
            transforms.ToTensor(),
            transforms.Lambda(lambda x: x * intensity)])

    def __len__(self):
        return len(self.data)

    def __getitem__(self, i):
        path, label = self.data[i], self.label[i]
        image = self.transform(Image.open(path))
        return {
            'image': image,
            'encoded_image': self.encoding(image),
            'label': label,
        }

class CategoriesSampler():
    """The class to generate episodic data"""
    def __init__(self, label, n_batch, n_cls, n_per):
        self.n_batch = n_batch
        self.n_cls = n_cls
        self.n_per = n_per + 1

        label = np.array(label)
        self.m_ind = []
        for i in range(max(label) + 1):
            ind = np.argwhere(label == i).reshape(-1)
            ind = torch.from_numpy(ind)
            self.m_ind.append(ind)

    def __len__(self):
        return self.n_batch
    def __iter__(self):
        for i_batch in range(self.n_batch):
            batch = []
            classes = torch.randperm(len(self.m_ind))[:self.n_cls]
            for c in classes:
                l = self.m_ind[c]
                pos = torch.randperm(len(l))[:self.n_per]
                batch.append(l[pos])
            batch = torch.stack(batch).t().reshape(-1)
            yield batch


from django.urls import path

from . import views

urlpatterns = [
    path("", views.welcome_page, name="welcome_page"),
    path("index", views.index, name="index"),
    path("login", views.login_view, name="login"),
    path("logout", views.logout_view, name="logout"),
    path("generar_pedido", views.generar_pedido_view, name="generar_pedido"),
    path("generar_turno", views.generar_turno_view, name="generar_turno"),
    path("editar_paciente", views.editar_paciente_view, name="editar_paciente"),
    path("turnos_med", views.turnos_med_view, name="turnos_med"),
    path("agregar_paciente", views.agregar_paciente_view, name="agregar_paciente"),
    path("turnos_hoy", views.turnos_hoy_view, name="turnos_hoy"),
    path("editar_turnos", views.editar_turnos_view, name="editar_turnos"),
    path("diagnosticar", views.diagnosticar_view, name="diagnosticar"),
    path("pacientes_med", views.pacientes_med_view, name="pacientes_med"),
    path("ver_pedidos", views.ver_pedidos_view, name="ver_pedidos"),
    path("reporte_1", views.reporte_1_view, name="reporte_1"),
    path("reporte_2", views.reporte_2_view, name="reporte_2"),
    path("reporte_3", views.reporte_3_view, name="reporte_3"),
    path("reporte_4", views.reporte_4_view, name="reporte_4"),
    path("taller", views.taller_view, name="taller"),
    path("ver_productos", views.ver_productos_view, name="ver_productos"),
    path("ver_pacientes", views.ver_pacientes_view, name="ver_pacientes"),
    path("nuevo_producto", views.nuevo_producto_view, name="nuevo_producto")
]


the_count = [1, 2, 3, 4, 5]
fruits = ['apples', 'oranges', 'pears', 'apricots']
change = [1, 'pennies', 2, 'dimes', 3, 'quarters']

print(fruits)

# this first kind of for-loop goes through a list
for numb3r in the_count:
    print(f"This is count {numb3r}")

# same as above
for fruit in fruits:
    print(f"A fruit of type: {fruit}")

# also we can go through mixed lists too
# notice how we have to use {} since we don't know what's in it
for i in change:
    print(f"I got", i(range([change-1])))

# we can also build lists; first start with an empty one
elements = []

# then use the range function to do 0 to 5 counts
for x in range(0, 7):
    print(f"Adding {x} to the list.")
    # append is a function that lists Understand
    elements.append(x)

# random:
# You have two options: Use 0 < x < 10 or 1 <= x < 10, which is classic notation
# or use x in range(1, 10).

# now we can print them out too
for x in elements:
    print(f"Element was: {x}")


from collections import defaultdict

class Graph:
    def __init__(self):
        self.graph = defaultdict(list)

    def add_edge(self, u, v):
        self.graph[u].append(v)

    def BFS(self, val):
        queue = list()
        visited = [False]*(max(self.graph) + 1)
        queue.append(val)
        visited[val] = True
        while queue:
            val = queue.pop(0)
            print(val, end = " ")
            for neighbour in self.graph[val]:
                if not visited[neighbour]:
                    queue.append(neighbour)
                    visited[neighbour] = True

g = Graph()
g.add_edge(9, 3)
g.add_edge(4, 2)
g.add_edge(25, 5)
g.add_edge(3, 9)
g.add_edge(9, 4)
g.add_edge(3, 5)
g.add_edge(4, 25)
g.BFS(3)


class Solution:
    def reverse(self, arr, start, end):
        while start <= end:
            arr[start], arr[end] = arr[end], arr[start]
            start += 1
            end -= 1

    def leftRotate(self, arr, k, n):
        # Your code goes here

        if k == 0:
            return

        k = k % n

        self.reverse(arr, 0, k - 1)
        self.reverse(arr, k, n - 1)

        self.reverse(arr, 0, n - 1)

#{
 # Driver Code Starts
if __name__ == '__main__':

    t=int(input())
    for _ in range(0,t):
        l=list(map(int,input().split()))
        n=l[0]
        k=l[1]
        a = list(map(int,input().split()))
        ob = Solution()
        ob.leftRotate(a,k,n)
        print(*a)
# } Driver Code Ends

Quick Left Rotation - GFG/quick-left-rotation.py


import matplotlib.pyplot as plt
import numpy as np
from matplotlib.colors import ListedColormap
from sklearn.datasets import make_circles, make_classification, make_moons
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier

h = .02
m = 200

names = [
    # "Nearest Neighbors",
    "Linear SVM",
    "RBF SVM",
    # "Gaussian Process",
    # "Decision Tree",
    # "Random Forest",
    "Neural Net",
    # "AdaBoost",
    # "Naive Bayes",
    # "QDA"
]

classifiers = [
    # KNeighborsClassifier(3),
    SVC(kernel="linear", C=0.025),
    SVC(gamma=2, C=1),
    # GaussianProcessClassifier(1.0 * RBF(1.0)),
    # DecisionTreeClassifier(max_depth=5),
    # RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
    MLPClassifier(alpha=1, max_iter=1000),
    # AdaBoostClassifier(),
    # GaussianNB(),
    # QuadraticDiscriminantAnalysis()
]

X, y = make_classification(n_samples=m, n_features=2, n_redundant=0, n_informative=2, random_state=1, n_clusters_per_class=1)
rng = np.random.RandomState(2)
X += 2 * rng.uniform(size=X.shape)
linearly_separable = (X, y)

datasets = [
    make_moons(n_samples=m, noise=0.2, random_state=0),
    make_circles(n_samples=m, noise=0.2, factor=0.5, random_state=1),
    linearly_separable
]

figure = plt.figure(figsize=(12, 9))
i = 1

with plt.style.context('Solarize_Light2'):
    for ds_cnt, ds in enumerate(datasets):

        X, y = ds
        X = StandardScaler().fit_transform(X)  # 标准化
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.4, random_state=42)  # 训练集、测试集

        x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
        y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
        xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))

        ax = plt.subplot(len(datasets), len(classifiers) + 1, i)
        if ds_cnt == 0:
            ax.set_title("Input data")
        ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, edgecolors='k')
        ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, alpha=0.6, edgecolors='k')
        ax.set_xlim(xx.min(), xx.max())
        ax.set_ylim(yy.min(), yy.max())
        ax.set_xticks(())
        ax.set_yticks(())
        i += 1

        for name, clf in zip(names, classifiers):
            ax = plt.subplot(len(datasets), len(classifiers) + 1, i)

            clf.fit(X_train, y_train)  # 训练模型
            score = clf.score(X_test, y_test)  # 测试

            if hasattr(clf, "decision_function"):
                Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
            else:
                Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]

            Z = Z.reshape(xx.shape)
            ax.contourf(xx, yy, Z, alpha=.8)

            ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, edgecolors='k')
            ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, edgecolors='k', alpha=0.6)

            ax.set_xlim(xx.min(), xx.max())
            ax.set_ylim(yy.min(), yy.max())
            ax.set_xticks(())
            ax.set_yticks(())
            if ds_cnt == 0:
                ax.set_title(name)
            ax.text(xx.max() - .3, yy.min() + .3, ('%.2f' % score).lstrip('0'), size=15, horizontalalignment='right')
            i += 1

plt.tight_layout()
plt.show()


from flask import Flask,request,render_template
from pymongo import MongoClient

app = Flask("task33")
client = MongoClient("mongodb://127.0.0.1:27017")

@app.route('/task33')
def task33():
	return render_template("index.html")

@app.route('/mgdata', methods=['POST'])
def mgdata():
	if request.method == 'POST':
		var = request.form.get('x')
		data= [ { "learner_name": var } ]
		client[arth][learner].insert(data)
	return var

app.run(debug=True)


# -*- coding:utf-8 -*-
# @Author: Jone Chiang
# @Date  : 2019/6/12 9:03
# @File  : db_utils
import re
import os
import calendar
import sqlite3
from datetime import datetime

cur_year = datetime.now().year
common_year = [-1, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
leap_year = [-1, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]

def get_all_days(s):
    """
    '12月30日~1月1日'
    :param s:
    :return:
    """
    begin, end = s.split('~')
    begin_0, begin_1 = re.findall('\d+', begin)
    end_0, end_1 = re.findall('\d+', end)
    begin_0, begin_1 = int(begin_0), int(begin_1)
    end_0, end_1 = int(end_0), int(end_1)

    # print(begin, end)

    date = []

    if begin_0 == end_0:  # 起止日期在同一个月
        days = list(range(begin_1, end_1+1))
        for day in days:
            day = str(begin_0) + '月' + str(day) + '日'
            date.append(day)

    elif begin_0 > end_0:  # 跨年 begin_0 12月 end_0 1月
        days_of_last_month = list(range(begin_1, begin_1 + leap_year[begin_0] - begin_1 + 1))
        for day in days_of_last_month:
            day = str(begin_0) + '月' + str(day) + '日'
            date.append(day)

        days_of_cur_month = list(range(1, end_0 + 1))
        for day in days_of_cur_month:
            day = str(end_0) + '月' + str(day) + '日'
            date.append(day)

    elif begin_0 < end_0:  # 起止日期不在同一个月
        if calendar.isleap(cur_year):
            days_of_last_month = list(range(begin_1, begin_1 + leap_year[begin_0] - begin_1 + 1))
            for day in days_of_last_month:
                day = str(begin_0) + '月' + str(day) + '日'
                date.append(day)

            days_of_cur_month = list(range(1, end_0 + 1))
            for day in days_of_cur_month:
                day = str(end_0) + '月' + str(day) + '日'
                date.append(day)
        else:
            days_of_last_month = list(range(begin_1, begin_1 + common_year[begin_0] - begin_1 + 1))
            for day in days_of_last_month:
                day = str(begin_0) + '月' + str(day) + '日'
                date.append(day)

            days_of_cur_month = list(range(1, end_1 + 1))
            for day in days_of_cur_month:
                day = str(end_0) + '月' + str(day) + '日'
                date.append(day)
    # print(date)
    return date

def get_all_days_2(s):
    res = re.findall('\d+月\d+日', s)
    print(res)
    return res

class DbUtils:
    def __init__(self, db_path):
        self.db_path = db_path

    def get_con(self):
        self.con = sqlite3.connect(self.db_path)
        self.cur = self.con.cursor()

    def close_db(self):
        if self.cur:
            self.cur.close()
        if self.con:
            self.con.close()

    def create_tab_holiday(self, year):
        """
        新建holiday表
        :return:
        """
        self.get_con()
        holiday_sql = """
            create table holiday_{}(id INTEGER primary key autoincrement, holiday_name varchar(50), holiday_period varchar(200), paid_leave varchar(200), days_off_num integer(2))
        """.format(year)
        self.cur.execute(holiday_sql)
        self.close_db()

    def analyze_holiday_period(self, year):
        """
        分析holiday表中的数据: 放假时间
        :return:
        """
        self.get_con()
        holiday_period_sql = """
            select holiday_period from holiday_{}
        """.format(year)
        self.cur.execute(holiday_period_sql)
        result = []
        results = self.cur.fetchall()
        for res in results:
            res0 = get_all_days(res[0])
            result.extend(res0)
        # print(result)

        path = './db/holiday_period_{}.txt'.format(cur_year)
        if os.path.exists(path):
            os.remove(path)
        for i in result:
            with open('./db/holiday_period_{}.txt'.format(cur_year), 'a+', encoding='utf-8') as f:
                f.write(i+'\n')

        self.close_db()

    def analyze_paid_leave(self, year):
        """
        分析holiday表中的数据: 调休上班日期
        :param year:
        :return:
        """
        self.get_con()
        paid_leave_sql = """
            select paid_leave from holiday_{}
        """.format(year)
        self.cur.execute(paid_leave_sql)
        result = []
        results = self.cur.fetchall()
        for res in results:
            res0 = get_all_days_2(res[0])
            result.extend(res0)
        # print(result)

        path = './db/paid_leave_{}.txt'.format(cur_year)
        if os.path.exists(path):
            os.remove(path)

        for i in result:
            with open('./db/paid_leave_{}.txt'.format(cur_year), 'a+', encoding='utf-8') as f:
                f.write(i+'\n')

        self.close_db()

    def get_holiday(self):
        with open('./db/holiday_period_{}.txt'.format(cur_year), 'r', encoding='utf-8') as f:
            res = f.readlines()
            res = [i.strip() for i in res]

        print(res)
        return res

    def get_workday(self):
        with open('./db/paid_leave_{}.txt'.format(cur_year), 'r', encoding='utf-8') as f:
            res = f.readlines()
            res = [i.strip() for i in res]

        print(res)
        return res

if __name__ == '__main__':
    cur_year = datetime.now().year
    db_util = DbUtils('./db/holiday.db')
    # db_util.create_tab_holiday(cur_year)
    # db_util.analyze_holiday_period(cur_year)
    # db_util.analyze_paid_leave(cur_year)

    # get_all_days('2月27日~3月5日')
    # # get_all_days('5月29日(除夕）~6月2日')

    # get_all_days_2('2月2日（周六）、2月3日（周日）上班')

    db_util.get_workday()


#----------TabSettings-Class----------#

class TabSettings:

    """
        Class: TabSettings
            A data storage class for transferring settings between correspnding tabs of differing

        File: pRigging/src/ui/tabsettings.py

        Contains:

        Imports:
            pymel.core as pm
            pRigging.src.riggingbase as prb
    """

    def __init__(self):

        """
            Method: __init__
                A method called when the class is instanciated, sets up the attributes

            Inputs:
        """

        self.m_baseName = ""
        self.m_tabName = ""
        self.m_limbName = ""
        self.m_doSideSpecify = True
        self.m_sideSpecifier = ""
        self.m_jntList = []
        self.m_numTwistJnts = 3
        self.m_doIK = True
        self.m_ikExt = "IK"
        self.m_doFK = True
        self.m_fkExt = "FK"
        self.m_jntExt = "JNT"
        self.m_ctrlExt = "CTRL"
        self.m_rigElement = 0

#----------END-TabSettings-Class----------#


#互斥锁
from threading import Thread, Lock
import time

g_num = 0

def test1():
    global g_num

    #上锁
    mutex.acquire()
    for i in range(1000000):
        #mutex.acquire()
        g_num+=1
        #mutex.release()

    #开锁
    mutex.release()

    print("-test1-g_num=%d-"%g_num)

def test2():
    global g_num

    #上锁
    mutex.acquire()
    for i in range(1000000):
        #mutex.acquire()
        g_num+=1
        #mutex.release()

    #开锁
    mutex.release()

    print("-test2-g_num=%d-"%g_num)

#创建互斥锁
mutex = Lock()  #默认是开锁状态

p1 = Thread(target=test1)
p1.start()

p2 = Thread(target=test2)
p2.start()


#This is part of our app.
from django.shortcuts import render, redirect, get_object_or_404
from .models import FlashCard, Deck
from django.contrib.auth.decorators import login_required
from .forms import CardForm, DeckForm

# @login_required
def homepage(request):
    # users = request.user.all()
    decks = Deck.objects.all()
    return render(request, "flashcards/home.html", {
        "decks": decks
    })

def flashcard(request, pk):
    deck = Deck.objects.get(pk=pk)
    flashcards = FlashCard.objects.all()

    return render(request, 'flashcards/home.html', {'deck': deck,'flashcards': flashcards, 'pk': pk})

def create_deck(request):
    if request.method == 'POST':
        form = DeckForm(request.POST)
        if form.is_valid():
            deck = form.save()
            return redirect('home')
    else:
        form = DeckForm()
        return render(request,'flashcards/add_deck.html', {'form': form})

def flashcard_new(request):
    if request.method == 'POST':
        form = CardForm(request.POST)
        if form.is_valid():
            flashcard = form.save()
            return redirect('home')
    else:
        form = CardForm()
        return render(request,'flashcards/flashcard_new.html', {'form': form})

def flashcard_edit(request, pk):
    flashcard = get_object_or_404(FlashCard, pk=pk)
    if request.method == 'POST':
        form = CardForm(request.POST, instance=flashcard)
        if form.is_valid():
            form.save()
            return redirect('home')
    else:
        form = CardForm(instance=flashcard)
        return render(request,'flashcards/flashcard_edit.html', {'form': form})

def flashcard_delete(request, pk):
    flashcard = get_object_or_404(FlashCard, pk=pk)
    flashcard.delete()
    return redirect('home')

def deck_detail(request, pk):
    deck = get_object_or_404(Deck, pk=pk)
    return render(request,'flashcards/deck_detail.html', {'deck': deck})

def deck_delete(request, pk):
    deck = get_object_or_404(Deck, pk=pk)
    deck.delete()
    return redirect('home')

# def add(request):
#     from random import randint

#     num_1 = randint(0,10)
#     num_2 = randint(0,10)

#     if request.method == "POST":
#         answer = request.POST['answer']
#         old_num_1 = request.POST['old_num_1']
#         old_num_2 = request.POST['old_num_2']

#         correct_answer = int(old_num_1) + int(old_num_2)
#         if int(answer) == correct_answer:
#             my_answer = "Correct! " + old_num_1 + " + " + old_num_2 + " = " + answer
#             color = "success"
#         else:
#             my_answer =  "Incorrect! " + old_num_1 + " + " + old_num_2 + " is not " + answer + " it is " + str(correct_answer)
#             color = "danger"

#         return render(request, 'add.html', {
#             'answer':answer,
#             'my_answer': my_answer,
#             'num_1':num_1,
#             'num_2':num_2,
#             'color':color,
#             })

#         return render(request, 'add.html', {
#             'num_1':num_1,
#             'num_2':num_2,
#             })

# def subtract(request):
#     return render(request, 'subtract.html', {})

# def multiply(request):
#     return render(request, 'multiply.html', {})

# def divide(request):
#     return render(request, 'divide.html', {})


from .accounts import relations
from .accounts.records import Record
from .images import Image
from .posts import Post
from .misc import abc
from .. import consts
from pymongo import MongoClient

client = MongoClient()
_db, _collection = consts.MONGO['ratings']
ratings_collection = client[_db][_collection]

def _rated(acct, item):
    ''' Return the vote information (or None if no such vote was given) '''
    data = ratings_collection.find_one({'rater': acct.id, 'item': item.id})
    return data

def _remove_vote(acct, item):
    ratings_collection.delete_one({'rater': acct.id, 'item': item.id})

def _save_vote(acct, item, value):
    ratings_collection.insert_one({'rater': acct.id, 'item': item.id, 'value': value})

def _update_vote(acct, item, new):
    ''' Change the vote (well, not exatly "change") '''
    _remove_vote(acct, item)
    _save_vote(acct, item, new)

def up(acct, item):
    if not isinstance(item, abc.Scorable):
        raise TypeError('We expect subclasses of "Scorable" here')

    if not isinstance(acct, Record):
        raise TypeError('We expect instances of "Record"')

    data = _rated(acct, item)

    if not data:
        _save_vote(acct, item, 1)
        item.score += 1
    elif data['value'] == -1:
        _update_vote(acct, item, 1)
        item.score += 2

def down(acct, item):
    if not isinstance(item, abc.Scorable):
        raise TypeError('We expect subclasses of "Scorable" here')

    if not isinstance(acct, Record):
        raise TypeError('We expect instances of "Record"')

    data = _rated(acct, item)

    if not data:
        _save_vote(acct, item, -1)
        item.score -= 1
    elif data['value'] != -1:
        _update_vote(acct, item, -1)
        item.score -= 2

def un(acct, item):
    if not isinstance(item, abc.Scorable):
        raise TypeError('We expect subclasses of "Scorable" here')

    if not isinstance(acct, Record):
        raise TypeError('We expect instances of "Record"')

    data = _rated(acct, item)

    if data:
        _remove_vote(acct, item)
        item.score -= data['value']


d={}
n=int(input())
def find_name(d,s):
    if s in d:
        return inp+"="+str(d.get(inp))
    else:
        return "Not found"
for i in range(n):
    inp=input().split()
    key=inp[0]
    value=int(inp[1])
    d[key]=value
list=[]
while True:
    try:
        inp=input().strip()
        list.append(find_name(d,inp))
    except:
        break
for i in list:
    print(i)


import imp
from transformers_local import BertModel, BertConfig
import torch.nn as nn
import torch.nn.functional as F
from data import KGTokenizer
import torch
import scipy.linalg
import scipy.spatial
from sklearn import metrics
import warnings
warnings.filterwarnings("ignore")
from qa_utils import TextEncoder, MLP
import numpy as np
from qa_utils import *

class KGBert(nn.Module):
    def __init__(self, tokenizer:KGTokenizer, args) -> None:
        super().__init__()

        self.configuration = BertConfig()
        self.configuration.vocab_size = len(tokenizer.token2id) + len(tokenizer.ent2id) + len(tokenizer.rel2id)
        self.configuration.vocab_size_token = len(tokenizer.token2id) # token embedding need ft
        self.configuration.vocab_size_ent = len(tokenizer.token2id) + len(tokenizer.ent2id)
        self.configuration.pad_token_id = tokenizer.token2id['[PAD]']
        self.configuration.hidden_size = args.hidden_size
        self.configuration.num_hidden_layers = args.num_hidden_layers
        self.configuration.intermediate_size = self.configuration.hidden_size*4
        self.configuration.num_attention_heads = args.num_attention_heads
        self.configuration.type_vocab_size = args.token_types # head_entity relation specific tail_entity
        # self.configuration.hidden_dropout_prob = 0.2 # default0.1

        self.encoder = BertModel(self.configuration)
        self.tokenizer = tokenizer
        self.cls0 = CLS_based_calssifier(self.configuration.hidden_size*2, 2)
        self.cls1 = MASK_based_calssifier(self.configuration.hidden_size, len(self.tokenizer.rel2id))
        # self.cls2 = MASK_based_calssifier(self.configuration.hidden_size, len(self.tokenizer.ent2id))
        self.map_mem = nn.Linear(self.configuration.hidden_size, self.configuration.hidden_size)

        # NLL(negative log likelihood) loss
        self.nll = nn.NLLLoss(ignore_index=-1)
        self.crossentrpy = nn.CrossEntropyLoss(ignore_index = -1)

        # MSE loss
        self.msel = nn.MSELoss(reduction='mean')
        self.msel_noreduction = nn.MSELoss(reduction='none')
        self.bcel = nn.BCELoss(reduction='mean')

    def forward(self, input, task_type, label, spe_g_indexs):
        outputs = self.encoder(**input)

        last_hidden_states = outputs.last_hidden_state
        pooling_states = outputs.pooler_output

        indexs = []
        for i in range(3):
            indexs.append(task_type==i)

        loss0, acc0 = self.loss_task0(last_hidden_states[indexs[0]],label[indexs[0]], spe_g_indexs)
        loss1, acc1 = self.loss_task1(last_hidden_states[indexs[1]],label[indexs[1]])
        loss2, acc2 = self.loss_task2(last_hidden_states[indexs[2]],label[indexs[2]])

        return loss0, loss1, loss2, acc0, acc1, acc2

    def loss_task0(self, last_hidden, label, spe_g_indexs):
        if len(label) == 0:
            return torch.tensor(0), torch.tensor(0)

        cls_emb = last_hidden[:,0]

        sep_g_emb = last_hidden[torch.tensor(range(len(spe_g_indexs))),spe_g_indexs]
        output = self.cls0(torch.cat((cls_emb, sep_g_emb), dim = 1))
        # output = self.cls0(cls_emb)
        label = label[:,0]
        loss = self.nll(output, label)

        # import pdb; pdb.set_trace()
        output2 = output.detach()
        label_pre = output2.argmax(dim=1)
        acc = torch.sum(label_pre == label)/len(label)

        return loss, acc

    def loss_task1(self, last_hidden, label):
        label_useful = label[label!=-1]

        if len(label_useful) == 0:
            return torch.tensor(0),torch.tensor(0)

        output = self.cls1(last_hidden)#[4, 97, 11] #label 0-40943

        loss = self.nll(output.transpose(1, 2), label)
        output2 = output.detach()
        label_pre = output2.argmax(dim=2)[label!=-1] #[64, 506]

        acc = torch.sum(label_pre == label_useful)/len(label_useful)

        return loss, acc

    def loss_task2(self, last_hidden, label):

        the_index = (label!=-1)
        the_label = label[the_index]

        if len(the_label) == 0:

            return torch.tensor(0),torch.tensor(0)

        the_hidden = self.map_mem(last_hidden[the_index])

        the_embed = self.encoder.embeddings.word_embeddings(the_label)

        if False:
            the_hidden = F.normalize(the_hidden, p=2, dim=1)
            the_embed = F.normalize(the_embed, p=2, dim=1)
            cos_sim = torch.mm(the_hidden, the_embed.permute(1,0))
            cos_sim = (cos_sim + 1)/ 2
        else:

            cos_sim = torch.mm(the_hidden, the_embed.permute(1,0))
            cos_sim = F.sigmoid(cos_sim)

        label_matrix = (the_label.unsqueeze(1) == the_label.unsqueeze(0)).float()

        pos_index = (label_matrix == 1)
        neg_index = (label_matrix == 0)

        cos_sim_pos = cos_sim[pos_index]
        cos_sim_neg = cos_sim[neg_index]

        loss_pos = self.bcel(cos_sim_pos, label_matrix[pos_index])
        loss_neg = self.bcel(cos_sim_neg, label_matrix[neg_index])

        loss = (loss_pos + loss_neg)/2

        acc_pos = (cos_sim_pos>0.5).sum()/len(cos_sim_pos)
        acc_neg = (cos_sim_neg<0.5).sum()/len(cos_sim_neg)

        acc = (acc_pos + acc_neg)/2

        return loss, acc

class CLS_based_calssifier(nn.Module):
    """
    triple task: degree, 3-class classification model : > < =
    """
    def __init__(self, hidden_size, num_class):
        """
        :param hidden: BERT model output size
        """
        super().__init__()
        self.linear = nn.Linear(hidden_size, num_class)
        self.softmax = nn.LogSoftmax(dim = -1)

    def forward(self, x): # bs * 768
        return self.softmax(self.linear(x))

class MASK_based_calssifier(nn.Module):
    """
    triple task: mask tail entity, total entity size-class classification
    """
    def __init__(self, hidden, num_class):
        """
        :param hidden: BERT model output size
        """
        super().__init__()
        self.linear = nn.Linear(hidden, num_class)
        self.softmax = nn.LogSoftmax(dim = -1)

    def forward(self, x):
        return self.softmax(self.linear(x))

class KGBert_down_tirplecls(KGBert):
    def __init__(self, tokenizer:KGTokenizer, args) -> None:
        super().__init__(tokenizer, args)

        self.triplecls = CLS_based_calssifier(self.configuration.hidden_size * 2, 2)

    def forward(self, input, label, task_index):
        outputs = self.encoder(**input)

        last_hidden_states = outputs.last_hidden_state

        loss, acc = self.loss_task_triplecls(last_hidden_states,label, task_index)
        return loss, acc

    def loss_task_triplecls(self, last_hidden, label, task_index):
        task_emb = last_hidden[torch.tensor(range(len(task_index))), task_index]
        cls_emb = last_hidden[:, 0]
        output = self.triplecls(torch.cat((cls_emb,task_emb), dim=1))
        loss_pos = self.nll(output[label==1], label[label==1])
        loss_neg = self.nll(output[label==0], label[label==0])
        loss = (loss_pos + loss_neg) / 2

        # import pdb; pdb.set_trace()
        label_pre = output.detach().argmax(dim=1)
        acc = metrics.accuracy_score(label.cpu(), label_pre.cpu())

        return loss, acc

    def predict(self, input, label, task_index):
        outputs = self.encoder(**input)

        last_hidden = outputs.last_hidden_state
        task_emb = last_hidden[torch.tensor(range(len(task_index))), task_index]
        cls_emb = last_hidden[:, 0]
        output = self.triplecls(torch.cat((cls_emb,task_emb), dim=1))

        loss = self.nll(output, label)

        label_pre = output.argmax(dim=1)

        return loss, label_pre.cpu().numpy(), label.cpu().numpy()

class KGBert_down_qa(nn.Module):
    def __init__(self, tokenizer: KGTokenizer, args):
        super().__init__()
        self.args = args
        self.text_encoder = TextEncoder(tokenizer.model_name)
        self.trans_encoder = KGBert_down_qa_trans(tokenizer, args)
        self.init_embed = self.trans_encoder.encoder.embeddings.word_embeddings.weight.data
        self.init_embed_roberta = self.text_encoder.module.embeddings.word_embeddings.weight.data

    def loss_task_qa(self, logits, labels):
        logits = logits.unsqueeze(1).reshape(-1, self.trans_encoder.tokenizer.num_choice)

        loss = self.trans_encoder.loss_func(logits, labels)

        return loss

    def qa2feat(self, batch_encoder_data):
        qa_feat_list = []
        for data in batch_encoder_data:
            sent_vecs, all_hidden_states = self.text_encoder(*data, layer_id=-1)
            qa_feat_list.append(sent_vecs)

        qa_feat_list = torch.cat(qa_feat_list)

        return qa_feat_list

    def forward(self, input, label, mask_index, f_index, istrain=True):
        feat = input['features']

        if self.args.train_part=='roberta':
            logits = self.trans_encoder.fc1(self.trans_encoder.dropout_fc(feat))
        else:
            input['features'] = self.trans_encoder.activation(self.trans_encoder.mapping(input['features']))
            outputs = self.trans_encoder.encoder(**input)
            # query
            if True:
                idx_range = torch.arange(f_index.shape[0]).to(f_index)
                feat_output = outputs.last_hidden_state[idx_range, f_index, :]
            # cls
            if False:
                cls_output = outputs.last_hidden_state[:, 0, :]

            if True:# pool as qa gnn

                the_mask = (1-((input['input_ids_ent']!=7).float() + (input['token_type_ids']==4).float())).bool()
                cls_output, pool_attn = self.trans_encoder.pooler_qagnn(feat, outputs.last_hidden_state, the_mask)

            if False:
                idx_range = torch.arange(f_index.shape[0]).to(f_index)
                cls_output = outputs.last_hidden_state[idx_range, mask_index, :]

            if False:
                # import pdb; pdb.set_trace()
                useful_token = (input['input_ids_ft']==1)
                input_mask_expanded = useful_token.unsqueeze(-1).expand(outputs.last_hidden_state.size()).float()
                t = outputs.last_hidden_state * input_mask_expanded
                sum_embeddings = torch.sum(t, 1) # 320 * 768
                sum_mask = useful_token.sum(1) #320
                sum_mask = torch.clamp(sum_mask, min=1e-9)
                cls_output = sum_embeddings/sum_mask.unsqueeze(1)

            concat = self.trans_encoder.dropout_fc(torch.cat((cls_output, feat_output, feat), 1))
            # concat = self.trans_encoder.dropout_fc(torch.cat((cls_output, feat), 1))
            # concat = self.trans_encoder.dropout_fc(cls_output)
            logits = self.trans_encoder.fc(concat)

        if istrain:
            loss = self.loss_task_qa(logits, label)
            return loss
        else:
            return logits

class KGBert_down_zsl_multi_pic(KGBert):
    def __init__(self, tokenizer:KGTokenizer, args) -> None:
        super().__init__(tokenizer, args)
        self.args = args
        self.mapping = nn.Linear(2048, self.configuration.hidden_size)
        self.mapping2 = nn.Linear(self.configuration.hidden_size, self.configuration.hidden_size)

        self.mapping2_token = nn.Linear(self.configuration.hidden_size, self.configuration.hidden_size)
        self.match_bicls = nn.Linear(self.configuration.hidden_size*2, 2)

        self.TokenCLSmapping = nn.Linear(self.configuration.hidden_size*2, tokenizer.attribute_kg.shape[0])

        self.match = self.TokenCLSmapping

        # self.ent_linear = nn.Linear(self.configuration.hidden_size, tokenizer.attribute_kg.shape[0]) # 50
        self.ent_linear = MLP(self.configuration.hidden_size,self.configuration.hidden_size, tokenizer.attribute_kg.shape[0], 0, 0.2, layer_norm=True)
        self.pic_linear = MLP(self.configuration.hidden_size,self.configuration.hidden_size, tokenizer.attribute_kg.shape[0], 0, 0.2, layer_norm=True)

        # self.attribute_layer = nn.Linear(85,50)

        # self.match2token_1sig = nn.Linear(self.configuration.hidden_size*2, 1)
        self.init_embed = self.encoder.embeddings.word_embeddings.weight.data
        self.bcel = nn.BCELoss(reduction='none')

    def forward(self, input, token2fcls, label, task_index, f_index, fcls, unseen_cls= None, seen_cls=None, is_train = True):
        input['features'] = self.mapping(input['features']) # 2048 --> 768

        outputs = self.encoder(**input)
        last_hidden_states = outputs.last_hidden_state

        loss, acc, loss_pos, loss_neg, pos_acc, neg_acc, all_scores = self.loss_task(last_hidden_states, token2fcls, label, task_index, f_index, input['features'], fcls, is_train)
        if is_train:
            return loss, loss_pos, loss_neg, pos_acc.item(), neg_acc.item(), acc.item()

        else:
            # import pdb; pdb.set_trace()
            all_scores = all_scores.reshape(-1,50,self.args.multi_pic) # 8 * 50 * 3
            true_label = fcls.reshape(-1,50,self.args.multi_pic)[:,0,:].cpu().numpy() #
            all_scores[:,seen_cls] -= self.args.lmbd # 8 * 40 * 15
            unseen_pre = unseen_cls[all_scores[:,unseen_cls].argmax(dim=1)].cpu().numpy()
            all_pre = all_scores.argmax(dim=1).cpu().numpy()
            seen_pre = seen_cls[all_scores[:,seen_cls].argmax(dim=1)].cpu().numpy()
            return all_pre.reshape(-1), unseen_pre.reshape(-1), seen_pre.reshape(-1), true_label.reshape(-1)

    def loss_task(self, last_hidden, token2fcls, label, task_index, f_index, pic_features, fcls, is_train):

        task_index = task_index.reshape(-1)
        label = label.reshape(-1)
        f_index = f_index.reshape(-1)
        fcls = fcls.reshape(-1)

        row_index = torch.tensor(range(last_hidden.shape[0])).unsqueeze(-1).repeat(1,self.args.multi_pic).reshape(-1)

        task_emb = last_hidden[row_index,task_index] # 3*8 768
        cls_emb = last_hidden[row_index,0]
        f_emb = last_hidden[row_index,f_index]

        #f_emb = self.mapping2(f_emb) # 768 --> 768
        #task_emb = self.mapping2_token(task_emb)
        #cls_emb = self.mapping2_token(cls_emb)

        # import pdb; pdb.set_trace()
        # f_emb = torch.mm(self.pic_linear(f_emb), self.tokenizer.attribute_kg.to(f_emb))
        # cls_emb = torch.mm(self.ent_linear(cls_emb), self.tokenizer.attribute_kg.to(cls_emb))
        cls_emb = self.ent_linear(cls_emb)
        f_emb = self.pic_linear(f_emb)
        output = torch.mul(cls_emb, f_emb).sum(dim=-1) # 960 = 64 * 15

        all_scores = F.sigmoid(output)

        label = label.float()
        pos_idx = label==1.0
        neg_idx = label==0.0
        neg_scores = all_scores[neg_idx]
        pos_scores = all_scores[pos_idx]
        loss_pos = self.bcel(pos_scores, label[pos_idx]).mean()
        # weight_neg = F.softmax(neg_scores * 2, dim = -1).detach()
        # loss_neg = (self.bcel(neg_scores, label[neg_idx]) * weight_neg).sum()
        loss_neg = self.bcel(neg_scores, label[neg_idx]).mean()

        loss = (loss_pos + loss_neg)/2

        neg_acc = (neg_scores < 0.5).sum()/(neg_idx.sum())
        pos_acc = (pos_scores >= 0.5).sum()/(pos_idx.sum())

        acc = (neg_acc + pos_acc)/2
        return loss, acc, loss_pos, loss_neg, pos_acc, neg_acc, all_scores


from torch import nn

from Modules.PosFusionEmbedding import PosFusionEmbedding
from Modules.TransformerEncoderLayer import TransformerEncoderLayer

class TransformerEncoder(nn.Module):
    def __init__(self, hidden_size, num_heads,
                 layer_preprocess_sequence, layer_postprocess_sequence,
                 dropout=None, scaled=True, ff_size=-1,
                 max_seq_len=-1, ff_activate='relu',
                 pe=None,
                 pe_ss=None, pe_ee=None,
                 use_pytorch_dropout=True, dataset='weibo'):
        super().__init__()

        self.use_pytorch_dropout = use_pytorch_dropout
        self.max_seq_len = max_seq_len
        self.hidden_size = hidden_size
        self.num_heads = num_heads
        self.layer_preprocess_sequence = layer_preprocess_sequence
        self.layer_postprocess_sequence = layer_postprocess_sequence
        self.scaled = scaled
        self.ff_activate = ff_activate
        self.dropout = dropout
        self.ff_size = ff_size

        self.pe = pe
        self.pe_ss = pe_ss
        self.pe_ee = pe_ee
        self.four_pos_fusion_embedding = PosFusionEmbedding(self.pe, self.pe_ss,
                                                            self.pe_ee, self.max_seq_len, self.hidden_size)

        self.transformer_layer = TransformerEncoderLayer(hidden_size, num_heads,
                                                         layer_preprocess_sequence,
                                                         layer_postprocess_sequence,
                                                         dropout, scaled, ff_size,
                                                         max_seq_len=max_seq_len,
                                                         ff_activate=ff_activate,
                                                         use_pytorch_dropout=True,
                                                         dataset=dataset,
                                                         )

    def forward(self, query, key, value, seq_len, lex_num=0, pos_s=None, pos_e=None):
        rel_pos_embedding = self.four_pos_fusion_embedding(pos_s, pos_e)
        # four_pos_fusion_embedding应该就是FLAT论文中Relative Position Encoding of Spans这一部分的实现了
        # 传入pos_s和pos_e，计算出这句话中所有词语的位置关系
        # 什么是span？文中说：Each span corresponds to a character or latent word and its position in the original lattice.
        # 可以理解为，span就是一段话的分词结果中的一个个"词组"。
        # 比如原文：重庆人民医院。这个的分词结果就是：[重，庆，人，民，医，院，重庆，重庆人，人民，民医，医院]，也就是11个span

        output = self.transformer_layer(query, key, value, seq_len, lex_num=lex_num,
                                        rel_pos_embedding=rel_pos_embedding)

        return output


#!/usr/bin/env python
# coding: utf-8

# In[2]:

from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from time import sleep
from os import path
import os
import pandas as pd
from tqdm.notebook import tqdm

chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_experimental_option('excludeSwitches', ['enable-logging'])
chrome_options.add_argument('--log-level=3')

# In[3]:

def coletar_dados_fundamentalista_status_invest():
    URL = 'https://statusinvest.com.br/acoes/busca-avancada'
    output = '/Users/jonathan_vieira/Downloads/statusinvest-busca-avancada.csv'

    if path.exists(output):
        print('... Removendo arquivo existente no diretório: {}'.format(output))
        os.remove(output)

    print('...Fazendo aquisição')
    driver = webdriver.Chrome(ChromeDriverManager().install(), options=chrome_options)
    #driver = webdriver.Chrome(executable_path='/Users/jonathan_vieira/chromedriver')
    driver.get(URL)
    sleep(1)

    print('...Buscando elemento')
    button_buscar =driver.find_element_by_xpath(
        '//div/button[contains(@class,"find")]')

    button_buscar.click()
    sleep(2)

    """
        print('Fechando anúncio')
        button_fechar = driver.find_element_by_class_name('btn-close)
        button_fechar.click()
        sleep(1)
    """

    print('...Baixando dados para o disco...')
    button_download=driver.find_element_by_xpath(
    '//div/a[contains(@class, "btn-download")]')

    button_download.click()
    sleep(2)

    if path.exists(output):
        df=pd.read_csv(output, sep=';', decimal=',', thousands='.')
        print('...Fechando Driver')
        driver.close()
        return df

    else:
        print('Arquivo não encontrado')

# In[4]:

acoes= coletar_dados_fundamentalista_status_invest()

# In[5]:

acoes.columns = [c.strip() for c in acoes.columns]

# In[6]:

filter_fundamentalista = (acoes['ROE']> 10) &                          (acoes['MARG. LIQUIDA'] >10) &                          (acoes['MARGEM EBIT'] >15) &                          (acoes['DY'] >4) &                          (acoes['P/L'] <20) & (acoes['P/L'] > 0) &                         (acoes['DIVIDA LIQUIDA / EBIT'] < 2) &                          (acoes['PEG Ratio'] < 1.5) & (acoes['PEG Ratio'] >0)

# In[7]:

acoes[filter_fundamentalista]

# In[8]:

#usando Request para coletar dados de Fundos imobiliarios

# In[9]:

import requests

# In[10]:

def colentando_fundos_imobiliarios_funds_explorer():
    url='https://www.fundsexplorer.com.br/ranking'
    page = requests.get(url)
    df=pd.read_html(page.content, encoding='utf-8')[0]

    df= df.replace({'%': '', 'R\$':'', '\.':'',',':'.'}, regex=True)

    columns=df.columns

    for c in columns:
        if df[c].dtype==object:
            df[c]=df[c].str.strip()
            try:
                df[c]=df[c].astype(float)
            except Exception as e:
                print('Erro: Coluna:{} -- {}'.format(c,e))
                pass
    df['P/VPA'] =df['P/VPA']/100
    df.fillna(0, inplace=True)

    return df

# In[11]:

fiis=colentando_fundos_imobiliarios_funds_explorer()

# In[12]:

fiis

# In[13]:

def oportunidade_media_setor(df_, setor='Shoppings', label_setor='Setor'):
    media_setor=df_.groupby('Setor').agg(['mean', 'std'])
    df_setor=df_[df_[label_setor].isin([setor])]

    filter_=             (df_setor['QuantidadeAtivos'] > 5) &             (df_setor['Liquidez Diária'] >5000) &             (df_setor['P/VPA'] <1.0) &             (df_setor['VacânciaFísica'] <10) &             (df_setor['DY (12M)Acumulado']>media_setor.loc[setor, ('DY (12M)Acumulado', 'mean')])
    print('média do setor Yield: {}'.format(media_setor.loc[setor, ('DY (12M)Acumulado', 'mean')]))
    print('média do setor p/VPA: {}'.format(media_setor.loc[setor, ('P/VPA', 'mean')]))
    print('média do setor Ativos: {}'.format(media_setor.loc[setor, ('QuantidadeAtivos', 'mean')]))

    return df_setor[filter_]

# In[14]:

fiis['Setor'].unique()

# In[15]:

oportunidade_media_setor(fiis, setor='Shoppings')

# In[16]:

fiis.groupby('Setor').agg(['median', 'mean'])['DY (12M)Acumulado']

# In[17]:

#coletando dados de cotacao usando Yfinance

# In[18]:

# In[19]:

import yfinance as yf

# In[20]:

symbol= 'TAEE11.SA'#IBV=F
ticker=yf.Ticker(symbol)
ticker.history().head()

# In[21]:

#coletando dados de cotacao usando InvestPy

# In[22]:

# In[23]:

import investpy as ip

# In[24]:

symbol='TAEE11'
ip.get_stock_historical_data(stock=symbol,
                            from_date='01/07/2021',
                            to_date='01/08/2021',
                            country='brazil').head()

# In[25]:

#obtendo as maiores altas e maiores baixas

# In[26]:

overview= ip.get_stocks_overview(country='Brazil')

# In[27]:

overview['change_percentage']= pd.to_numeric(overview['change_percentage'].str.replace('%',''))

# In[28]:

overview.sort_values('change_percentage', ascending=True).iloc[:10]

# In[29]:

overview.sort_values('change_percentage', ascending=False).iloc[:10]

# In[30]:

#entendendo a analise técnica

# In[31]:

from pandas_datareader import data

# In[32]:

start_date='2020-01-01'
end_date='2021-08-01'

ibov=yf.Ticker('MGLU3.SA').history(start=start_date, end_date=end_date)

# In[33]:

#calculando rentabilidade

# In[34]:

ibov['pct_change']=ibov['Close'].pct_change()*100

# In[35]:

#calculando medias moveis

# In[36]:

ibov['mv_7d']=ibov['Close'].rolling(7).mean()
ibov['mv_21d']=ibov['Close'].rolling(21).mean()
ibov['mv_50d']=ibov['Close'].rolling(50).mean()
ibov['mv_200d']=ibov['Close'].rolling(200).mean()

# In[37]:

def bandas_de_bolling(data, period=20, std_factor=2):
    data['std']=data['Close'].rolling(period).std()
    data['mean']=data['Close'].rolling(period).mean()
    data['banda_superior']=data['mean']+data['std']* std_factor
    data['banda_inferior']=data['mean']-data['std']* std_factor

# In[38]:

bandas_de_bolling(ibov, period=20, std_factor=2)

# In[39]:

#visualizar dados

# In[40]:

import plotly.graph_objs as go

# In[41]:

def plotCandleStick(df_, name='ticket', showlegend=True, lines=[]):
    trace1 ={
        'x':df_.index,
        'open':df_.Open,
        'close':df_.Close,
        'high':df_.High,
        'low':df_.Low,
        'type':'candlestick',
        'name':name,
        'showlegend':showlegend
    }

    data= [trace1]
    layout=go.Layout()

    fig= go.Figure(data=data, layout=layout)

    if len(lines)>0:
        for c in lines:
            fig.add_trace(
            go.Scatter(x=list(df_.index),
                      y=df_[c],
                      mode='lines',
                      name=c))
    return fig

# In[42]:

plotCandleStick(ibov,
                showlegend=False,
                lines=['mv_7d','mv_21d','mv_50d','mv_200d','banda_superior','banda_inferior','mean'])

# In[43]:

import mplfinance as mpf

# In[44]:

filter_date= '2021-01-01'
bollinger = mpf.make_addplot(ibov.loc[filter_date:,['banda_superior', 'banda_inferior', 'mean']])
mpf.plot(ibov[filter_date:], addplot=bollinger, type='candle', mav=(7, 50), volume=True)

# In[45]:

mpf.plot(ibov['2021-05-01':'2021-08-01'], type='candle')

# In[46]:

mpf.plot(ibov['2021-05-01':'2021-08-01'], type='candle', mav=(7,21), volume=True)

# In[47]:

#Bot no Telegram

# In[48]:

class Bot():
    def __init__(self, token, chat_id=None):
        self.token=token
        self.chat_id=chat_id
        self.url='https://api.telegram.org/bot{}'.format(token)

    def send_message(self, msg,
                     disable_web_page_preview=False,
                     disable_notification=False,
                     parse_mode=['HTML', 'Markdown']):
        try:
            assert self.chat_id is not None, 'ERRO:chat_id is None, please set a chat_id'
            data= {'chat_id':self.chat_id, 'text':msg, 'disable_web_page_preview':disable_web_page_preview,
                  'disable_notification': disable_notification, 'parse_mode':parse_mode[0]}
            url=self.url + '/sendMessage'
            response=requests.post(url, data)
            return response
        except Exception as e:
            print('ERRO send_message:request Não Enviada: {}'.format(url))
            print(e)
            raise e

    def setChatId(self, chat_id):
        self.chat_id=chat_id

# In[49]:

token='1931963301:AAEKxzfDMcsFouIFyA49hjyJfYnucFgadF8'

# In[50]:

JFinancebot= Bot(token=token)
JFinancebot.setChatId("-511572903")

# In[51]:

JFinancebot.send_message(msg="oi")

# In[52]:

#Gerenciando Carteira

# In[53]:

def search_opportunities():
    acoes=coletar_dados_fundamentalista_status_invest()
    acoes.columns=[c.strip() for c in acoes.columns]
    filter_fundamentalista = (acoes['ROE']> 10) &                          (acoes['MARG. LIQUIDA'] >10) &                          (acoes['MARGEM EBIT'] >15) &                          (acoes['DY'] >4) &                          (acoes['P/L'] <20) & (acoes['P/L'] > 0) &                         (acoes['DIVIDA LIQUIDA / EBIT'] < 2) &                          (acoes['PEG Ratio'] < 1.5) & (acoes['PEG Ratio'] >0)
    tickers=acoes[filter_fundamentalista]['TICKER'].unique()
    cotacoes=[]

    for t in tqdm(tickers):
        ticker = yf.Ticker(t+'.SA')
        aux=ticker.history(period='10mo', interval='1h')
        if aux.shape[0] >0:
            aux['ticker']=t
            cotacoes.append(aux)
    df_cotacoes =pd.concat(cotacoes)

    group_cotacoes =df_cotacoes.groupby('ticker')
    df_cotacoes_diario=group_cotacoes.resample('D').mean()
    df_cotacoes_diario.dropna(inplace=True)

    df_cotacoes_diario['ma_7d'] =df_cotacoes_diario['Close'].rolling(7).mean()
    df_cotacoes_diario['ma_21d'] =df_cotacoes_diario['Close'].rolling(21).mean()
    df_cotacoes_diario['ma_50d'] =df_cotacoes_diario['Close'].rolling(50).mean()
    df_cotacoes_diario['ma_200d'] =df_cotacoes_diario['Close'].rolling(200).mean()

    period=20
    std_factor=2
    df_cotacoes_diario['std']=df_cotacoes_diario['Close'].rolling(period).std()
    df_cotacoes_diario['mean']=df_cotacoes_diario['Close'].rolling(period).mean()
    df_cotacoes_diario['banda_superior']=df_cotacoes_diario['mean']+df_cotacoes_diario['std']* std_factor
    df_cotacoes_diario['banda_inferior']=df_cotacoes_diario['mean']-df_cotacoes_diario['std']* std_factor

    last_cotacao=df_cotacoes_diario.groupby('ticker').last()

    filter_oportunidades =(last_cotacao['Close']<last_cotacao['banda_inferior'])
    return last_cotacao[filter_oportunidades]

# In[54]:

opportunities=search_opportunities()

# In[55]:

opportunities.reset_index(inplace=True)

# In[56]:

opportunities

# In[57]:

link= 'https://statusinvest.com.br/acoes/'
msg_opportunities = ["#{}".format(link, row['ticker'].lower(), row['ticker'])for i, row in opportunities.iterrows()]

# In[58]:

msg = '\n'.join(msg_opportunities)

# In[59]:

JFinancebot.send_message('{}'.format(msg))

# In[60]:

#agendar as tarefas

# In[61]:

import schedule

# In[62]:

def search_opportunities():
    print('...Buscando por Oportunidades')
    acoes=coletar_dados_fundamentalista_status_invest()
    acoes.columns=[c.strip() for c in acoes.columns]
    filter_fundamentalista = (acoes['ROE']> 10) &                          (acoes['MARG. LIQUIDA'] >10) &                          (acoes['MARGEM EBIT'] >15) &                          (acoes['DY'] >4) &                          (acoes['P/L'] <20) & (acoes['P/L'] > 0) &                         (acoes['DIVIDA LIQUIDA / EBIT'] < 2) &                          (acoes['PEG Ratio'] < 1.5) & (acoes['PEG Ratio'] >0)
    tickers=acoes[filter_fundamentalista]['TICKER'].unique()
    cotacoes=[]

    for t in tqdm(tickers):
        ticker = yf.Ticker(t+'.SA')
        aux=ticker.history(period='10mo', interval='1h')
        if aux.shape[0] >0:
            aux['ticker']=t
            cotacoes.append(aux)

    df_cotacoes =pd.concat(cotacoes)

    print('...Extraindo médias móveis')

    group_cotacoes =df_cotacoes.groupby('ticker')
    df_cotacoes_diario=group_cotacoes.resample('D').mean()
    df_cotacoes_diario.dropna(inplace=True)

    df_cotacoes_diario['ma_7d'] =df_cotacoes_diario['Close'].rolling(7).mean()
    df_cotacoes_diario['ma_21d'] =df_cotacoes_diario['Close'].rolling(21).mean()
    df_cotacoes_diario['ma_50d'] =df_cotacoes_diario['Close'].rolling(50).mean()
    df_cotacoes_diario['ma_200d'] =df_cotacoes_diario['Close'].rolling(200).mean()

    print('...Criando Pandas de Bollinger')
    period=20
    std_factor=2
    df_cotacoes_diario['std']=df_cotacoes_diario['Close'].rolling(period).std()
    df_cotacoes_diario['mean']=df_cotacoes_diario['Close'].rolling(period).mean()
    df_cotacoes_diario['banda_superior']=df_cotacoes_diario['mean']+df_cotacoes_diario['std']* std_factor
    df_cotacoes_diario['banda_inferior']=df_cotacoes_diario['mean']-df_cotacoes_diario['std']* std_factor

    print('...Filtrando dados baseado na analise técnica')
    last_cotacao=df_cotacoes_diario.groupby('ticker').last()

    filter_oportunidades =(last_cotacao['Close']<last_cotacao['banda_inferior'])
    last_cotacao[filter_oportunidades]
    last_cotacao.reset_index(inplace=True)

    print('... Enviando mensagem no telegram')
    link= 'https://statusinvest.com.br/acoes/'
    msg_opportunities = ["#{}".format(link, row['ticker'].lower(), row['ticker'])for i, row in opportunities.iterrows()]
    msg = '\n'.join(msg_opportunities)
    JFinancebot.send_message('{}'.format(msg))

# In[63]:

import time

# In[64]:

time_to_msgs=['10:00', '11:00', '12:00','13:00', '14:00', '15:00', '16:00', '17:00', '18:00', '19:00', '12:45']

for h in time_to_msgs:
    schedule.every().monday.at(h).do(search_opportunities)
    schedule.every().tuesday.at(h).do(search_opportunities)
    schedule.every().wednesday.at(h).do(search_opportunities)
    schedule.every().thursday.at(h).do(search_opportunities)
    schedule.every().friday.at(h).do(search_opportunities)
    schedule.every().saturday.at(h).do(search_opportunities)
    schedule.every().sunday.at(h).do(search_opportunities)

while True:
    schedule.run_pending()
    time.sleep(10)

# In[ ]:


#!/usr/bin/env python3

# Version 1.0
# Author Alexis Blanchet-Cohen
# Date: 24/02/2014

import argparse
import os
import shutil

parser = argparse.ArgumentParser(description='Make project directory structure.')
parser.add_argument("-n", "--name", help="Name of the project.", required=True)
parser.add_argument("-c", "--config_file", help="Configuration file to be copied to Analysis/scripts/config_project.txt")
args = parser.parse_args()

# Get absolute path to configuration file.
if args.config_file != None:
    config_file = os.path.abspath(args.config_file)

# Create project directories, if they do not exist yet, and change to main project directory.
if not os.path.exists(args.name):
    os.mkdir(args.name)
os.chdir(args.name)

# Create output directories, if they do not exist yet..
if not os.path.exists("FASTQ/untrimmed"):
    os.makedirs("FASTQ/untrimmed")

if not os.path.exists("Analysis/scripts"):
    os.makedirs("Analysis/scripts")

shutil.copy(config_file, "Analysis/scripts/config_project.txt")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
pnlp-4_english-4_modeling_with_random_forest-refactored.py
pnlp-4_text_classification-english_sentiment_analysis-...

This is a refactorized version.
There're some minor changes in the variable names and so on.

Source: Python Natural Language Processing
Advanced machine learning and deep learning techniques for natural language processing
Jalaj Thanaki

파이썬 자연어 처리의 이론과 실제
효율적인 자연어 처리를 위한 머신 러닝과 딥러닝 구현하기
04. 텍스트분류 > 01. 영어 텍스트 분류
pp.146~212

* Dataset
IMDB Movie Review

Bag of Words Meets Bags of Popcorn
https://www.kaggle.com/c/word2vec-nlp-tutorial
"""
import os

import numpy as np
import pandas as pd

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier

#%%##################
# Directory & Files #
#####################
dir_data_in = '../data_in/'
dir_data_out = '../data_out/'

if not os.path.exists( dir_data_out ):
    os.makerdirs( dir_data_out )

file_train_clean_data = 'train_clean.csv'
file_test_clean_data  = 'test_clean.csv'
file_result_output    = 'movie_review-bag_of_words-random_forest-result.csv'

#%%##############
# Load the data #
#################
# The preprocessed data was saved to a csv file
file = dir_data_out + file_train_clean_data
train_data = pd.read_csv( file )
print(f'Read {file}...')

file = dir_data_out + file_test_clean_data
print(f'Reading in {file}...')
test_data  = pd.read_csv( file )

reviews    = list( train_data['review'] )
sentiments = list( train_data['sentiment'] )

y = np.array( sentiments )

random_seed      = 42
test_split_ratio = 0.2

#%%#################################
# Vectorizing with CountVectorizer #
####################################
vectorizer          = CountVectorizer( analyzer = 'word', max_features=5000 )
train_data_features = vectorizer.fit_transform( reviews )  # x = train_data_features

#%%###################
# Training the Model #
######################
# Split training and testing data
train_input, test_input, train_label, test_label = train_test_split( train_data_features, y, test_size=test_split_ratio, random_state=random_seed )
# x_train, x_eval, y_train, y_eval = train_input, eval_input, train_label, eval_label
# train_input, eval_input, train_label, eval_label = train_input, test_input, train_label, test_label

# Modeling with Random Forest
forest = RandomForestClassifier( n_estimators=100 )
forest.fit( train_input, train_label )

#predicted = lgs.predict( x_eval )
accuracy  = forest.score( test_input, test_label )

print(f'accuracy = {accuracy}')
# accuracy = 0.8596

#%%##################
# Testing the Model #
#####################
test_reviews = test_data['review']  # Better to do list( test_data['review'] )?
ids          = test_data['id']  # Better to do list( test_data['id'] )?

test_data_feature_vect = vectorizer.transform( test_reviews )
predictions            = forest.predict( test_data_feature_vect )
print( predictions )

ids = test_data['id']  # Better to do list( test_data['id'] )?

#%%##################
# Saving the Result #
#####################

# From dictionary to dataframe
# TODO: Do I actually need list()?
result    = {'id': list(ids), 'sentiment': list(predictions) }
result_df = pd.DataFrame( result )

file = dir_data_out + file_result_output
result_df.to_csv( file , index=False, quoting=3)
print(f'Saved to {file}...')


import datetime
import unittest

from core.helper import DateTool

class TestHelper(unittest.TestCase):

    def test_datetool(self):
        date1_forms = ['2019. 3. 4', '2019.3.4',  '2019-03-04', '20190304']
        date2_forms = ['2019. 3.14', '2019.3.14', '2019-03-14', '20190314']
        date1_expected = DateTool.to_strfdate(datetime.datetime(2019, 3, 4))
        date2_expected = DateTool.to_strfdate(datetime.datetime(2019, 3, 14))
        for form in date1_forms:
            rv = DateTool.to_strfdate(DateTool.to_datetime(form))
            self.assertEqual(rv, date1_expected)
        for form in date2_forms:
            rv = DateTool.to_strfdate(DateTool.to_datetime(form))
            self.assertEqual(rv, date2_expected)


dict1 = {'Nike': 'Shoes', 'HP': "Computer", 'Mahindra': "Tractor"}
dict1['Mahindra']
dict1['Nike']
dict1['HP']
# adding
dict1['Tupperware'] = 'Bottles'
print(dict1)
# removing
del dict1['Nike']
print(dict1)

dict1.pop('Mahindra')
print(dict1)

# Mapping two lists to a dictionary:
keys = ["a", "b", "c"]
values = [1, 2, 3]
mydict = dict(zip(keys, values))
mydict


import warnings

warnings.filterwarnings("ignore")

import pandas as pd
import numpy as np
from IPython import display
from datetime import datetime, timedelta
import datetime as dt
import pickle

from meta import config
from meta.data_processor import DataProcessor
from main import check_and_make_directories
from meta.data_processors.binance import Binance
from talib.abstract import MACD, RSI, CCI, DX

from sklearn import metrics
from sklearn.metrics import classification_report
from sklearn.model_selection import train_test_split

# from imblearn.over_sampling import SMOTE

from sklearn.preprocessing import MinMaxScaler
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader, WeightedRandomSampler
from IPython.display import display, HTML

import os
from typing import List
from argparse import ArgumentParser
from meta import config
from meta.config_tickers import DOW_30_TICKER
from meta.config import (
    DATA_SAVE_DIR,
    TRAINED_MODEL_DIR,
    TENSORBOARD_LOG_DIR,
    RESULTS_DIR,
    INDICATORS,
    TRAIN_START_DATE,
    TRAIN_END_DATE,
    TEST_START_DATE,
    TEST_END_DATE,
    TRADE_START_DATE,
    TRADE_END_DATE,
    ERL_PARAMS,
    RLlib_PARAMS,
    SAC_PARAMS,
    ALPACA_API_KEY,
    ALPACA_API_SECRET,
    ALPACA_API_BASE_URL,
)

pd.options.display.max_columns = None

print("ALL Modules have been imported!")

# %% md

### Create folders

# %%

import os

"""
use check_and_make_directories() to replace the following
if not os.path.exists("./datasets"):
    os.makedirs("./datasets")
if not os.path.exists("./trained_models"):
    os.makedirs("./trained_models")
if not os.path.exists("./tensorboard_log"):
    os.makedirs("./tensorboard_log")
if not os.path.exists("./results"):
    os.makedirs("./results")
"""

check_and_make_directories(
    [DATA_SAVE_DIR, TRAINED_MODEL_DIR, TENSORBOARD_LOG_DIR, RESULTS_DIR]
)

# %% md

### Download data, cleaning and feature engineering

# %%

ticker_list = ["BTCUSDT"]
TIME_INTERVAL = "1h"
# start_date = '2015-01-01 00:00:00'
# end_date = '2020-01-01 00:00:00'

TRAIN_START_DATE = "2015-01-01"
TRAIN_END_DATE = "2019-08-01"
TRADE_START_DATE = "2019-08-01"
TRADE_END_DATE = "2020-01-03"
technical_indicator_list = [
    "open",
    "high",
    "low",
    "close",
    "volume",
    "macd",
    "macd_signal",
    "macd_hist",
    "rsi",
    "cci",
    "dx",
]

if_vix = False
# download and clean

p = DataProcessor(
    data_source="binance",
    start_date=TRAIN_START_DATE,
    end_date=TRADE_END_DATE,
    time_interval=TIME_INTERVAL,
)
p.download_data(ticker_list=ticker_list)
p.clean_data()
df = p.dataframe
print(f"p.dataframe: {p.dataframe}")

def add_technical_indicator(df, tech_indicator_list):
    # print('Adding self-defined technical indicators is NOT supported yet.')
    # print('Use default: MACD, RSI, CCI, DX.')

    final_df = pd.DataFrame()
    for i in df.tic.unique():
        tic_df = df[df.tic == i].copy()
        tic_df["rsi"] = RSI(tic_df["close"], timeperiod=14)
        tic_df["macd"], tic_df["macd_signal"], tic_df["macd_hist"] = MACD(
            tic_df["close"], fastperiod=12, slowperiod=26, signalperiod=9
        )
        tic_df["cci"] = CCI(
            tic_df["high"], tic_df["low"], tic_df["close"], timeperiod=14
        )
        tic_df["dx"] = DX(tic_df["high"], tic_df["low"], tic_df["close"], timeperiod=14)
        final_df = final_df.append(tic_df)
    return final_df

processed_df = add_technical_indicator(df, technical_indicator_list)
print(f"processed_df: {processed_df.head()}")

# Drop unecessary columns and make time as index
processed_df.index = pd.to_datetime(processed_df.time)
processed_df.drop("time", inplace=True, axis=1)
print(processed_df.tail(20))

# IMPORTANT: Make sure that pd.Timedelta() is according to the time_interval to get the volatility for that time interval
# if TIME_INTERVAL == '5m':
#     Delta = pd.Timedelta(minutes=5)
# elif TIME_INTERVAL == '1h':
#     Delta = pd.Timedelta(hours=1)
# else:
#     raise ValueError('Timeframe not supported yet, please manually add!')

def get_vol(prices, span=100):
    # 1. compute returns of the form p[t]/p[t-1] - 1
    # 1.1 find the timestamps of p[t-1] values
    # price.index as datetime format
    df0 = prices.pct_change()
    # 2. estimate rolling standard deviation
    df0 = df0.ewm(span=span).std()
    return df0

data_ohlcv = processed_df.assign(volatility=get_vol(processed_df.close)).dropna()
print("data_ohlcv: ", data_ohlcv.head())

##Adding Path Dependency: Triple-Barrier Method
###The labeling schema is defined as follows:
# y = 2 : top barrier is hit first
# y = 1 : right barrier is hit first
# y = 0 : bottom barrier is hit first###
t_final = 25

def get_barriers(
    daily_volatility=data_ohlcv["volatility"],
    t_final=25,
    upper_lower_multipliers=[2, 2],
    price=data_ohlcv["close"],
):
    prices = price[daily_volatility.index]
    # create a container
    barriers = pd.DataFrame(
        columns=[
            "days_passed",
            "price",
            "vert_barrier",
            "top_barrier",
            "bottom_barrier",
        ],
        index=daily_volatility.index,
    )
    for day, vol in daily_volatility.iteritems():
        days_passed = len(daily_volatility.loc[daily_volatility.index[0] : day])
        # set the vertical barrier
        if days_passed + t_final < len(daily_volatility.index) and t_final != 0:
            vert_barrier = daily_volatility.index[days_passed + t_final]
        else:
            vert_barrier = np.nan
        # set the top barrier
        if upper_lower_multipliers[0] > 0:
            top_barrier = (
                prices.loc[day] + prices.loc[day] * upper_lower_multipliers[0] * vol
            )
        else:
            # set it to NaNs
            top_barrier = pd.Series(index=prices.index)
        # set the bottom barrier
        if upper_lower_multipliers[1] > 0:
            bottom_barrier = (
                prices.loc[day] - prices.loc[day] * upper_lower_multipliers[1] * vol
            )
        else:
            # set it to NaNs
            bottom_barrier = pd.Series(index=prices.index)

        barriers.loc[
            day,
            [
                "days_passed",
                "price",
                "vert_barrier",
                "top_barrier",
                "bottom_barrier",
            ],
        ] = (
            days_passed,
            prices.loc[day],
            vert_barrier,
            top_barrier,
            bottom_barrier,
        )

    return barriers

barriers = get_barriers()
print("barriers: ", barriers.head())

####Function to get label for the dataset (0, 1, 2)
# 0: hit the stoploss
# 1: hit the time out
# 2: hit the profit take
# The part in this function (commented out), allows for easy conversion to a regression analysis (currently it is classification).
# If one changes the labels to (-1, 0, 1), and change the hit on the vertical barrier to the function stated below.
# That will make hitting the profit take barrier 1, the vertical barrier a range from (-1, 1), and the stoploss barrier -1.
# This is a continuos space then.

def get_labels():

    """
    start: first day of the window
    end:last day of the window
    price_initial: first day stock price
    price_final:last day stock price
    top_barrier: profit taking limit
    bottom_barrier:stop loss limt
    condition_pt:top_barrier touching conditon
    condition_sl:bottom_barrier touching conditon
    """
    barriers["label_barrier"] = None
    for i in range(len(barriers.index)):
        start = barriers.index[i]
        end = barriers.vert_barrier[i]
        if pd.notna(end):
            # assign the initial and final price
            # price_initial = barriers.price[start]
            # price_final = barriers.price[end]
            # assign the top and bottom barriers
            top_barrier = barriers.top_barrier[i]
            bottom_barrier = barriers.bottom_barrier[i]

            # set the profit taking and stop loss conditons
            condition_pt = (barriers.price[start:end] >= top_barrier).any()
            condition_sl = (barriers.price[start:end] <= bottom_barrier).any()

            # assign the labels
            if condition_pt:
                barriers["label_barrier"][i] = 2
            elif condition_sl:
                barriers["label_barrier"][i] = 0
            else:
                # Change to regression analysis by switching labels (-1, 0, 1)
                # and uncommenting the alternative function for vert barrier
                barriers["label_barrier"][i] = 1
                # barriers['label_barrier'][i] = max(
                #           [(price_final - price_initial)/
                #             (top_barrier - price_initial), \
                #             (price_final - price_initial)/ \
                #             (price_initial - bottom_barrier)],\
                #             key=abs)
    return

get_labels()
print("barriers after labeling: ", barriers.head())
# Merge the barriers with the main dataset and drop the last t_final + 1 barriers (as they are too close to the end)

data_ohlcv = data_ohlcv.merge(
    barriers[["vert_barrier", "top_barrier", "bottom_barrier", "label_barrier"]],
    left_on="time",
    right_on="time",
)
data_ohlcv.drop(data_ohlcv.tail(t_final + 1).index, inplace=True)
print("data_ohlcv after labeling: ", data_ohlcv.head())
# Count barrier hits ( 0 = stoploss, 1 = timeout, 2 = profit take)
print(pd.Series(data_ohlcv["label_barrier"]).value_counts())

###Copying the Neural Network function from AI4Finance's ActorPPO agent
# https://github.com/AI4Finance-Foundation/ElegantRL/blob/master/elegantrl/agents/net.py
data_ohlcv = data_ohlcv.drop(
    ["vert_barrier", "top_barrier", "bottom_barrier", "adjusted_close", "tic"],
    axis=1,
)

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()

        self.state_dim = 12  # all the features
        self.mid_dim = 2**10  # net dimension
        self.action_dim = 3  # output (sell/nothing/buy)

        # make a copy of the model in ActorPPO (activation function in forward function)

        # Original initial layers
        self.fc1 = nn.Linear(self.state_dim, self.mid_dim)
        self.relu1 = nn.ReLU()
        self.fc2 = nn.Linear(self.mid_dim, self.mid_dim)

        # Original residual layers
        self.relu2 = nn.ReLU()
        self.fc3 = nn.Linear(self.mid_dim, self.mid_dim)
        self.hw1 = nn.Hardswish()
        self.fc_out = nn.Linear(self.mid_dim, self.action_dim)

    def forward(self, x):
        x = x.float()

        # Original initial layers
        x = self.fc1(x)
        x = self.relu1(x)
        x = self.fc2(x)

        # Original residual layers
        x = self.relu2(x)
        x = self.fc3(x)
        x = self.hw1(x)
        x = self.fc_out(x)
        return x

model_NN1 = Net()
print(model_NN1)

class ClassifierDataset(Dataset):
    def __init__(self, X_data, y_data):
        self.X_data = X_data
        self.y_data = y_data

    def __getitem__(self, index):
        return self.X_data[index], self.y_data[index]

    def __len__(self):
        return len(self.X_data)

# Set constants
batch_size = 16
epochs = 300

# Reinitiating data here
data = data_ohlcv

X = data[
    [
        "open",
        "high",
        "low",
        "close",
        "volume",
        "rsi",
        "macd",
        "macd_signal",
        "macd_hist",
        "cci",
        "dx",
        "volatility",
    ]
].values
y = np.squeeze(data[["label_barrier"]].values).astype(int)

# Split into train+val and test
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, stratify=y, random_state=69
)

# Normalize input
scaler = MinMaxScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

# Convert to numpy arrays
X_train, y_train = np.array(X_train), np.array(y_train)
X_test, y_test = np.array(X_test), np.array(y_test)

# initialize sets and convet them to Pytorch dataloader sets
train_dataset = ClassifierDataset(
    torch.from_numpy(X_train).float(),
    torch.from_numpy(y_train.astype(int)).long(),
)
test_dataset = ClassifierDataset(
    torch.from_numpy(X_test).float(),
    torch.from_numpy(y_test.astype(int)).long(),
)

train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size)

test_loader = DataLoader(dataset=test_dataset, batch_size=1)
# Check GPU
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)

# Set optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model_NN1.parameters(), lr=0.0001)

# Train function
def train(fold, model, device, trainloader, optimizer, epoch):
    model.train()
    correct_train = 0
    correct_this_batch_train = 0
    total_train_loss = 0
    for batch_idx, (data, target) in enumerate(train_loader):
        data, target = data.to(device), target.to(device)
        optimizer.zero_grad()
        output = model(data)
        train_loss = criterion(output, target.flatten())
        train_loss.backward()
        optimizer.step()

        if batch_idx % 100 == 0:
            print(
                "Train Fold/Epoch: {}/{} [{}/{} ({:.0f}%)]\ttrain_loss: {:.6f}".format(
                    fold,
                    epoch,
                    batch_idx * len(data),
                    len(train_loader.dataset),
                    100.0 * batch_idx / len(train_loader),
                    train_loss.item(),
                )
            )

        # Measure accuracy on train set
        total_train_loss += train_loss.item()
        _, y_pred_tags_train = torch.max(output, dim=1)
        correct_this_batch_train = y_pred_tags_train.eq(
            target.flatten().view_as(y_pred_tags_train)
        )
        correct_train += correct_this_batch_train.sum().item()

    return correct_train, train_loss

# Test function
def test(fold, model, device, test_loader, correct_train, train_loss):
    model.eval()
    test_loss = 0
    correct = 0
    with torch.no_grad():
        for data, target in test_loader:
            data, target = data.to(device), target.to(device)
            output = model(data)
            test_loss += criterion(output, target.flatten()).item()  # sum up batch loss

            # Measure accuracy on test set
            _, y_pred_tags = torch.max(output, dim=1)
            correct_this_batch = y_pred_tags.eq(target.flatten().view_as(y_pred_tags))
            correct += correct_this_batch.sum().item()

    test_loss /= len(test_loader.dataset)
    train_loss /= len(train_loader.dataset)

    # Print train accuracy for epoch
    # TODO: still a bug in summed up batch train loss
    print(
        "\nTrain set for fold {}: Average train_loss: {:.4f}, Accuracy: {}/{} ({:.5f}%)".format(
            fold,
            train_loss,
            correct_train,
            len(train_loader.dataset),
            100 * correct_train / len(train_loader.dataset),
        )
    )

    # Print test result for epoch
    print(
        "Test set for fold {}:  Average test_loss:  {:.4f}, Accuracy: {}/{} ({:.5f}%)\n".format(
            fold,
            test_loss,
            correct,
            len(test_loader.dataset),
            100 * correct / len(test_loader.dataset),
        )
    )

model_NN1.to(device)

# State fold (no PurgedKFold build yet, ignore this)
# took about 1hour to train when epochs=300
epochs = 100
fold = 0
for epoch in range(1, epochs + 1):
    correct_train, train_loss = train(
        fold, model_NN1, device, train_loader, optimizer, epoch
    )
    test(fold, model_NN1, device, test_loader, correct_train, train_loss)

# Save model to disk and save in your own files to save you some time

# from google.colab import files

filename = "model_NN1"
out = open(filename, "wb")

with open(filename + ".pkl", "wb") as fid:
    pickle.dump(model_NN1, fid)

# load pickle file
with open(filename + ".pkl", "rb") as fid:
    model_NN1 = pickle.load(fid)

with torch.no_grad():
    # Show accuracy on test set
    model_NN1.eval()

    # predict proba
    y_pred_nn1_proba = model_NN1(torch.from_numpy(X_test).float().to(device))
    y_pred_nn1 = torch.argmax(y_pred_nn1_proba, dim=1)
    y_pred_nn1 = y_pred_nn1.cpu().detach().numpy()

# print predction values
print("labels in prediction:", np.unique(y_pred_nn1), "\n")

# print report
label_names = ["long", "no bet", "short"]
print(classification_report(y_test.astype(int), y_pred_nn1, target_names=label_names))

def perturbation_rank(model, x, y, names):
    errors = []

    X_saved = x
    y = y.flatten()

    with torch.no_grad():
        model.eval()
        for i in range(x.shape[1]):
            # Convert to numpy, shuffle, convert back to tensor, predict
            x = x.detach().numpy()
            np.random.shuffle(x[:, i])
            x = torch.from_numpy(x).float().to(device)
            pred = model(x)

            # log_loss requires (classification target, probabilities)
            pred = pred.cpu().detach().numpy()
            error = metrics.log_loss(y, pred)
            errors.append(error)

            # Reset x to saved tensor matrix
            x = X_saved

    max_error = np.max(errors)
    importance = [e / max_error for e in errors]

    data = {"name": names, "error": errors, "importance": importance}
    result = pd.DataFrame(data, columns=["name", "error", "importance"])
    result.sort_values(by=["importance"], ascending=[0], inplace=True)
    result.reset_index(inplace=True, drop=True)
    return result

names = list(data_ohlcv.columns)
names.remove("label_barrier")
rank = perturbation_rank(
    model_NN1,
    torch.from_numpy(X_test).float(),
    torch.from_numpy(y_test.astype(int)).long(),
    names,
)

print(display(rank))


'''
Variavel1 = []
variavel2 = [['nome', idade] ['nome', idade]]
                0        1       0       1
                indice 0         indice 1

teste = []
teste.append('Gustavo')
teste.append(40)
galera_1 = []
galera_1.append(teste[:])
teste[0] = 'Maria'
teste[1] = 22
galera_1.append(teste)
print(galera_1)
galera_2 = [['Adamos', 29], ['Pedro', 15], ['João', 25], ['Alesson', 19]]
print(galera_2[0])
print(galera_2[0][1])
print('=' * 30)
for p in galera_2:
    print(f'{p[0]} tem {p[1]} anos de idade')
    '''
galera = []
dado = []
totalmai = totalmeno = 0
for c in range (0,3):
    dado.append(str(input('Digite nome: ')))
    dado.append(int(input('Digite idade: ')))
    galera.append(dado[:])
    dado.clear()
print(galera,'\n')

for p in galera:
    if p[1] >= 21:
        print(f'{p[0]} é maior de idade')
        totalmai +=1
    else:
        print(f'{p[0]} é menor de idade')
        totalmeno += 1
print(f'Temos {totalmai} maiores e {totalmeno} menores de idade')


import io
import time
import random
from media.RqMediaManager import RqMediaManager
import tomesd
import torch
from diffusers import DiffusionPipeline
# , utils
from PIL import Image
from media.MediaManager import MediaManager
# from pictures.PictureMaker import PictureMaker

class LocalSdPictureMaker():

    def __init__(self, rq_mgr:RqMediaManager = RqMediaManager()):
        self._rq_mgr = rq_mgr

    def make_picture(self, prompt_dict: dict):
        save_img = False

        prompt, neg_prompt = self.create_prompt(prompt_dict)

        base = DiffusionPipeline.from_pretrained(
            "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16, variant="fp16", use_safetensors=True
        )
        base.enable_model_cpu_offload()
        tomesd.apply_patch(base, ratio=0.5)

        refiner = DiffusionPipeline.from_pretrained(
            "stabilityai/stable-diffusion-xl-refiner-1.0",
            text_encoder_2=base.text_encoder_2,
            vae=base.vae,
            torch_dtype=torch.float16,
            use_safetensors=True,
            variant="fp16",
        )

        refiner.enable_model_cpu_offload()
        n_steps = 25 #default 25
        high_noise_frac = 0.8

        image = base(
            prompt=prompt,
            negative_prompt=neg_prompt,
            num_inference_steps=n_steps,
            denoising_end=high_noise_frac,
            output_type="latent",
            height = 768, #default 768
            width = 768,
        ).images

        image = refiner(
            prompt=prompt,
            num_inference_steps=n_steps,
            denoising_start=high_noise_frac,
            image=image,
        ).images[0]

        with io.BytesIO() as output:
            image.save(output, format="PNG")
            image_bytes = output.getvalue()
        if save_img:
            file_name = time.strftime("%Y%m%d_%H%M%S") + ''.join([str(random.randint(1, 9)) for _ in range(5)]) + ".png"
            image.save(file_name)

        if "rq_id" in prompt_dict:
            rq_id= prompt_dict["rq_id"]
            # return self._rq_mgr.file_write(rq_id, ".png", image_bytes, "image/png")
            img = Image.open(io.BytesIO(image_bytes))
            jpeg_buffer = io.BytesIO()
            img.save(jpeg_buffer, format="JPEG")
            jpeg_image_bytes = jpeg_buffer.getvalue()
            return self._rq_mgr.file_write(rq_id, ".jpg", jpeg_image_bytes, "image/jpeg")

        url = MediaManager().bytes_to_png_url(image_bytes)
        return url

    def create_prompt(self,prompt_dict: dict):
        prompt_types = ["positive","negative","style"]
        prompt_data = dict()
        for prompt_type in prompt_types:
            text_key = f"{prompt_type}_prompt_text"
            if text_key in prompt_dict:
                prompt_data[text_key] = prompt_dict[text_key]

        pos_text = prompt_data.get("positive_prompt_text")
        neg_text = prompt_data.get("negative_prompt_text")
        style_text = prompt_data.get("style_prompt_text")

        if pos_text == None:
            raise Exception("PictureMaker: No Positive Prompt")

        prompt = pos_text if style_text == None else pos_text + " in the style of " + style_text
        neg_prompt = neg_text

        return (prompt,neg_prompt)


from sqlalchemy import Column, create_engine
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import scoped_session, sessionmaker

from pg_bitemporal.sqlalchemy.base import CurrentBase, HistoryBase, to_history_table

engine = create_engine(
    "postgresql://example:example@localhost:5432/example", convert_unicode=True
)
db_session = scoped_session(
    sessionmaker(autocommit=False, autoflush=False, bind=engine)
)

class OrderMixin:
    key_fields_and_equality_operators = [("order_id", "=")]
    order_id = Column(UUID)

    def __repr__(self):
        return f"row: {self.row_id}, order: {self.order_id}, app: {self.app_period}, txn: {self.txn_period}"

class Order(OrderMixin, CurrentBase):
    pass

OrderHistory = to_history_table(current_model_cls=Order, mixin_classes=[OrderMixin])

CurrentBase.metadata.create_all(engine)
HistoryBase.metadata.create_all(engine)