import tkinter as tk
from tkinter import messagebox
import pyodbc

class CrudApp:
    def __init__(self, root):
        self.root = root
        self.root.title("CRUD App")

        self.id_var = tk.StringVar()
        self.nome_var = tk.StringVar()
        self.endereco_var = tk.StringVar()
        self.telefone_var = tk.StringVar()
        self.email_var = tk.StringVar()

        # Frames
        frame_form = tk.Frame(self.root)
        frame_form.pack(pady=20)

        frame_listbox = tk.Frame(self.root)
        frame_listbox.pack()

        # Labels e Entry para o formulário
        tk.Label(frame_form, text="ID:").grid(row=0, column=0, sticky="w")
        tk.Entry(frame_form, textvariable=self.id_var, state="readonly").grid(row=0, column=1)

        tk.Label(frame_form, text="Nome:").grid(row=1, column=0, sticky="w")
        tk.Entry(frame_form, textvariable=self.nome_var).grid(row=1, column=1)

        tk.Label(frame_form, text="Endereço:").grid(row=2, column=0, sticky="w")
        tk.Entry(frame_form, textvariable=self.endereco_var).grid(row=2, column=1)

        tk.Label(frame_form, text="Telefone:").grid(row=3, column=0, sticky="w")
        tk.Entry(frame_form, textvariable=self.telefone_var).grid(row=3, column=1)

        tk.Label(frame_form, text="Email:").grid(row=4, column=0, sticky="w")
        tk.Entry(frame_form, textvariable=self.email_var).grid(row=4, column=1)

        # Botões
        tk.Button(frame_form, text="Novo", command=self.novo_registro).grid(row=5, column=0, pady=10)
        tk.Button(frame_form, text="Salvar", command=self.salvar_registro).grid(row=5, column=1, pady=10)
        tk.Button(frame_form, text="Excluir", command=self.excluir_registro).grid(row=5, column=2, pady=10)

        # Listbox para mostrar os registros
        self.listbox = tk.Listbox(frame_listbox, width=50, height=10)
        self.listbox.pack(pady=20)
        self.listbox.bind("<<ListboxSelect>>", self.preencher_campos)

        # Conectar ao SQL Server
        self.conn = pyodbc.connect('Driver={SQL Server};'
                                   'Server=LAPTOP-5M9EP0AR\SQLEXPRESS;'
                                   'Database=PythonSQL;'
                                   'Trusted_Connection=yes;')

        # Carregar dados na Listbox
        self.carregar_dados()

    def carregar_dados(self):
        cursor = self.conn.cursor()
        cursor.execute("SELECT * FROM TblClientes")
        rows = cursor.fetchall()

        self.listbox.delete(0, tk.END)

        for row in rows:
            self.listbox.insert(tk.END, f"{row.Nome} - {row.Telefone}")

    def preencher_campos(self, event):
        selected_index = self.listbox.curselection()
        if selected_index:
            selected_data = self.listbox.get(selected_index[0])
            # Você precisará implementar a lógica para obter o ID do registro selecionado
            # e buscar os detalhes desse registro no banco de dados.

            # Exemplo:
            # id_selecionado = obter_id_do_registro(selected_data)
            # dados_registro = obter_dados_do_banco(id_selecionado)
            # self.id_var.set(dados_registro['id'])
            # self.nome_var.set(dados_registro['nome'])
            # ... e assim por diante.

    def novo_registro(self):
        self.limpar_campos()

    def salvar_registro(self):
        # Lógica para validar os campos (por exemplo, verificar se todos estão preenchidos)
        if not self.campos_validos():
            messagebox.showerror("Erro", "Todos os campos devem ser preenchidos.")
            return

        # Lógica para inserir ou atualizar no banco de dados
        # ...

        # Após salvar, carregue os dados novamente na Listbox
        self.carregar_dados()

        messagebox.showinfo("Sucesso", "Registro salvo com sucesso.")

    def excluir_registro(self):
        # Verificar se o usuário realmente deseja excluir
        resposta = messagebox.askyesno("Confirmação", "Deseja realmente excluir este registro?")
        if resposta == tk.YES:
            # Lógica para excluir no banco de dados
            # ...

            # Após excluir, carregue os dados novamente na Listbox
            self.carregar_dados()

            messagebox.showinfo("Sucesso", "Registro excluído com sucesso.")

    def campos_validos(self):
        # Lógica para validar campos (por exemplo, verificar se estão preenchidos)
        return bool(self.nome_var.get() and self.endereco_var.get() and
                    self.telefone_var.get() and self.email_var.get())

    def limpar_campos(self):
        self.id_var.set("")
        self.nome_var.set("")
        self.endereco_var.set("")
        self.telefone_var.set("")
        self.email_var.set("")

root = tk.Tk()
app = CrudApp(root)
root.mainloop()

Treinando Python 13-10-2023_Sistema crud com base no sql server/app.py


# CNN image emotion classifier
# argv[0]: cnn_test.py
# argv[1]: test.csv

import sys
import numpy as np

# Settings for reproducibility
import os, random
os.environ['PYTHONHASHSEED'] = '0'
random.seed(12345)
np.random.seed(15)

# Import Keras package
from keras.models import Sequential, load_model
from keras.layers import Dense, Activation
from keras.layers import Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.utils import np_utils, plot_model

def load_train(path):
    # Return training data in np.array
    f = open(path, 'r')
    f.readline()  # drop column name

    x_train=[]
    y_train=[]
    for line in f:
        line = line.strip('\n').split(',')
        y = int(line[0])
        x = list(map(int,line[1].split(' ')))

        y_train.append(y)
        x_train.append(x)

    f.close()
    return np.array(x_train), np.array(y_train)

def load_test(path):
    # Return training data in np.array
    f = open(path, 'r')
    f.readline()  # drop column names

    idx=[]
    x_test=[]
    for line in f:
        line = line.strip('\n').split(',')
        i = line[0]
        x = list(map(int,line[1].split(' ')))

        idx.append(i)
        x_test.append(x)

    f.close()
    return idx, np.array(x_test)

def normalize1D(x_train, x_test):
    # Normalization on all x's
    x_all = np.concatenate((x_train, x_test)).astype(float)
    mu = sum(x_all) / len(x_all)
    sigma = np.std(x_all, axis=0)

    for i in range(len(x_all)):
        x_all[i]=(x_all[i]-mu)/sigma
    return x_all[0:len(x_train)], x_all[len(x_train):]

def hist_eq(arr, number_bins=256):
    # Histogram equalization on single image
    # Get image histogram
    arr = arr.astype(np.float32)
    hist, bins = np.histogram(arr, number_bins, density=True)
    cdf = hist.cumsum() # CDF
    cdf = 255 * cdf / cdf[-1]

    # use linear interpolation of cdf to find new pixel values
    arr_eq = np.interp(arr, bins[:-1], cdf)

    return arr_eq

def reshape_data(x):
    num_data=x.shape[0]
    x_r = x.reshape((num_data,48,48,1))
    return x_r

def main(argv):
    model_path = './model.h5'
    idx, x_test = load_test(argv[1])

    # Reshape Data
    x_test = reshape_data(x_test)

    # Model parameters
    num_data = x_test.shape[0]
    num_classes = 7
    input_shape = (48,48,1)  # 48*48 with single channel

    # Load trained model
    sys.stderr.write('Load trained model...\n')
    model = load_model(model_path)

    y_test = model.predict_classes(x_test, verbose=0)
    print('id,label')
    for i, l in enumerate(y_test):
        print('%s,%d' % (idx[i],l))

if __name__ == '__main__':
    main(sys.argv)


import streamlit as sl
import RPi.GPIO as GPIO
from time import sleep

GPIO.setmode(GPIO.BCM) # GPIO Numbers instead of board numbers

button = 14
GPIO.setup(button, GPIO.OUT) # GPIO Assign mode

def press():
    sl.write('Switching...')
    GPIO.output(button, GPIO.LOW) # switch
    sleep(1)

def main():
    sl.title("USB Swithcer")
    switch = sl.button('Switch')
    while(switch):
        press()
        GPIO.output(button, GPIO.HIGH)
        switch = False
        # print(switch)

if __name__ == '__main__':
    main()


import warnings
warnings.filterwarnings('ignore')
import pandas as pd
from sklearn.model_selection import train_test_split
from xgboost import XGBClassifier
from sklearn.multiclass import OneVsRestClassifier
from sklearn.metrics import cohen_kappa_score

def process_data_for_model():
    data = pd.read_csv("data/processed/data.csv")

    y = data["AdoptionSpeed"]
    data = data.drop(
        columns=["AdoptionSpeed"])

    X_train, X_test, Y_train, Y_test = train_test_split(data, y, test_size=0.2, random_state=1)

    return X_train, X_test, Y_train, Y_test

def train_and_test_models(X_train, X_test, Y_train, Y_test):
    model1 = XGBClassifier(learning_rate=0.02, objective='multi:softmax',
                        silent=True, nthread=1, num_class=5,
                        max_depth=15, gamma=0.5, colsample_bytree=1.0, min_child_weight=5,
                         subsample=0.8, n_estimators=200)
    model = OneVsRestClassifier(model1)
    model = model.fit(X_train, Y_train)

    acc = model.score(X_test, Y_test)
    pred = model.predict(X_test)

    return acc, pred

X_train, X_test, Y_train, Y_test = process_data_for_model()
acc, pred = train_and_test_models(X_train, X_test, Y_train, Y_test)

print(acc)

Y_test = Y_test.values

kappa = cohen_kappa_score(Y_test, pred)
print(kappa)


import csv
from difflib import SequenceMatcher
from itertools import chain

import matplotlib.pyplot as plt
from unidecode import unidecode
import networkx as nx

class EstacaoMetro:
    def __init__(self, id, nome, linha, lat, lon):
        self.id = id
        self.nome = nome
        self.linha = linha
        self.lat = float(lat)
        self.lon = float(lon)

    def get_id(self):
        return self.id

    def get_nome(self):
        return self.nome

    def get_linha(self):
        return self.linha

    def get_lat(self):
        return self.lat

    def get_lon(self):
        return self.lon

    def get_coordenadas(self):
        return self.lat, self.lon

    def __str__(self):
        return f"EstacaoMetro(id={self.id}, nome={self.nome}, linha={self.linha}, lat={self.lat}, lon={self.lon})"

def criar_estacoes(lista_estacoes_file="resources/estacoes.csv", estacoes_coordenadas_file='resources/stops.txt'):
    lista_estacoes = []
    estacoes_coordenadas = []
    final_list = []

    with open(lista_estacoes_file, 'r', encoding='utf-8') as csvfile:
        reader = csv.reader(csvfile, delimiter=';')
        next(reader)
        for row in reader:
            lista_estacoes.append([row[0], row[1]])

    with open(estacoes_coordenadas_file, 'r', encoding='utf-8') as txfile:
        reader = csv.reader(txfile)
        next(reader)
        for row in reader:
            row[1] = row[1].replace('"', '')
            estacoes_coordenadas.append([row[0], row[1], row[3], row[4]])

        for i in range(len(lista_estacoes)):
            for j in range(len(estacoes_coordenadas)):
                if similar(lista_estacoes[i][1], estacoes_coordenadas[j][1]) > 0.8:
                    final_list.append(
                        EstacaoMetro(estacoes_coordenadas[j][0], lista_estacoes[i][1], lista_estacoes[i][0],
                                     estacoes_coordenadas[j][2],
                                     estacoes_coordenadas[j][3]))
                    break

    return final_list

def print_estacoes(estacoes):
    for i in range(len(estacoes)):
        print(estacoes[i])

def similar(a, b):
    return SequenceMatcher(None, normalize(a), normalize(b)).ratio()

def normalize(text):
    return unidecode(text).lower()

def build_station_network(filename='resources/estacoes_edges_resolved.csv'):
    t_map = nx.Graph()

    with open(filename, 'r', encoding='utf-8') as csvfile:
        reader = csv.reader(csvfile, delimiter=',')
        next(reader)
        for row in reader:
            (source, destination, time, color) = (row[0], row[1], row[2], row[3])
            color = color.replace('\n', '')
            t_map.add_edge(source, destination, weight=float(time), color=color)

    return t_map

def draw_network(G, edges, color='black', new_plot=True, title='Metro SP', label=None):
    locations = {}
    nodes = list(chain(*edges))
    for station in criar_estacoes():
        locations[station.get_id()] = station.get_coordenadas()

    if new_plot:
        plt.figure()
        plt.title(title)
    if label is not None:
        nx.draw_networkx_labels(G, pos=locations, labels=label, font_size=6)
    nx.draw_networkx_edges(G, pos=locations, edge_color=color, edgelist=edges)
    nx.draw_networkx_nodes(G, pos=locations, node_color='black',
                           node_size=20, nodelist=nodes)
    plt.show()

def all_paths(G, source, target, cutoff=None):
    paths = []
    for path in nx.all_simple_paths(G, source=source, target=target, cutoff=cutoff):
        paths.append(path)
    return paths

def path_length(G, x):
    length = 0
    for i in range(len(x) - 1):
        length += G[x[i]][x[i + 1]]['weight']
    return length

def shortest_paths(G, cutoff=None):
    shortest_paths_list = []

    for source in G.nodes():
        for target in G.nodes():
            if source != target:
                paths = all_paths(G, source, target, cutoff)
                shortest_path = min(paths, key=lambda x: path_length(G, x))
                shortest_paths_list.append(shortest_path)

    with open('resources/shortest_paths.csv', 'w', encoding='utf-8', newline='') as csvfile:
        writer = csv.writer(csvfile, delimiter=',')
        writer.writerow(['source', 'target', 'path'])
        for path in shortest_paths_list:
            writer.writerow([path[0], path[-1], path])


from itertools import combinations
import tensorflow as tf
from base64 import decodebytes
from io import BytesIO
from skimage.measure import regionprops
from skimage.measure import label as mlabel
from tensorflow.keras.models import load_model
from tensorflow.keras.applications.resnet50 import preprocess_input
from skimage.io import imread
import numpy as np

# import segmentation as seg
import matplotlib.pyplot as plt
from gramex.config import random_string, variables
import geopandas as gpd
from shapely.geometry import Polygon

def img_from_b64(s):
    s = s.split(",")[1]
    bytedata = decodebytes(s.encode())
    return imread(BytesIO(bytedata))

def plot_attention(
    image,
    prediction,
    windows_shape,
    prob_threshold=0.5,
    prob_alpha=0.5,
    cmap=plt.cm.Reds,
    show=False,
):
    prob_max = tf.reduce_max(prediction, axis=1).numpy().reshape(*windows_shape)
    fig, ax = plt.subplots()
    ax.imshow(image, extent=[0, image.shape[1], 0, image.shape[0]])
    if prob_threshold > 0:
        mask = prob_max > prob_threshold
    else:
        mask = prob_max
    ax.imshow(
        mask, alpha=prob_alpha, cmap=cmap, extent=[0, image.shape[1], 0, image.shape[0]]
    )
    ax.set_axis_off()
    if show:
        plt.show()
    return mask

def get_bboxes(mask, threshold=0.7, plot=False):
    bbox = []
    for i in range(mask.shape[-1]):
        rp = regionprops(mlabel(mask[..., i].numpy().astype(int)))
        class_box = []
        for region in rp:
            class_box.append(region.bbox)
            if plot:
                minrow, mincol, maxrow, maxcol = region.bbox
                plt.hlines(minrow, mincol, maxcol)
                plt.hlines(maxrow, mincol, maxcol)
                plt.vlines(mincol, minrow, maxrow)
                plt.vlines(maxcol, minrow, maxrow)
        bbox.append(class_box)
        if plot:
            plt.show()
    return bbox

def compare_intersection(x, y):
    i = x.geometry.intersection(y.geometry).area
    smaller = min([x, y], key=lambda g: g.geometry.area)
    if i >= (smaller.geometry.area / 2):
        return smaller.name
    return False

def drop_overlapping_classes(boxes):
    d = []
    for box, label in zip(boxes, variables["LENC"]):
        for top, left, bottom, right in box:
            d.append(
                {
                    "label": label,
                    "geometry": Polygon(
                        [(left, top), (right, top), (right, bottom), (left, bottom)]
                    ),
                }
            )
    d = gpd.GeoDataFrame(d)
    to_drop = []
    for i, j in combinations(d.index, 2):
        smaller = compare_intersection(d.iloc[i], d.iloc[j])
        if smaller:
            to_drop.append(smaller)
    to_drop = set(to_drop)
    d.drop(to_drop, axis=0, inplace=True)
    dropped = []
    for label in variables['LENC']:
        xdf = d[d['label'] == label]
        bound = xdf.geometry.bounds.values
        dropped.append(np.c_[bound[:, :2][:, ::-1], bound[:, 2:][:, ::-1]])
    return dropped

def get_pre_annotations(x, model, bbox_threshold=0.7, plot=False):
    payload = []
    x.shape = (1,) + x.shape
    x = preprocess_input(x)
    height, width, boxes = model.predict(x, batch_size=32)
    boxes = drop_overlapping_classes(boxes)
    for i, _boxes in enumerate(boxes):
        if i != 0:  # Don't predict bars for now
            if len(_boxes) > 0:
                for rowmin, colmin, rowmax, colmax in _boxes:
                    payload.append(
                        {
                            "value": {
                                "x": colmin / width * 100,
                                "y": rowmin / height * 100,
                                "width": (colmax - colmin) / width * 100,
                                "height": (rowmax - rowmin) / height * 100,
                                "rotation": 0,
                                "rectanglelabels": [variables["LENC"][i]],
                            },
                            "id": random_string(6),
                            "from_name": "tag",
                            "to_name": "img",
                            "type": "rectanglelabels",
                        }
                    )
    return payload

if __name__ == "__main__":
    vgg = load_model("vgg16-validated-five-classes.h5")
    x = imread("/tmp/medistrava.png")
    get_bboxes(x, vgg)


from tkinter import *
from tkinter import ttk
#from tkinter import font

class App:
    def __init__(self):
        self.ventana = Tk()
        self.ventana.title("Colores")
        self.ventana.geometry("600x550")
        self.ventana.resizable(False, False)
        # Variable del color #
        self.letras = {
            10: "A",
            11: "B",
            12: "C",
            13: "D",
            14: "E",
            15: "F"
        }
        self.rgb = { # valores iniciales #
            "rojo": 0,
            "verde": 0,
            "azul": 0
        }

        self.hex = { # valores iniciales #
            "rojo": ["F","F"],
            "verde": ["0","0"],
            "azul": ["0","0"]
        }

        self.colores_niveles = {
            "color_nivel1": 255, #Rojo
            "color_nivel2": 510, #Amarillo
            "color_nivel3": 765, #Verde
            "color_nivel4": 1020, #Azul clarito
            "color_nivel5": 1275, #Azul
            "color_nivel6": 1530, #Morado
            "color_nivel7": 1785 #Rojo final
        }

        self.key_list = list(self.letras.keys())
        self.val_list = list(self.letras.values())

    def calcularRGB(self, valor_escala):
        color_estatico = ["rojo", "verde", "verde", "azul", "azul", "rojo"]
        color_variable = ["verde", "rojo", "azul", "verde", "rojo", "azul"]
        color_pausa = ["azul", "azul", "rojo", "rojo", "verde", "verde", "azul"]
        val_niveles = list(self.colores_niveles.values())

        if valor_escala == self.colores_niveles["color_nivel7"]:
            self.rgb["azul"] = 0
        else:
            for i in range(len(val_niveles)):
                nivel_actual = val_niveles[i]
                try:
                    nivel_superior = val_niveles[i+1]
                except:
                    pass

                if valor_escala >= nivel_actual and valor_escala <= nivel_superior:
                    self.rgb[color_estatico[i]] = 255 #se mantiene el valor del color que no se mueve#
                    self.rgb[color_pausa[i]] = 0

                if valor_escala >= nivel_actual and valor_escala < nivel_superior:
                    if nivel_actual == 510 or nivel_actual == 1020 or nivel_actual == 1530:
                        calculo = valor_escala - nivel_actual
                        calculo -= 255
                        self.rgb[color_variable[i]] = -calculo #Disminuye el color variable
                    else:
                        calculo = valor_escala - nivel_actual
                        self.rgb[color_variable[i]] = calculo #Aumenta el color variable

    def calcularHEX(self):
        key_hex = list(self.hex.keys())
        for i in range(len(key_hex)):
            #print(int(self.hex[key_hex[i]][0]))

            cociente = int((self.rgb[key_hex[i]] / 16))
            residuo = int((self.rgb[key_hex[i]] % 16))

            # EL 0 ES PORQUE ESTÁ PRIMERO EN LA LISTA, ESTAS SON LAS DECENAS #
            if cociente in self.letras:
                self.hex[key_hex[i]][0] = self.letras[cociente]
            else:
                self.hex[key_hex[i]][0] = str(cociente)

            # EL 1 ES PORQUE ESTÁ DESPUÉS EN LA LISTA, ESTAS LAS UNIDADES #
            if residuo in self.letras:
                self.hex[key_hex[i]][1] = self.letras[residuo]
            else:
                self.hex[key_hex[i]][1] = str(residuo)

    def calcular(self, valor_escala):
        valor_escala = int(valor_escala)
        #print(valor_escala)
        self.calcularRGB(valor_escala)
        self.calcularHEX()

        self.c_rgb.delete(0, END)
        self.c_hex.delete(0, END)
        val_rgb = list(self.rgb.values())
        val_hex = list(self.hex.values())
        val_hex = val_hex[0]+val_hex[1]+val_hex[2]
        val_hex = "".join(val_hex)

        self.c_rgb.insert(0, val_rgb)
        self.c_hex.insert(0, val_hex)

        self.frame_paleta_colores.config(bg="#"+val_hex)

    def interfaz(self):
        # FRAME PALETA DEL TONO DE COLOR A ELEGIR CON EL CURSOR #
        self.frame_paleta_colores = Frame(self.ventana)
        self.frame_paleta_colores.config(bg="#000", width=400, height=400)
        self.frame_paleta_colores.place(x=10, y=10)

        # FRAME OPCIONES USUARIO #
        frame_config_usuario = Frame(self.ventana)
        frame_config_usuario.config(bg="#fff", width=580, height=120)
        frame_config_usuario.place(x=10, y=450)

        # FRAME PARA LA BARRA ESCALABLE #
        frame_barra_escalable = Frame(self.ventana)
        frame_barra_escalable.config(width=50, height=400)
        frame_barra_escalable.place(x=480, y=0)

        # Barra escalable #
        # 1,785 = 7 veces 255, suben y bajan los valores #
        self.barra_escalable = Scale(frame_barra_escalable)
        self.barra_escalable.config(from_=255, to=1785, length=400, showvalue=0, resolution=1,command=self.calcular)
        self.barra_escalable.place(x=0, y=0)

        # ETIQUETAS DE TEXTO #
        txt_hex = Label(frame_config_usuario, text="HEX")
        txt_hex.config(bg="#fff", fg="#4B4B4B", font=("Arial", 12, "bold"))
        txt_hex.place(x=10, y=50)

        txt_rgb = Label(frame_config_usuario, text="RGB")
        txt_rgb.config(bg="#fff", fg="#4B4B4B", font=("Arial", 12, "bold"))
        txt_rgb.place(x=200, y=50)

        # CUADROS DE TEXTO #
        self.c_hex = ttk.Entry(frame_config_usuario)
        self.c_hex.place(x=50, y=52)

        self.c_rgb = ttk.Entry(frame_config_usuario)
        self.c_rgb.place(x=240, y=52)

        # EVENTOS Y SUS FUNCIONES #
        def escala_abajo(event):
            valor = self.barra_escalable.get()
            self.barra_escalable.set(valor+1)

        def escala_arriba(event):
            valor = self.barra_escalable.get()
            self.barra_escalable.set(valor-1)

        def hex_manual(evento):
            """ ESTA FUNCIÓN NO ESTÁ BIEN HECHA, SÓLO ERA PARA PROBAR Y USARLA EN UN FUTURO """
            valor = self.c_hex.get()
            barra_escalable.set(valor)

        # PARA BAJAR Y SUBIR LA BARRA DESDE LAS FLECHAS DEL TECLADO #
        self.ventana.bind("<Up>", escala_arriba)
        self.ventana.bind("<Down>", escala_abajo)
        #self.ventana.bind("<Left>", x)
        # PARA PASAR UN CÓDIGO MANUALMENTE #
        self.c_hex.bind("<Return>", hex_manual)

        self.ventana.mainloop()

if __name__ == "__main__":
    App().interfaz()


import requests
from pandas import DataFrame
from bs4 import BeautifulSoup
import re
from datetime import datetime
import os
import urllib.request
import pandas as pd
import numpy as np

def main(): # 제목

    total_list_title = []
    total_list_date = []

    start = 11
    while True:

        if start != 1921 :

            # 삼성
            url = 'https://search.naver.com/search.naver?where=news&sm=tab_pge&query=%EC%82%BC%EC%84%B1%EC%A0%84%EC%9E%90&sort=0&photo=3&field=0&pd=3&ds=2021.02.01&de=2021.02.28&cluster_rank=24&mynews=0&office_type=0&office_section_code=0&news_office_checked=&nso=so:r,p:from20210201to20210228,a:all&start={}'

            headers = {'User-Agent': 'Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57'}

            response = requests.get(url.format(start))

            soup = BeautifulSoup(response.text, 'html.parser')

            # 기사 헤드라인 추출
            title_list = [title['title'] for title in soup.find_all('a', attrs={'class' : 'news_tit'})]

            # 날짜 추출
            dates = [date.get_text() for date in soup.find_all('span', attrs={'class':'info'})]
            date_list = dates[1::2] # 다른 부분까지 같이 추출되기 때문에

            total_list_title.append(title_list)
            total_list_date.append(date_list)

            start += 10
            print('크롤링 중.......')

        else:
            print('크롤링 완료!')

            total_list_date = [element for array in total_list_date for element in array]
            total_list_title = [element for array in total_list_title for element in array]
            print(total_list_date)

            df = pd.DataFrame({'title':total_list_title, 'date':total_list_date}, columns=['title', 'date'])

            # DataFrame으로 변환 후 데이터 전처리
            def clean_text(text): # 특수 문자 제거
                cleaned_text = re.sub('[-=+,#/\?:^$.@*\"*~&%`!\\`|\(\)\[\]\<\>`\'...>]', '', text)
                cleaned_text = re.sub("[^가-힣ㄱ-ㅎㅏ-ㅣ\\s]", " ", text)
                return cleaned_text

            df['title'] = df['title'].dropna(axis=0).apply(lambda x : clean_text(x))
            df['date'] = df['date'].dropna(axis=0)

            df = df.sort_values(by=['date'], axis=0) # 날짜 순으로 정렬

            print(df)

            # 엑셀 파일로 데이터 프레임 저장
            date_time = str(datetime.now())
            date_time = date_time[:date_time.rfind(':')].replace(' ', '_')
            date_time = date_time.replace(':','시') + '분'

            folder_path = os.getcwd()
            xlsx_file_name = '네이버뉴스_2월{}.xlsx'.format(date_time)

            df.to_excel(xlsx_file_name)

            print('엑셀 저장 완료 | 경로 : {}\\{}'.format(folder_path, xlsx_file_name))
            os.startfile(folder_path)

            break

main()


import torch
from torchvision.models import resnet50
from DeformableConv import DeformableConv2d
from plugplay import *
from PEG import *

def testDeformableConv():
    print("start of test LeFF")
    testx = torch.randn(8, 32, 123, 25)
    print("input shape ", testx.shape)
    deformConv = DeformableConv2d(in_channels=32, out_channels=64, kernel_size=1, stride=1, padding=0)
    output = deformConv(testx)
    print("output shape ", output.shape)

def testProjAttention():
    testx = torch.randn(8, 3, 25, 25)
    print("input shape ", testx.shape)
    #s3_proj_kernel = 3 s3_kv_proj_stride = 2, s3_heads = 4, s3_depth = 10,
    projAttention = ProjAttention(dim = 3, proj_kernel = 3, kv_proj_stride = 2, heads = 4, dim_head = 64, dropout = 0.)
    print("proj attention created ")
    output = projAttention(testx)
    print("output shape ", output.shape)

def testLeFF():
    print("start of test LeFF")
    testx = torch.randn(8, 196, 3)
    print("input shape ", testx.shape)
    leFF = LeFF(dim = 3, scale = 4, depth_kernel = 3, h =14,w=14)
    print("leFF created ")
    output = leFF(testx)
    print("output shape ", output.shape)

def testPEG():
    print("start of test PEG")
    testx = torch.randn(8, 400, 256)
    print("input shape ", testx.shape)
    peg = PEG(256)
    output = peg(testx, 20, 20)
    print("output shape ", output.shape)

if __name__ == "__main__":
    print("This is the start of main program")
    testDeformableConv()
    print("=============================================")
    testProjAttention()
    print("=============================================")
    testLeFF()
    print("=============================================")
    testPEG()

    testX = torch.randn(400, 2, 256)  #=> 2, 256, 20,20
    testX = testX.permute(1, 2, 0)   #Now 2,256,400
    B, C, HW = testX.shape
    H = 20
    W = 20
    testX = testX.view(B, C, H, W)
    print("testX shape:", testX.shape)
    testX = testX.view(B, C, -1)
    print("testX shape:", testX.shape)


# Anthony Krivonos
# Oct 29, 2018
# src/algorithms/top_movers_no_day_trades.py

# Global Imports
import numpy as np
import math

# Local Imports
from utility import *
from enums import *
from mathematics import *

from algorithms.__algorithm import *

# Abstract: Algorithm employing a top movers tactic.

class TopMoversNoDayTradesAlgorithm(Algorithm):

    # __init__:Void
    # param query:Query => Query object for API access.
    # param sec_interval:Integer => Time interval in seconds for event handling.
    def __init__(self, query, portfolio, sec_interval = 900, test = False, cash = 0.00):

        # Initialize properties
        self.buy_range = (0.00, 5.00)

        # Call super.__init__
        Algorithm.__init__(self, query, portfolio, sec_interval, name = "Top Movers, No Day Trades", buy_range = self.buy_range, test = test, cash = cash)

        self.perform_buy_sell()

    #
    # Event Functions
    #

    # on_market_will_open:Void
    # param cash:Float => User's buying power.
    # param prices:{String:Float}? => Map of symbols to ask prices.
    # NOTE: Called an hour before the market opens.
    def on_market_will_open(self, cash = None, prices = None):
        Algorithm.on_market_will_open(self, cash, prices)

        self.perform_buy_sell()
        pass

    # on_market_open:Void
    # param cash:Float => User's buying power.
    # param prices:{String:Float}? => Map of symbols to ask prices.
    # NOTE: Called exactly when the market opens.
    def on_market_open(self, cash = None, prices = None):
        Algorithm.on_market_open(self, cash, prices)
        pass

    # while_market_open:Void
    # param cash:Float => User's buying power.
    # param prices:{String:Float}? => Map of symbols to ask prices.
    # NOTE: Called on an interval while market is open.
    def while_market_open(self, cash = None, prices = None):
        Algorithm.while_market_open(self, cash, prices)

        self.perform_buy_sell()
        pass

    # on_market_close:Void
    # param cash:Float => User's buying power.
    # param prices:{String:Float}? => Map of symbols to ask prices.
    # NOTE: Called exactly when the market closes.
    def on_market_close(self, cash = None, prices = None):
        Algorithm.on_market_close(self, cash, prices)

        self.perform_buy_sell()
        pass

    #
    # Algorithm
    #

    # perform_buy_sell:Void
    # NOTE: Algorithm works like this:
    #   - Assign "purchase propensities" depending on the following criteria:
    #   - ROUND 1: Concavity of price graph
    #   - ROUND 2: Most recent rate of change in the price graph
    #   - ROUND 3: Last closing price of the stock
    #   - Sell bottom 2/3 of performers
    #   - Buy top 1/3 of performers by ratio
    def perform_buy_sell(self):

        Algorithm.log(self, "Executing perform_buy_sell:")

        # The percentage of the user's total equity to use for this algorithm
        USER_CASH_PERCENTAGE = 0.6

        # Weight of each round of propensity calculation
        ROUND_1_WEIGHT = 1.1
        ROUND_2_WEIGHT = 1.7
        ROUND_3_WEIGHT = 1.3

        Algorithm.log(self, "Cash percentage: " + str(USER_CASH_PERCENTAGE))
        Algorithm.log(self, "Round 1 weight: " + str(ROUND_1_WEIGHT))
        Algorithm.log(self, "Round 2 weight: " + str(ROUND_2_WEIGHT))
        Algorithm.log(self, "Round 3 weight: " + str(ROUND_3_WEIGHT))

        symbols_to_analyze = []
        symbol_quantity_map = {}

        for quote in self.portfolio.get_quotes():
            symbols_to_analyze.append(quote.symbol)
            symbol_quantity_map[quote.symbol] = quote.count

        for symbol in self.query.get_by_tag(Tag.TOP_MOVERS):
            if symbol not in symbol_quantity_map:
                symbols_to_analyze.append(symbol)
                symbol_quantity_map[symbol] = 0

        # Store symbol count
        symbol_count = len(symbols_to_analyze)

        # Store tuples for symbols to multiple different values

        # symbol: quintuple
        symbol_quintuple_map = {}
        # (symbol, quintuple)
        symbol_quintuples = []

        # Store tuples for polynomial evaluation
        # (symbol, value)
        symbol_first_deriv = []
        symbol_second_deriv = []

        # Propensity to buy each stock
        # symbol: propensity
        symbol_purchase_propensity = {}

        # Loop through symbols
        for symbol in symbols_to_analyze:

            # Default purchase propensity = 0
            symbol_purchase_propensity[symbol] = 0

            # Get historicals over past week
            symbol_historical = self.query.get_history(symbol, Span.TEN_MINUTE, Span.WEEK)

            # Get historical quintuples per symbol
            quintuples = Utility.get_quintuples_from_historicals(symbol_historical)

            # Store the list of quintuples in the first map
            symbol_quintuples.append((symbol, quintuples))
            symbol_quintuple_map[symbol] = quintuples

            # Keep track of x and y values to find the polynomial of best fit
            x = []
            y = []
            for quintuple in quintuples:
                x.append(quintuple[Quintuple.TIME.value])
                y.append(quintuple[Quintuple.CLOSE.value])

            # Store the polynomial (deg 2), first derivative (deg 1), and second derivative (deg 1)
            symbol_poly = Math.poly(x, y, 2)

            # First derivative
            symbol_first_deriv.append((symbol, Math.eval(Math.deriv(symbol_poly, 1), x[-1])))

            # Second derivative
            symbol_second_deriv.append((symbol, Math.eval(Math.deriv(symbol_poly, 2), x[-1])))

        # Store quintuples by open price, ascending
        symbol_quintuples = sorted(symbol_quintuples, key=lambda pair: pair[1][-1][Quintuple.CLOSE.value])

        # Sort lists of derivatives, descending
        symbol_first_deriv = sorted(symbol_first_deriv, key=lambda pair: pair[1], reverse=True)
        symbol_second_deriv = sorted(symbol_second_deriv, key=lambda pair: pair[1], reverse=True)

        # Assign 1st round of purchase propensities, by second derivative
        factor = ROUND_1_WEIGHT ** symbol_count
        for pair in symbol_second_deriv:
            symbol_purchase_propensity[pair[0]] += factor
            factor /= ROUND_1_WEIGHT

        # Assign 2nd round of purchase propensities, by first derivative
        factor = ROUND_2_WEIGHT ** symbol_count
        for pair in symbol_first_deriv:
            symbol_purchase_propensity[pair[0]] += factor
            factor /= ROUND_2_WEIGHT

        # Assign 3rd round of purchase propensities, by open price
        factor = ROUND_3_WEIGHT ** symbol_count
        for pair in symbol_quintuples:
            symbol_purchase_propensity[pair[0]] += factor
            factor /= ROUND_3_WEIGHT

        # Convert the list of propensities into an array of tuples
        symbol_propensity_list = []
        for symbol, propensity in symbol_purchase_propensity.items():
            symbol_propensity_list.append((symbol, propensity))

        # Sort the propensity list by order propensity
        symbol_propensity_list = sorted(symbol_propensity_list, key=lambda pair: pair[1], reverse=True)

        #
        # First Execution - Sell bottom 2/3 performers
        #

        # Count the quotes to sell
        bad_performer_count = round(symbol_count * 2 / 3)
        bad_performer_list = symbol_propensity_list[-1 * bad_performer_count:symbol_count]

        Algorithm.log(self, "Bad performers: " + str(bad_performer_list))

        # Sell each poor performer
        for pair in bad_performer_list:
            symbol = pair[0]
            quantity = symbol_quantity_map[symbol]
            limit = symbol_quintuple_map[symbol][-1][Quintuple.LOW.value]
            if quantity > 0.0:
                did_sell = Algorithm.sell(self, symbol, quantity, limit=limit)

        #
        # Second Execution - Buy top 1/4 performers
        #

        # Count the quotes to buy
        good_performer_count = symbol_count - bad_performer_count
        good_performer_list = symbol_propensity_list[0:good_performer_count]

        Algorithm.log(self, "Good performers: " + str(good_performer_list))

        user_cash = USER_CASH_PERCENTAGE * self.cash

        # Determine quantity of each stock to buy
        # Add a third value to the good_performer tuple, which is the amount we're able to spend on that stock
        total_propensity = 0
        for performer in good_performer_list:
            total_propensity += performer[1]
        for i, performer in enumerate(good_performer_list):
            amount_to_spend = (performer[1] / total_propensity) * user_cash
            good_performer_list[i] = (performer[0], performer[1], amount_to_spend)
            pass

        # Buy each good performer
        for triple in good_performer_list:
            symbol = triple[0]
            quantity = round(triple[2] / symbol_quintuple_map[symbol][-1][Quintuple.HIGH.value])
            limit = symbol_quintuple_map[symbol][-1][Quintuple.LOW.value]
            if quantity > 0.0:
                did_buy = Algorithm.buy(self, symbol, quantity, limit=limit)

        Algorithm.log(self, "Finished run of perform_buy_sell")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jun 15 11:23:24 2018

@author: simonwang
"""

#####################################################################################################################
#   CS 6375.003 - Assignment 3, Neural Network Programming
#   This is a starter code in Python 3.6 for a 2-hidden-layer neural network.
#   You need to have numpy and pandas installed before running this code.
#   Below are the meaning of symbols:
#   train - training dataset - can be a link to a URL or a local file
#         - you can assume the last column will the label column
#   train - test dataset - can be a link to a URL or a local file
#         - you can assume the last column will the label column
#   h1 - number of neurons in the first hidden layer
#   h2 - number of neurons in the second hidden layer
#   X - vector of features for each instance
#   y - output for each instance
#   w01, delta01, X01 - weights, updates and outputs for connection from layer 0 (input) to layer 1 (first hidden)
#   w12, delata12, X12 - weights, updates and outputs for connection from layer 1 (first hidden) to layer 2 (second hidden)
#   w23, delta23, X23 - weights, updates and outputs for connection from layer 2 (second hidden) to layer 3 (output layer)
#
#   You need to complete all TODO marked sections
#   You are free to modify this code in any way you want, but need to mention it in the README file.
#
#####################################################################################################################
from sklearn.metrics import accuracy_score
from sklearn import preprocessing
from sklearn.preprocessing import normalize
import numpy as np
import pandas as pd
import pickle
import sys

ww=[1,2,3,4]
ww[1]=0
ww[2]=0
ww[3]=0

class NeuralNet:
    def __init__(self, train, header = True, h1 = 4, h2 = 2):
        np.random.seed(1)
        # train refers to the training dataset
        # test refers to the testing dataset
        # h1 and h2 represent the number of nodes in 1st and 2nd hidden layers

        #read and mark na
        raw_input = pd.read_csv(train, skipinitialspace=True, index_col=False, na_values=['?'], na_filter='?')
        # TODO: Remember to implement the preprocess method

        #clean data
        train_dataset, test_dataset = self.preprocess(raw_input)

        #train data
        ncols = len(train_dataset.columns)
        nrows = len(train_dataset.index)
        self.X = train_dataset.iloc[:, 0:(ncols -1)].values.reshape(nrows, ncols-1)
        self.y = train_dataset.iloc[:, (ncols-1)].values.reshape(nrows, 1)

        #test data
        ncols1 = len(test_dataset.columns)
        nrows1 = len(test_dataset.index)
        self.X1 = test_dataset.iloc[:, 0:(ncols1 -1)].values.reshape(nrows1, ncols1-1)
        self.y1 = test_dataset.iloc[:, (ncols1-1)].values.reshape(nrows1, 1)

        #One hot encoder for y
        # 1. INSTANTIATE
        enc = preprocessing.OneHotEncoder()
        # 2. FIT
        enc.fit(self.y)
        enc.fit(self.y1)
        # 3. Transform
        self.y = enc.transform(self.y).toarray()
        self.y1 = enc.transform(self.y1).toarray()

        #
        # Find number of input and output layers from the dataset
        #
        #how many attributes x1, x2, ..., xn
        input_layer_size = len(self.X[0])

        #how many class
        if not isinstance(self.y[0], np.ndarray):
            output_layer_size = 1
        else:
            output_layer_size = len(self.y[0])

        # assign random weights to matrices in network
        # number of weights connecting layers = (no. of nodes in previous layer) x (no. of nodes in following layer)
        self.w01 = 2 * np.random.random((input_layer_size, h1)) - 1
        self.X01 = self.X #all the examples
        self.delta01 = np.zeros((input_layer_size, h1))
        self.w12 = 2 * np.random.random((h1, h2)) - 1
        self.X12 = np.zeros((len(self.X), h1))
        self.delta12 = np.zeros((h1, h2))
        self.w23 = 2 * np.random.random((h2, output_layer_size)) - 1
        self.X23 = np.zeros((len(self.X), h2))
        self.delta23 = np.zeros((h2, output_layer_size))
        self.deltaOut = np.zeros((output_layer_size, 1))
    #
    # TODO: I have coded the sigmoid activation function, you need to do the same for tanh and ReLu
    #

    def __activation(self, x, activation):
        if activation == "sigmoid":
            self.__sigmoid(self, x)
        if activation == "tanh":
            self.__tanh(self, x)
        if activation == "Relu":
            self.__Relu(self, x)

    def __activation_derivative(self, x, activation):
        if activation == "sigmoid":
            self.__sigmoid_derivative(self, x)
        if activation == "tanh":
            self.__tanh_derivative(self, x)
        if activation == "Relu":
            self.__Relu_derivative(self, x)

    #
    # Sigmoid
    #

    def __sigmoid(self, x):
        return 1 / (1 + np.exp(-x))

    # derivative of sigmoid function, indicates confidence about existing weight
    def __sigmoid_derivative(self, x):
        return x * (1 - x)

    #
    # tanh
    #
    def __tanh(self, x):
        return (np.exp(x) - np.exp(-x)) / (np.exp(x) + np.exp(-x))

    # derivative of tanh function, indicates confidence about existing weight
    def __tanh_derivative(self, x):
        return 1-np.power(x, 2)

    #
    # ReLu
    #
    def __Relu(self, x):
        return np.maximum(x, 0)

    # derivative of Relu function
    def __Relu_derivative(self, x):
        return 1 * (x > 0)

    #
    # TODO: Write code for pre-processing the dataset, which would include standardization, normalization,
    #   categorical to numerical, etc
    #

    def preprocess(self,df):
        #drop na
        df = df.dropna()
        print(df)
        column_names = df.columns.values
        for i in column_names:
            df = df[~df[i].isin(['?'])]

        #transfer str to int
        column_names = df.columns.values
        for column in column_names:
            col_data = df[column];
            unique = set(col_data) #set data into number
            metadata = dict()
            #key, value
            for i, value in enumerate(unique):
                #store each value's transfer value
                metadata[value] = i
            #modified string to int
            df = df.applymap(lambda s: metadata.get(s) if s in metadata else s)
            #print(df)

        ncols = len(df.columns)
        nrows = len(df.index)
        X = df.iloc[:, 0:(ncols -1)].values.reshape(nrows, ncols-1)
        #print(train_dataset.iloc[:, 0:(ncols -1)].values)
        Y = df.iloc[:, (ncols-1)].values.reshape(nrows, 1)
        normalized_X = normalize(X, axis=0)
        #standarized_X = preprocessing.scale(X)
        train_length=int(0.8*nrows)
        test_length=int(0.2*nrows)
        X_train=normalized_X[:train_length]
        X_test=normalized_X[test_length:]
        y_train=Y[:train_length]
        y_test=Y[test_length:]
        df1=pd.DataFrame(X_train);
        df2=pd.DataFrame(y_train);
        dfTrain=pd.concat([df1, df2], axis=1);
        df3=pd.DataFrame(X_test);
        df4=pd.DataFrame(y_test);
        dfTest=pd.concat([df3, df4], axis=1);
        #print(df)

        return dfTrain, dfTest

    def preprocess2(self,x):
        return x

    # Below is the training function

    def train(self, max_iterations, learning_rate, activation):
        for iteration in range(max_iterations):
            #Forward
            out = self.forward_pass(activation, "") #count the output
            error = 0.5 * np.power((out - self.y), 2) #E=1/2 SUM(o-t)^2

            #Backpropagation
            self.backward_pass(out, activation)
            update_layer2 = learning_rate * self.X23.T.dot(self.deltaOut)
            update_layer1 = learning_rate * self.X12.T.dot(self.delta23)
            update_input = learning_rate * self.X01.T.dot(self.delta12)

            self.w23 += update_layer2
            self.w12 += update_layer1
            self.w01 += update_input

        print("After " + str(max_iterations) + " iterations, the total error is " + str(np.sum(error)))
        print("The final weight vectors are (starting from input to output layers)")
        print(self.w01)
        print(self.w12)
        print(self.w23)

        #save training weight
        #print(self.y)
        ww[0]=self.w01
        ww[1]=self.w12
        ww[2]=self.w23
        #print("out")
        #print(out)
        afile = open(r'ww.pkl', 'wb')
        pickle.dump(ww, afile)
        afile.close()

        #encode the out
        out = self.reconstrct(out)

        #print("size of")
        #print(self.y.shape)
        #print(out.shape)
        #print("encode out")
        #print(out)
        print("Train accuracy")
        print(accuracy_score(self.y, out))
        print()

     #encode the output
    def reconstrct(self, out):
        ncols = out.shape[1]
        nrows = out.shape[0]
        for i in range(nrows):
            max = 0
            idx = 0
            for j in range(ncols):
                if out[i][j] > max:
                    max = out[i][j]
                    idx = j
                out[i][j] = 0

            out[i][idx] = 1

        return out;

    def forward_pass(self, activation, predict):

        #use training weight to test
        if(len(predict) != 0):
            #read file
            print("Reading weight from file:")
            file2 = open(r'ww.pkl', "rb")
            ww = pickle.load(file2)
            print(ww)
            file2.close();
            if activation == "sigmoid" :
                # pass our inputs through our neural network
                in1 = np.dot(self.X, ww[0])
                self.X12 = self.__sigmoid(in1)
                in2 = np.dot(self.X12, ww[1])
                self.X23 = self.__sigmoid(in2)
                in3 = np.dot(self.X23, ww[2])
                out = self.__sigmoid(in3)
            if activation == "tanh" :
                # pass our inputs through our neural network
                in1 = np.dot(self.X, ww[0])
                self.X12 = self.__sigmoid(in1)
                in2 = np.dot(self.X12, ww[1])
                self.X23 = self.__sigmoid(in2)
                in3 = np.dot(self.X23, ww[2])
                out = self.__tanh(in3)
            if activation == "Relu" :
                # pass our inputs through our neural network
                in1 = np.dot(self.X, ww[0])
                self.X12 = self.__sigmoid(in1)
                in2 = np.dot(self.X12, ww[1])
                self.X23 = self.__sigmoid(in2)
                in3 = np.dot(self.X23, ww[2])
                out = self.__Relu(in3)

        else:
            if activation == "sigmoid" :
                # pass our inputs through our neural network
                in1 = np.dot(self.X, self.w01)
                self.X12 = self.__tanh(in1)
                in2 = np.dot(self.X12, self.w12)
                self.X23 = self.__tanh(in2)
                in3 = np.dot(self.X23, self.w23)
                out = self.__sigmoid(in3)
            if activation == "tanh" :
                # pass our inputs through our neural network
                in1 = np.dot(self.X, self.w01 )
                self.X12 = self.__tanh(in1)
                in2 = np.dot(self.X12, self.w12)
                self.X23 = self.__tanh(in2)
                in3 = np.dot(self.X23, self.w23)
                out = self.__tanh(in3)
            if activation == "Relu" :
                # pass our inputs through our neural network
                in1 = np.dot(self.X, self.w01 )
                self.X12 = self.__Relu(in1) #int is array
                in2 = np.dot(self.X12, self.w12)
                self.X23 = self.__Relu(in2)
                in3 = np.dot(self.X23, self.w23)
                out = self.__Relu(in3)

        return out

    def backward_pass(self, out, activation):
        # pass our inputs through our neural network
        self.compute_output_delta(out, activation)
        self.compute_hidden_layer2_delta(activation)
        self.compute_hidden_layer1_delta(activation)

    # TODO: Implement other activation functions
    #__activation
    def compute_output_delta(self, out, activation):
        if activation == "sigmoid":
            delta_output = (self.y - out) * (self.__sigmoid_derivative(out))
        if activation == "tanh":
            delta_output = (self.y - out) * (self.__tanh_derivative(out))
        if activation == "Relu":
            delta_output = (self.y - out) * (self.__Relu_derivative(out))

        self.deltaOut = delta_output

    # TODO: Implement other activation functions
    #__activation_derivative
    def compute_hidden_layer2_delta(self, activation):
        if activation == "sigmoid":
            delta_hidden_layer2 = (self.deltaOut.dot(self.w23.T)) * (self.__sigmoid_derivative(self.X23))
        if activation == "tanh":
            delta_hidden_layer2 = (self.deltaOut.dot(self.w23.T)) * (self.__tanh_derivative(self.X23))
        if activation == "Relu":
            delta_hidden_layer2 = (self.deltaOut.dot(self.w23.T)) * (self.__Relu_derivative(self.X23))

        self.delta23 = delta_hidden_layer2

    # TODO: Implement other activation functions

    def compute_hidden_layer1_delta(self, activation):
        if activation == "sigmoid":
            delta_hidden_layer1 = (self.delta23.dot(self.w12.T)) * (self.__sigmoid_derivative(self.X12))
        if activation == "tanh":
            delta_hidden_layer1 = (self.delta23.dot(self.w12.T)) * (self.__tanh_derivative(self.X12))
        if activation == "Relu":
            delta_hidden_layer1 = (self.delta23.dot(self.w12.T)) * (self.__Relu_derivative(self.X12))

        self.delta12 = delta_hidden_layer1

    # TODO: Implement other activation functions

    def compute_input_layer_delta(self, activation):
        if activation == "sigmoid":
            delta_input_layer = np.multiply(self.__sigmoid_derivative(self.X01), self.delta01.dot(self.w01.T))
        if activation == "tanh":
            delta_input_layer = np.multiply(self.__tanh_derivative(self.X01), self.delta01.dot(self.w01.T))
        if activation == "Relu":
            delta_input_layer = np.multiply(self.__Relu_derivative(self.X01), self.delta01.dot(self.w01.T))

        self.delta01 = delta_input_layer

    # TODO: Implement the predict function for applying the trained model on the  test dataset.
    # You can assume that the test dataset has the same format as the training dataset
    # You have to output the test error from this function

    def predict(self):
        self.X = self.X1

        #Activation
        if act == '1':
            out = self.forward_pass("sigmoid", "1")
        elif act == '2':
            out = self.forward_pass("tanh", "1")
        elif act == '3':
            out = self.forward_pass("Relu", "1")

        #reconstruct
        out = self.reconstrct(out)
        print("Test accuracy")
        print(accuracy_score(self.y1, out))
        return 0

if __name__ == "__main__":
    dataset = sys.argv[1];
    act = sys.argv[2];
    neural_network = NeuralNet(dataset)

    #Activation
    if act == '1':
        print("sigmoid:")
        neural_network.train(max_iterations = 1000, learning_rate = 0.055, activation = "sigmoid")
    elif act == '2':
         print("tanh:")
         neural_network.train(max_iterations = 1000, learning_rate = 0.05, activation = "tanh")
    elif act == '3':
         print("Relu:")
         neural_network.train(max_iterations = 1000, learning_rate = 0.05, activation = "Relu")

    #predict
    testError = neural_network.predict()


import pandas as pd
import numpy as np
from scipy.sparse import csr_matrix, save_npz, load_npz
import sys

# Use python3
if sys.version_info[0]==2:
    sys.exit("Use python3")
file_name = "../data/lastfm-dataset-1K/userid-timestamp-artid-artname-traid-traname.tsv" # whole
file_name = "../data/halfid_20%_train.tsv"
df = pd.read_csv(file_name, sep='\t',header=None, quoting=3,
                 names=['userid', 'timestamp', 'artid', 'artname', 'traid', 'traname'],
                 dtype=object)

sys.version_info
cnt_df = pd.crosstab(df.userid,df.traid) # use traid or traname
del df
print ("release df")

np.save('sp_info_usr.npy', cnt_df.index ) # Save info
np.save('sp_info_tra.npy', cnt_df.columns ) # Save info
print ("save sp_info")

save_npz('sparse_matrix.npz',csr_matrix(cnt_df.values)) # Save sp
print ("save sparse_matrix.npz")


import yaml
import os
from pathlib import Path
from typing import Dict

PROJECT_ROOT = Path(__file__).parent.parent.absolute()

def load_yaml(source: str) -> Dict:
    with open(os.path.join(PROJECT_ROOT, source)) as resource:
        return yaml.safe_load(resource)

class Resources:
    def __init__(self, *paths: str):
        self.data = {}

        for path in paths:
            self.data.update(load_yaml(path))

    def __getitem__(self, key):
        return self.data[key]


import cv2 as cv
import numpy as np
from matplotlib import pyplot as plt

img = cv.imread("C:\\Users\\megan\\Desktop\\Curso Udemy - Visao computacional com Python e OpenCV\\Parte 3\\Imagens\\colorida.jpg")
azul, verde, vermelho = cv.split(img)

plt.hist(azul.ravel(), 256, [0,256])
plt.figure()
plt.hist(verde.ravel(), 256, [0,256])
plt.figure()
plt.hist(vermelho.ravel(), 256,[0,256])
plt.show()


import sys, logging
sys.path.insert(0, '../')
from planet_wars import issue_order
from math import sqrt

def reinforce_defense(state):

    enemy_attacks = [fleet for fleet in state.enemy_fleets() # get list of enemy attacks to our planets
                     if any(fleet.destination_planet == planet.ID for planet in state.my_planets())]

    for attacking_fleet in enemy_attacks: # for each enemy fleet

        target_planet = None # get destination planet of fleet
        for planet in state.my_planets():
            if planet.ID is attacking_fleet.destination_planet:
                target_planet = planet
                break

        if target_planet is not None: # if we get the target_planet successfully

            if not (any(fleet.destination_planet == planet.ID for fleet in state.my_fleets())):

                ship_difference = attacking_fleet.num_ships - (target_planet.num_ships * planet.growth_rate)
                if ship_difference > 0: # if fleet can take the planet
                    for planet in state.my_planets(): # for each planet we own
                        if planet.ID is not attacking_fleet.destination_planet: # check if current planet is not the one being attacked
                            if state.distance(planet.ID, target_planet.ID) <= attacking_fleet.turns_remaining: # check if current planet can reach target planet in time
                                if planet.num_ships > ship_difference:
                                    return issue_order(state, planet.ID, target_planet.ID, ship_difference)

    return False

# ===================================================================================================================

def attack_weakest_enemy_planet(state):

    weakest_planets = [planet for planet in state.enemy_planets()
                        if not any(fleet.destination_planet == planet.ID for fleet in state.my_fleets())]
    weakest_planets.sort(key=lambda p: p.num_ships)

    if not weakest_planets: # check if we have a legal source
        return False

    our_planets = [planet for planet in state.my_planets()
                      if not any(fleet.destination_planet == planet.ID for fleet in state.enemy_fleets())]
    our_planets.sort(key=lambda our_planet: state.distance(weakest_planets[0].ID, our_planet.ID))

    if len(our_planets) == 0:
        return False

    for i in range(len(our_planets)): # for each of our planets, sorted by army
        required_ships = weakest_planets[0].num_ships + \
            state.distance(our_planets[i].ID, weakest_planets[0].ID) * weakest_planets[0].growth_rate + 1
        if our_planets[i].num_ships > required_ships:
            return issue_order(state, our_planets[i].ID, weakest_planets[0].ID, required_ships)

    return False

# ===================================================================================================================

def spread_to_optimal_neutral_planet(state):

    strongest_planet = max(state.my_planets(), key=lambda p: p.num_ships, default=None)

    if not strongest_planet: # check if we have a legal source
        return False

    weakest_planets = [planet for planet in state.neutral_planets() # get and sort weakest planets
                        if not any(fleet.destination_planet == planet.ID for fleet in state.my_fleets())]
    weakest_planets.sort(key=lambda p: p.num_ships)

    closest_planets = [planet for planet in state.neutral_planets()
                        if not any(fleet.destination_planet == planet.ID for fleet in state.my_fleets())]
    closest_planets.sort(key=lambda closest_planet: state.distance(strongest_planet.ID, closest_planet.ID))
    del closest_planets[5:]

    for i in range(len(weakest_planets)): # for each neutral planet, sorted by army
        if not i > int(len(weakest_planets)):
            if weakest_planets[i] in closest_planets:
                required_ships = weakest_planets[i].num_ships + 1
                if strongest_planet.num_ships > required_ships:
                    return issue_order(state, strongest_planet.ID, weakest_planets[i].ID, required_ships)

    return False


"""
environment is 3.3

implementation of a Page class and a BTree class
"""
from bs4 import BeautifulSoup

import pickle
import copy
import re

#  BTree order (M) => M - 1 keys per node
BTORDER = 1000
p = re.compile(r'[^;.,!:\&\-\"\/\/]')   # remove punctuation marks
js = re.compile(r'\bvar |\bfunction\(') # remove javascript

class Entry(object):
    def __init__(self):
        self.key = 'word'     # a word
        self.values = set()   # a set of urls
        self.next = None      # a key-value or key-next pair

class Page(object):
    """
    page: a node in the BTree
    """
    def __init__(self, bottom, pagenum):
        self.bottom = bottom               # True: page is external
        self.keylist = [None] * BTORDER    # stores up to M - 1 key
        self.m = 0                         # number of keys stored on page
        self.name = 'BTreeNode' + str(pagenum) + '.txt'

    def close(self):
        """
        write a page in memory to disk
        """
        with open(self.name, 'wb') as fp:
            pickle.dump(self, fp)

    def addValue(self, key, value):
        for e in self.keylist[:self.m]:
            if e.key == key:
                e.values.add(value)

    def add(self, key, value, nextpage):
        """
        key-value: invoked by an external page
        key-next: invoked by an internal page
        maintains sorted order
        """
        t = Entry()
        t.key = key
        if value:
            t.values.add(value)
        else:
            t.next = nextpage

        # insert t into keylist
        if self.m == 0:
            self.keylist[0] = t
        else:
            self._insert(t)
        self.m += 1

    def _insert(self, entry):
        def binary_search(entry):
            lo = 0
            hi = self.m - 1
            while lo <= hi:
                mid = lo + (hi - lo) // 2
                if entry.key < self.keylist[mid].key:
                    hi = mid - 1
                elif entry.key > self.keylist[mid].key:
                    lo = mid + 1
                else:
                    return mid
            return lo
        i = binary_search(entry)

        # shift to right
        for j in range(self.m, i, -1):
            self.keylist[j] = self.keylist[j - 1]
        # insert here
        self.keylist[i] = entry

    def isExternal(self):
        """
        True if self is external, False otherwise
        """
        return self.bottom

    def contains(self, key):
        """
        True if page isExternal and page contains key
        False otherwise
        """
        for e in self.keylist[:self.m]:
            if key == e.key:
                return True
        return False

    def getNext(self, key):
        """
        returns the root page of the subtree that could contain key
        returns an internal page one level lower in the tree
        """
        for i in range(self.m):
            if i + 1 == self.m or key < self.keylist[i + 1].key:
                pagename = self.keylist[i].next.name
                return openPage(pagename)
        return

    def isFull(self):
        """
        True is page contains M elements
        """
        return self.m == BTORDER

    def split(self, pagenum):
        """
        splits a full page into 2 halves:
        lefthalf: M/2 items (original page)
        righthalf: M/2 items (a newly created page; brought into internal memory)
        returns (to its parent page) a reference to righthalf
        """
        righthalf = Page(self.bottom, pagenum)
        righthalf.keylist[:BTORDER // 2] = \
            copy.deepcopy(self.keylist[BTORDER // 2:])
        righthalf.m = BTORDER // 2

        lefthalf = self
        lefthalf.m = BTORDER // 2
        lefthalf.close()

        return righthalf

    def splitRoot(self, pagenum):
        """
        split root and create a new root page
        """
        righthalf = Page(True, pagenum)
        righthalf.keylist[:BTORDER // 2] = \
            copy.deepcopy(self.keylist[BTORDER // 2:])
        righthalf.m = BTORDER // 2

        lefthalf = self
        lefthalf.m = BTORDER // 2

        # add lefthalf and righthalf pages to new root page

        new_page = Page(False, pagenum + 1)

        tRIGHT = Entry()
        tRIGHT.key = righthalf.keylist[0].key
        tRIGHT.next = righthalf

        tLEFT = Entry()
        tLEFT.key = lefthalf.keylist[0].key
        tLEFT.next = lefthalf

        new_page.keylist[0] = tLEFT
        new_page.keylist[1] = tRIGHT
        new_page.m = 2

        # close lefthalf page, righthalf page
        lefthalf.close()
        righthalf.close()

        return new_page

class BTreeST(object):
    """
    balanced M order B-tree
    underlying data structure for a reverse web index
    each node (page) is stored as a file on disk
    word => file that contains word
    """

    def __init__(self):
        self.root = Page(True, 0)
        self.numpages = 1
        self.N = 0     #  number of words indexed

    def size(self):
        return self.numpages, self.N

    def contains(self, key):
        """
        True if key is in tree
        False otherwise
        """
        def helper_contains(p, key):
            if p.isExternal():
                return p.contains(key)
            else:
                return helper_contains(p.next(key), key)
        return helper_contains(self.root, key)

    def get(self, key):
        """
        if key in ST: return value
        else return None
        """
        def helper_get(page, key):
            """
            return value associated with key
            """
            # page is external
            if page.isExternal():
                for e in page.keylist[:page.m]:
                    if key == e.key:
                        return e.values
                return
            else:
                nextpage = page.getNext(key)
                return helper_get(nextpage, key)
        return helper_get(self.root, key)

    def add(self, r):
        """
        add r to BTree
        r : a response obj from a GET request
        """
        soup = BeautifulSoup(r.text, "lxml")
        list_of_words = []

        for string in soup.stripped_strings:
            if string and not js.search(string):
                words = (word for word in string.split() if p.match(word) if word not in list_of_words)
                for word in words:
                    list_of_words.append(word)
        list_of_words.sort()

        def helper_add(page, word):
            """
            page: stored as a file on disk
            list_of_words: list of keys added to a page
            """
            #  external page
            if page.isExternal():
                if not page.contains(word):
                    page.add(word, r.url, None)
                    self.N += 1  # counts number of words
                else:
                    page.addValue(word, r.url)
                return

            # internal page
            nextpage = page.getNext(word)  # returns a new, open page
            if not nextpage:
                return("file not found")
            helper_add(nextpage, word)
            if nextpage.isFull():
                # numpages: next page number to be used
                new_page = nextpage.split(self.numpages)
                self.numpages += 1  # accounts for new righthalf

                # link to new page
                page.add(new_page.keylist[0].key, value=None, nextpage=new_page)
                new_page.close()
            nextpage.close()
            return

        for word in list_of_words:
            helper_add(self.root, word)
            if self.root.isFull():
                # split root
                self.root = self.splitRoot()
                self.numpages += 2
                print(self.root.m)  # debug

    def splitRoot(self):
        """
        split root and create a new root page
        """
        return self.root.splitRoot(self.numpages)

def openPage(filename):
    """
    bring an external page into internal memory
    """
    with open(filename, "rb") as f:
        page = pickle.load(f)
    return page


#James Current

# magic
def makeDict(f):
#value to be returned
    d = {}
#grab all the lines from the file
    lines = f.readlines()

    for i in lines:
#key and value stored on this line, key at 0, value at 1
        knv = i[:len(i)-1].split(',')

#set them up in our map
        d[knv[0]] = knv[1]

    return d

# grab all the files we need
fd1 = open('firstDict.csv', 'r')
fd2 = open('secondDict.csv', 'r')
f = open('pretendItsAWebSite.csv', 'w')

# recreate the maps stored in each file
d1 = makeDict(fd1)
d2 = makeDict(fd2)

# write each key and the matching values from d1 and d2 to a file
for i in d1:
    f.write(i + ',' + d1[i] + ',' + d2[i] + '\n')

# clean up
fd1.close()
fd2.close()
f.close()


__author__ = 'hafid'

import sys
import  time

class ProgressBar:
    def __init__(self, max):
        self.max = max
        self.maxBar = 50;
        self.timeStart = time.time()
        self.timeElapsed = 0
        self.tempElapsedSeconds = time.time()
        self.start()

    def update(self, progress):
        if progress <=0 or self.max <=0 :
            return

        status = ""
        self.timeElapsed = self.timer(self.timeStart, time.time())
        progress = float(progress) / float(self.max)
        block = int(round(self.max * progress)) / (self.max / self.maxBar)

        elapsedSeconds = time.time() - self.tempElapsedSeconds
        if (elapsedSeconds > 1) or (progress == 1):
            self.tempElapsedSeconds = time.time()

            text = "\rPercent: [{0} {1}] [{2}% {3}]{4}"\
                .format( "#"*int(block),
                 "-" * (self.maxBar - int(block)),
                round(progress*100, 1), self.timeElapsed, status)
            sys.stdout.write(text)
            sys.stdout.flush()
    def end(self):
        self.update(self.max)
        print("")
    def start(self):
        try :
            self.update(0)
        except :
            pass
    def timer(self, start, end):
         hours, rem = divmod(end-start, 3600)
         minutes, seconds = divmod(rem, 60)
         return ("{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds))


import typing
from typing import Any, Optional, Union

import more_itertools as mit
import rich
import rich.table
import wandb
import lib_constant

class WandbLoggingState:
    def __init__(self, any_tokenizer, split, also_print):
        self._any_tokenizer = any_tokenizer
        self._table = wandb.Table(columns=["batch_idx", "ref_inputs", "clean_output", "raw_output"])
        self._table_just_clean_gen = wandb.Table(columns=["clean_output"])
        self._split = split
        self._also_print = also_print

    def log(
            self,
            *,
            batch_idx,
            batch,
            raw_output_tokens,
            clean_output_tokens,
        ):

        raw_output_text   = self._any_tokenizer.batch_decode(
            raw_output_tokens,   skip_special_tokens=False)
        clean_output_text = self._any_tokenizer.batch_decode(
            clean_output_tokens, skip_special_tokens=True)

        if self._also_print:
            self._rich_table = rich.table.Table("batch_idx", "ref_inputs", "clean_output", show_lines=True)

        for clean_text, raw_text, sample in mit.zip_equal(
            clean_output_text, raw_output_text, batch,
        ):
            ref_sp = sample["ref_fs_scratchpad_gen_query_detok_not_skip"]
            end_of_few_shots = ref_sp.rfind("Q:")
            sample_specific = ref_sp[end_of_few_shots:]

            self._table.add_data(
                batch_idx,
                sample_specific,
                clean_text,
                raw_output_text
            )
            self._table_just_clean_gen.add_data(clean_text)

            if self._also_print:
                self._rich_table.add_row(
                    str(batch_idx),
                    rich.markup.escape(sample_specific),
                    rich.markup.escape(clean_text),
                    # rich.markup.escape(raw_text),
                )

        rich.print(self._rich_table)

        try:
            wandb.log({f"{lib_constant.WANDB_NAMESPACE}/{self._split}/batch": self._table})
            wandb.log({f"{lib_constant.WANDB_NAMESPACE}/{self._split}/just_clean_gen": self._table_just_clean_gen})
        except Exception as e:
            rich.print(f"[red bold]{e}")
            rich.print(f"Wandb logging failed. Continuing without logging.")

    __call__ = log


import requests
import json

port = 5005
url = f'http://127.0.01:{port}/'

def get_body():
    return {'input': 'This Is The sTrInG we want in lowEr CAsE.'}

def main():
    body = get_body()
    print('This is the input to our API.')
    print(body)

    # We have to serialize the body of our post request before we can send.
    body_str = json.dumps(body)

    # Don't believe me? Uncomment the line below this one!
    # res = requests.post(url, data=body)

    # This is us passing the serialized body.
    res = requests.post(url, data=body_str)
    print('This is the output from the API.')
    print(res.json())

if __name__ == '__main__':
    main()


import numpy as np
import pandas as pd

import cupy as cp

from .conv import *
from .core import iSeq
from .sig import *
from .wtx import *

def open_csv(filepath, n_null=100, random_seed=None):
    np.random.seed(random_seed)
    x = pd.read_csv(filepath, header=None).values[:, :-100]
    t, n_cell = x.shape
    x /= x.sum(axis=0)
    null = np.zeros((t, n_null))
    pool = x.reshape(-1)
    for i in range(n_null):
        null[:, i] = np.random.choice(pool, size=t)
        null[:, i] /= null[:, i].sum()
    y = np.hstack((x, null))
    y[y == 0] = y[y>0].min()

    return y

def open_csv_nonull(filepath):
    x = pd.read_csv(filepath, header=None).values
    x[x == 0] = x[x > 0].min()
    return x

def solve_middle(V, k, l, z_th=0.001, tolerance=1e-7, n_iter=[30, 30, 10, 30], comp_rate=0.3, random_seed=None):

    n, t = V.shape
    V1 = cp.array([V[:, i*l:(i+1)*l].mean(axis=1) for i in range(int(cp.ceil(t/l)))]).T
    model = iSeq(cp.asarray(V1), k, 2, max_iter=n_iter[0], z_th=z_th, tolerance=tolerance, random_seed=random_seed)
    for i in range(n_iter[0]):
        model._update_W(True)
        model._update_H(True)
    h1 = model.H.copy()
    w1 = model.W.copy()

    t1 = V1.shape[1]
    t1_true = int(cp.ceil(t1 * comp_rate))
    t1_index = cp.array([False for _ in range(t1)])
    count = 0
    tmp = model.H.argsort(axis=1)[:, ::-1]

    for i in range(t1):
        for j in range(k):
            ij = tmp[j, i]
            if ij > 0 and ~t1_index[ij-1]:
                t1_index[ij-1] = True
                count += 1
            if ij > 1 and ~t1_index[ij-2]:
                t1_index[ij-2] = True
                count += 1
            if ij < t1 and ~t1_index[ij]:
                t1_index[ij] = True
                count += 1
        if count >= t1_true:
            break

    V2 = cp.asarray(V)[:, cp.array([t1_index for _ in range(l)]).T.reshape(-1)[:t]]

    model = iSeq(cp.asarray(V2), k, l, max_iter=n_iter[1], z_th=z_th, tolerance=tolerance, random_seed=random_seed+1)
    model.W = cp.zeros_like(model.W)
    model.W += w1.max(axis=2)[:, :, cp.newaxis]

    count = 0
    for i, b in enumerate(t1_index):
        if b:
            if count == 0:
                model.H[:, :l-1] = h1[:, i:i+1]
                count += 1
            model.H[:, l*count-1:l*(count+1)-1] = h1[:, i+1:i+2]
            count += 1

    w_sum = model.W.sum(axis=2).sum(axis=0)
    model.H *= w_sum.reshape(-1, 1)
    model.W /= w_sum.reshape(1, -1, 1)

    model.solve()

    return model


#!/usr/bin/env python3.6
# coding=utf-8
'''

Deep Learning Models for concept identification

@author: Chunchuan Lyu (chunchuan.lv@gmail.com)
@since: 2018-05-30
'''

import torch
import torch.nn as nn
from modules.helper_module import data_dropout
from torch.nn.utils.rnn import PackedSequence
from utility.constants import *
import numpy as np
from torch.autograd import Variable

class SentenceEncoder(nn.Module):
    def __init__(self, opt, embs):
        self.layers = opt.txt_enlayers
        self.num_directions = 2 if opt.brnn else 1
        assert opt.txt_rnn_size % self.num_directions == 0
        self.hidden_size = opt.txt_rnn_size // self.num_directions
    #    inputSize = opt.word_dim*2 + opt.lemma_dim + opt.pos_dim +opt.ner_dim
        inputSize = embs["word_fix_lut"].embedding_dim +  embs["lemma_lut"].embedding_dim\
                    +embs["pos_lut"].embedding_dim + embs["ner_lut"].embedding_dim

        super(SentenceEncoder, self).__init__()
        self.rnn = nn.LSTM(inputSize, self.hidden_size,
                        num_layers=self.layers,
                        dropout=opt.dropout,
                        bidirectional=opt.brnn)

        self.lemma_lut = embs["lemma_lut"]

        self.word_fix_lut  = embs["word_fix_lut"]

        self.pos_lut = embs["pos_lut"]

        self.ner_lut = embs["ner_lut"]

        self.drop_emb = nn.Dropout(opt.dropout)
        self.alpha = opt.alpha

        if opt.cuda:
            self.rnn.cuda()

    def forward(self, packed_input: PackedSequence,hidden=None):
    #input: pack(data x n_feature ,batch_size)
        input = packed_input.data
        if self.alpha and self.training:
            input = data_dropout(input,self.alpha)

        word_fix_embed = self.word_fix_lut(input[:,TXT_WORD])
        lemma_emb = self.lemma_lut(input[:,TXT_LEMMA])
        pos_emb = self.pos_lut(input[:,TXT_POS])
        ner_emb = self.ner_lut(input[:,TXT_NER])

        emb = self.drop_emb(torch.cat([lemma_emb,pos_emb,ner_emb],1))#  data,embed
        emb = torch.cat([word_fix_embed,emb],1)#  data,embed
        emb =  PackedSequence(emb, packed_input.batch_sizes)
        outputs, hidden_t = self.rnn(emb, hidden)

        return  outputs

class Node_emblayer(nn.Module):  # 继承nn.Module  yang 20190205
    def __init__(self,opt,in_features,out_features):
        super(Node_emblayer, self).__init__()  # 等价于nn.Module.__init__(self)

        self.inputfeature = in_features
        self.outfeature = out_features
        self.fc = nn.Linear(in_features, out_features)
        self.bc = nn.Linear(in_features, out_features)
        self.edge = nn.Linear(3*out_features, out_features)
        self.relu = nn.ReLU(inplace=True)
        # self.transform = nn.Sequential(
        #     nn.Dropout(opt.dropout),
        #     nn.Linear(self.outfeature,self.outfeature,bias = self.outfeature))
        self.K = opt.Node_K

    def forward(self, fw_input,bw_input, edge_embs, fw_adj, bw_adj,fw_edgeid,bw_edgeid,fw_edgedep,bw_edgedep):

        fw_adjmask = fw_adj.ge(0)
        bw_adjmask = bw_adj.ge(0)
        fw_edgmask = fw_edgeid.ge(0)
        bw_edgmask = bw_edgeid.ge(0)
        fw_edgdepmask = fw_edgedep.ge(0)
        bw_edgdepmask = bw_edgedep.ge(0)
        edgout = torch.masked_select(fw_edgedep[:, 0], fw_edgdepmask[:, 0])

        for k in range(self.K):
            alpha = 1-0.1*k
            fw_hidden = Variable(torch.zeros(len(bw_input), self.inputfeature))
            bw_hidden = Variable(torch.zeros(len(bw_input), self.inputfeature))

            if self.cuda:
                fw_hidden = fw_hidden.cuda()
                bw_hidden = bw_hidden.cuda()

            for i in range(len(fw_input)):
                indices = Variable(torch.masked_select(fw_adj[i,:], fw_adjmask[i,:]).cuda())
                # indices = Variable(fw_adj[i,0:fw_end[i]].cuda())
                adj_x = torch.index_select(fw_input, 0, indices)

                out = torch.masked_select(fw_edgeid[i,:], fw_edgmask[i,:])
                if len(out)>0:
                    # print("out",out)
                    indices = Variable(out.cuda())
                    adj_x1 = torch.index_select(edge_embs, 0, indices)
                    adj_x = torch.cat((adj_x1, adj_x), 0)
                adj_x = torch.mean(adj_x, dim=0)
                # if k < self.K - 1:
                #     adj_x = torch.mean(adj_x, dim=0)
                # else:
                #     adj_x = self.transform(adj_x)
                #     adj_x = self.relu(adj_x)
                #     adj_x = torch.max(adj_x, dim=0)[0]  # torch.mean(adj_x, dim=0)
                fw_hidden[i,:] = torch.cat((fw_input[i, :], adj_x), -1).view(1, -1)

                indices = Variable(torch.masked_select(bw_adj[i,:], bw_adjmask[i,:]).cuda())
                adj_x = torch.index_select(bw_input, 0, indices)

                out = torch.masked_select(bw_edgeid[i, :], bw_edgmask[i, :])
                if len(out)>0:
                    indices = Variable(out.cuda())
                    adj_x1 = torch.index_select(edge_embs, 0, indices)
                    adj_x = torch.cat((adj_x1, adj_x), 0)
                adj_x = torch.mean(adj_x, dim=0)
                # if k < self.K - 1:
                #     adj_x = torch.mean(adj_x, dim=0)
                # else:
                #     adj_x = self.transform(adj_x)
                #     adj_x = self.relu(adj_x)
                #     adj_x = torch.max(adj_x, dim=0)[0]  # torch.mean(adj_x, dim=0)
                bw_hidden[i,:] = torch.cat((bw_input[i, :], adj_x), -1).view(1, -1)

            fw_hidden = self.fc(fw_hidden)
            bw_hidden = self.bc(bw_hidden)

            if k < self.K - 1:
                bw_hidden = self.relu(bw_hidden)
                fw_hidden = self.relu(fw_hidden)

            fw_input = fw_hidden
            bw_input = bw_hidden

            if len(edgout)>0:
                edge_hidden = Variable(torch.zeros(len(edgout), 3*self.outfeature))
                if self.cuda:
                    edge_hidden = edge_hidden.cuda()

                kk = 0
                for i in range(len(bw_edgedep)):
                    edge_dep = edge_embs[i,:].clone()

                    out = torch.masked_select(fw_edgedep[i, :], fw_edgdepmask[i, :])
                    if len(out)>0:

                        indices = Variable(out.cuda())
                        fadj_x = torch.index_select(fw_input, 0, indices)
                        fadj_x = torch.mean(fadj_x, dim=0)
                        # if k<self.K-1:
                        #     fadj_x = torch.mean(fadj_x, dim=0)
                        # else:
                        #     fadj_x = self.transform(fadj_x)
                        #     fadj_x = self.relu(fadj_x)
                        #     fadj_x = torch.max(fadj_x, dim=0)[0]  # torch.mean(adj_x, dim=0)
                        edge_dep = torch.cat((edge_dep, fadj_x), -1)

                        out = torch.masked_select(bw_edgedep[i, :], bw_edgdepmask[i, :])
                        indices = Variable(out.cuda())
                        badj_x = torch.index_select(bw_input, 0, indices)
                        badj_x = torch.mean(badj_x, dim=0)
                        # if k<self.K-1:
                        #     badj_x = torch.mean(badj_x, dim=0)
                        # else:
                        #     badj_x = self.transform(badj_x)
                        #     badj_x = self.relu(badj_x)
                        #     badj_x = torch.max(badj_x, dim=0)[0]  # torch.mean(adj_x, dim=0)
                        edge_dep = torch.cat((edge_dep, badj_x), -1).view(1,-1)
                        # print("edge_dep",edge_dep)
                        edge_hidden[kk, :] = edge_dep.view(1, -1)
                        kk = kk+1

                edge_hidden = self.edge(edge_hidden)
                edge_hidden = self.relu(edge_hidden)

            if k<self.K-1:
                out_edge_hidden = Variable(torch.zeros(len(bw_edgedep), self.outfeature))
                if self.cuda:
                    out_edge_hidden = out_edge_hidden.cuda()
                kk = 0
                for i in range (len(bw_edgedep)):
                    if len(torch.masked_select(fw_edgedep[i, :], fw_edgdepmask[i, :])) > 0:
                        out_edge_hidden[i, :]= edge_hidden[kk,:].clone()
                        kk = kk+1
                    else:
                        out_edge_hidden[i, :] = edge_embs[i, :].clone()
                edge_embs = out_edge_hidden
        return fw_input, bw_input  # fw_output,bw_output

class Node_embedding(nn.Module):  # 继承nn.Module  yang 20190205
    def __init__(self, opt,embs):
        super(Node_embedding, self).__init__()  # 等价于nn.Module.__init__(self)
        self.out_features = opt.txt_rnn_size
        # self.edge_embs = nn.Parameter(torch.randn(opt.dep_len, opt.txt_rnn_size))
        self.gc1 = Node_emblayer(opt,2 * opt.txt_rnn_size,opt.txt_rnn_size)
        self.edge_embs = embs["dep_lut"]
        self.dep_len = opt.dep_len
        # self.gc2 = Node_emblayer2(2 * opt.txt_rnn_size, opt.txt_rnn_size)
        if opt.cuda:
            self.cuda()

    # def node_sence(self, fw_input,bw_input, edge_embs, fw_adj, bw_adj,bw_end,fw_end,fw_edgeid,bw_edgeid,bw_edgeend,fw_edgeend):
    #
    #     fw_hidden = Variable(torch.zeros(len(bw_input), self.out_features))
    #     bw_hidden = Variable(torch.zeros(len(bw_input), self.out_features))
    #
    #     if self.cuda:
    #         fw_hidden = fw_hidden.cuda()
    #         bw_hidden = bw_hidden.cuda()
    #
    #     for i in range(len(fw_input)):
    #         indices = Variable(fw_adj[i, 0:fw_end[i]].cuda())
    #         adj_x = torch.index_select(fw_input, 0, indices)
    #         if fw_edgeend[i]>0:
    #             indices = Variable(fw_edgeid[i, 0:fw_edgeend[i]].cuda())
    #             adj_x1 = torch.index_select(edge_embs, 0, indices)
    #             adj_x = torch.cat((adj_x1, adj_x), 0)
    #
    #         adj_x = torch.cat((fw_input[i,:].view(1,-1), adj_x), 0)
    #         # adj_x = torch.max(adj_x, dim=0)[0]
    #         adj_x = torch.mean(adj_x, dim=0)
    #         fw_hidden[i,:] = adj_x.view(1, -1)
    #         # fw_hidden[i, :] = fw_input[i, :]
    #
    #         indices = Variable(bw_adj[i, 0:bw_end[i]].cuda())
    #         adj_x = torch.index_select(bw_input, 0, indices)
    #
    #         if fw_edgeend[i]>0:
    #             indices = Variable(bw_edgeid[i, 0:bw_edgeend[i]].cuda())
    #             adj_x1 = torch.index_select(edge_embs, 0, indices)
    #             adj_x = torch.cat((adj_x1, adj_x), 0)
    #         adj_x = torch.cat((bw_input[i,:].view(1,-1), adj_x), 0)
    #         # adj_x = torch.max(adj_x, dim=0)[0]
    #         adj_x = torch.mean(adj_x, dim=0)
    #         bw_hidden[i,:] = adj_x.view(1, -1)
    #
    #     return fw_hidden,bw_hidden

    def forward(self, packed_input: PackedSequence,fw_adj,bw_adj,fw_edgeid,bw_edgeid,fw_edgedep,bw_edgedep):

        fw_input = packed_input.data
        bw_input = packed_input.data
        embs_ind = Variable(torch.linspace(0,self.dep_len-1,self.dep_len).long())
        if self.cuda:
            embs_ind = embs_ind.cuda()
        edge_embs = self.edge_embs(embs_ind)
        fw_output, bw_output = self.gc1(fw_input, bw_input, edge_embs, fw_adj, bw_adj,fw_edgeid,bw_edgeid,fw_edgedep,bw_edgedep)

        node_embedding = PackedSequence(torch.cat([fw_output, bw_output], dim=1), packed_input.batch_sizes)
        return node_embedding

class Concept_Classifier(nn.Module):

    def __init__(self, opt, embs):
        super(Concept_Classifier, self).__init__()
        self.txt_rnn_size = 2*opt.txt_rnn_size

        self.n_cat = embs["cat_lut"].num_embeddings
        self.n_high = embs["high_lut"].num_embeddings
        self.n_aux = embs["aux_lut"].num_embeddings

        self.cat_score =nn.Sequential(
            nn.Dropout(opt.dropout),
            nn.Linear(self.txt_rnn_size,self.n_cat,bias = opt.cat_bias))

        self.le_score =nn.Sequential(
            nn.Dropout(opt.dropout),
            nn.Linear(self.txt_rnn_size,self.n_high+1,bias = opt.lemma_bias))

        self.ner_score =nn.Sequential(
            nn.Dropout(opt.dropout),
            nn.Linear(self.txt_rnn_size,self.n_aux,bias = opt.cat_bias))

        self.t = 1
        self.sm =  nn.Softmax(dim=1)
        if opt.cuda:
            self.cuda()

    def forward(self, src_enc ):
        '''
            src_enc: pack(data x txt_rnn_size ,batch_size)
           src_le:  pack(data x 1 ,batch_size)

           out:  (datax n_cat, batch_size),    (data x n_high+1,batch_size)
        '''

        assert isinstance(src_enc,PackedSequence)

     #   high_embs = self.high_lut.weight.expand(le_score.size(0),self.n_high,self.dim)
      #  le_self_embs = self.lemma_lut(src_le.data).unsqueeze(1)
      #  le_emb = torch.cat([high_embs,le_self_embs],dim=1) #data x high+1 x dim

        pre_enc =src_enc.data

        cat_score = self.cat_score(pre_enc) #  n_data x n_cat
        ner_score = self.ner_score(pre_enc)#  n_data x n_cat
        le_score = self.le_score (src_enc.data)
        le_prob = self.sm(le_score)
        cat_prob = self.sm(cat_score)
        ner_prob = self.sm(ner_score)
        batch_sizes = src_enc.batch_sizes
        return   PackedSequence(cat_prob,batch_sizes),PackedSequence(le_prob,batch_sizes),PackedSequence(ner_prob,batch_sizes)

class ConceptIdentifier(nn.Module):
    #could share encoder with other model
    def __init__(self, opt,embs,encoder = None):
        super(ConceptIdentifier, self).__init__()
        if encoder:
            self.encoder = encoder
        else:
            self.encoder = SentenceEncoder( opt, embs)
        self.generator = Concept_Classifier( opt, embs)
        self.node_embedding = Node_embedding(opt,embs)  # yang

    def forward(self, srcBatch,fw_adjs,bw_adjs,fw_edgeid,bw_edgeid,fw_edgedep,bw_edgedep):
        src_enc = self.encoder(srcBatch)
        node_embedding = self.node_embedding(src_enc,fw_adjs,bw_adjs,fw_edgeid,bw_edgeid,fw_edgedep,bw_edgedep)

        probBatch = self.generator(node_embedding)
        return probBatch,node_embedding


from flask.helpers import flash
from app.helpers.decorators import user_has
from app.models import Configuration, Location
from app.models.location import Coordinate
from app.views import blueprint
from app.forms import LocationForm
from flask import render_template, redirect, url_for, request
from flask_login import login_required
from flask import jsonify

@blueprint.route('/location_list', methods=['POST', 'GET'])
@login_required
@user_has('punto_encuentro_index')
def location_list():
    per_page = Configuration.get_num_elems()
    order_by = 'name asc' if Configuration.is_ascendent() else 'name desc'

    unpublished = bool(request.form.get('unpublished'))

    # del campo hidden del template
    page = request.form.get('page')
    current_page = int(page) if page else 1

    search_location = request.form.get('search')
    if search_location:
        locs = Location.search_location(search_location, order_by, not unpublished).paginate(
            current_page, per_page, error_out=False)
    else:
        locs = Location.all_locations(order_by, not unpublished).paginate(
            current_page, per_page, error_out=False)
    if not len(locs.items) and search_location:
        flash("No hubo coincidencias en su búsqueda", "warning")

    return render_template("locations/location-list.html", locations=locs, unpublished=unpublished)

@blueprint.route('/locations/<int:id>/', methods=['GET'])
@login_required
@user_has('punto_encuentro_show')
def location_view(id):
    loc = Location.get_by_id(id)
    return render_template('locations/location-view.html', loc=loc)

@blueprint.route('/locations/<int:id>/edit', methods=['GET', 'POST'])
@login_required
@user_has('punto_encuentro_update')
def location_edit(id):
    loc = Location.get_by_id(id)
    form = LocationForm(obj=loc)

    if form.is_submitted():
        if form.validate():
            form.populate_obj(loc)
            loc.save()
            flash("cambios registrados con éxito", 'success')
            return redirect(url_for('blueprint.location_list'))
        return redirect(url_for('blueprint.location_edit', id=id))
    return render_template('locations/location-edit.html', form=form, id=id)

@blueprint.route('/locations/register', methods=['GET', 'POST'])
@login_required
@user_has('punto_encuentro_new')
def register_location():
    form = LocationForm()
    if form.is_submitted():
        if form.validate():
            location = Location()
            form.populate_obj(location)  # Copies matching attributes from form onto location
            location.save()
            flash("Punto de encuentro creado exitosamente", 'success')
            return redirect(url_for('blueprint.location_list'))
        return redirect(url_for('blueprint.register_location'))
    return render_template('locations/add_location.html', form=form)

@blueprint.route('/locations/<int:id>/delete')
@login_required
@user_has('punto_encuentro_destroy')
def delete_location(id):
    Location.delete(id)
    flash("Punto de encuentro eliminado satisfactoriamente", "success")
    return redirect(url_for("blueprint.location_list"))

@blueprint.route('/locations/all/coordinates', methods=['GET'])
@login_required
@user_has('punto_encuentro_show')
def location_coordinates():
    locations = Location.all()
    list = []
    for loc in locations:
        list.append(Coordinate(latitude=loc.latitude, longitude=loc.longitude))
    return jsonify(results=list)


import pymongo
from datetime import datetime, timezone, timedelta
import pytz
import os
import argparse
import re
import pandas as pd
import numpy as np

if __name__ == "__main__":
    print("Hello, World")
    df = pd.DataFrame({"A": ["foo", "foo", "foo", "foo", "foo",
                          "bar", "bar", "bar", "bar"],
                    "B": ["one", "one", "one", "two", "two",
                          "one", "one", "two", "two"],
                    "C": ["small", "large", "large", "small",
                          "small", "large", "small", "small",
                          "large"],
                    "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],
                    "E": [2, 4, 5, 5, 6, 6, 8, 9, 9],
                    "F": ["a", "a", "a", "a", "a", "a", "a", "a", "a"]
                    })
    print(df)
    print("")
    table = pd.pivot_table(df, values='D', index=['A', 'B'], columns=['C'], aggfunc=np.sum, fill_value=0)
    table_count = pd.pivot_table(
        df, values='D', index=['A', 'B'], columns=['C'], aggfunc="count", fill_value=0, margins=True)

    print(table)
    print("")
    print(table_count)


import torch
import torch.nn as nn
import numpy as np

class LabelSmoothing(nn.Module):
    """
    NLL loss with label smoothing.
    """
    def __init__(self, smoothing=0.0):
        """
        Constructor for the LabelSmoothing module.
        :param smoothing: label smoothing factor
        """
        super(LabelSmoothing, self).__init__()
        self.confidence = 1.0 - smoothing
        self.smoothing = smoothing

    def forward(self, x, target):
        logprobs = torch.nn.functional.log_softmax(x, dim=-1)

        nll_loss = -logprobs.gather(dim=-1, index=target.unsqueeze(1))
        nll_loss = nll_loss.squeeze(1)
        smooth_loss = -logprobs.mean(dim=-1)
        loss = self.confidence * nll_loss + self.smoothing * smooth_loss
        return loss.mean()

class BCEWithLogitsLoss(nn.Module):
    def __init__(self, weight=None, size_average=None, reduce=None, reduction='mean', pos_weight=None, num_classes=64):
        super(BCEWithLogitsLoss, self).__init__()
        self.num_classes = num_classes
        self.criterion = nn.BCEWithLogitsLoss(weight=weight,
                                              size_average=size_average,
                                              reduce=reduce,
                                              reduction=reduction,
                                              pos_weight=pos_weight)
    def forward(self, input, target):
        target_onehot = F.one_hot(target, num_classes=self.num_classes)
        return self.criterion(input, target_onehot)

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def adjust_learning_rate(epoch, opt, optimizer):
    """Sets the learning rate to the initial LR decayed by decay rate every steep step"""
    steps = np.sum(epoch > np.asarray(opt.lr_decay_epochs))
    if steps > 0:
        new_lr = opt.learning_rate * (opt.lr_decay_rate ** steps)
        for param_group in optimizer.param_groups:
            param_group['lr'] = new_lr

def accuracy(output, target, topk=(1,)):
    """Computes the accuracy over the k top predictions for the specified values of k"""
    with torch.no_grad():
        maxk = max(topk)
        batch_size = target.size(0)

        _, pred = output.topk(maxk, 1, True, True)
        pred = pred.t()
        correct = pred.eq(target.view(1, -1).expand_as(pred))

        res = []
        for k in topk:
            correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
            res.append(correct_k.mul_(100.0 / batch_size))
        return res


def build(bld):
    bld.new_task_gen(
            features='slice2cpp cxx cstaticlib pcgen',
            source=bld.glob('*.ice'),
            target='tartarus-slice-SysDB',
            slice_includes='..',
            headers_path='${INCLUDEDIR}/Tartarus/SysDB',
            includes='. ..',
            pc_filename = 'tartarus-slice-sysdb.pc',
            pc_env_defines = 'includedir',
            pc_name = 'tartarus-slice-sysdb',
            pc_description = 'Tartarus library for using SysDB service',
            pc_requires = ['tartarus-slice-core = ${version}'],
            pc_libs = '-ltartarus-slice-sysdb',
            pc_cflags='-I${includedir}/Tartarus')


from vng_api_common.utils import get_uuid_from_path
from zgw_consumers.client import UnknownService
from zgw_consumers.models import Service

def create_remote_zaakbesluit(besluit_url: str, zaak_url: str) -> dict:
    client = Service.get_client(zaak_url)
    if client is None:
        raise UnknownService(f"{zaak_url} API should be added to Service model")

    zaak_uuid = get_uuid_from_path(zaak_url)
    body = {"besluit": besluit_url}

    response = client.create("zaakbesluit", data=body, zaak_uuid=zaak_uuid)

    return response

def delete_remote_zaakbesluit(zaakbesluit_url: str) -> None:
    client = Service.get_client(zaakbesluit_url)
    if client is None:
        raise UnknownService(f"{zaakbesluit_url} API should be added to Service model")

    client.delete("zaakbesluit", zaakbesluit_url)


import collectd
import psutil

def configer(cfg):
    """Configure"""
    collectd.debug('Configuring stuff with: key = %s' %(cfg.values))
    for child in cfg.children:
        collectd.debug('Configuring params name = %s, value = %s' %(child.key, str(child.values)))

def initer():
    """Initer"""
    collectd.debug('Initializing stuff ...')

def dispatch_value(value_type, value):
    if not value:
        return

    metric = collectd.Values()
    metric.plugin = 'virtual_memory'
    metric.type = 'virtual_memory'
    metric.type_instance = value_type
    metric.values = [value]
    metric.dispatch()

def reader(input_data=None):
    """Read data and dispatch"""
    memories = psutil.virtual_memory()
    for key in ['total', 'available', 'percent', 'used', 'active', 'inactive', 'buffers', 'cached', 'shared']:
        dispatch_value(key, getattr(memories, key))

collectd.register_config(configer)
collectd.register_init(initer)
collectd.register_read(reader)