import numpy as np
import matplotlib.pyplot as plt
from matplotlib import style
style.use('ggplot')

class Support_Vector_Machine:
    # visualization => if we want to graph the data
    def __init__(self, visualization=True):
        self.visualization = visualization
        self.colors = {1: 'r', -1: 'b'}
        if self.visualization:
            self.fig = plt.figure()
            self.ax = self.fig.add_subplot(1, 1, 1)

    # train
    def fit(self, data):
        self.data = data

        # { ||w||: [w, b] }
        opt_dic = {}

        transforms = [
            [1, 1],
            [-1, 1],
            [-1, -1],
            [1, -1]
        ]

        all_data = []
        for yi in self.data:
            for featureset in self.data[yi]:
                for feature in featureset:
                    all_data.append(feature)

        self.max_feature_value = max(all_data)
        self.min_feature_value = min(all_data)
        all_data = None

        # yi(xi.w + b) = 1
        # we know we have found a rly good value for w and b when for both classes, the value is close to 1

        # step size for the search of minimum (bowl drawing)
        step_sizes = [self.max_feature_value * 0.1,
                      self.max_feature_value * 0.01,
                      # point of expense (deeper = very expensive but better results)
                      self.max_feature_value * 0.005]

        # b don't need as much optimization as w, so multiply it (less expensive)
        b_range_multiple = 1
        b_multiple = 5
        latest_optimum = self.max_feature_value * 10

        for step in step_sizes:
            w = np.array([latest_optimum, latest_optimum])
            optimized = False
            while not optimized:
                for b in np.arange(-1 * (self.max_feature_value * b_range_multiple), self.max_feature_value * b_range_multiple, step * b_multiple):
                    for transformation in transforms:
                        w_t = w * transformation
                        found_option = True
                        # i is the class in the dict
                        for i in self.data:
                            for xi in self.data[i]:
                                # yi(xi.w + b) >= 1
                                yi = i
                                if not yi*(np.dot(w_t, xi) + b) >= 1:
                                    found_option = False
                        if found_option:
                            # linalg.norm = magnitude of the vector
                            opt_dic[np.linalg.norm(w_t)] = [w_t, b]

                if w[0] < 0:
                    optimized = True
                    print('Optimized a step.')
                else:
                    w = w - step

            norms = sorted([n for n in opt_dic])
            opt_choice = opt_dic[norms[0]]

            self.w = opt_choice[0]
            self.b = opt_choice[1]

    def predict(self, features):
        # sign of (x.w + b)
        classfication = np.sign(np.dot(np.array(features), self.w) + self.b)
        if classfication != 0 and self.visualization:
            self.ax.scatter(features[0], features[1], s=200, marker='*', c=self.colors[classfication])

        return classfication

    def visualize(self):
        for i in data_dict:
            for x in data_dict[i]:
                self.ax.scatter(x[0], x[1], s=100, color=self.colors[i])

        # x.w + b = v = hyperplane
        # positive support vector = 1
        # negative support vector = -1
        # decision boudary = 0
        def hyperplane(x, w, b, v):
            return (-w[0] * x - b + v) / w[1]

        # positive support vector hyperplane : (w.x + b) = 1
        psv1 = hyperplane(self.min_feature_value, self.w, self.b, 1)
        psv2 = hyperplane(self.max_feature_value, self.w, self.b, 1)
        self.ax.plot([self.min_feature_value, self.max_feature_value], [psv1, psv2], 'k')

        # negative support vector hyperplane : (w.x + b) = -1
        nsv1 = hyperplane(self.min_feature_value, self.w, self.b, -1)
        nsv2 = hyperplane(self.max_feature_value, self.w, self.b, -1)
        self.ax.plot([self.min_feature_value, self.max_feature_value], [nsv1, nsv2], 'k')

        # decision boudary hyperplane : (w.x + b) = 0
        db1 = hyperplane(self.min_feature_value, self.w, self.b, 0)
        db2 = hyperplane(self.max_feature_value, self.w, self.b, 0)
        self.ax.plot([self.min_feature_value, self.max_feature_value], [db1, db2], 'y--')

        plt.savefig('svm.svg')

data_dict = {
    -1: np.array([
        [1, 7],
        [2, 8],
        [3, 8]
    ]),
    1: np.array([
        [5, 1],
        [6, -1],
        [7, 3]
    ])
}

svm = Support_Vector_Machine()
svm.fit(data=data_dict)

predict_us = [
    [0, 10],
    [3, 4],
    [3, 5],
    [5, 5],
    [5, 6],
    [6, -5],
    [5, 8]
]

for p in predict_us:
    svm.predict(p)

svm.visualize()


#!/usr/bin/python3
# -*- coding: utf-8 -*-

import os, sys, re
import click
from html import parser
from urllib import request, response

def get_reviry_list(url, separator="
", match="""a href="/index.php/14-mimopstruhove-reviry/"""):
    output = []
    urlparser = request.urlparse(url)
    with request.urlopen(url) as basepage:
        htmlpage = basepage.read().decode()
        for item in htmlpage.split(separator):
            if match in item:
                rematch = re.search('href="([^"]+)"', item)
                if rematch:
                    output.append(f"{urlparser.scheme}://{urlparser.netloc}{rematch.groups()[0]}")
    return output

def parser_revir_page(url):
    with request.urlopen(url) as page:
        readed = page.read().decode()
        sep = """
"""
        sep2 = """
"""
        tmp = readed.split(sep)[1].split(sep2)[0]
        tmp = re.sub(r"
", "\n", tmp)
        tmp = re.sub(r"", " ", tmp)
        tmp = re.sub(r"<.?br[^>]+>", "\n", tmp)
        tmp = re.sub(r"<[^>]+>", " ", tmp)
        tmp = re.sub(r"&[^;]+;", "", tmp)
        return tmp

def strip_list(inp):
    output = []
    for item in inp:
        if item is not None:
            output.append(item.strip())

def revir_data(text):
    jmeno = ""
    jmeno_add = ""
    GPS = {}
    for line in text.split("\n"):
        #print(">>:", line, file=sys.stderr)
        stripped_line = line.strip()
        if not jmeno:
            jmeno = stripped_line
            jmeno_add = jmeno
            print("REVIR:", jmeno, file=sys.stderr)
            continue

        # podrevir
        search = re.search('^\d+\..*ha', stripped_line)
        if search:
            jmeno_add = stripped_line
            print("NAME:", jmeno_add, file=sys.stderr)
            continue

        # reka
        search = re.search('^GPS.*Z:(.+)K:(.+)', stripped_line)
        if search:
            zacatek = search.groups()[0].split("N")
            konec = search.groups()[1].split("N")
            GPS[jmeno_add] = (convert(zacatek[0]), convert(zacatek[1]), convert(konec[0]), convert(konec[1]))
            print("adding reka", zacatek, konec, file=sys.stderr)
            continue

        # rybnik
        search = re.search('^GPS(.*)', stripped_line)
        if search:
            zacatek = search.group(0).split("N")
            if len(zacatek) < 2:
                raise Exception(f"BAD GPS {zacatek}")
            GPS[jmeno_add] = (convert(zacatek[0]), convert(zacatek[1]))
            print("adding rybnik", zacatek)
            continue

        # misto ci zakaz
        search_all = re.findall(r'GPS\S*\s+([^N]+N\S*\s+[^E]+E)', line)
        for search in search_all:
            #print(">>>>>>>>>>>", search)
            zacatek = search.split("N")

            GPS[f"ZAKAZ ci RYBNIK: {search}"] = (convert(zacatek[0]), convert(zacatek[1]))
            print("ZAKAZ ci RYBNIK", zacatek, file=sys.stderr)
            continue
    return jmeno, GPS

def convert(coor):
    output = 0
    found = re.search(r"(\d+)\D+(\d+)\D+([0-9.]+)", coor)
    if found:
        output = float(found.group(1)) + (float(found.group(2)) / 60.0) + (float(found.group(3))) / 3600.0
    else:
        found = re.search(r"(\d+\.\d+)", coor)
        if found:
            output = float(found.group(1))
        else:
            print(sys.stderr, "ERROR convert coordinates:", coor, file=sys.stderr)
    return output

def output_to_file(filename, reviry):
    with open(filename, mode="w") as outfile:
        intro="""<?xml version="1.0" encoding="UTF-8"?>
                 <kml xmlns="http://www.opengis.net/kml/2.2">
                <Document>
    """
        modnum=1
        colors=["ff0000", "ffff00", "ff00ff", "990000", "99ff00", "9900ff", "ff9900", "ff0099", "550000", "55ff00", "5500ff"]
        for foo in range(0, modnum):
            intro +="""

                    """.format(foo, colors[foo])

        outfile.write(intro)

        count=0
        for revir_name, revir_data in reviry.items():
            count += 1
            for k, v in revir_data['GPS'].items():
                outfile.write('<Placemark>\n')
                outfile.write(f'<name>{revir_name}</name>')
                description = revir_data["data"]
                outfile.write(f'<description>\nJmeno: {k}\n\n{description}</description>')
                if "ZAKAZ" in k:
                    outfile.write('\n'.format(count % modnum))
                    outfile.write('<Point> <coordinates>{0},{1}\n</coordinates> </Point>\n'.format(v[1], v[0]))
                elif len(v) == 2:
                    outfile.write('\n'.format(count%modnum))
                    outfile.write('<Point> <coordinates>{0},{1}\n</coordinates> </Point>\n'.format(v[1], v[0]))
                elif len(v) == 4:
                    outfile.write('\n'.format(count%modnum))
                    outfile.write('<LineString><tessellate>1</tessellate><coordinates>{0},{1}\n{2},{3}\n</coordinates> </LineString>\n'.format(v[1], v[0], v[3], v[2]))
                outfile.write('</Placemark>')
        outfile.write('</Document>\n')
        outfile.write('</kml>')

@click.command()
@click.option("--url", "-u", default="http://mrsbrno.cz/index.php/33-seznam-reviru/142-seznam-rybarskych-reviru-mimopstruhovych")
@click.option("--outputfile", "-o", default="outfile.kml")
@click.option("--quiet", "-q", is_flag=True, default=False)
def parser(url, outputfile, quiet):
    reviry = {}
    for item in get_reviry_list(url):
        text = parser_revir_page(item)
        jmeno, GPS = revir_data(text)
        reviry[jmeno] = {'GPS': GPS, 'data': text}
    output_to_file(outputfile, reviry)

if __name__ == '__main__':
    parser()


import sys
import app.DataPreProcessor as dpp
import app.VideoInput as vIp
import app.CNN as cnn
import numpy as np
import app.ConfusionMatrix as cm
import app.Tester as Tester
import app.LiveCam as lcm
import playsound

class Start():
      # argumentList = sys.argv
    def loadDataSets(self):
        # playsound.playsound('beep.mp3', True)
        print("Starting .............................................................")
        dataPreProcessor = dpp.DataPreProcessor()
        self.training_data = dataPreProcessor.create_training_data()
        print("Training Data processed ! ............................................")
        return self.training_data

    def loadModel(self):
        cnn_algo = cnn.ConvolutionalNeuralNetwork()
        cnn_model = cnn_algo.create_model(self.training_data)
        print("Model Loaded ! .......................................................")
        return cnn_model

    def confusionMatrix(self, cnn_model):
        print("Confusion Matrix .....................................................")
        con_mat, class_w_acc =  cm.confusion_matrix_fn(cnn_model)
        return con_mat

    def videoInputTest(self,cnn_model,fileUrl):
        print("Video Input .........................................................")
        videoInput = vIp.VideoInput()
        videoInput.start_capture(cnn_model,fileUrl)
        return "Done"

    def testOnUnknown(self, cnn_model):
        print("Testing Unknown .....................................................")
        tester = Tester.Tester(cnn_model)
        tester.test()
        return "Done"

    def liveInputTest(self ,cnn_model):
        print("Live Input ..........................................................")
        livecam = lcm.LiveCam()
        livecam.start_capture(cnn_model)
        return "Done"


import frida
import base64
import time
import os
import sys
import time
import signal

output_folder = sys.argv[1]
if not os.path.exists(output_folder):
    os.mkdir(output_folder)

pid = (sys.argv[2])

mypid = os.getpid()

class QEntry(object):
    def __init__(self):
        self.filename = ""
        self.size = 0
        self.num = 0
        self.was_fuzzed = False
        self.exec_us = 0
        self.time = 0
        self.new_cov = False
        self.next = None

class Queue(object):
    def __init__(self):
        self.size = 0
        self.start = None
        self.cur = None
        self.top = None

    def add(self, buf, exec_us, new_cov, stage):
        q = QEntry()
        q.filename = os.path.join(output_folder, "id_%d_%s" % (self.size, stage))
        if new_cov:
            q.filename += "_cov"
        q.num = self.size
        q.exec_us = exec_us
        q.new_cov = new_cov
        q.time = int(time.time())
        q.size = len(buf)
        with open(q.filename, "wb") as f:
            f.write(buf)
        self.size += 1
        if self.top:
            self.top.next = q
            self.top = q
        else:
            self.start = q
            self.top = q

    def get(self):
        if self.cur is None:
            self.cur = self.start
        elif self.cur.next is None:
            self.cur = self.start
        else:
            q = self.cur.next
            self.cur = q
        return self.cur

queue = Queue()
start_time = 0
last_path = 0

def readable_time(t):
    h = t // 60 // 60
    m = t // 60 - h * 60
    s = t - m * 60
    return "%dh-%dm-%ds" % (h, m, s)

def status_screen(status):
    global queue, pid
    t = time.time()
    TERM_HOME = "\x1b[H"
    TERM_CLEAR = TERM_HOME + "\x1b[2J"
    print (TERM_CLEAR)
    print (" |=------------=[ frida-fuzz ]=------------|")
    print ("   target app       :", pid)
    print ("   output folder    :", output_folder)
    print ("   uptime           :", readable_time(t - start_time))
    print ("   last path        :", readable_time(last_path - start_time))
    print ("   queue size       :", queue.size)
    print ("   last stage       :", status["stage"])
    print ("   current testcase :", "<init>" if queue.cur is None else os.path.basename(queue.cur.filename))
    print ("   total executions :", status["total_execs"])
    print ("   execution speed  : %d/sec" % (status["total_execs"] / (t - start_time)))
    print (" |=----------------------------------------|\n")

with open("frida-fuzz-agent.js") as f:
    code = f.read()

device = frida.get_usb_device()
session = device.attach(pid)
#session = frida.attach(pid)
session.enable_jit()

script = session.create_script(code)

def on_interesting(message, data):
    global queue, last_path
    exec_us = message["exec_us"]
    new_cov = message["new_cov"]
    stage = message["stage"]
    last_path = time.time()
    queue.add(data, exec_us, new_cov, stage)

def on_next(message, data):
    global queue
    q = queue.get()
    with open(q.filename, "rb") as f:
        buf = f.read()
    script.post({
      "type": "input",
      "buf": buf.hex(),
      "was_fuzzed": q.was_fuzzed,
    })

def on_crash(message, data):
    global queue, script, session
    print ("\n"*2 + " ************** CRASH FOUND! **************")
    print ("    type:", message["err"]["type"])
    if "memory" in message["err"]:
        print ("    %s at:" % message["err"]["memory"]["operation"], message["err"]["memory"]["address"])
    print ("")
    t = int(time.time())
    name = os.path.join(output_folder, "crash_%s_%s_%d" % (message["stage"], message["err"]["type"], t))
    #name = os.path.join(output_folder, "crash_%d" % t)
    print (" >> Saving at %s" % name)
    with open(name, "wb") as f:
        f.write(data)
    print (" >> Press Control-C to exit...")
    script.unload()
    session.detach()

def on_stats(message, data):
    status_screen(message)

def on_message(message, data):
    if message["type"] == "error":
        print (" ************** FUZZER ERROR! **************")
        print ("  line %d: %s" % (message["lineNumber"], message["description"]))
        print ("  JS stacktrace:\n")
        print (message["stack"])
        print ("")
        print (" >> Press Control-C to exit...")
        script.unload()
        session.detach()
    msg = message["payload"]
    if msg["event"] == "interesting":
        on_interesting(msg, data)
        on_stats(msg, data)
    elif msg["event"] == "next":
        on_next(msg, data)
        on_stats(msg, data)
    elif msg["event"] == "crash":
        on_crash(msg, data)
    elif msg["event"] == "stats":
        on_stats(msg, data)

script.on('message', on_message)

script.load()

def signal_handler(sig, frame):
    print (" >> Exiting...")
    try:
        script.unload()
        session.detach()
    except: pass
    os._exit (0)
signal.signal(signal.SIGINT, signal_handler)

start_time = int(time.time())

last_path = start_time
queue.add(b"0000", 0, True, "init")

try:
    script.exports.loop()
except frida.InvalidOperationError as e:
    print (e)
    exit (1)


from __future__ import annotations

from django.urls import path

from .views import TriggerView, auth_check

app_name = "zapier_triggers"

urlpatterns = [
    # GET - the auth check URL
    path("auth/", auth_check, name="auth_check"),
    # GET - the "list" trigger endpoint
    path("<str:trigger>/", TriggerView.as_view(), name="list"),
    # POST - the "subscribe" endopint
    path("<str:trigger>/subscriptions/", TriggerView.as_view(), name="subscribe"),
    # DELETE - "unsubscribe" endpoint
    path(
        "<str:trigger>/subscriptions/<uuid:subscription_id>/",
        TriggerView.as_view(),
        name="unsubscribe",
    ),
]


from memawka import settings
from koparka_memow.settings import SETTINGS as FB_SETTINGS
import requests
import os

class TokenManager:

    @classmethod
    def _read_token(cls):
        with open(os.path.join(os.path.join(settings.BASE_DIR, 'koparka_memow'),
                               'access_token.txt'), 'r') as f:
            return f.read()

    @classmethod
    def _save_token(cls, token):
        with open(os.path.join(os.path.join(settings.BASE_DIR, 'koparka_memow'),
                               'access_token.txt'), 'w') as f:
            f.write(token)

    @classmethod
    def get_token(cls):
        try:
            return cls._read_token()

        except FileNotFoundError as e:

            cls._save_token(token='')
            print("insert a token into",
                  os.path.join(os.path.join(settings.BASE_DIR, 'koparka_memow'), 'access_token.txt')
                  )
            return None

    @classmethod
    def refresh_token(cls, token=None):
        if token is None:
            token = cls.get_token()
        link = (
            f'https://graph.facebook.com/oauth/access_token?'
            f'grant_type=fb_exchange_token&'
            f'client_id={FB_SETTINGS["app-id"]}&'
            f'client_secret={FB_SETTINGS["app-secret"]}&'
            f'fb_exchange_token={token}'
        )

        response = requests.get(link)
        cls._save_token(response.json()['access_token'])

if __name__ == '__main__':
    TokenManager.refresh_token()


from tkinter import *
from tkinter import filedialog
from tkinter.messagebox import showinfo
from tkinter.ttk import *
from openpyxl import *
from openpyxl.utils import get_column_letter
from os import walk
import os
import xlrd

class GUI:
    def __init__(self):
        self.window = Tk()

        self.window.title("Μετατροπή αρχείων xls σε xlsx")
        self.window.resizable(False, False)
        self.create_widgets()

    def getFilesDirName(self):
        dName = filedialog.askdirectory(initialdir="./data/", title="Επιλέξτε τον φάκελο με τα αρχεία προς μετατροπή")

        if dName == "":
            return

        self.filesDirName.set(dName)
        self.btnRun.configure(state='normal')

    def setColsWidth(self, ws):
        column_widths = []
        for row in ws.iter_rows():
            for i, cell in enumerate(row):
                try:
                    column_widths[i] = max(column_widths[i], len(str(cell.value)))
                except IndexError:
                    column_widths.append(len(str(cell.value)))

        for i, column_width in enumerate(column_widths):
            ws.column_dimensions[get_column_letter(i + 1)].width = column_width * 1.23

    def convertXls(self):
        inputDirectory = self.filesDirName.get()

        for path, dirs, files in walk(inputDirectory):
            for file in files:
                if file[-4:] == ".xls":
                    book_xls = xlrd.open_workbook(os.path.join(path, file))
                    book_xlsx = Workbook()

                    sheet_names = book_xls.sheet_names()
                    for sheet_index, sheet_name in enumerate(sheet_names):
                        sheet_xls = book_xls.sheet_by_name(sheet_name)
                        if sheet_index == 0:
                            sheet_xlsx = book_xlsx.active
                            sheet_xlsx.title = sheet_name
                        else:
                            sheet_xlsx = book_xlsx.create_sheet(title=sheet_name)

                        for row in range(0, sheet_xls.nrows):
                            for col in range(0, sheet_xls.ncols):
                                sheet_xlsx.cell(row=row + 1, column=col + 1).value = sheet_xls.cell_value(row, col)

                        self.setColsWidth(sheet_xlsx)

                    book_xlsx.save(os.path.join(path, file + 'x'))

    def run(self):
        self.btnRun.configure(state='disabled')
        self.convertXls()
        showinfo(title="Ολοκλήρωση εκτέλεσης", message="Η μετροπή των αρχείων ολοκληρώθηκε.")

    def create_widgets(self):
        self.fData = Frame(self.window)

        self.lData = Label(self.fData, text="Φάκελος με αρχεία\nπρος μετατροπή:")
        self.lData.grid(column=0, row=0, padx=10, pady=10, sticky=E)

        self.filesDirName = StringVar()
        self.ntrFilesDirName = Entry(self.fData, width=100, state='readonly', textvariable=self.filesDirName)
        self.ntrFilesDirName.grid(column=1, row=0, padx=10, pady=10, sticky=W)

        self.btnOpenData = Button(self.fData, text="Επιλέξτε φάκελο...", command=self.getFilesDirName)
        self.btnOpenData.grid(column=2, row=0, padx=10, pady=10)

        self.btnRun = Button(self.fData, text="Εκτέλεση", command=self.run, state='disabled')
        self.btnRun.grid(column=1, row=10, padx=10, pady=10)

        self.fData.pack()

gui = GUI()
gui.window.mainloop()


################
# José Lambrechts - @alt165
# UNRN Andina - Introducción a la Ingenieria en Computación
################
"""
2. Estadísticas

Implementar una función que obtenga los máximos, mínimos y promedio
de una secuencia con números, retornando los valores como una tuple.

Sin utilizar lazos for o las funciones integradas del lenguaje que
procesan secuencias.
"""

import random

def obtiene_maximo(secuencia):
    """obtiene_secuencia(lista[int..int]) -> int
    """
    maximo = secuencia[0]
    i = 1
    while i < len(secuencia):
        if maximo < secuencia[i]:
            maximo = secuencia[i]
        i = i + 1
    return maximo

def obtiene_minimo(secuencia):
    """obtiene_secuencia(lista[int..int]) -> int
    """
    minimo = secuencia[0]
    i = 1
    while i < len(secuencia):
        if minimo > secuencia[i]:
            minimo = secuencia[i]
        i = i + 1
    return minimo

def obtiene_promedio(secuencia):
    """obtiene_secuencia(lista[int..int]) -> float
    """
    total = 0
    i = 0
    while i < len(secuencia):
        total = total + secuencia[i]
        i = i + 1
    return total / i

def obtiene_secuencia(rango_minimo, rango_maximo, tamano):
    """obtiene_secuencia(int, int, int) -> list
    esta funcion devuelve una secuencia de enteros
    """
    i = 0
    secuencia = []
    while i < tamano:
        numero = random.randint(rango_minimo, rango_maximo)
        secuencia.append(numero)
        i = i + 1
    return secuencia

def principal():
    """
    Esta función es la que se encarga de la parte 'interactiva' del ejercicio
    (La entrada, la llamada al algoritmo y la salida)
    """
    lista = obtiene_secuencia(-10, 10, 5)
    maximo = obtiene_maximo(lista)
    minimo = obtiene_minimo(lista)
    promedio = obtiene_promedio(lista)
    print(f"En la lista {lista}")
    print(f"El máximo es: {maximo}")
    print(f"El mínimo es: {minimo}")
    print(f"El promedio es: {promedio:.2}")

if __name__ == "__main__":
    principal()


import requests

with open('input.txt', 'r') as inp:
    for line in inp:
        number = line.strip()
        # print(number)

        url = f'http://numbersapi.com/{number}/math'
        print(url)
        params = {
            'json': 'true'
        }
        res = requests.get(url, params=params)
        # print(res.status_code)
        # print(res.headers['Content-Type'])
        # print(res.json())
        data = res.json()
        if data['found']:
            print('Interesting')
        else:
            print('Boring')


import sys
import os
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtMultimediaWidgets import QVideoWidget

import resources

class Ui_NewProject(object):
    def setupUi(self, NewProject):
        NewProject.setObjectName("NewProject")
        NewProject.resize(736, 625)
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap(":/plugins/VideoGis/icon.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        NewProject.setWindowIcon(icon)
        self.gridLayout_2 = QtWidgets.QGridLayout(NewProject)
        self.gridLayout_2.setObjectName("gridLayout_2")
        self.gridLayout = QtWidgets.QGridLayout()
        self.gridLayout.setObjectName("gridLayout")
        self.horizontalLayout = QtWidgets.QHBoxLayout()
        self.horizontalLayout.setObjectName("horizontalLayout")
        self.video_frame_2 = QVideoWidget(NewProject)
        p = self.video_frame_2.palette()
        p.setColor(QtGui.QPalette.Window, QtCore.Qt.black)
        self.video_frame_2.setPalette(p)
        self.video_frame_2.setAttribute(QtCore.Qt.WA_OpaquePaintEvent)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.video_frame_2.sizePolicy().hasHeightForWidth())
        self.video_frame_2.setSizePolicy(sizePolicy)
        self.video_frame_2.setStyleSheet("background-color: rgb(0, 0, 10);")
        self.video_frame_2.setObjectName("video_frame_2")

        # self.scroll = QtWidgets.QScrollArea()#+
        # self.scroll.setWidget(self.video_frame_2)#+
        # self.video_frame_2.setGeometry(0,0,736,625)
        # self.horizontalLayout.addWidget(self.scroll)

        self.horizontalLayout.addWidget(self.video_frame_2)

        self.gridLayout.addLayout(self.horizontalLayout, 0, 0, 1, 16)
        self.horizontalSlider = QtWidgets.QSlider(NewProject)
        self.horizontalSlider.setOrientation(QtCore.Qt.Horizontal)
        self.horizontalSlider.setObjectName("horizontalSlider")
        self.gridLayout.addWidget(self.horizontalSlider, 1, 0, 1, 16)
        self.replayPlay_pushButton = QtWidgets.QPushButton(NewProject)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.replayPlay_pushButton.sizePolicy().hasHeightForWidth())
        self.replayPlay_pushButton.setSizePolicy(sizePolicy)
        self.replayPlay_pushButton.setCheckable(False)
        self.replayPlay_pushButton.setChecked(False)
        self.replayPlay_pushButton.setObjectName("replayPlay_pushButton")
        self.gridLayout.addWidget(self.replayPlay_pushButton, 3, 1, 1, 1)
        self.replayPosition_label = QtWidgets.QLabel(NewProject)
        self.replayPosition_label.setObjectName("replayPosition_label")
        self.gridLayout.addWidget(self.replayPosition_label, 3, 4, 1, 1)
        self.muteButton = QtWidgets.QToolButton(NewProject)
        self.muteButton.setText("")
        self.muteButton.setObjectName("muteButton")
        self.gridLayout.addWidget(self.muteButton, 3, 2, 1, 1)
        self.comboBox = QtWidgets.QComboBox(NewProject)
        self.comboBox.setObjectName("comboBox")
        self.gridLayout.addWidget(self.comboBox, 3, 15, 1, 1)
        self.pushButton_2 = QtWidgets.QPushButton(NewProject)
        self.pushButton_2.setObjectName("pushButton_2")
        self.gridLayout.addWidget(self.pushButton_2, 3, 13, 1, 1)
        self.toolButton_3 = QtWidgets.QToolButton(NewProject)
        icon1 = QtGui.QIcon()
        icon1.addPixmap(QtGui.QPixmap(":/plugins/VideoGis/mIconFormSelect.svg"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.toolButton_3.setIcon(icon1)
        self.toolButton_3.setObjectName("toolButton_3")
        self.gridLayout.addWidget(self.toolButton_3, 3, 11, 1, 1)

        # <<
        self.toolButton = QtWidgets.QToolButton(NewProject)
        icon5 = QtGui.QIcon()
        icon5.addPixmap(QtGui.QPixmap(":/VgisIcon/mActionAtlasPrev.svg"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.toolButton.setIcon(icon5)
        self.toolButton.setObjectName("toolButton")
        self.gridLayout.addWidget(self.toolButton, 3, 6, 1, 1)

        # >>
        self.toolButton_2 = QtWidgets.QToolButton(NewProject)
        icon2 = QtGui.QIcon()
        icon2.addPixmap(QtGui.QPixmap(":/VgisIcon/mActionAtlasNext.svg"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.toolButton_2.setIcon(icon2)
        self.toolButton_2.setObjectName("toolButton_2")
        self.gridLayout.addWidget(self.toolButton_2, 3, 9, 1, 1)

        # <<<
        self.toolButton_6 = QtWidgets.QToolButton(NewProject)
        icon2 = QtGui.QIcon()
        icon2.addPixmap(QtGui.QPixmap(":/VgisIcon/mActionAtlasNext.svg"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.toolButton_6.setIcon(icon2)
        self.toolButton_6.setObjectName("toolButton_6")
        self.gridLayout.addWidget(self.toolButton_6, 3, 5, 1, 1)

        # >>>
        self.toolButton_5 = QtWidgets.QToolButton(NewProject)
        icon2 = QtGui.QIcon()
        icon2.addPixmap(QtGui.QPixmap(":/VgisIcon/mActionAtlasNext.svg"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.toolButton_5.setIcon(icon2)
        self.toolButton_5.setObjectName("toolButton_5")
        self.gridLayout.addWidget(self.toolButton_5, 3, 10, 1, 1)

        # >
        self.SkipFortoolButton_8 = QtWidgets.QToolButton(NewProject)
        self.SkipFortoolButton_8.setStyleSheet("")
        icon3 = QtGui.QIcon()
        icon3.addPixmap(QtGui.QPixmap(":/VgisIcon/mActionArrowRight.svg"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.SkipFortoolButton_8.setIcon(icon3)
        self.SkipFortoolButton_8.setObjectName("SkipFortoolButton_8")
        self.gridLayout.addWidget(self.SkipFortoolButton_8, 3, 8, 1, 1)

        self.SkipBacktoolButton_7 = QtWidgets.QToolButton(NewProject)
        self.SkipBacktoolButton_7.setStyleSheet("")
        icon4 = QtGui.QIcon()
        icon4.addPixmap(QtGui.QPixmap(":/VgisIcon/mActionArrowLeft.svg"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.SkipBacktoolButton_7.setIcon(icon4)
        self.SkipBacktoolButton_7.setObjectName("SkipBacktoolButton_7")
        self.gridLayout.addWidget(self.SkipBacktoolButton_7, 3, 7, 1, 1)

        self.pushButton = QtWidgets.QPushButton(NewProject)
        self.pushButton.setObjectName("pushButton")
        self.gridLayout.addWidget(self.pushButton, 3, 0, 1, 1)
        self.toolButton_4 = QtWidgets.QToolButton(NewProject)
        self.toolButton_4.setObjectName("toolButton_4")
        self.gridLayout.addWidget(self.toolButton_4, 3, 12, 1, 1)
        self.checkBox = QtWidgets.QCheckBox(NewProject)
        self.checkBox.setObjectName("checkBox")
        self.gridLayout.addWidget(self.checkBox, 3, 14, 1, 1)
        self.gridLayout_2.addLayout(self.gridLayout, 0, 0, 1, 1)

        self.retranslateUi(NewProject)
        QtCore.QMetaObject.connectSlotsByName(NewProject)

    def retranslateUi(self, NewProject):
        _translate = QtCore.QCoreApplication.translate
        NewProject.setWindowTitle(_translate("NewProject", "Video UAV Tracker - New Project"))
        self.replayPlay_pushButton.setText(_translate("NewProject", "Play/Pause"))
        self.replayPosition_label.setText(_translate("NewProject", "-:- / -:-"))
        self.pushButton_2.setToolTip(_translate("NewProject", "
Synchronize actual video frame with selected GPS time
"))
        self.comboBox.setToolTip(_translate("NewProject", "
 GPS time
"))
        self.pushButton_2.setText(_translate("NewProject", "Synchronize!"))
        self.toolButton_3.setToolTip(_translate("NewProject", "
Add point shape database to project
"))
        #self.toolButton_3.setText(_translate("NewProject", "DB"))

        self.toolButton_2.setToolTip(_translate("NewProject", "
Next second
"))
        self.toolButton_2.setText(_translate("NewProject", ">>"))

        self.toolButton_6.setToolTip(_translate("NewProject", "
Previous 1min
"))
        self.toolButton_6.setText(_translate("NewProject", "<<<"))

        self.toolButton_5.setToolTip(_translate("NewProject", "
Next 1min
"))
        self.toolButton_5.setText(_translate("NewProject", ">>>"))

        self.SkipFortoolButton_8.setToolTip(_translate("NewProject", "
Next frame
"))
        self.SkipFortoolButton_8.setText(_translate("NewProject", ">"))
        self.SkipBacktoolButton_7.setToolTip(_translate("NewProject", "
Previous frame
"))
        self.SkipBacktoolButton_7.setText(_translate("NewProject", "<"))
        self.toolButton.setToolTip(_translate("NewProject", "
Previous second
"))
        self.toolButton.setText(_translate("NewProject", "<<"))
        self.pushButton.setToolTip(_translate("NewProject", "
Select video and relative gpx
"))
        self.pushButton.setText(_translate("NewProject", "Select Video and GPX"))
        self.toolButton_4.setText(_translate("NewProject", "3D"))
        self.checkBox.setToolTip(_translate("NewProject","
Activate Pixel value conversion and display (see README)
"))


import asyncio
import itertools as it
import os
import random
import time

async def makeitem(size: int = 5) -> str:
    return os.urandom(size).hex()

async def randsleep(caller=None) -> None:
    """Sleep for a random amount of time between 0 and 2 seconds.

    Example::

        asyncio.run(randsleep(caller="Producer 1"))
    """
    i = random.randint(0, 2)
    if caller:
        print(f"{caller} sleeping for {i} seconds.")
    await asyncio.sleep(i)

async def produce(name: int, q: asyncio.Queue) -> None:
    """Produce random data and put it into the queue.

    Example::

        async def main():
            q = asyncio.Queue()
            # have 2 producers
            await asyncio.gather(produce(1, q), produce(2, q))
            #print out q current size
            print(f"q size: {q.qsize()}")

        asyncio.run(main())
    """
    n = random.randint(0, 10)
    print(f"Producer {name} will produce {n} items.")
    for _ in it.repeat(None, n):  # Synchronous loop for each single producer
        await randsleep(caller=f"Producer {name}")
        i = await makeitem()
        t = time.perf_counter()
        await q.put((i, t))
        print(f"Producer {name} added <{i}> to queue.")

async def consume(name: int, q: asyncio.Queue) -> None:
    """Produce random data and put it into the queue.

    Example::

        async def main():
            q = asyncio.Queue()
            # put 2 items into the queue
            await q.put((1, time.perf_counter()))
            await q.put((2, time.perf_counter()))
            # 2 consumer as task
            consumers = [asyncio.create_task(consume(1, q)),
                            asyncio.create_task(consume(2, q))]
            # wait for the queue to be empty
            await q.join()
            # cancel the consumers
            for c in consumers:
                c.cancel()
            #print out q current size
            print(f"q size: {q.qsize()}")

        asyncio.run(main())
    """
    while True:
        await randsleep(caller=f"Consumer {name}")
        i, t = await q.get()
        now = time.perf_counter()
        print(f"Consumer {name} got element <{i}>" f" in {now-t:0.5f} seconds.")
        print(f"q size: {q.qsize()}")
        q.task_done()

async def main(nprod: int, ncon: int):
    q = asyncio.Queue()
    producers = [asyncio.create_task(produce(n, q)) for n in range(nprod)]
    consumers = [asyncio.create_task(consume(n, q)) for n in range(ncon)]
    await asyncio.gather(*producers)
    await q.join()  # Implicitly awaits consumers, too
    for c in consumers:
        c.cancel()

if __name__ == "__main__":
    import argparse

    random.seed(444)
    parser = argparse.ArgumentParser()
    parser.add_argument("-p", "--nprod", type=int, default=3)
    parser.add_argument("-c", "--ncon", type=int, default=4)
    ns = parser.parse_args()
    start = time.perf_counter()
    asyncio.run(main(**ns.__dict__))
    elapsed = time.perf_counter() - start
    print(f"Program completed in {elapsed:0.5f} seconds.")


# import statsapi as mlbstats
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import pybaseball as pbb
from datetime import date
from matplotlib import gridspec
from matplotlib import gridspec

df_sch = pd.read_csv('mets_2023.csv')
df_sch.pop('Unnamed: 0') #artifact from saving csv in other code

other_team = [] # list of other teams played

for index, row in df_sch.iterrows():

    if row['home_id'] == 121:
        other_team.append(row['away_name'])
    else:
        other_team.append(row['home_name'])

df_sch['other_team'] = other_team

team_set = set(other_team) #only want unique values of other teams

df_teams = pd.DataFrame({'team': [t for t in team_set]})

df_teams['won'] = np.zeros(len(team_set))
df_teams['lost'] = np.zeros(len(team_set))
df_teams['incomplete'] = np.zeros(len(team_set))
df_teams['total'] = np.zeros(len(team_set))
df_teams['inc_home'] = np.zeros(len(team_set))
df_teams['inc_away'] = np.zeros(len(team_set))

### Not optimal method, should optimize to list comprehension in future version
for index, row in df_sch.iterrows():
    if row['status'] == 'Scheduled':
        df_teams.loc[df_teams['team']==row['other_team'], ['incomplete']] += 1
        if row['other_team'] == row['away_name']:
            df_teams.loc[df_teams['team']==row['other_team'], ['inc_home']] += 1
        else:
            df_teams.loc[df_teams['team']==row['other_team'], ['inc_away']] += 1

    elif row['winning_team'] == 'New York Mets':
        df_teams.loc[df_teams['team']==row['other_team'], ['won']] += 1

    else:
        df_teams.loc[df_teams['team']==row['other_team'], ['lost']] += 1

    df_teams.loc[df_teams['team']==row['other_team'], ['total']] += 1

### Preseason messes up pybaseball.standings:
try:
    standings = pd.concat(pbb.standings(), ignore_index=True)
    team_rec = [f"{standings.loc[standings['Tm']==team, ['W']].values[0][0]}-{standings.loc[standings['Tm']==team, ['L']].values[0][0]}" for team in df_teams['team']]
except:
    team_rec = ["0-0" for team in df_teams['team']]

nicknames = [] ## Adding city names crowds up the labels too much
for i,t in enumerate(df_teams['team']):
    if t.split()[-1] == "Sox": #Accounting for Boston/Chicago Sox
        nicknames.append(f"{t.split()[-2]+ ' ' + t.split()[-1] } ({team_rec[i]})")
    else:
        nicknames.append(f"{t.split()[-1]} ({team_rec[i]})")
df_teams['team'] = nicknames

### Order for the teams to be plotted in, consistent throughout the season
df_teams.sort_values(['total', 'team'], ascending=[False, True], inplace=True, ignore_index=True)

_, won, lost, incomplete, total, inc_home, inc_away = df_teams.sum().values

won = round(won)
lost = round(lost)
incomplete = round(incomplete)
incomplete_home = round(inc_home)
incomplete_away = round(inc_away)

### Trying new way due to 50% more teams:
fig = plt.figure(tight_layout=True)
gs = gridspec.GridSpec(2, 2, height_ratios=[1,12])

w = 0.9 #main bar width
x = [np.arange(15), np.arange(14)]

### Total season progress bar:
ax = fig.add_subplot(gs[0, :])

if won != 0:
    wins   = ax.barh(0, won, w, label='won',
                        color="cornflowerblue", edgecolor='k')
    ax.text(won/2, 0, round(won) ,weight='bold', va='center', ha='center',
                    size=18, fontname='serif')
if lost != 0:
    losses = ax.barh(0, lost, w, left=won,
                        label='lost', color="darkorange", edgecolor='k')
    ax.text(won+lost/2, 0, round(lost), weight='bold', va='center', ha='center',
                    size=18, fontname='serif')
if inc_home != 0:
    home   = ax.barh(0, inc_home, w, left=won+lost,
                        label="incomplete home", color='silver', edgecolor='k')
    ax.text(won+lost+inc_home/2, 0, round(inc_home),
                    weight='bold', va='center', ha='center',
                    size=18, fontname='serif')

if inc_away != 0:
    away   = ax.barh(0, inc_away, w, left=won+lost+inc_home,
                        label="incomplete away", color='k', edgecolor='k')
    ax.text(won+lost+inc_home+inc_away/2, 0, round(inc_away),
                    weight='bold', va='center', ha='center', color='w',
                    size=18, fontname='serif')

ax.axis('off')
ax.set_title('Total Season:', size=20, weight='bold',
             fontname='serif')

ax.set_xlim(0, 162)

### Individual team progress bars, broken into two plots
for i in range(2):
    ax = fig.add_subplot(gs[1, i])
    wins       = ax.barh(x[i], df_teams['won'][i::2], w, label='won',
                        color="cornflowerblue", edgecolor='k')

    losses     = ax.barh(x[i], df_teams['lost'][i::2], w, left=df_teams['won'][i::2],
                        label='lost', color="darkorange", edgecolor='k')

    home       = ax.barh(x[i], df_teams['inc_home'][i::2], w,
                          left=df_teams['won'][i::2]+df_teams['lost'][i::2],
                        label="incomplete home", color='silver', edgecolor='k')

    away       = ax.barh(x[i], df_teams['inc_away'][i::2], w,
                         left=df_teams['won'][i::2]+df_teams['lost'][i::2] + df_teams['inc_home'][i::2],
                        label="incomplete away", color='k', edgecolor='k')

    for j in x[i]:
        if wins.datavalues[j] != 0: #Ignore values that are currently 0
            ax.text(wins.datavalues[j]/2, x[i][j], int(wins.datavalues[j]),
                    weight='bold', va='center', ha='center',
                    size=18, fontname='serif')

        if losses.datavalues[j] != 0:
            ax.text(wins.datavalues[j] + losses.datavalues[i]/2, x[i][j], int(losses.datavalues[j]),
                    weight='bold', va='center', ha='center',
                    size=18, fontname='serif')

        if home.datavalues[j] != 0:
            ax.text(wins.datavalues[j] + losses.datavalues[j] + home.datavalues[j]/2, x[i][j], int(home.datavalues[j]),
                    weight='bold', va='center', ha='center',
                    size=18, fontname='serif')

        if away.datavalues[j] != 0:
            ax.text(wins.datavalues[j] + losses.datavalues[j] + home.datavalues[j] + away.datavalues[j]/2, x[i][j], int(away.datavalues[j]),
                    weight='bold', va='center', ha='center',
                    size=18, fontname='serif', color='w')

    ax.set_ylim(min(x[0])-w/2, max(x[0])+w/2)
    ax.invert_yaxis()
    ax.set_xticks(range(0, 21, 5), fontfamily='serif', minor=False)
    ax.set_xticks(range(0, 21, 1), fontfamily='serif', minor=True)
    ax.tick_params('x', labelsize='large')
    ax.set_yticks(ticks = x[i], labels=df_teams['team'][i::2], size=18, fontfamily='serif')
    ax.set_xlim(0, df_teams['total'].max())

handles, labels = ax.get_legend_handles_labels()
fig.legend(handles, labels, loc='upper center', ncol=4, bbox_to_anchor=(0.5, 0.05),
           prop={'family': "serif", 'size':'x-large', 'weight': 'bold'}, framealpha=0)

fig.suptitle(f"Mets Current Season Series Progress\n{date.today()} ({won}-{lost}, {incomplete} GR)"
             , size=24, weight='bold', fontname='serif', y=0.9, va='bottom')

plt.show()


r = float(input()) #smaller radius
R = float(input()) #larger radius

n = int((0.83* (R**2)/(r**2)) - 1.9)

area = (n*(r**2)/R**2)*100
area = format(area,'.2f')

print(n,'smaller circles cover',area+'% of the larger circle')


#!/usr/bin/env python

import curses

def simple_curses_menu(prompt, choices):
  def impl(stdscr, prompt, choices):
    curses.use_default_colors()
    selected = 0
    ch = curses.KEY_RESIZE
    while ch != ord('\n'):
      if ch == curses.KEY_DOWN:
        selected = min(selected + 1, len(choices) - 1)
      elif ch == curses.KEY_UP:
        selected = max(0, selected - 1)
      (height, width) = stdscr.getmaxyx()
      stdscr.addnstr(0, 0, prompt, width, curses.A_BOLD)
      for (i, choice) in enumerate(choices):
        if i + 1 < height:
          stdscr.addnstr(1 + i, 0, choice, width)
          stdscr.clrtoeol()
      number_string = "({0}/{1})".format(selected + 1, len(choices))
      stdscr.addnstr(min(height-1, selected+1), 0, choices[selected], width - len(number_string), curses.A_REVERSE)
      stdscr.addstr(min(height-1, selected+1), width - len(number_string), number_string)
      ch = stdscr.getch()
    stdscr.clear()
    return selected
  return curses.wrapper(impl, prompt, choices)


from math import log
from preProcess import preProcess
from sklearn.model_selection import KFold
from sklearn.metrics import confusion_matrix
from collections import Counter
from testResults import testResults
import numpy as np

class node:
    def __init__(self, attribute=None, depth=None):
        # Add a new node with a particular attribute value and depth
        self.depth = depth
        self.children = [None] * 10
        self.attribute = attribute
        self.definiteLabel = None

    def assign_child(self, attribute_value, child):
        # Assigning children at particular attribute value
        self.children[attribute_value] = child

    def assign_label(self, label):
        # Assigning a definite label when the node is a leaf node
        self.definiteLabel = label

class decisionTree:
    def __init__(self, criterion='entropy', max_depth=None, min_samples_split=2):
        # Initialize a decision tree with parameters
        # max_depth refers to maximum depth of the tree
        # min_samples_built refers to the minimum samples needed to build a subtree
        self.criterion = criterion
        self.max_depth = max_depth
        self.min_samples_split = min_samples_split
        self.tree_built = False

    def find_entropy(self, dataset):
        # finds the entropy in a particular dataset using shannonEntropy measure
        numEntries = len(dataset)
        labelCounts = Counter()
        for i in range(numEntries):
            labelCounts[dataset[i][-1]] += 1
        shannonEntropy = 0
        for label in labelCounts:
            prob = float(labelCounts[label])/numEntries
            shannonEntropy -= prob*log(prob, 2)
        return shannonEntropy

    def perform_split(self, dataset, axis, value):
        # performs split at a particular axis of the dataset for a particular value and returns newDataset
        newDataset = []
        listDataset = list(dataset)
        for sample in listDataset:
            if sample[axis] == value:
                reducedFeatureVector = list(sample[:axis])
                reducedFeatureVector.extend(list(sample[axis+1:]))
                newDataset.append(reducedFeatureVector)
        return np.array(newDataset)

    def chooseBestFeatureToSplit(self, dataset):
        # finds the best attribute to split at based on the entropy value
        numFeatures = len(dataset[0])-1
        baseEntropy = self.find_entropy(dataset)
        bestInfoGain = 0
        bestFeature = -1
        for i in range(numFeatures):
            entropyVal = 0
            for value in range(1, 11):
                subDataset = self.perform_split(dataset, i, value)
                prob = len(subDataset) / float(len(dataset))
                entropyVal += prob*self.find_entropy(subDataset)
            infoGain = baseEntropy - entropyVal
            if infoGain > bestInfoGain:
                bestInfoGain = infoGain
                bestFeature = i
        return bestFeature

    def returnClassProbabilities(self, dataset):
        # returns the probabilites of 2 classes of the dataset
        sampleCount = len(dataset)
        labelCounts = Counter()
        for i in range(sampleCount):
            if dataset[i][-1] == 2:
                labelCounts[2] += 1
            else:
                labelCounts[4] += 1
        return [labelCounts[2]/float(sampleCount), labelCounts[4]/float(sampleCount)]

    def recursiveBuild(self, dataNode, dataset):
        # recursively builds the nodes
        probs = self.returnClassProbabilities(dataset)
        probs = sorted(self.returnClassProbabilities(dataset), reverse=True)
        # for the following conditions, the node stops building further and assigns class with highest count as definitiveLabel
        if probs[0] > 90 or dataNode.depth == self.max_depth or len(dataset)<self.min_samples_split:
            dataNode.assign_label(probs[0])
            return

        dataNode.labelSamples = [2,4][np.argmax(probs)]
        for i in range(1, 11):
            # performing n-way split
            splitDataset = self.perform_split(dataset, dataNode.attribute, i)
            if len(splitDataset):
                best_feature = self.chooseBestFeatureToSplit(splitDataset)
                # if bestFeature isn't found, we assign that node as a leaf node
                if best_feature==-1:
                    subProbs = self.returnClassProbabilities(splitDataset)
                    dataNode.children[i-1] = node(depth = dataNode.depth+1)
                    dataNode.children[i-1].assign_label([2,4][np.argmax(subProbs)])
                # else we assign the best feature as the attribute to the child node and recursively build it
                else:
                    dataNode.children[i-1] = node(attribute=best_feature, depth = dataNode.depth+1)
                    self.recursiveBuild(dataNode.children[i-1], splitDataset)

    def fit(self, dataset):
        # fits the dataset to the model and recursively starts building the decision tree
        self.tree_built = True
        self.root = node(attribute = self.chooseBestFeatureToSplit(dataset), depth=0)
        self.recursiveBuild(self.root, dataset)

    def predict(self, groups):
        # predicts for group of samples
        results = []
        for group in groups:
            results.append(self.predict_single(group))
        return np.array(results)

    def predict_single(self, x):
        # predicts for single sample
        available_features = list(x)
        if self.tree_built:
            pres_node = self.root
            while pres_node.definiteLabel == None:
                attribute = pres_node.attribute
                if pres_node.children[available_features[attribute]-1]:
                    pres_node = pres_node.children[available_features[attribute]-1]
                    available_features.pop(attribute)
                else: return pres_node.labelSamples
        return pres_node.definiteLabel

if __name__ == '__main__':
    preProcessData = preProcess()
    preProcessData.handle_missing_values()
    preProcessData.handle_highly_correlated_features()
    df = preProcessData.return_df()
    DT = decisionTree()
    kFold = KFold(6, True, 1)
    values = df.values
    for train, test in kFold.split(values):
        print("Taking %d train datapoints" % len(train))
        train_x = values[train]
        test_x, test_y = values[test][:,:-1], values[test][:,-1]
        DT.fit(train_x)
        pred_y = DT.predict(test_x)
        results = testResults(pred_y, test_y)
        print("Accuracy of model is ", results.return_accuracy())
        print('F Score of model is ', results.return_fscore())
        print("Confusion Matrix for the model :\n", confusion_matrix(test_y, pred_y))
        print()
        # print(DT.predict_single(test_x[0]), test_y[0])


"""
This file reads the CSV file containing the lyrics data by chunks.
The path to the CSV file should be ./Data/song_lyrics.csv
For each row the data is processed.
This file saves the processed data chunks in the folders: Data/InputData & Data/OutputData

Content of the saved files:
    Data/InputData -> npy files containing a matrix of the input data.
    Data/InputDataNotNorm -> npy files containing a matrix of the non normalized input data.
    Data/OutputData -> npy files containing a matrix of the output data.
"""

import nltk, re, string, logging
import numpy as np
import pandas as pd
import gensim.downloader as api
from nltk.corpus import stopwords
import time
import threading
import os

START_FROM = int(input("Start from: "))

word2vec_model = api.load("word2vec-google-news-300")
print("---- Loading of Word2Vec model completed")

def data_clean(text):
    """
    Cleans the input text by removing metadata, stopwords, and punctuation.

    Args:
    text (str): The input text to be cleaned.

    Returns:
    list: A list of cleaned words.
    """

    text = re.sub(r'\[.*?\]', '', text) # Remove metadata from text using regular expressions
    text.replace('\n',' ')

    words = nltk.tokenize.wordpunct_tokenize(text)

    text_result = []
    for word in words: # for each word in the tokenized list of words from the sentence
        if word.lower() not in stopwords.words('english') and word[0] not in string.punctuation and word.lower() not in text_result: # if the word not a stop word, word not already in vocab and word not punctuation
            text_result.append(word.lower())

    return text_result

def process_data(start_row=START_FROM, chunk_size=250000, verbose=True):
    empty_songs_counter = 0
    if verbose:
        print(f"---- Chunk size = {chunk_size}")

    # Define the vectors for "rap", "pop" and others
    pop_vector =        np.array([1,0,0,0,0,0])
    rap_vector =        np.array([0,1,0,0,0,0])
    rock_vector =       np.array([0,0,1,0,0,0])
    rb_vector =         np.array([0,0,0,1,0,0])
    country_vector =    np.array([0,0,0,0,1,0])
    other_vector =      np.array([0,0,0,0,0,1])

    # Define the counter for the number of songs processed
    count = start_row
    progress = start_row
    if verbose:
        print(f"---- Starting from {count}")

    timed = time.time()

    # Iterate through the CSV file in chunks
    for chunk in pd.read_csv('./Data/song_lyrics.csv', skiprows=range(1, start_row), chunksize=chunk_size):

        # declare arrays of input and output
        input_data = []
        input_data_non_normalized = []
        output_data = []

        # update counters
        count+= chunk_size

        # for each row (song)
        for index, row in chunk.iterrows():

            # Show progress
            progress+=1
            if progress%500==0:
                if verbose:
                    print(f"Progress -> {progress} || Time taken -> {round(time.time()-timed,2)}")
                    timed = time.time()

            # Process each row

            if row['language']=="en": # if the lyrics are in english

                ##_________________Input processing_________________
                sum_word_vecs = np.zeros(300) # initialise sum of word vectors
                total_number_words = 0 # number of words transformed to vectors

                for word_to_vectorize in data_clean(row['lyrics']): # for each word in the cleaned, tokenized list from the lyrics
                    # print(word_to_vectorize)
                    try: # vectorise the word
                        sum_word_vecs+= word2vec_model[word_to_vectorize]
                        total_number_words+=1
                    except Exception as e:
                        # print(f"failed on: {word_to_vectorize} error: {e}")
                        pass

                if total_number_words>=10: # if the lyrics contain more than 10 words

                    tot_vec = sum_word_vecs/total_number_words # mean of all the vectors
                    input_data_non_normalized.append(tot_vec)
                    input_data.append(tot_vec/np.linalg.norm(tot_vec)) # append the normalized vector to the inpu_data list
                    ##____________________________________________________________________

                    ##_________________Output processing_________________
                    # process output data
                    tag = row['tag']
                    if tag == 'rap':
                        output_data.append(rap_vector)
                    elif tag == 'pop':
                        output_data.append(pop_vector)
                    elif tag == 'rock':
                        output_data.append(rock_vector)
                    elif tag == 'rb':
                        output_data.append(rb_vector)
                    elif tag == 'country':
                        output_data.append(country_vector)
                    else:
                        output_data.append(other_vector)
                    ##____________________________________________________________________
                else:
                    print(f"Not enough words: {progress}, found only {total_number_words}")
                    empty_songs_counter +=1

        np.save(f'{save_directories[0]}/I_data_chunk_{count}.npy', np.array(input_data)) # save input data matrix
        np.save(f'{save_directories[1]}/I_data_chunk_{count}.npy', np.array(input_data_non_normalized)) # save input data matrix
        np.save(f'{save_directories[2]}/O_data_chunk_{count}.npy', np.array(output_data)) # save output data matrix

        print(f"Number of rows processed: from {start_row} to {count} = {(count-start_row)}") # print how many rows have been processed
        print(f"Number of empty songs encountered: {empty_songs_counter}")

    print("DONE. CHUNK COMPLETED!!!")

# Helper function to calculate the starting indexes for multithreading the whole set
def get_starting_indexes(goal=5000000, chunksize=250000, threads=5):
    starting_indexes = []

    adder = goal/threads
    i=0

    while (threads*i) < goal:
        k = i
        indexes = []
        while k < goal:
            indexes.append(int(k))
            k += adder
        i += chunksize
        starting_indexes.append(indexes)

    return starting_indexes

# ______________________________EXECUTION OF THE FUNCTIONS________________________________________________
# Directories where all the output will be saved
save_directories = ['Data/InputData_Complete',
                   'Data/InputData_Complete_NotNorm',
                   'Data/OutputData_Complete']
for directory in save_directories:
    os.makedirs(directory, exist_ok=True)

# Call function wich will process only the output and start at the row given as global argument
# process_data(chunk_size=2, verbose=True)

# ________________________________________________________________________________________________________
# SINGLE THREAD, MULTIPLE TERMINALS:
chunksize = 50000
threads = 4

starting_2d = get_starting_indexes(chunksize=chunksize, threads=threads)
starting_2d = np.array(starting_2d)
starting_2d = np.transpose(starting_2d)

for starting_index in starting_2d[START_FROM]:
    process_data(start_row=starting_index, chunk_size=chunksize, verbose=True)
# ________________________________________________________________________________________________________

# ________________________________________________________________________________________________________
# Multi threaded Approach
# chunksize = 50000
# threads = 8

# starting_2d = get_starting_indexes(chunksize=chunksize, threads=threads)

# # for serie in starting_2d:
# def process_data_threaded(start_row):
#     process_data(start_row=start_row, chunk_size=chunksize, verbose=False)

# threads = []

# for starting_index in serie:
#     thread = threading.Thread(target=process_data_threaded, args=(starting_index,))
#     threads.append(thread)
#     thread.start()

# # Wait for all threads to finish
# for thread in threads:
#     thread.join()

# print("\n\n!!!DONE!!! \n\n", serie, "\n\n\n")
# ________________________________________________________________________________________________________


#!/usr/bin/env python
import cv2
import numpy as np
import os
import glob
if __name__ == '__main__':
    filepath = 'video.flv'

    npzfile = np.load('calibr_parameters.npz')
    mtxstr, diststr = npzfile.files
    mtx = npzfile[mtxstr]
    dist = npzfile[diststr]

    cap = cv2.VideoCapture(filepath)
    if not cap.isOpened():
        print("VideoCapture failed...")
    else:
        print("Video opened! ...")
    fps = cap.get(cv2.CAP_PROP_FPS)
    print("FPS:", fps)

    ret, frame = cap.read()         # Read first frame to get info about the resolution
    h,w = frame.shape[:2]
    newcameramtx, roi = cv2.getOptimalNewCameraMatrix(mtx,dist,(w,h),0,(w,h))
    x,y,w,h = roi
    out = cv2.VideoWriter('undistorted.avi', cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), 15, (w, h))

    dst = cv2.undistort(frame, mtx, dist, None, newcameramtx)
    dst = dst[y:y+h, x:x+w]
    out.write(dst)

    while cap.isOpened():
        ret, frame = cap.read()
        if np.shape(frame) == ():
            continue

        dst = cv2.undistort(frame, mtx, dist, None, newcameramtx)
        dst = dst[y:y+h, x:x+w]
        out.write(dst)
        #cv2.imshow("video", dst)
        #k = cv2.waitKey(30) & 0xff                   # if 'Esc' is pressed then quit
        #if k == 27:
        #    break

    cv2.destroyAllWindows()
    out.release()
    cap.release()


import os
import time
import pickle
import json

import numpy as np
from copy import deepcopy
from tqdm import tqdm
import tensorflow as tf
# from dmn.attention_gru_cell import AttentionGRUCell
from dmn.attention_gru_cell import AttentionGRUCell
from tensorflow.contrib.cudnn_rnn.python.ops import cudnn_rnn_ops

# from dmn.dmn_fasttext.config import Config
from dmn.dmn_fasttext.config import Config
# config = Config()

def _add_gradient_noise(t, stddev=1e-3, name=None):
    """Adds gradient noise as described in http://arxiv.org/abs/1511.06807
    The input Tensor `t` should be a gradient.
    The output will be `t` + gaussian noise.
    0.001 was said to be a good fixed value for memory networks."""
    # with tf.op_scope([t, stddev], name, "add_gradient_noise") as name:
    with tf.name_scope(name, "add_gradient_noise", [t, stddev]) as name:
        t = tf.convert_to_tensor(t, name="t")
        gn = tf.random_normal(tf.shape(t), stddev=stddev)
        return tf.add(t, gn, name=name)

# from https://github.com/domluna/memn2n

def _position_encoding(sentence_size, embedding_size):
    """Position encoding described in section 4.1 in "End to End Memory Networks" (http://arxiv.org/pdf/1503.08895v5.pdf)"""
    encoding = np.ones((embedding_size, sentence_size), dtype=np.float32)
    ls = sentence_size + 1
    le = embedding_size + 1
    for i in range(1, le):
        for j in range(1, ls):
            encoding[i - 1, j - 1] = (i - (le - 1) / 2) * (j - (ls - 1) / 2)
    encoding = 1 + 4 * encoding / embedding_size / sentence_size
    return np.transpose(encoding)

class DMN_PLUS(object):
    """FUCKING DMN_PLUS MODEL"""

    def _load_metadata(self):
        self.max_input_len = self.metadata['max_input_len']
        self.max_q_len = self.metadata['max_q_len']
        self.max_sen_len = self.metadata['max_sen_len']
        self.max_mask_len = self.metadata['max_mask_len']
        self.num_supporting_facts = self.metadata['num_supporting_facts']
        self.candidate_size = self.metadata['candidate_size']
        self.candid2idx = self.metadata['candid2idx']
        self.idx2candid = self.metadata['idx2candid']
        self.w2idx = self.metadata['w2idx']
        self.idx2w = self.metadata['idx2w']
        self.vocab_size = self.metadata['vocab_size']

    def _create_placeholders(self):
        """add data and placeholder to graph"""

        if self.config.word:
            self.question_placeholder = tf.placeholder(
                tf.float32, shape=(None, self.max_q_len, self.config.embed_size), name='questions')
            # self.question_placeholder = tf.placeholder(
            #         tf.float32, shape=(None, None, None), name='questions')
            self.question_len_placeholder = tf.placeholder(
                tf.int32, shape=(None,), name='question_lens')

            self.input_placeholder = tf.placeholder(tf.float32, shape=(
                None, self.max_input_len, self.max_sen_len, self.config.embed_size), name='inputs')
            # self.input_placeholder = tf.placeholder(tf.float32, shape=(
            #     None, None, None, self.config.embed_size), name='inputs')
            self.input_len_placeholder = tf.placeholder(
                tf.int32, shape=(None,), name='input_lens')

            self.embeddings = None
        else:
            self.question_placeholder = tf.placeholder(
                tf.int32, shape=(None, self.max_q_len), name='questions')
            self.question_len_placeholder = tf.placeholder(
                tf.int32, shape=(None,), name='question_lens')

            self.input_placeholder = tf.placeholder(tf.int32, shape=(
                None, self.max_input_len, self.max_sen_len), name='inputs')
            self.input_len_placeholder = tf.placeholder(
                tf.int32, shape=(None,), name='input_lens')

            self.embeddings = tf.Variable(tf.random_uniform(
                [self.vocab_size, self.config.embed_size], -1.0, 1.0),
                trainable=True, dtype=tf.float32, name='embeddings')
            self.embedding_placeholder = tf.placeholder(
                tf.float32, [None, None])
            self.embedding_init = self.embeddings.assign(
                self.embedding_placeholder)

        if self.config.multi_label:
            self.answer_placeholder = tf.placeholder(
                tf.float32, shape=(None, self.candidate_size), name='answers')
        else:
            self.answer_placeholder = tf.placeholder(
                tf.int32, shape=(None,), name='answers')

        self.rel_label_placeholder = tf.placeholder(tf.int32, shape=(
            None, self.num_supporting_facts), name='rel_labels')
        self.dropout_placeholder = tf.placeholder(tf.float32, name='dropout')

    def _create_loss(self, output):
        """Calculate loss"""
        # optional strong supervision of attention with supporting facts
        gate_loss = 0
        if self.config.strong_supervision:
            for i, att in enumerate(self.attentions):
                labels = tf.gather(tf.transpose(self.rel_label_placeholder), 0)
                gate_loss += tf.reduce_sum(
                    tf.nn.sparse_softmax_cross_entropy_with_logits(logits=att, labels=labels))

        # loss = self.config.beta * tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits(
        #     logits=output, labels=self.answer_placeholder)) + gate_loss
        if self.config.multi_label:
            # multi targets
            loss = self.config.beta * tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(
                logits=output, labels=self.answer_placeholder)) + gate_loss
        else:
            loss = self.config.beta * tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits(
                logits=output, labels=self.answer_placeholder)) + gate_loss

        # stackoverflow :https://stackoverflow.com/questions/33712178/tensorflow-nan-bug
        # cross_entropy = - tf.reduce_sum(self.answer_placeholder *
        #                                 tf.log(tf.clip_by_value(output, 1e-10, 1.0)))
        # loss = self.config.beta * tf.reduce_sum(cross_entropy) + gate_loss

        # add l2 regularization for all variables except biases
        for v in tf.trainable_variables():
            if not 'bias' in v.name.lower():
                loss += self.config.l2 * tf.nn.l2_loss(v)

        # tf.summary.scalar('loss', loss)

        return loss

    def _create_training_op(self, loss):
        """Calculate and apply gradients"""
        opt = tf.train.AdamOptimizer(
            learning_rate=self.config.lr, epsilon=1e-8)
        gvs = opt.compute_gradients(loss)

        # optionally cap and noise gradients to regularize
        if self.config.cap_grads:
            gvs = [(tf.clip_by_norm(grad, self.config.max_grad_val), var)
                   for grad, var in gvs]
        if self.config.noisy_grads:
            gvs = [(_add_gradient_noise(grad), var) for grad, var in gvs]

        train_op = opt.apply_gradients(gvs)
        return train_op

    def get_question_representation(self, embeddings):
        """Get question vectors via embedding and GRU"""
        if embeddings:
            questions = tf.nn.embedding_lookup(
                embeddings, self.question_placeholder)
        else:
            questions = self.question_placeholder

        gru_cell = tf.contrib.rnn.GRUCell(self.config.hidden_size)
        _, q_vec = tf.nn.dynamic_rnn(gru_cell,
                                     questions,
                                     dtype=np.float32,
                                     sequence_length=self.question_len_placeholder
                                     )
        return q_vec

    def get_input_representation(self, embeddings):
        """Get fact (sentence) vectors via embedding, positional encoding and bi-directional GRU"""
        # get word vectors from embedding
        if embeddings:
            inputs = tf.nn.embedding_lookup(embeddings, self.input_placeholder)
        else:
            inputs = self.input_placeholder

        # use encoding to get sentence representation
        inputs = tf.reduce_sum(inputs * self.encoding, 2)

        forward_gru_cell = tf.contrib.rnn.GRUCell(self.config.hidden_size)
        # backward_gru_cell = tf.contrib.rnn.GRUCell(self.config.hidden_size)
        # outputs, _ = tf.nn.bidirectional_dynamic_rnn(
        #     forward_gru_cell,
        #     backward_gru_cell,
        #     inputs,
        #     dtype=np.float32,
        #     sequence_length=self.input_len_placeholder
        # )

        outputs, _ = tf.nn.dynamic_rnn(
                forward_gru_cell,
                inputs,
                dtype=np.float32,
                sequence_length=self.input_len_placeholder
        )

        # f<-> = f-> + f<-
        # fact_vecs = tf.reduce_sum(tf.stack(outputs), axis=0)
        # apply dropout
        fact_vecs = tf.nn.dropout(outputs, self.dropout_placeholder)

        return fact_vecs

    def get_attention(self, q_vec, prev_memory, fact_vec, reuse):
        """Use question vector and previous memory to create scalar attention for current fact"""
        with tf.variable_scope("attention", reuse=reuse):
            features = [fact_vec * q_vec,
                        fact_vec * prev_memory,
                        tf.abs(fact_vec - q_vec),
                        tf.abs(fact_vec - prev_memory)]

            feature_vec = tf.concat(features, 1)

            attention = tf.contrib.layers.fully_connected(feature_vec,
                                                          self.config.embed_size,
                                                          activation_fn=tf.nn.tanh,
                                                          reuse=reuse, scope="fc1")

            attention = tf.contrib.layers.fully_connected(attention,
                                                          1,
                                                          activation_fn=None,
                                                          reuse=reuse, scope="fc2")

        return attention

    def generate_episode(self, memory, q_vec, fact_vecs, hop_index):
        """Generate episode by applying attention to current fact vectors through a modified GRU"""

        attentions = [tf.squeeze(
            self.get_attention(q_vec, memory, fv, bool(hop_index) or bool(i)), axis=1)
            for i, fv in enumerate(tf.unstack(fact_vecs, axis=1))]

        attentions = tf.transpose(tf.stack(attentions))
        self.attentions.append(attentions)
        attentions = tf.nn.softmax(attentions)
        attentions = tf.expand_dims(attentions, axis=-1)

        reuse = True if hop_index > 0 else False

        # concatenate fact vectors and attentions for input into attGRU
        gru_inputs = tf.concat([fact_vecs, attentions], 2)

        with tf.variable_scope('attention_gru', reuse=reuse):
            _, episode = tf.nn.dynamic_rnn(AttentionGRUCell(self.config.hidden_size),
                                           gru_inputs,
                                           dtype=np.float32,
                                           sequence_length=self.input_len_placeholder
                                           )

        return episode

    def add_answer_module(self, rnn_output, q_vec):
        """Linear softmax answer module"""

        rnn_output = tf.nn.dropout(rnn_output, self.dropout_placeholder)

        output = tf.layers.dense(tf.concat([rnn_output, q_vec], 1),
                                 self.candidate_size,
                                 activation=None)

        return output

    def _inference(self):
        """Performs inference on the DMN model"""
        # input fusion module
        with tf.variable_scope("question", initializer=tf.contrib.layers.xavier_initializer()):
            print('==> get question representation')
            q_vec = self.get_question_representation(self.embeddings)

        with tf.variable_scope("input", initializer=tf.contrib.layers.xavier_initializer()):
            print('==> get input representation')
            fact_vecs = self.get_input_representation(self.embeddings)

        # keep track of attentions for possible strong supervision
        self.attentions = []

        # memory module
        with tf.variable_scope("memory", initializer=tf.contrib.layers.xavier_initializer()):
            print('==> build episodic memory')

            # generate n_hops episodes
            prev_memory = q_vec

            for i in range(self.config.num_hops):
                # get a new episode
                print('==> generating episode', i)
                episode = self.generate_episode(
                    prev_memory, q_vec, fact_vecs, i)

                # untied weights for memory update
                with tf.variable_scope("hop_%d" % i):
                    prev_memory = tf.layers.dense(tf.concat([prev_memory, episode, q_vec], 1),
                                                  self.config.hidden_size,
                                                  activation=tf.nn.relu)

            output = prev_memory

        # pass memory module output through linear answer module
        with tf.variable_scope("answer", initializer=tf.contrib.layers.xavier_initializer()):
            output = self.add_answer_module(output, q_vec)

        return output

    def get_predictions(self, output):
        if self.config.multi_label:
            # multi targets
            predict_by_value = tf.nn.top_k(
                output, k=self.config.top_k, name="predict_op")
            predict_proba_op = tf.nn.softmax(output, name="predict_proba_op")
            pred = tf.nn.top_k(
                predict_proba_op, k=self.config.top_k, name="top_predict_proba_op")
        else:
            predict_proba_op = tf.nn.softmax(output)
            predict_proba_top_op = tf.nn.top_k(
                predict_proba_op, k=self.config.top_k, name='top_predict_proba_op')
            pred = tf.argmax(predict_proba_op, 1, name='pred')

        # predict_proba_op = tf.nn.softmax(output, name="predict_proba_op")
        # preds = tf.nn.top_k(
        #     predict_proba_op, k=self.config.top_k, name="top_predict_proba_op")
        return pred, predict_proba_top_op

    def predict(self, session, inputs, input_lens, max_sen_len, questions, q_lens):
        feed = {
            self.question_placeholder: questions,
            self.input_placeholder: inputs,
            self.question_len_placeholder: q_lens,
            self.input_len_placeholder: input_lens,
            self.dropout_placeholder: self.config.dropout
        }
        pred, prob_top = session.run(
            [self.pred, self.prob_top_k], feed_dict=feed)
        return pred, prob_top

    def __init__(self, config, metadata):
        self.config = config
        self.metadata = metadata
        self._load_metadata()
        self.encoding = _position_encoding(
            self.max_sen_len, self.config.embed_size)
        self._create_placeholders()
        self.output = self._inference()
        self.pred, self.prob_top_k = self.get_predictions(self.output)
        # self.pred = self.get_predictions(self.output)
        self.calculate_loss = self._create_loss(self.output)
        self.train_step = self._create_training_op(self.calculate_loss)
        # self.merged = tf.summary.merge_all()


from __future__ import division
from model.yolo_util import parse_cfg, get_test_input
import torch
import torch.nn as nn

class EmptyLayer(nn.Module):
    def __init__(self):
        super(EmptyLayer, self).__init__()

class DetectionLayer(nn.Module):
    def __init__(self, anchors):
        super(DetectionLayer, self).__init__()
        self.anchors = anchors

def create_modules(blocks):
    '''
    create net structure
    '''
    net_info = blocks[0]
    module_list = nn.ModuleList()
    prev_filters = 3
    output_filters = []

    for index, x in enumerate(blocks[1:]):
        module = nn.Sequential()
        if (x["type"] == "convolutional"):
            activation = x["activation"]
            try:
                batch_normalize = int(x["batch_normalize"])
                bias = False
            except:
                batch_normalize = 0
                bias = True

            filters = int(x["filters"])
            padding = int(x["pad"])
            kernel_size = int(x["size"])
            stride = int(x["stride"])

            if padding:
                pad = (kernel_size - 1) // 2
            else:
                pad = 0

            conv = nn.Conv2d(prev_filters, filters,
                             kernel_size, stride, pad, bias=bias)
            module.add_module("conv_{0}".format(index), conv)

            if batch_normalize:
                bn = nn.BatchNorm2d(filters)
                module.add_module("batch_norm_{0}".format(index), bn)

            if activation == "leaky":
                activn = nn.LeakyReLU(0.1, inplace=True)
                module.add_module("leaky_{0}".format(index), activn)

        elif (x["type"] == "upsample"):
            stride = int(x["stride"])
            upsample = nn.Upsample(scale_factor=stride, mode="nearest")
            module.add_module("upsample_{0}".format(index), upsample)

        elif (x["type"] == "route"):
            x["layers"] = x["layers"].split(',')
            start = int(x["layers"][0])
            try:
                end = int(x["layers"][1])
            except:
                end = 0
            if start > 0:
                start = start - index
            if end > 0:
                end = end - index
            route = EmptyLayer()
            module.add_module("route_{0}".format(index), route)
            if end < 0:
                filters = output_filters[index +
                                         start] + output_filters[index + end]
            else:
                filters = output_filters[index + start]

        elif x["type"] == "shortcut":
            shortcut = EmptyLayer()
            module.add_module("shortcut_{}".format(index), shortcut)

        elif x["type"] == "yolo":
            mask = x["mask"].split(",")
            mask = [int(x) for x in mask]

            anchors = x["anchors"].split(",")
            anchors = [int(a) for a in anchors]
            anchors = [(anchors[i], anchors[i + 1])
                       for i in range(0, len(anchors), 2)]
            anchors = [anchors[i] for i in mask]

            detection = DetectionLayer(anchors)
            module.add_module("Detection_{}".format(index), detection)

        module_list.append(module)
        prev_filters = filters
        output_filters.append(filters)

    return (net_info, module_list)

class Darknet(nn.Module):
    '''
    Darknet model
    '''

    def __init__(self, cfgfile):
        super(Darknet, self).__init__()
        self.blocks = parse_cfg(cfgfile)
        self.net_info, self.module_list = create_modules(self.blocks)

    def forward(self, x):
        '''
        forward
        '''
        modules = self.blocks[1:]
        outputs = {}
        detection = []

        for i, module in enumerate(modules):
            module_type = (module["type"])

            if module_type == "convolutional" or module_type == "upsample":
                x = self.module_list[i](x)

            elif module_type == "route":
                layers = module["layers"]
                layers = [int(a) for a in layers]

                if (layers[0]) > 0:
                    layers[0] = layers[0] - i
                if len(layers) == 1:
                    x = outputs[i + (layers[0])]
                else:
                    if (layers[1]) > 0:
                        layers[1] = layers[1] - i

                    map1 = outputs[i + layers[0]]
                    map2 = outputs[i + layers[1]]
                    x = torch.cat((map1, map2), 1)

            elif module_type == "shortcut":
                from_ = int(module["from"])
                x = outputs[i - 1] + outputs[i + from_]

            elif module_type == "yolo":
                '''
                anchors ==>
                [(116, 90), (156, 198), (373, 326)]
                [(30, 61), (62, 45), (59, 119)]
                [(10, 13), (16, 30), (33, 23)]
                '''

                # x is output of the model
                # x ==> torch.float32 requires_grad=True

                # x = x.data
                detection.append(x)
                '''
                x ==>
                torch.Size([bs, 255, 13, 13])
                torch.Size([bs, 255, 26, 26])
                torch.Size([bs, 255, 52, 52])
                '''

            outputs[i] = x

        # [bs, 10647, 15]
        return detection

    def weight_init(self):
        '''
        initialize weight
        '''
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.xavier_normal_(m.weight)
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

# model test
if __name__ == "__main__":
    model = Darknet("yolov3.cfg")
    model.load_weights("weight/yo.pth")
    inp = get_test_input()
    pred = model(inp)
    print(pred)


import logging

from automation_infra.utils.waiter import wait_for_predicate, wait_for_predicate_nothrow
from automation_infra.plugins.ssh_direct import SshDirect, SSHCalledProcessError
from pytest_automation_infra.helpers import hardware_config
from devops_automation_infra.plugins.power import Power
from devops_automation_infra.plugins.gravity import Gravity
from devops_automation_infra.plugins.k8s import K8s
from devops_automation_infra.utils.k8s_utils import create_deployment_with_replicas
from devops_automation_infra.utils.health_check import host_is_active

def all_deployments_pods_alive(host, deployment_name):
    deployment_pod_ips = host.K8s.get_deployments_pod_internal_ips(deployment_name)
    for pod_ip in deployment_pod_ips:
        res = pod_is_live(host, ip=pod_ip, port=8080)
        return res == "Hello World!"

def pod_is_live(host, ip, port, timeout=10):
    try:
        res = host.SshDirect.execute(f"curl --max-time {timeout} http://{ip}:{port}")
        return res
    except SSHCalledProcessError as e:
        logging.error(e.output)

@hardware_config(hardware={"host1": {}, "host2": {}})
def test_cluster_network_master_restart(base_config,
                                        clean_up_all_deployments_and_svcs,
                                        amount_of_replicas=100,
                                        docker_image_name='gcr.io/hello-minikube-zero-install/hello-node',
                                        deployment_name="test"):
    # Clean up before test starts
    base_config.hosts.host1.SshDirect.connect(timeout=60)
    create_deployment_with_replicas(base_config.hosts.host1, deployment_name, docker_image_name, amount_of_replicas)
    base_config.hosts.host1.Power.reboot()
    # Check host has started again
    wait_for_predicate_nothrow(lambda: host_is_active(base_config.hosts.host1.ip), timeout=60)
    base_config.hosts.host1.SshDirect.connect(timeout=60)
    wait_for_predicate(lambda: base_config.hosts.host1.Gravity.is_cluster_healthy(),
                       timeout=120,
                       interval=5)
    wait_for_predicate_nothrow(lambda: all_deployments_pods_alive(base_config.hosts.host1, deployment_name),
                       timeout=300,
                       interval=10)


import tensorflow as tf
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tensorflow import keras
from keras.layers import Dense, Conv2D, Flatten, MaxPooling2D, BatchNormalization, Dropout
from keras.models import Sequential
from keras.callbacks import ReduceLROnPlateau
from keras.utils import to_categorical
import seaborn as sns
from sklearn.model_selection import train_test_split

## importing data
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')

type(train)
train.head()
#shape of data imported
train.shape, test.shape

train.iloc[:,0].head()

#converting to the data to array
train_data = train.iloc[:,1:]
train_labels = train.iloc[:,0]

#one hot encoding for train_labels
train_labels = to_categorical(train_labels, num_classes = 10)
train_labels.shape

train_data = np.array(train_data)
test = np.array(test)

train_data = train_data.astype('float32')
train_labels = train_labels.astype('int32')
test = test.astype('float32')
train_data = train_data / 255.0
test = test / 255.0

# plotting an image
img = train_data[999].reshape(28,28)
plt.imshow(img, cmap = 'bone')

#plotting graphs for the train_labels
sns.countplot(train['label'])

#train_labels data
train_labels

#splitting data to training and validation data
x_train, x_val, y_train, y_val = train_test_split(train_data,train_labels, random_state = 0, shuffle = True)
x_train.shape, x_val.shape

#converting the data to keras required dimensions
x_train = x_train.reshape(x_train.shape[0], img_rows,img_cols, 1)
y_train = np.array(y_train)
y_train = to_categorical(y_train)

x_train.shape, y_train.shape

#validation data
x_val = x_val.reshape(x_val.shape[0],img_rows, img_cols, 1)
y_val = np.array(y_val)
y_val = to_categorical(y_val, num_classes = 10)

x_val.shape, y_val.shape

# Conv2D MODEL
img_rows, img_cols = 28,28
img_shape = (img_rows, img_cols, 1)

model = Sequential()

model.add(Conv2D(128, (3,3), activation = 'relu', kernel_initializer = 'he_normal', input_shape = img_shape))
model.add(Conv2D(128, (3,3), activation = 'relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Dropout(0.3))

model.add(Conv2D(256, (3,3), activation = 'relu'))
model.add(BatchNormalization())
model.add(Conv2D(256, (3,3), activation = 'relu'))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Dropout(0.3))

model.add(Flatten())

model.add(Dense(512, activation = 'relu'))
model.add(Dense(10, activation = 'softmax'))

#COMPILING MODEL
model.compile(loss = 'categorical_crossentropy', optimizer = 'adam', metrics = ['accuracy'])

# Fitting the MODEL

model.fit(x_train,y_train, epochs = 1, batch_size = 120, validation_data =(x_val,y_val), verbose = 1)


import requests
import json

def get_all():
    r = requests.get("http://localhost:8983/solr/newcore/select?q=*:*")
    print("Solr status " + str(r.status_code))
    js = r.json()
    # print("All")
    li = js["response"]["docs"]
    # for k, v in js.items():
    #    print(k)
    #    print(v)
    return li

def get_docs_default(url_to_use):
    # r = requests.get("http://localhost:8983/solr/newcore/select?q=*:*")
    url_full = url_to_use + "/newcore/select?q=*:*"
    r = requests.get(url_full)
    print("Solr status " + str(r.status_code))
    js = r.json()
    # print("Docs")
    li = js["response"]["docs"]
    # for l in li:
    #    print(l)
    return li

def get_docs_max(url_to_use, default=10000):
    # r = requests.get("http://localhost:8983/solr/newcore/select?q=*:*&rows=" + str(default))
    url_full = url_to_use + "/newcore/select?q=*:*&rows=" + str(default)
    r = requests.get(url_full)
    print("Solr status " + str(r.status_code))
    js = r.json()
    # print("Docs")
    li = js["response"]["docs"]
    # for l in li:
    #    print(l)
    return li

def get_info():
    print("http://lucene.apache.org/solr/guide/7_5/the-standard-query-parser.html")

def main():
    print("Get job")
    # get_all()
    # get_info()
    # get_docs_max()

if __name__ == "__main__":
    main()


import numpy as np
import warnings
from sfsidb import constants
import glob

def deprecation(message):
    warnings.warn(message, stacklevel=3)

def create_motion_name(test_name, sensor_code, code_suffix=""):
    """
    Builds the full name of the file

    :param test_name: str, test name
    :param sensor_code: str, a sensor code (e.g. ACCX-UB1-L2C-M)
    :param code_suffix: str, suffix
    :return:
    """
    return "%s-%s-%s" % (test_name, sensor_code, code_suffix)

def get_sensor_code_by_number(si, mtype, sensor_number, quiet=False):
    """
    Given a sensor number, get the full sensor code (e.g. ACCX-UB1-L2C-M)

    :param si: dict, sensor index json dictionary
    :param mtype: str, sensor type
    :param sensor_number: int, number of sensor
    :param quiet: bool, if true then return None if not found
    :return: str or None, sensor_code: a sensor code (e.g. ACCX-UB1-L2C-M)
    """
    try:
        if 'Orientation' in si[mtype][sensor_number]:
            orientation = si[mtype][sensor_number]['Orientation']
        else:
            orientation = ""
        return "%s%s-%s-%s-%s" % (mtype,
                                orientation,
                                si[mtype][sensor_number]['X-CODE'],
                                si[mtype][sensor_number]['Y-CODE'],
                                si[mtype][sensor_number]['Z-CODE'])
    except KeyError:
        if quiet:
            return None
        raise

def get_mtype_and_number_from_code(si, sensor_code):
    """
    Given a sensor sensor_code, get motion type and sensor number

    :param si: dict, sensor index json dictionary
    :param sensor_code: str, a sensor code (e.g. ACCX-UB1-L2C-M)
    :return:
    """
    mtype_and_ory, x, y, z = sensor_code.split("-")
    if mtype_and_ory[-1] in "XYZ" and "ACCX" not in si:  # Need to support old sensor_file.json files.
        mtype = mtype_and_ory[:-1]
    else:
        mtype = mtype_and_ory
    for m_number in si[mtype]:
        cc = get_sensor_code_by_number(si, mtype, m_number)
        if cc == sensor_code:
            return mtype, m_number
    return None, None

def get_all_sensor_codes(si, wild_sensor_code):
    """
    Get all sensor sensor_codes that match a wild sensor code

    :param si: dict, sensor index json dictionary
    :param wild_sensor_code: str, a sensor code with "*" for wildcards (e.g. ACCX-*-L2C-*)
    :return:
    """
    mtype_and_ory, x, y, z = wild_sensor_code.split("-")
    if mtype_and_ory == "*":
        mtypes = list(si)
    elif mtype_and_ory[-1] in "XYZ" and "ACCX" not in si:  # Need to support old sensor_file.json files.
        mtypes = [mtype_and_ory[:-1]]
    else:
        mtypes = [mtype_and_ory]

    all_sensor_codes = []
    for mtype in mtypes:
        for m_number in si[mtype]:
            if x in ["*", si[mtype][m_number]['X-CODE']] and \
                    y in ["*", si[mtype][m_number]['Y-CODE']] and \
                    z in ["*", si[mtype][m_number]['Z-CODE']]:
                cc = get_sensor_code_by_number(si, mtype, m_number)
                all_sensor_codes.append(cc)

    return all_sensor_codes

def load_record(ffp, dbset, quiet=False):
    deprecation('Deprecated, switch to load_record_and_time, load_record_and_dt')
    # raise Warning("Deprecated, switch to load_record_and_time, load_record_and_dt")
    if quiet:
        try:
            data = np.loadtxt(ffp + dbset.SENSOR_FILE_TYPE,
                              dtype='float',
                              delimiter=dbset.SENSOR_DATA_DELIMITER,
                              skiprows=dbset.SENSOR_DATA_SKIP_ROWS)
        except FileNotFoundError:
            print("File not found: ", ffp + dbset.SENSOR_FILE_TYPE)
            return None, None
        except IOError:
            print("File not found: ", ffp + dbset.SENSOR_FILE_TYPE)
            return None, None
    else:
        data = np.loadtxt(ffp + dbset.SENSOR_FILE_TYPE,
                              dtype='float',
                              delimiter=dbset.SENSOR_DATA_DELIMITER,
                              skiprows=dbset.SENSOR_DATA_SKIP_ROWS)
    time = data[:, 0]
    dt = time[1] - time[0]
    series = data[:, 1]
    return series, time

def get_available_sensor_codes(ffp, local_path_ext, wild_sensor_code, dbset):
    file_name = dbset.create_file_name("*", wild_sensor_code)
    full_wild_file_path = ffp + local_path_ext + file_name
    files = glob.glob(full_wild_file_path)
    files.sort()
    import re
    compiled = re.compile(wild_sensor_code)

    for ff in range(len(files)):
        ms = compiled.match(files[ff])
        # files[ff] = ms
        # sname = files[ff].split(local_path_ext)[-1]
        # sname = sname.split()
        # files[ff].replace(files[ff])
    return files

def load_record_only(db_fp, local_path_ext, test_name, sensor_code, dbset, quiet=False, first=True):
    """
    Finds the file and returns the time series of values

    :param db_fp: str, Database root directory
    :param local_path_ext: str, local path to sensor file from database root directory
    :param test_name: str, name of test used as prefix of file name
    :param sensor_code: str, a sensor code (e.g. ACCX-UB1-L2C-M)
    :param dbset: module, A database set module from sfsidb.sets
    :param quiet: bool, if True then return None
    :return:
    """
    folder_path = db_fp + local_path_ext
    rec_and_dt = dbset.wild_load_record_and_dt(folder_path, test_name, sensor_code, quiet, first)
    if rec_and_dt is None and quiet:
        return None
    return rec_and_dt[0]

def load_record_and_time(db_fp, local_path_ext, test_name, sensor_code, dbset, quiet=False, first=True):
    """
    Finds the file and returns the time series of values and the time series

    :param db_fp: str, Database root directory
    :param local_path_ext: str, local path to sensor file from database root directory
    :param test_name: str, name of test used as prefix of file name
    :param sensor_code: str, a sensor code (e.g. ACCX-UB1-L2C-M)
    :param dbset: module, A database set module from sfsidb.sets
    :param quiet: bool, if True then return None
    :return:
    """
    folder_path = db_fp + local_path_ext
    if first:
        rec, dt = dbset.wild_load_record_and_dt(folder_path, test_name, sensor_code, quiet, first=first)
        if rec is None and quiet:
            return None, None
        time = np.arange(1, len(rec) + 1) * dt
        return rec, time
    else:
        recs, dts = dbset.wild_load_record_and_dt(folder_path, test_name, sensor_code, quiet, first)
        times = []
        for i, dt in enumerate(dts):
            time = np.arange(1, len(recs[i]) + 1) * dt
            times.append(time)
        return recs, times

def load_record_and_dt(db_fp, local_path_ext, test_name, sensor_code, dbset, quiet=False, first=True):
    """
    Finds the file and returns the time series of values and the time step

    :param db_fp: str, Database root directory
    :param local_path_ext: str, local path to sensor file from database root directory
    :param test_name: str, name of test used as prefix of file name
    :param sensor_code: str, a sensor code (e.g. ACCX-UB1-L2C-M)
    :param dbset: module, A database set module from sfsidb.sets
    :param quiet: bool, if True then return None
    :return:
    """
    folder_path = db_fp + local_path_ext
    return dbset.wild_load_record_and_dt(folder_path, test_name, sensor_code, quiet, first)

def sensor_code_to_name(sensor_code, part="sensor"):
    """
    Converts a sensor code into written english.

    E.g. ACCX-UB1-L2C-M = Horizontal acceleration

    :param sensor_code: str, a sensor code (e.g. ACCX-UB1-L2C-M)
    :param part: str, what part of the code to convert
    :return: str
    """
    mtype_and_ory, x, y, z = sensor_code.split("-")

    if part == "sensor":
        return constants.sensor_type_codes[mtype_and_ory]
    elif part == "xloc":
        return constants.x_locations[x]
    elif part == "yloc":
        return constants.y_locations[y]