from tkinter import *
from tkinter import filedialog
from tkinter import messagebox
import os
import csv

csv_wordbank = r"C:\Users\adami\Documents\GitHub\global-tax-expense-handler\version_2\data\savedCSVWordbank.csv"
script_data_path = r'C:\Users\Ahmed\OneDrive\Desktop\global-tax-expense-handler\all_data\script_data'
myTxtFile = r"C:\Users\Ahmed\OneDrive\Desktop\global-tax-expense-handler\all_data\script_data\managed.csv"

class expense_maker:
	def __init__(self):
		self.numOfExpenseType = 0
		self.expenseDict = {}
	def addExpenseType(self,type):
		self.expenseDict[type] = []
	def addExpense(self,type,expense):
		if expense != '':
			self.expenseDict[type].append(expense)
	def listExpenseTypes(self):
		expenseTypes = []
		for x in self.expenseDict.keys():
			expenseTypes.append(x)
		return expenseTypes
	def listAllExpenses(self):
		for expenseTypes in self.expenseDict.keys():
			print(f'{expenseTypes} : {self.expenseDict[expenseTypes]}')
	def clearDict(self):
		self.expenseDict.clear()
	def saveToFile(self,newFilePath,fileName):
		largestLen = self.largestExpenseList()
		fList = []
		firstRow = self.listExpenseTypes()
		fList.append(firstRow)

		for lineNum in range(largestLen):
			# print(lineNum+1)
			row = []

			for expenseType in fList[0]:
				# print(expenseType)
				if lineNum+1 <= len(self.expenseDict[expenseType]):
					row.append(self.expenseDict[expenseType][lineNum])
				else:
					row.append('')
			fList.append(row)

		for x in fList:
			pass
			# print(x)
		# flist.append(", ".join, self.expenseDict.keys())

		with open(os.path.join(newFilePath,fileName),"w+") as text_file:
			for line in fList:
				lineText = ','.join(line)
				text_file.write(f'{lineText}\n')
			pass

	def readFile(self,path):
		self.clearDict()
		with open(path,mode='r') as csv_file:
			csv_reader = csv.reader(csv_file,delimiter=',')
			line_count = 0

			expenseTypesX  = []
			column = 0
			for row in csv_reader:
				if line_count == 0:
					for cell in row:
						self.addExpenseType(cell)
						expenseTypesX.append(cell)
				else:
					for expenseType in self.listExpenseTypes():
						if not row[column].isspace():
							self.addExpense(expenseType,row[column])
						column += 1
					column = 0
				line_count+= 1

	def largestExpenseList(self):
		largestListLen = 0
		for expenseType in self.expenseDict.keys():
			if len(self.expenseDict[expenseType]) > largestListLen:
				largestListLen = len(self.expenseDict[expenseType])
		return largestListLen

	def inputCatagoryColumnToCsv(self,column,statementPath,newPath):
		fileRowList = []
		with open(statementPath,'r') as csv_input:
			csv_reader_input = csv.reader(csv_input,delimiter=',')
			count = 0
			# print(column)
			for row in csv_reader_input:
				row.append(column[count])
				fileRowList.append(row)
				count +=1
		with open(newPath,'w') as csv_output:
			for row in fileRowList:
				csv_output.write(f"{','.join(row)}\n")

	def getCatagorylist(self,path,newStatementPath,newStatementName):
		with open(path,mode='r') as csv_file:
			csv_reader = csv.reader(csv_file,delimiter=',')

			line_count = 0
			expenseTypeList = ['Expense Type']
			payeeColumn = 0
			undefinedExpenses = []

			for row in csv_reader:
				if line_count == 0:
					payeeColumn= row.index("Payee" or "payee")
					line_count+=1
				else:
					# print(row[payeeColumn])
					exp = self.isDictInValue(row[payeeColumn].lower())
					# print(f'{row[payeeColumn]} is {exp}')
					if exp != False:
						expenseTypeList.append(exp)
					else:
						expenseTypeList.append("Undefined")
						undefinedExpenses.append(row[payeeColumn])

		self.inputCatagoryColumnToCsv(expenseTypeList,path,os.path.join(newStatementPath,newStatementName))
		return undefinedExpenses
	def isDictInValue(self,value):
		for key in self.expenseDict.keys():
			for expense in self.expenseDict[key]:
				if expense.lower() in value.lower():
					return key
		return False

	def isValueInDict(self,value):
		for key in self.expenseDict.keys():
			for expense in self.expenseDict[key]:
				if expense.lower() == value.lower():
					return True
		return False
class expenseListbox():
	def __init__(self,surface):
		self.my_frame = Frame(surface) #underlying frame
		self.my_scrollbar = Scrollbar(self.my_frame,orient=VERTICAL) #scrollbar
		self.my_listbox = Listbox(self.my_frame,yscrollcomman=self.my_scrollbar,height=50,width=50,exportselection=False) #listbox
		self.my_scrollbar.config(command=self.my_listbox.yview)
		self.currentlySelected = ''
	def render(self):
		self.my_scrollbar.pack(side=RIGHT,fill=Y)
		self.my_frame.pack()
		self.my_listbox.pack()

	def addList(self,myList):
		for item in myList:
			self.my_listbox.insert(END,item)
	def bind(self,func):
		self.my_listbox.bind('<<ListboxSelect>>', func)
	def clear(self):
		self.my_listbox.delete(0,END)
	def getSelection(self):
		value = self.my_listbox.curselection()[0]
		return self.my_listbox.get(value)

#function for updating second listbox
def updateSecondLb(event):
	expensesListBox.clear()
	key = expenseTypeListBox.getSelection()
	expensesListBox.addList(myExpenses.expenseDict[key])
	expensesLabel.config(text=key)
	expenseTypeListBox.currentlySelected = key

def addExpenseType():
	newKey = expenseType_entry.get()

	if newKey in myExpenses.expenseDict.keys():
		messagebox.showinfo("Error","Type already in system")
	elif newKey != '' and newKey not in myExpenses.expenseDict.keys():
		myExpenses.addExpenseType(newKey)
		expenseTypeListBox.clear()
		expenseTypeListBox.addList(myExpenses.listExpenseTypes())
		expenseType_entry.delete(0,END)

def deleteExpenseType():
	keyToDelete = expenseTypeListBox.getSelection()
	if keyToDelete in myExpenses.listExpenseTypes():
		del myExpenses.expenseDict[keyToDelete]

	expensesListBox.clear()
	expenseTypeListBox.clear()
	expenseTypeListBox.addList(myExpenses.listExpenseTypes())

def addExpense_listbox():
	newExpense = expense_entry.get()
	try:
		curExpenseType = expenseTypeListBox.getSelection()
		print(curExpenseType)

		if myExpenses.isValueInDict(newExpense):
			messagebox.showinfo("Error","Value already in system")
		elif newExpense != '':
			myExpenses.addExpense(curExpenseType,newExpense)

		expensesListBox.clear()
		expensesListBox.addList(myExpenses.expenseDict[curExpenseType])
		expense_entry.delete(0,END)
	except IndexError:
		messagebox.showinfo("Error","Please Select an Expense Type")

	expense_entry.delete(0,END)

def deleteExpense():
	selection = expensesListBox.my_listbox.curselection()
	count = 0
	if selection:
		# print(expensesListBox.getSelection())
		for x in myExpenses.expenseDict[expenseTypeListBox.currentlySelected]:
			if x == expensesListBox.getSelection():
				del myExpenses.expenseDict[expenseTypeListBox.currentlySelected][count]
				# print(myExpenses.expenseDict[expenseTypeListBox.currentlySelected][x])
				break
			count+=1
	expensesListBox.my_listbox.delete(count)

def save():
	myExpenses.saveToFile(script_data_path,'savedCSVWordbank.csv')

def csv_ColumnRead(path,columnNum):
    lineArr = []
    with open(path) as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',')

        for row in csv_reader:
            lineArr.append(row[columnNum])

        csv_file.close()
    return lineArr

myExpenses = expense_maker()
myExpenses.readFile(csv_wordbank)

root = Tk()
root.title('ListBox Class')
# root.geometry("400x400")

#expense types frame
expenseTypeFrame = Frame(root)
expenseTypeFrame.pack(side=LEFT)

#label for expense types
expenseTypeLabel = Label(expenseTypeFrame,text="Expense Types")
expenseTypeLabel.pack(side=TOP)

# button to delete a expense type
deleteExpenseType_button = Button(expenseTypeFrame,text='Delete Expense Type',command=deleteExpenseType)
deleteExpenseType_button.pack(side=BOTTOM)

#add expense type button
addExpenseType_button = Button(expenseTypeFrame,text='Add Expense Type',command=addExpenseType)
addExpenseType_button.pack(side=BOTTOM)

#entry to input expense type
expenseType_entry = Entry(expenseTypeFrame)
expenseType_entry.pack(side=BOTTOM)

#listbox for expense types
expenseTypeListBox = expenseListbox(expenseTypeFrame)
expenseTypeListBox.render()
expenseTypeListBox.addList(myExpenses.expenseDict.keys())

# #frame for unsorted expense list
# unsortedExpenseFrame = Frame(root)
# unsortedExpenseFrame.pack(side=LEFT)

# #label for unsorted expenseList
# unsortedExpenseLabel = Label(unsortedExpenseFrame,text='Unsorted List')
# unsortedExpenseLabel.pack(side=TOP)

# #unsorted listbox
# unsortedListbox = expenseListbox(unsortedExpenseFrame)
# unsortedListbox.render()
# unsortedListbox.addList(csv_ColumnRead(myTxtFile,0))

#frame for expenses
expensesFrame = Frame(root)
expensesFrame.pack(side=LEFT)

#expenses label
expensesLabel = Label(expensesFrame,text="Expenses")
expensesLabel.pack(side=TOP)
#expenses lb
expensesListBox = expenseListbox(expensesFrame)
expensesListBox.render()
expenseTypeListBox.bind(updateSecondLb)

#save button
save_button = Button(expensesFrame,text='Save Word Bank',command=save)
save_button.pack(side=BOTTOM)

#delete expense button
delete_expense_button = Button(expensesFrame, text='Delete Expense', command=deleteExpense)
delete_expense_button.pack(side=BOTTOM)

#add expense button
add_expense = Button(expensesFrame,text='Add Expense',command=addExpense_listbox)
add_expense.pack(side=BOTTOM)

#entry to enter in
expense_entry = Entry(expensesFrame)
expense_entry.pack(side=BOTTOM)

#Undefined List
undefinedFrame = Frame(root)
undefinedFrame.pack(side = LEFT)

#label
undefinedLabel = Label(undefinedFrame,text="Undefined Expenses")
undefinedLabel.pack(side=TOP)

#undefined expenses listbox
undefinedListbox = expenseListbox(undefinedFrame)
undefinedListbox.render()

#function that handles choosing a file
root.filename=''
def chooseFile(label):
	root.filename = filedialog.askopenfilename(initialdir="../all_data/script_data",title="Select A CSV Statement",filetypes=(("csv files", "*.csv"),("all files","*.*")))
	print(root.filename)
	label.config(text = f'File to sort is:{root.filename}')

#choose statement csv to analyse
selectedFile = ''
fileLabel = Label(undefinedFrame,text='File to sort is: ')
statementFileChooser = Button(undefinedFrame,text='Choose File',command=lambda:chooseFile(fileLabel))
statementFileChooser.pack(side = BOTTOM)

#show which file is selected
fileLabel.pack(side=BOTTOM)

def addExpenseTypeColumn(file,label,listbox,expenses):
	if file != '':
		base = os.path.basename(file)
		base = base.split(".")[0]
		base = f'{base}_sorted.csv'
		undefinedExpenses = myExpenses.getCatagorylist(file,script_data_path,base)
		listbox.clear()
		listbox.add_list(undefinedExpenses)

		label.config(text='Sorted! Open in Excel to Create a Pivot Table',fg='green')
	else:
		messagebox.showinfo("Error", "Please Select A Valid CSV File")

#button  to analyse
sortButton = Button(undefinedFrame,text="Add Expense Type Column",command= lambda:addExpenseTypeColumn(root.filename,fileLabel,undefinedListbox,myExpenses))
sortButton.pack(side=BOTTOM)
root.mainloop()


from qgis.PyQt.QtCore import pyqtSignal, QCoreApplication
from qgis.PyQt.QtWidgets import QWidget, QTreeView, QMenu, QLabel

from qgis.core import Qgis, QgsProject, QgsMessageLog
from qgis.gui import QgsMessageBar, QgsMessageBarItem

from .db_model import DBModel, PluginItem
from .db_plugins.plugin import DBPlugin, Schema, Table
from .db_plugins.vlayers.plugin import LTable

class DBTree(QTreeView):
    selectedItemChanged = pyqtSignal(object)

    def __init__(self, mainWindow):
        QTreeView.__init__(self, mainWindow)
        self.mainWindow = mainWindow

        self.setModel(DBModel(self))
        self.setHeaderHidden(True)
        self.setEditTriggers(QTreeView.EditKeyPressed | QTreeView.SelectedClicked)

        self.setDragEnabled(True)
        self.setAcceptDrops(True)
        self.setDropIndicatorShown(True)

        self.doubleClicked.connect(self.addLayer)
        self.selectionModel().currentChanged.connect(self.currentItemChanged)
        self.expanded.connect(self.itemChanged)
        self.collapsed.connect(self.itemChanged)
        self.model().dataChanged.connect(self.modelDataChanged)
        self.model().notPopulated.connect(self.collapse)

    def refreshItem(self, item=None):
        if item is None:
            item = self.currentItem()
            if item is None:
                return
        self.model().refreshItem(item)

    def showSystemTables(self, show):
        pass

    def currentItem(self):
        indexes = self.selectedIndexes()
        if len(indexes) <= 0:
            return
        return self.model().getItem(indexes[0])

    def currentDatabase(self):
        item = self.currentItem()
        if item is None:
            return

        if isinstance(item, (DBPlugin, Schema, Table)):
            return item.database()
        return None

    def currentSchema(self):
        item = self.currentItem()
        if item is None:
            return

        if isinstance(item, (Schema, Table)):
            return item.schema()
        return None

    def currentTable(self):
        item = self.currentItem()

        if isinstance(item, Table):
            return item
        return None

    def newConnection(self):
        index = self.currentIndex()
        if not index.isValid() or not isinstance(index.internalPointer(), PluginItem):
            return
        item = self.currentItem()
        self.mainWindow.invokeCallback(item.addConnectionActionSlot, index)

    def itemChanged(self, index):
        self.setCurrentIndex(index)
        self.selectedItemChanged.emit(self.currentItem())

    def modelDataChanged(self, indexFrom, indexTo):
        self.itemChanged(indexTo)

    def currentItemChanged(self, current, previous):
        self.itemChanged(current)

    def contextMenuEvent(self, ev):
        index = self.indexAt(ev.pos())
        if not index.isValid():
            return

        if index != self.currentIndex():
            self.itemChanged(index)

        item = self.currentItem()

        menu = QMenu(self)

        if isinstance(item, (Table, Schema)) and not isinstance(item, LTable):
            menu.addAction(QCoreApplication.translate("DBTree", "Rename…"), self.rename)
            menu.addAction(QCoreApplication.translate("DBTree", "Delete…"), self.delete)

            if isinstance(item, Table) and item.canBeAddedToCanvas():
                menu.addSeparator()
                menu.addAction(self.tr("Add to Canvas"), self.addLayer)
                item.addExtraContextMenuEntries(menu)

        elif isinstance(item, DBPlugin):
            if item.database() is not None:
                menu.addAction(self.tr("Re-connect"), self.reconnect)
            menu.addAction(self.tr("Remove"), self.delete)

        elif not index.parent().isValid() and item.typeName() in ("spatialite", "gpkg"):
            menu.addAction(QCoreApplication.translate("DBTree", "New Connection…"), self.newConnection)

        if not menu.isEmpty():
            menu.exec_(ev.globalPos())

        menu.deleteLater()

    def rename(self):
        item = self.currentItem()
        if isinstance(item, (Table, Schema)):
            self.edit(self.currentIndex())

    def delete(self):
        item = self.currentItem()
        if isinstance(item, (Table, Schema)):
            self.mainWindow.invokeCallback(item.database().deleteActionSlot)
        elif isinstance(item, DBPlugin):
            self.mainWindow.invokeCallback(item.removeActionSlot)

    def addLayer(self):
        table = self.currentTable()
        if table is not None:
            layer = table.toMapLayer(table.geometryType())
            layers = QgsProject.instance().addMapLayers([layer])
            if len(layers) != 1:
                QgsMessageLog.logMessage(
                    self.tr("%1 is an invalid layer - not loaded").replace("%1", layer.publicSource()))
                msgLabel = QLabel(self.tr(
                    "%1 is an invalid layer and cannot be loaded. Please check the message log for further info.").replace(
                    "%1", layer.publicSource()), self.mainWindow.infoBar)
                msgLabel.setWordWrap(True)
                msgLabel.linkActivated.connect(self.mainWindow.iface.mainWindow().findChild(QWidget, "MessageLog").show)
                msgLabel.linkActivated.connect(self.mainWindow.iface.mainWindow().raise_)
                self.mainWindow.infoBar.pushItem(QgsMessageBarItem(msgLabel, Qgis.Warning))

    def reconnect(self):
        db = self.currentDatabase()
        if db is not None:
            self.mainWindow.invokeCallback(db.reconnectActionSlot)


## rbmgr [genaf]
##
## genaf rbmgr script will override both rhombus' rbmgr
## this script will be called by run.py
## all parameters has been set by run.py

import sys, transaction

from rhombus.lib.utils import cout, cerr, cexit
from rhombus.scripts.rbmgr import ( init_argparser as rhombus_init_argparser,
                                    do_rbmgr as rhombus_do_rbmgr,
                                    main as rhombus_main,
                                    get_dbhandler)
from rhombus.scripts import setup_settings
#from fatools.scripts.dbmgr import ( init_argparser as fatools_init_argparser,
#                                    do_dbmgr as fatools_do_dbmgr )

def init_argparser( parser = None ):

    if parser is None:
        import argparse
        p = argparse.ArgumentParser('[genaf] - rbmgr', conflict_handler='resolve')
    else:
        p = parser

    # update our argparser
    p = rhombus_init_argparser( p )

    # provide our entry here
    return p

def main(args):

    cerr('[genaf] rbmgr main()')

    rhombus_main( args )

    return

    settings = setup_settings( args )

    if any( (args.exportuserclass, args.exportgroup, args.exportenumkey) ):
        do_dbmgr( args, settings )

    elif not args.rollback and not args.test and (args.commit or args.initdb):
        with transaction.manager:
            do_dbmgr( args, settings )
            cerr('** COMMIT database **')

    else:
        cerr('** WARNING -- running without database COMMIT **')
        if not args.rollback and not args.test:
            keys = input('Do you want to continue [y/n]: ')
            if keys.lower()[0] != 'y':
                sys.exit(1)
        do_dbmgr( args, settings )

def do_dbmgr(args, settings, dbh=None):

    if dbh is None:
        dbh = get_dbhandler(settings, initial = args.initdb)
    print(dbh)

    if not rhombus_do_dbmgr(args, settings, dbh):
        cerr('ERR - command unrecognized!')


import os
import pickle

import numpy as np
import pandas as pd
from flask import Flask, jsonify, request

from patient_count import con_rec_dec

app = Flask(__name__)

current_path = os.getcwd()
pickle_path = os.path.join(current_path, "assets", "covid.pkl")
print(pickle_path)
classifier = pickle.load(open(pickle_path, "rb"))
count_data_path = os.path.join(current_path, "assets", "districts.csv")
counts = pd.read_csv(count_data_path)

@app.route("/api/covid_test", methods=["POST"])
def predict_risk():
    data = request.get_json()
    print(data)
    sex = data["sex"]
    pneumonia = data["pneumonia"]
    age = data["age"]
    pregnancy = data["pregnancy"]
    diabetes = data["diabetes"]
    copd = data["copd"]
    asthma = data["asthma"]
    hypertension = data["hypertension"]
    other_disease = data["other_disease"]
    cardiovascular = data["cardiovascular"]
    obesity = data["obesity"]
    renal_chronic = data["renal_chronic"]
    tobacco = data["tobacco"]
    contact_other_covid = data["contact_other_covid"]

    cols = [
        "sex",
        "pneumonia",
        "age",
        "pregnancy",
        "diabetes",
        "copd",
        "asthma",
        "hypertension",
        "other_disease",
        "cardiovascular",
        "obesity",
        "renal_chronic",
        "tobacco",
        "contact_other_covid",
    ]

    test = pd.DataFrame(
        [
            [
                sex,
                pneumonia,
                age,
                pregnancy,
                diabetes,
                copd,
                asthma,
                hypertension,
                other_disease,
                cardiovascular,
                obesity,
                renal_chronic,
                tobacco,
                contact_other_covid,
            ]
        ],
        columns=cols,
    )

    pred = classifier.predict_proba(test)
    result = pred[0][1]

    if result < 0.25:
        return jsonify("Low Risk")
    elif (result >= 0.25) and (result < 0.5):
        return jsonify("Medium Risk")

    elif (result >= 0.5) and (result < 0.75):
        return jsonify("High Risk")

    else:
        return jsonify("Vulnerable Risk")

@app.route("/api/patient_count", methods=["POST"])
def patient_count_dist():
    data = request.get_json()
    # print(data)
    date = str(data["date"])
    district = str(data["district"])
    try:
        confirmed, recovered, deceased = con_rec_dec(counts, date, district)
        patient_details = f"Confirmed_cases= {confirmed}, Recovered_cases={recovered}, Deceased_cases= {deceased}"
        return jsonify(patient_details)
    except:
        return jsonify("Date or District input Error")

if __name__ == "__main__":
    app.run(debug=True)


import struct
import numpy as np

class Stl:
    dtype = np.dtype([
        ('normals',np.float32, (3,)),
        ('v0',np.float32,(3,)),
        ('v1',np.float32,(3,)),
        ('attr','u2',(1,)),
        ])

    def __init__(self,header,data):
        self.header = header
        self.data = data

    @classmethod
    def from_file(cls, filename, mode='rb'):
        with open(filename,mode) as fh:
            header = fh.read(80)
            size, = struct.unpack("@i", fh.read(4))
            data = np.fromfile(fh, dtype=cls.dtype, count=size)
            return Stl(header,data)

    def to_file(self, filename,mode='wb'):
        with open(filename, mode) as fh:
            fh.write(self.header)
            fh.write(struct.pack("@i",self.data.size))
            self.data.tofile(fh)

if __name__ == '__main__':
    stl = Stl.from_file('test.stl')

    stl.data['v0'][:, 0] += 1
    stl.data['v1'][:, 0] += 1
    stl.data['v2'][:, 0] += 1

    stl.to_file('test2.stl')


import configparser
import os
import codecs
from sys import platform as _platform
import dbf
import csv

def main():
    """Конвертер файлов импорта формата csv в формат dbf"""
    folder_in = 'in/csv'
    folder_in_dct = "%s/dct" % folder_in
    folder_out = 'out/dbf'
    folder_out_dct = "%s/dct" % folder_out

    config = configparser.ConfigParser()
    config.read('config.ini')
    if config.get("csv_to_dbf", "delimiter") == ";":
        delimiter = ';'
    elif config.get("csv_to_dbf", "delimiter") == ",":
        delimiter = ','
    elif config.get("csv_to_dbf", "delimiter") == "tab":
        delimiter = '\t'
    else:
        print("Разделитель в файле config.ini не указан. Разделителем будет ';'")
        delimiter = ';'

    if not os.path.exists(folder_out):
        os.makedirs(folder_out)
    if not os.path.exists(folder_out_dct):
        os.makedirs(folder_out_dct)

    files = [f for f in os.listdir(folder_in) if os.path.isfile("%s/%s" % (folder_in, f))]

    for fl in files:
        name, extension_in = fl.rsplit('.', 1)

        # Проверка, что файл имеет расширение csv и наличие словаря к файлу
        if extension_in == "csv" and os.path.isfile("%s/%s.dct" % (folder_in_dct, name)):
            reader = csv.reader(open('%s/%s.%s' % (folder_in, name, extension_in)), delimiter=delimiter, quotechar='"')
            # Замена разделителя на запятую(требуется для работы from_csv)
            with(open("%s/output.csv" % folder_out, 'w', newline="")) as infile:
                writer = csv.writer(infile, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL)
                for row in reader:
                    if row:
                        writer.writerow(row)
            # Удаление пустых строк
            with(codecs.open("%s/output.csv" % folder_out, 'r', "cp1251")) as infile:
                with(codecs.open("%s/output2.csv" % folder_out, 'w', "cp1251")) as infile2:
                    infile2.write(infile.read().rstrip())

            csvfile = "%s/output2.csv" % folder_out
            dbf_outfile = "%s/%s.dbf" % (folder_out, name)
            dctfile = "%s/%s.dct" % (folder_in_dct, name)
            dct_outfile = "%s/%s.dct" % (folder_out_dct, name)

            converter(csvfile, dctfile, dbf_outfile, dct_outfile)
            os.remove("%s/output.csv" % folder_out)
            os.remove("%s/output2.csv" % folder_out)

    # Открыть в проводнике ОС
    if _platform == "linux" or _platform == "linux2":
        # Требует наличия xdg-open
        os.system('xdg-open "%s"' % folder_out)
    elif _platform == "darwin":
        os.system('open "%s"' % folder_out)
    elif _platform == "win32":
        os.startfile(folder_out.replace("/", "\\"))

def converter(csvfile, dctfile, dbf_outfile, dct_outfile):
    print(csvfile)

    some_table = from_csv(csvfile=csvfile, encoding="cp1251", to_disk=False, filename=dbf_outfile, field_names=None, extra_fields=None, dbf_type="db3", memo_size=64, min_field_size=1)

    with(codecs.open(dctfile, 'r', "cp1251")) as infile:
        with(codecs.open(dct_outfile, 'w', "cp1251")) as outfile:
            i = 0
            f = 0
            for line in infile.readlines():
                if i == 2:
                    print(line.replace("csv", "dbf"))
                    outfile.write(line.replace("csv", "dbf"))
                elif i > 4:
                    print("%s=${F%d,M}" % (line.split("=", 1)[0], f))
                    outfile.write("%s=${F%d,M}\r\n" % (line.split("=", 1)[0], f))
                    f += 1
                else:
                    print(line)
                    outfile.write(line)
                i += 1

"""
функция кусок from_csv - изменённый код из пакета https://pypi.python.org/pypi/dbf
Изменена кодировка с cp1252 на cp1251
=========
Copyright
=========
    - Portions copyright: 2008-2012 Ad-Mail, Inc -- All rights reserved.
    - Portions copyright: 2012-2013 Ethan Furman -- All rights reserved.
    - Author: Ethan Furman
    - Contact: ethan@stoneleaf.us

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    - Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    - Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    - Neither the name of Ad-Mail, Inc nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
def from_csv(csvfile, to_disk=False, filename=None, field_names=None, extra_fields=None,
             dbf_type='db3', memo_size=64, min_field_size=1,
             encoding=None, errors=None):

    """
    creates a Character table from a csv file
    to_disk will create a table with the same name
    filename will be used if provided
    field_names default to f0, f1, f2, etc, unless specified (list)
    extra_fields can be used to add additional fields -- should be normal field specifiers (list)
    """
    with codecs.open(csvfile, 'r', encoding='cp1251', errors=errors) as fd:
        reader = csv.reader(fd)
        if field_names:
            if isinstance(field_names, dbf.basestring):
                field_names = field_names.split()
            if ' ' not in field_names[0]:
                field_names = ['%s M' % fn for fn in field_names]
        else:
            field_names = ['f0 M']
        if filename:
            to_disk = True
        else:
            filename = os.path.splitext(csvfile)[0]
        if to_disk:
            csv_table = dbf.Table(filename, [field_names[0]], dbf_type=dbf_type, memo_size=memo_size, codepage=encoding)
        else:
            csv_table = dbf.Table(':memory:', [field_names[0]], dbf_type=dbf_type, memo_size=memo_size,
                              codepage=encoding, on_disk=False)
        csv_table.open()
        fields_so_far = 1
        while reader:
            try:
                row = next(reader)
            except UnicodeEncodeError:
                row = ['']
            except StopIteration:
                break
            while fields_so_far < len(row):
                if fields_so_far == len(field_names):
                    field_names.append('f%d M' % fields_so_far)
                csv_table.add_fields(field_names[fields_so_far])
                fields_so_far += 1
            csv_table.append(tuple(row))
        if extra_fields:
            csv_table.add_fields(extra_fields)
        csv_table.close()
        return csv_table

if __name__ == '__main__':
    main()


#!/usr/bin/env python
import numpy as np
import base64

class LFSR():
    def __init__(self, fpoly=[5,2], initstate='ones', verbose=False):
        if isinstance(initstate, str):
            if initstate=='ones':
                initstate = np.ones(np.max(fpoly))
            elif initstate=='random':
                initstate = np.random.randint(0,2,np.max(fpoly))
            else:
                raise Exception('Unknown initial state')

        self.initstate = initstate
        self.fpoly  = fpoly
        self.state  = initstate.astype(int)
        self.count  = 0
        self.seq    = np.array(-1)
        self.outbit = -1
        self.feedbackbit = -1
        self.verbose = verbose
        self.M = self.initstate.shape[0]
        self.expectedPeriod = 2**self.M -1
        self.fpoly.sort(reverse=True)
        feed = ' '
        for i in range(len(self.fpoly)):
            feed = feed + 'x^'+str(self.fpoly[i])+' + '
        feed = feed + '1'
        self.feedpoly = feed

    def set(self,fpoly,state='ones'):
        self.__init__(fpoly=fpoly,initstate=state)

    def reset(self):
        self.__init__(initstate=self.initstate,fpoly=self.fpoly)

    def changeFpoly(self, newfpoly, reset=False):
        newfpoly.sort(reverse=True)
        self.fpoly =newfpoly
        feed = ' '
        for i in range(len(self.fpoly)):
            feed = feed + 'x^'+str(self.fpoly[i])+' + '
        feed = feed + '1'
        self.feedpoly = feed

        self.check()
        if reset:
            self.reset()

    def next(self):
        b = np.logical_xor(self.state[self.fpoly[0]-1],self.state[self.fpoly[1]-1])
        if len(self.fpoly)>2:
            for i in range(2,len(self.fpoly)):
                b = np.logical_xor(self.state[self.fpoly[i]-1],b)

        self.state = np.roll(self.state, 1)
        self.state[0] = b*1
        self.feedbackbit = b*1
        if self.count==0:
            self.seq = self.state[-1]
        else:
            self.seq = np.append(self.seq,self.state[-1])
        self.outbit = self.state[-1]
        self.count +=1
        if self.verbose:
            print('S: ',self.state)
        return self.state[-1]

    def runFullCycle(self):
        for i in range(self.expectedPeriod):
            self.next()
        return self.seq

    def runKCycle(self,k):
        tempseq =np.ones(k)*-1
        for i in range(k):
            tempseq[i] = self.next()

        return tempseq

def main():
    print("Reading flag...")
    with open("flag.txt") as f:
        flag = f.read().strip()

    flag = f"Well done! You got the flag. The flag is {flag}".encode('ascii')

    with open("state.txt") as f:
        state1 = np.array(list(map(int, f.readline().split(','))))
        state2 = np.array(list(map(int, f.readline().split(','))))

    print(f"state 1 size: {len(state1)}")
    print(f"state 2 size: {len(state2)}")

    fpoly1 = [8,4,1]
    fpoly2 = [13,6,3]

    A = LFSR(fpoly=fpoly1, initstate =state1)
    B = LFSR(fpoly=fpoly2, initstate =state2)

    size = len(flag)
    xord_byte_array = bytearray(len(flag))

    # Flag is encrypted by xor-ing with keystream
    # Keystream = (LFSR A output + LFSR B output) % 256
    print("Encrypting flag...")
    for i in range(size):
        newbyte1=0
        newbyte2=0
        for x in range(8):
            A.next()
            B.next()
            newbyte1 += A.outbit*pow(2,x)
            newbyte2 += B.outbit*pow(2,x)

        xorbyte=(newbyte1+newbyte2)%256
        xord_byte_array[i] = flag[i] ^ xorbyte

    encoded = base64.b64encode(xord_byte_array)
    print(f"Safely encrypted flag: {encoded.decode('ascii')}")

if __name__ == '__main__':
    main()


from fastai.vision import unet_learner, models, Learner

from fast_radiology.losses import generalized_dice_loss
from fast_radiology.models.unet import ClassicUNet
from artificial_contrast.const import (
    FREQS_LIMIT_WINDOWS,
    FREQS_NO_LIMIT_WINDOWS,
    SIMPLE_MULTIPLE_WINDOWS,
    SIMPLE_WINDOW_1CHANNEL_CLASSIC_UNET,
    SIMPLE_WINDOW_3CHANNEL_CLASSIC_UNET,
    SIMPLE_WINDOW_SMALL,
)

def get_learner(
    data, metrics=None, model_save_path='models', loss_func=None, pretrained=True, experiment_name=None
):
    if metrics is None:
        metrics = []
    if loss_func is None:
        loss_func = generalized_dice_loss

    if experiment_name is None or experiment_name in [
        FREQS_LIMIT_WINDOWS, FREQS_NO_LIMIT_WINDOWS, SIMPLE_MULTIPLE_WINDOWS, SIMPLE_WINDOW_SMALL
    ]:
        return unet_learner(
            data,
            models.resnet18,
            metrics=metrics,
            self_attention=True,
            loss_func=loss_func,
            path=model_save_path,
            pretrained=pretrained,
        )
    elif experiment_name in [
        SIMPLE_WINDOW_1CHANNEL_CLASSIC_UNET, SIMPLE_WINDOW_3CHANNEL_CLASSIC_UNET
    ]:
        if experiment_name == SIMPLE_WINDOW_1CHANNEL_CLASSIC_UNET:
            n_channels = 1
        elif experiment_name == SIMPLE_WINDOW_3CHANNEL_CLASSIC_UNET:
            n_channels = 3

        return Learner(
            data,
            ClassicUNet(n_channels=n_channels, n_classes=2),
            metrics=metrics,
            loss_func=loss_func,
            path=model_save_path,
        )
    else:
        raise RuntimeError('Incorrect experiment_name')


# -*- coding: utf-8 -*-

#Install Libraries
from bs4 import BeautifulSoup
import urllib2
import re
import csv
import itertools

#Add path to the file test.csv
writer=csv.writer(open('Path to crawled speaker data.csv','wb'))
x=[]
html = urllib2.urlopen("https://speakers.acm.org/")
soup = BeautifulSoup(html)
for h in soup.findAll("div", {"id": "topics-list"}):
  for a in h.findAll("a", {"href": True}):
    x.append("https://speakers.acm.org"+a["href"])

#https://speakers.acm.org/topics/applied-computing type page
for i in range(len(x)):
    html = urllib2.urlopen(x[i])
    soup = BeautifulSoup(html)
    for link in soup.findAll('a', attrs={'href': re.compile("\.\./speakers/")}):
        a=link.get('href')
        a=a.replace("..", "http://speakers.acm.org")
        a=(re.sub(r"\s+", "", a))
        a=a.encode('utf-8')
        a=a.replace("ö", "o")

        #Speaker page
        try:
            html = urllib2.urlopen(a)
        except urllib2.HTTPError as e:
            if e.getcode() == 404:
                continue
            raise

        #Using beautiful soup to crawl data
        soup = BeautifulSoup(html)
        for h in soup.findAll("li", {"class": "current"}):
            speaker_name=h.text
            speaker_name=speaker_name.encode('utf-8')
            writer.writerow([speaker_name])

        for h in soup.findAll("div", {"class": "speakers"}):
            for a in h.findAll("span"):
                b=a.text
                speaker_loc=re.sub(r"\s*Based in\s*", "", str(b))
                speaker_loc=re.sub(r"\s*$", "", str(speaker_loc))
                writer.writerow([speaker_loc])
                break


import numpy as np

from __context__ import src
from src import mathutils as mu

def estimateHomographies(allDetections: list):
    """
    Input:
        allDetections -- list of tuples (one for each view).
                Each tuple is (Xa, Xb), a set of sensor points
                and model points respectively

    Output:
        Hs -- list of homographies, one for each view
    """
    Hs = []
    for Xa, Xb in allDetections:
        H = estimateHomography(Xa[:,:2], Xb[:,:2])
        Hs.append(H)
    return Hs

def estimateHomography(Xa: np.ndarray, Xb: np.ndarray):
    """
    Estimate homography using DLT
    Inputs:
        Xa -- 2D points in sensor
        Xb -- 2D model points
    Output:
        aHb -- homography matrix which relates the model plane (points Xb)
                to the sensor plane (points Xa)
    Rearrange into the formulation:
        M * h = 0
    M represents the model and sensor point correspondences
    h is a vector representation of the homography aHb we are trying to find:
        h = (h11, h12, h13, h21, h22, h23, h31, h32, h33).T

    Prior to constructing M, the points Xa and Xb need to be 'normalized'
    so that the results of SVD are more well behaved.
    """
    mu.validateShape(Xa.shape, (None, 2))
    mu.validateShape(Xb.shape, (None, 2))
    Na = computeNormalizationMatrix(Xa)
    Nb = computeNormalizationMatrix(Xb)
    N = Xa.shape[0]
    M = np.zeros((2*N, 9))
    for i in range(N):
        ui, vi = mu.unhom(Na @ mu.hom(Xa[i]))
        Xi, Yi = mu.unhom(Nb @ mu.hom(Xb[i]))
        M[2*i,:]   = (-Xi, -Yi, -1,   0,   0,  0, ui * Xi, ui * Yi, ui)
        M[2*i+1,:] = (  0,   0,  0, -Xi, -Yi, -1, vi * Xi, vi * Yi, vi)
    U, S, V_T = np.linalg.svd(M)
    h = V_T[-1]
    Hp = h.reshape(3,3)
    H = np.linalg.inv(Na) @ Hp @ Nb
    H /= H[2,2]
    return H

def computeNormalizationMatrix(X):
    """
    Compute a matrix M which maps a set of points X to their 'normalized'
    form Xnorm, i.e.

        Xnorm = unhom(M * hom(X))

    where the mean Euclidean distance of the of points in Xnorm is sqrt(2)
    and the centroid of the points is the origin.

    For more on why this is necessary, see 'Multiple View Geometry in Computer Vision,
    2nd edition', Hartley & Zisserman, §4.4.4, pg 108.
    """
    Xmean = np.mean(X, axis=0)
    M1 = np.array([
        [1, 0, -Xmean[0]],
        [0, 1, -Xmean[1]],
        [0, 0, 1],
    ])
    Xshifted = X - Xmean
    Xmagnitudes = np.linalg.norm(Xshifted, axis=1)
    meanMagnitude = np.mean(Xmagnitudes)
    scaleFactor = np.sqrt(2) / meanMagnitude
    M2 = np.array([
        [scaleFactor, 0, 0],
        [0, scaleFactor, 0],
        [0, 0, 1],
    ])
    M = M2 @ M1
    return M

def computeIntrinsicMatrix(Hs: list):
    """
    Compute the intrinsic matrix from a list of homographies

    Inputs:
        Hs -- list of homographies

    Output:
        A -- intrinsic camera matrix

    From the Burger paper, use equations 96 - 98 to solve for vector b = (B0, B1, B2, B3, B4, B5)^T
    and then compute the intrinsic matrix and return.

    Notes:
        H = [h0 h1 h2] = lambda * A * [r0 r1 t]

        By leveraging that r0 and r1 are orthonormal, we get:

            h0^T * (A^-1)^T * A^-1 * h1 = 0
            h0^T * (A^-1)^T * A^-1 * h0 = h1^T * (A^-1)^T * A^-1 * h1

        B = (A^-1)^T * A^-1, where B = [B0 B1 B3]
                                       [B1 B2 B4]
                                       [B3 B4 B5]

        simplifying:
            h0^T * B * h1 = 0
            h0^T * B * h0 - h1^T * B * h1 = 0

        letting b = (B0, B1, B2, B3, B4, B5)^T

        we reformulate the h^T * B * h form:
            hp^T * B * hq = vecpq(H) * b

            with vec(H, p, q) = (
                    H0p * H0q,
                    H0p * H1q + H1p * H0q,
                    H1p * H1q,
                    H2p * H0q + H0p * H2q,
                    H2p * H1q + H1p * H2q,
                    H2p * H2q,
                )

        so we can rewrite our system of equations for a single homography as:

            [        vec(H, 0, 1)        ] * b = 0
            [vec(H, 0, 0) - vec(H, 1, 1) ]

        Now we can stack these terms in the left matrix for each homography to create
        a matrix V of size (2*N, 6) and then solve for b with SVD.

            V * b = 0
    """
    N = len(Hs)
    V = np.zeros((2*N, 6))
    for i in range(N):
        H = Hs[i]
        V[2*i,:]   = vecHomography(H, 0, 1)
        V[2*i+1,:] = vecHomography(H, 0, 0) - vecHomography(H, 1, 1)
    U, S, V_T = np.linalg.svd(V)
    b = tuple(V_T[-1])

    A = computeIntrinsicMatrixFrombCholesky(b)
    if np.sum(np.isnan(A)) > 0:
        raise ValueError(f"Computed intrinsic matrix contains NaN: \n{A}")
    return A

def vecHomography(H: np.ndarray, p: int, q: int):
    """
    Input:
        H -- 3x3 homography matrix
        p -- first column index
        q -- second column index

    Output:
        v -- 1x6 vector containing components of H based on the
             indices p and q which represent columns of the homography H

    Notes:
        This format of v allows the product to be used in homogenous
        form to solve for the values
        of a matrix which is a product of the intrinsic parameters (B).

        Implements equation 96 of Burger
    """
    values = (
        H[0,p] * H[0,q],
        H[0,p] * H[1,q] + H[1,p] * H[0,q],
        H[1,p] * H[1,q],
        H[2,p] * H[0,q] + H[0,p] * H[2,q],
        H[2,p] * H[1,q] + H[1,p] * H[2,q],
        H[2,p] * H[2,q],
    )
    v = np.array(values).reshape(1, 6)
    return v

def computeIntrinsicMatrixFrombClosedFormBurger(b):
    """
    Computes the intrinsic matrix from the vector b using the closed
    form solution given in Burger, equations 99 - 104.

    Input:
        b -- vector made up of (B0, B1, B2, B3, B4, B5)^T

    Output:
        A -- intrinsic matrix
    """
    B0, B1, B2, B3, B4, B5 = b

    # eqs 104, 105
    w = B0*B2*B5 - B1**2*B5 - B0*B4**2 + 2*B1*B3*B4 - B2*B3**2
    d = B0*B2 - B1**2

    α = np.sqrt(w / (d * B0))          # eq 99
    β = np.sqrt(w / d**2 * B0)         # eq 100
    γ = np.sqrt(w / (d**2 * B0)) * B1  # eq 101
    uc = (B1*B4 - B2*B3) / d           # eq 102
    vc = (B1*B3 - B0*B4) / d           # eq 103

    A = np.array([
        [α, γ, uc],
        [0, β, vc],
        [0, 0,  1],
    ])
    return A

def computeIntrinsicMatrixFrombClosedFormZhang(b):
    """
    Computes the intrinsic matrix from the vector b using the closed
    form solution given in Burger, equations 99 - 104.

    Input:
        b -- vector made up of (B0, B1, B2, B3, B4, B5)^T

    Output:
        A -- intrinsic matrix
    """
    B = matrixBfromVector(b)
    B11 = B[0,0]
    B12 = B[1,0]
    B13 = B[2,0]
    B22 = B[1,1]
    B23 = B[1,2]
    B33 = B[2,2]

    v0 = (B12 * B13 - B11 * B23) / (B11 * B22 - B12**2)
    λ = B33 - (B13**2 + v0 * (B12 * B13 - B11 * B23)) / B11
    α = np.sqrt(λ / B11)
    β = np.sqrt((λ * B11) / (B11 * B22 - B12**2))
    γ = -B12 * α**2 * β / λ
    u0 = γ * v0 / β - B13 * α**2 / λ

    A = np.array([
        [α, γ, u0],
        [0, β, v0],
        [0, 0,  1],
    ])
    return A

def matrixBfromVector(b):
    B0, B1, B2, B3, B4, B5 = b
    B = np.array([
        [B0, B1, B3],
        [B1, B2, B4],
        [B3, B4, B5],
    ])
    return B

def computeIntrinsicMatrixFrombCholesky(b: tuple):
    """
    Computes the intrinsic matrix from the vector b using
    Cholesky decomposition.

    Input:
        b -- vector made up of (B0, B1, B2, B3, B4, B5)^T

    Output:
        A -- intrinsic matrix

    Notes:
        Recall,

        B = (A^-1)^T * A^-1, where B = [B0 B1 B3]
                                       [B1 B2 B4]
                                       [B3 B4 B5]
        let L = (A^-1)^T
        then B = L * L^T
        L = Chol(B)

        and
        A = (L^T)^-1
    """
    B0, B1, B2, B3, B4, B5 = b
    # ensure B is positive semi-definite
    sign = +1
    if B0 < 0 or B2 < 0 or B5 < 0:
        sign = -1
    B = sign * np.array([
        [B0, B1, B3],
        [B1, B2, B4],
        [B3, B4, B5],
    ])
    L = np.linalg.cholesky(B)
    A = np.linalg.inv(L.T)
    A /= A[2,2]
    return A

def computeExtrinsics(Hs: list, A: np.ndarray):
    """
    Input:
        Hs -- list of homographies
        A -- intrinsic matrix

    Output:
        worldToCameraTransforms -- list of transform matrices from world to camera
    """
    Ainv = np.linalg.inv(A)
    worldToCameraTransforms = []
    for H in Hs:
        h0 = H[:,0]
        h1 = H[:,1]
        h2 = H[:,2]

        λ = np.linalg.norm(Ainv @ h0)

        r0 = (1 / λ) * Ainv @ h0
        r1 = (1 / λ) * Ainv @ h1
        r2 = np.cross(r0, r1)

        t = (1 / λ) * Ainv @ h2

        # Q is not in SO(3)
        Q = np.hstack((mu.col(r0), mu.col(r1), mu.col(r2)))

        # R is in SO(3)
        R = approximateRotationMatrix(Q)
        transformWorldToCamera = mu.poseFromRT(R, t)
        worldToCameraTransforms.append(transformWorldToCamera)
    return worldToCameraTransforms

def approximateRotationMatrix(Q: np.ndarray):
    """
    Input:
        Q -- a 3x3 matrix which is close to a rotation matrix

    Output:
        R -- a 3x3 rotation matrix which is in SO(3)

    Method from Zhang paper, Appendix C

    Notes:
        Find R which minimizes the frobenius_norm(R - Q) subject to R^T * R = I

        frobenius_norm(R - Q) = trace((R - Q)^T * (R - Q))
                              = 3 + trace(Q^T * Q) - 2*trace(R^T * Q)

        so equivalently, maximize trace(R^T * Q)

        let
            U, S, V^T = svd(Q)

        we define
            Z = V^T * R^T * U
        ==>
            trace(R^T * Q)
            trace(R^T * U * S * V^T)
            trace(V^T * R^T * U * S)
            trace(Z * S)
    """
    U, S, V_T = np.linalg.svd(Q)
    R = U @ V_T
    return R


import json
import os

from PyQt5.QtWidgets import QMainWindow, QFileDialog, QMessageBox

from ComparatorUi import Ui_Comparator
from utils.Config import Config
from utils.comparator import compare

class CompareTool(QMainWindow, Ui_Comparator):

    def __init__(self, parent=None):
        super(CompareTool, self).__init__(parent=parent)
        self.setupUi(self)

        self.textEdit.textChanged.connect(self.compare)
        self.textEdit_2.textChanged.connect(self.compare)
        self.pushButton_2.clicked.connect(self.openDest)

    def openDest(self):
        try:
            config = Config.readConfig()
            lastOpenDir = config['lastOpenDir']
        except:
            lastOpenDir = './'
        file, type = QFileDialog.getOpenFileName(self, 'Open a JSon file', lastOpenDir)
        if file:
            dir = os.path.dirname(file)
            Config.writeConfig({'lastOpenDir': dir})
            try:
                f = open(file, 'r', encoding='utf-8')
                text = f.read()
                data = json.load(f)
                self.textEdit_2.setPlainText(text)
                f.close()
            except:
                QMessageBox.warning(self, 'Error', 'Not JSon')

    def setSource(self, text):
        self.textEdit.setPlainText(text)

    def compare(self):
        text1 = self.textEdit.toPlainText()
        text2 = self.textEdit_2.toPlainText()
        if not text1 or not text2:
            return

        data1 = json.loads(text1, encoding='utf-8')
        data2 = json.loads(text2, encoding='utf-8')
        isSame, err = compare(data1, data2)
        if isSame:
            self.textBrowser.setVisible(False)
            self.statusBar.showMessage('Same JSon')
        else:
            promote = 'Note：\n\tThis comparator only compares the keys and the type of values, ignors all the real value\n\n'
            self.textBrowser.setText(promote + err)
            self.textBrowser.setVisible(True)
            self.statusBar.showMessage('Not same JSon')


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
@author: Florian Timm
@version: 2017.11.27
"""

import sqlite3
from vdFile import VdFile
from vdPoint import VdPoint

class VdSQLite(VdFile):

    """ class for writing data to sqlite database """

    def _open(self, filename):
        """
        opens a new db file
        :param filename: name of db file
        :type filename: str
        """
        filename = self._make_filename("db", filename)
        print(filename)
        self.__db = sqlite3.connect(filename)
        self.__cursor = self.__db.cursor()
        self.__cursor.execute("CREATE TABLE IF NOT EXISTS raw_data ("
                              "id INTEGER PRIMARY KEY AUTOINCREMENT,"
                              "time FLOAT,"
                              "azimuth FLOAT,"
                              "vertical FLOAT,"
                              "distance FLOAT,"
                              "reflection INTEGER)")
        self.__db.commit()

    def write(self):
        """ writes data to database """
        insert = []
        for p in self.writing_queue:
            insert.append((p.time, p.azimuth, p.vertical,
                           p.distance, p.reflection))

        self.__cursor.executemany("INSERT INTO raw_data ("
                                  "time, azimuth, vertical, "
                                  "distance, reflection) "
                                  "VALUES (?, ?, ?, ?, ?)", insert)
        self.__db.commit()
        self.clear_writing_queue()

    def close(self):
        """ close database """
        self.__db.close()


import time
import e32
import unittest
import e32calendar
import test.test_support

class CalendarTest(unittest.TestCase):
    week = 7 * 24 * 60 * 60
    day = 24 * 60 * 60
    hour = 60 * 60
    minute = 60
    now = 1243664756  # Sat, 30 May 2009 06:25:56 GMT

    def setUp(self):
        self.db = e32calendar.open('c:cal_test_db.cdb', 'n')

    def tearDown(self):
        self.db = e32calendar.open('c:cal_test_db.cdb', 'c')

        for id in self.db:
            del self.db[id]
        del self.db

    def create_calendar_entry(self, entry_type, entry_content):
        if entry_type == 'appointment':
            new_entry = self.db.add_appointment()
        elif entry_type == 'anniversary':
            new_entry = self.db.add_anniversary()
        elif entry_type == 'event':
            new_entry = self.db.add_event()
        elif entry_type == 'todo':
            new_entry = self.db.add_todo()
        elif entry_type == 'reminder':
            new_entry = self.db.add_reminder()

        new_entry.set_time(self.now + self.week, self.now + self.week +
                           self.hour)
        new_entry.alarm = self.now + self.week - 5 * self.minute
        new_entry.content = entry_content
        new_entry.location = 'right here right now'
        new_entry.replication = 'private'
        new_entry.priority = 1
        new_entry.commit()

        return new_entry

    def test_getitem_delitem(self):
        # Retrieve and delete the calendar entry using the __getitem__ and
        # __delitem__ members of CalendarDb class
        entry_content = 'test_getitem'
        new_entry = self.create_calendar_entry('appointment', entry_content)
        id = iter(self.db).next()
        stored_entry = self.db.__getitem__(id)
        self.assertEqual(stored_entry.content, entry_content)
        self.db.__delitem__(id)
        self.assertRaises(RuntimeError, lambda: self.db.__getitem__(id))

    def test_repeat_types(self):
        # Create different types of entries with repeat rules and then verify
        # that the rules are applied properly
        new_entry_1 = self.create_calendar_entry('appointment', 'daily')
        new_entry_2 = self.create_calendar_entry('anniversary',
                                                 'monthly_by_dates')
        new_entry_3 = self.create_calendar_entry('event', 'monthly_by_days')
        new_entry_4 = self.create_calendar_entry('todo', 'yearly_by_date')
        new_entry_5 = self.create_calendar_entry('reminder', 'yearly_by_day')
        new_entry_6 = self.create_calendar_entry('appointment', 'weekly')
        new_entry_7 = self.create_calendar_entry('anniversary', 'weekly2')

        repeat_1 = {'type': 'daily',
                  'start': new_entry_1.start_time,
                  'end': new_entry_1.start_time + self.week - self.day,
                  'interval': 2,  # on every second day.
                  'exceptions': []}

        # set monthly repeat (by dates) for 90 days.
        repeat_2 = {'type': 'monthly_by_dates',
                    # Sat, 06 Jun 2009 06:25:56 GMT
                    'start': new_entry_2.start_time,
                    # Thu, 03 Sep 2009 06:25:56 GMT
                    'end': new_entry_2.start_time + 90 * self.day - self.day,
                    # set the repeat occur 10th and 20th day of the month.
                    'days': [9, 19],
                    'exceptions': []}
                          # Wed, 10 Jun 2009 06:25:56 GMT
        repeat_2_value = {'start': 1244615156.0,
                          # Thu, 20 Aug 2009 06:25:56 GMT
                          'end': 1250749556.0,
                          'interval': 1,
                          'days': [9, 19],
                          'exceptions': [],
                          'type': 'monthly_by_dates'}

        # set monthly repeat (by days) for 90 days.
        repeat_3 = {'type': 'monthly_by_days',
                    # Sat, 06 Jun 2009 06:25:56 GMT
                    'start': new_entry_3.start_time,
                    # Thu, 03 Sep 2009 06:25:56 GMT
                    'end': new_entry_3.start_time + 90 * self.day - self.day,
                     # second tuesday and last friday of the month.
                    'days': [{'week': 1, 'day': 1}, {'week': 4, 'day': 4}],
                    'exceptions': []}
                          # Tue, 09 Jun 2009 06:25:56 GMT
        repeat_3_value = {'start': 1244528756.0,
                          # Fri, 28 Aug 2009 06:25:56
                          'end': 1251440756.0,
                          'interval': 1,
                          'days': [{'week': 4, 'day': 4},
                                   {'week': 1, 'day': 1}],
                          'exceptions': [],
                          'type': 'monthly_by_days'}

        # set yearly repeat (by date) for 3 years.
        repeat_4 = {'type': 'yearly_by_date',
                    # Sat, 06 Jun 2009 07:25:56 GMT
                    'start': new_entry_4.start_time,
                     # Mon, 04 Jun 2012 07:25:56 GMT
                    'end': new_entry_4.start_time + 3 * 365 * self.day - \
                                                                      self.day,
                    'exceptions': []}
                          # Sat, 06 Jun 2009 07:25:56 GMT
        repeat_4_value = {'start': 1244273156.0,
                          # Mon, 06 Jun 2011 07:25:56 GMT
                          'end': 1307345156.0,
                          'type': 'yearly_by_date',
                          'interval': 1,
                          'exceptions': []}

        # set yearly repeat (by day) on third thursday of june
        # during time interval new_entry.start_time -->
        # new_entry.start_time + 3 * 365 * day - day.
        repeat_5 = {'type': 'yearly_by_day',
                    # Sat, 06 Jun 2009 06:25:56 GMT
                    'start': new_entry_5.start_time,
                     # Mon, 04 Jun 2012 06:25:56 GMT
                    'end': new_entry_5.start_time + 3 * 365 * self.day - \
                                                                      self.day,
                    'days': {'day': 3, 'week': 2, 'month': 5},
                    'exceptions': []}
                          # Thu, 18 Jun 2009 06:25:56 GMT
        repeat_5_value = {'start': 1245306356.0,
                          # Thu, 21 Jun 2012 06:25:56 GMT
                          'end': 1340259956.0,
                          'interval': 1,
                          'days': {'week': 2, 'day': 3, 'month': 5},
                          'exceptions': [],
                          'type': 'yearly_by_day'}

        # repeat on tuesdays and thursdays every second week except on the
        # exception dates ('exceptions').
        repeat_6 = {'type': 'weekly',
                    # Sat, 06 Jun 2009 06:25:56 GMT
                    'start': new_entry_6.start_time,
                    # Sat, 15 Aug 2009 06:25:56 GMT
                    'end': new_entry_6.start_time + 10 * self.week,
                    'days': [1, 3],  # repeat on tuesday and thursday.
                    # 'exceptions':
                    # [1244874356.0, Sat, 13 Jun 2009 06:25:56 GMT
                    #  1244960756.0, Sun, 14 Jun 2009 06:25:56 GMT
                    #  1245047156.0, Mon, 15 Jun 2009 06:25:56 GMT
                    #  1245133556.0, Tue, 16 Jun 2009 06:25:56 GMT
                    #  1245219956.0, Wed, 17 Jun 2009 06:25:56 GMT
                    #  1245306356.0, Thu, 18 Jun 2009 06:25:56 GMT
                    #  1245392756.0] Fri, 19 Jun 2009 06:25:56 GMT
                    # --------------------------------------------
                    # no repeats on these days.
                    'exceptions': list([new_entry_6.start_time + self.week +
                                        i * self.day for i in range(7)]),
                    # repeat every second week.
                    'interval': 2}
                          # Tue, 16 Jun 2009 06:25:56 GMT
        repeat_6_value = {'start': 1245133556.0,
                          # Thu, 13 Aug 2009 06:25:56 GMT
                          'end': 1250144756.0,
                          'interval': 2,
                          'days': [1, 3],
                          # Tue, 16 Jun 2009 06:25:56 GMT &
                          # Thu, 18 Jun 2009 06:25:56 GMT
                          'exceptions': [1245133556.0, 1245306356.0],
                          'type': 'weekly'}

        # make it repeat weekly for 4 weeks.
        repeat_7 = {'type': 'weekly',
                    # which days in a week (monday, tuesday)
                    'days': [0, 1],
                    # Sat, 06 Jun 2009 06:25:56 GMT
                    'start': new_entry_7.start_time,
                    # Fri, 03 Jul 2009 06:25:56 GMT
                    'end': new_entry_7.start_time + 4 * self.week - self.day,
                    'exceptions': []}
                          # Mon, 08 Jun 2009 06:25:56 GMT
        repeat_7_value = {'start': 1244442356.0,
                          # Tue, 30 Jun 2009 06:25:56 GMT
                          'end': 1246343156.0,
                          'interval': 1,
                          'days': [0, 1],
                          'exceptions': [],
                          'type': 'weekly'}
        new_entry_1.begin()
        new_entry_1.set_repeat(repeat_1)
        new_entry_1.commit()

        new_entry_2.begin()
        new_entry_2.set_repeat(repeat_2)
        new_entry_2.commit()

        new_entry_3.begin()
        new_entry_3.set_repeat(repeat_3)
        new_entry_3.commit()

        new_entry_4.begin()
        new_entry_4.set_repeat(repeat_4)
        new_entry_4.commit()

        new_entry_5.begin()
        new_entry_5.set_repeat(repeat_5)
        new_entry_5.commit()

        new_entry_6.begin()
        new_entry_6.set_repeat(repeat_6)
        new_entry_6.commit()

        new_entry_7.begin()
        new_entry_7.set_repeat(repeat_7)
        new_entry_7.commit()

        self.assertEqual(repeat_1, new_entry_1.get_repeat())
        self.assertEqual(repeat_2_value, new_entry_2.get_repeat())
        self.assertEqual(repeat_3_value, new_entry_3.get_repeat())
        self.assertEqual(repeat_4_value, new_entry_4.get_repeat())
        self.assertEqual(repeat_5_value, new_entry_5.get_repeat())
        self.assertEqual(repeat_6_value, new_entry_6.get_repeat())
        self.assertEqual(repeat_7_value, new_entry_7.get_repeat())

    def test_export_import(self):
        # Exports an appointment entry and then imports the same and checks
        # the Entry ID
        db_id = []
        new_entry = self.create_calendar_entry('appointment', 'import/export')
        db_id.append(iter(self.db).next())
        vcals = self.db.export_vcalendars(tuple(db_id))
        imported_id = self.db.import_vcalendars(vcals)
        self.assertEqual(imported_id, db_id)

    def test_autocommit_rollback(self):
        # Autocommit and rollback feature tested by changing the entry
        # content
        new_entry = self.create_calendar_entry('appointment', 'autocommit')

        for id in self.db:
            entry = self.db[id]

        # autocommit is now on since the content has changed in the database.
        entry.content = 'TEXT I'
        self.assertEqual(self.db[entry.id].content, entry.content)

        # autocommit is now off..since the content has not changed in the
        # database.
        entry.begin()
        entry.content = 'TEXT II'
        self.assertNotEqual(self.db[entry.id].content, entry.content)

        # Now the changes are saved (and autocommit is set on again) since the
        # content has changed in the database.
        entry.commit()
        self.assertEqual(self.db[entry.id].content, entry.content)

        entry.begin()
        entry.content = 'TEXT III'
        # Revert the content back to 'TEXT II'
        entry.rollback()
        self.assertEqual(self.db[entry.id].content, 'TEXT II')

    def test_instances(self):
        # Create daily and monthly instances and then search for the same
        # using find_instances
        start_time = time.mktime((2008, 5, 31, 8, 0, 0, 0, 0, 0))
        end_time = time.mktime((2008, 5, 31, 9, 0, 0, 0, 0, 0))

        new_entry = self.create_calendar_entry('appointment', 'daily instance')

        # Appointment Entry created and committed for may31
        new_entry.begin()
        new_entry.set_time(start_time, end_time)
        new_entry.alarm = start_time - 5 * self.minute
        new_entry.commit()

        may30 = time.mktime((2008, 5, 30, 0, 0, 0, 0, 0, 0))
        may31 = time.mktime((2008, 5, 31, 0, 0, 0, 0, 0, 0))
        june30 = time.mktime((2008, 6, 30, 0, 0, 0, 0, 0, 0))

        # Daily instance 2 & 3 should not match any entries
        daily_instances_1 = self.db.daily_instances(may31)
        daily_instances_2 = self.db.daily_instances(may30)
        daily_instances_3 = self.db.daily_instances(may31, events=1)

        entry_id = iter(self.db).next()

        self.assertEqual(len(daily_instances_1), 1)
        self.assertEqual(entry_id, daily_instances_1[0]['id'])
        self.assertNotEqual(len(daily_instances_2), 1)
        self.assertNotEqual(len(daily_instances_3), 1)

        # Monthly instance 2 & 3 should not match any entries
        monthly_instances_1 = self.db.monthly_instances(may31)
        monthly_instances_2 = self.db.monthly_instances(june30)
        monthly_instances_3 = self.db.monthly_instances(may31, events=1,
                                                        todos=1)

        self.assertEqual(len(monthly_instances_1), 1)
        self.assertEqual(entry_id, monthly_instances_1[0]['id'])
        self.assertNotEqual(len(daily_instances_2), 1)
        self.assertNotEqual(len(daily_instances_3), 1)

        # Searching for 'instance' event and 'month' should return '[]'
        instances_found_1 = self.db.find_instances(may30, june30, u'instance')
        instances_found_2 = self.db.find_instances(may30, june30, u'instance',
                                                   events=1)
        instances_found_3 = self.db.find_instances(may30, june30, u'month')

        self.assertEqual(len(instances_found_1), 1)
        self.assertEqual(entry_id, instances_found_1[0]['id'])
        self.assertNotEqual(len(instances_found_2), 1)
        self.assertNotEqual(len(instances_found_3), 1)

def test_main():
    if e32.in_emulator():
        raise test.test_support.TestSkipped("Cannot test on emulator in " +
                                            "textshell mode")
    test.test_support.run_unittest(CalendarTest)

if __name__ == "__main__":
    test_main()


# https://www.codewars.com/kata/529bf0e9bdf7657179000008

def valid_solution(board):
    regex=[1,2,3,4,5,6,7,8,9]
    for lst in board:
        if regex != sorted(lst):return False
    for i in range(9):
        if regex != sorted([board[nb][i] for nb in range(9)]):return False
    for y in range(0,9,3):
        for x in range(0,9,3):
            lst = []
            for ix in range(3):
                for iy in range(3):
                    lst.append(board[x+ix][y+iy])
            if sorted(lst) != regex:return False
    return True

Python/4 kyu/Sudoku Solution Validator.py


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2020/4/29 23:20
# @Author : Mr. Zhou Chuang
# @Site :
# @File : 接雨水.py
# @Software: PyCharm
from typing import List

class Solution:
    def trap(self, height: List[int]) -> int:
        # 1. 暴力法
        # 能接多少雨水与左右两侧最近的比它高的有关
        # if not height:
        #     return 0
        # n = len(height)
        # max_left = [0] * n
        # max_right = [0] * n
        # max_left[0] = height[0]
        # max_right[-1] = height[-1]
        #
        # for i in range(1, n):
        #     max_left[i] = max(height[i], max_left[i-1])  # max_left处的值要么为本身，要么为前一个max_left
        # for i in range(n - 1 - 1, -1, -1):
        #     max_right[i] = max(height[i], max_right[i+1])
        # res = 0
        # for i in range(n):
        #     res += min(max_left[i], max_right[i])-height[i]
        #
        # return res

        # 2. 双指针
        # if not height:
        #     return 0
        # left = 0
        # right = len(height) - 1
        # res = 0
        # # 记录左右两边最大值
        # left_max = height[left]
        # right_max = height[right]
        # while left < right:
        #     if height[left] < height[right]:
        #         if left_max > height[left]:
        #             res += left_max - height[left]
        #         else:
        #             left_max = height[left]  # 更新 left_max
        #         left += 1
        #     else:
        #         if right_max > height[right]:
        #             res += right_max - height[right]
        #         else:
        #             right_max = height[right]  # 更新right_max
        #         right -= 1
        # return res

        length = len(height)
        if length < 3: return 0
        res, idx = 0, 0
        stack = []
        while idx < length:
            while len(stack) > 0 and height[idx] > height[stack[-1]]:
                top = stack.pop()  # index of the last element in the stack
                if len(stack) == 0:
                    break
                h = min(height[stack[-1]], height[idx]) - height[top]
                dist = idx - stack[-1] - 1
                res += (dist * h)
            stack.append(idx)
            idx += 1
        return res

if __name__ == '__main__':
    Sol = Solution()
    ans = Sol.trap([0, 1, 0, 2, 1, 0, 1, 3, 2, 1, 2, 1])
    print(ans)

算法学习/栈、队列/接雨水.py


import math
import random

from openrazer.client.devices.keyboard import RazerKeyboard
from openrazer.client.fx import Frame
from ore.core.program import KeyboardProgram
from ore.core.utils import ROWS, COLS
from ore.registry.registry import register_effect

@register_effect
class Wave(KeyboardProgram):

    def get_name(self):
        return "Wave"

    def get_cli_name(self):
        return "wave"

    def get_author(self):
        return "explodes"

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.time = 0
        self.random = random.Random()

    def init(self, kb: RazerKeyboard):
        kb.brightness = 100

    def draw(self, kb: RazerKeyboard, matrix: Frame, frame_num: int, dt: float):
        RPS = 1  # revolutions per second
        CPS = 0.25  # color revolutions per second
        self.time += COLS * dt * RPS

        theta = math.pi / ROWS
        offset = theta * self.time

        for col in range(COLS):
            mid_point = int((math.sin(theta * col + offset) + 1) / 2 * (ROWS + 1))

            for y in range(0, ROWS):
                if ROWS <= y < 0:
                    continue

                if y < mid_point:
                    color_wave = (math.sin(self.time * CPS) + 1) / 2
                    matrix[y, col] = (255 * color_wave, 10 * y, 20 * color_wave)
                else:
                    color_wave = (math.sin(self.time * CPS / 2) + 1) / 2
                    matrix[y, col] = (0, 50 + 50 * color_wave, ROWS * 5 - 5 * y)

if __name__ == "__main__":
    effect = Wave()
    effect.start()


import os

import torch
from torch import nn
from torch.nn import functional as F
from torch.utils.data import DataLoader, random_split

import pytorch_lightning as pl
from pytorch_lightning.metrics.functional import accuracy

# self attention with lstm
class AttentionModel(pl.LightningModule):
    def __init__(self, trial,batch_size,input_dim, output_dim):
        super(AttentionModel, self).__init__()

        self.batch_size = batch_size
        self.input_dim = input_dim
        self.output_dim = output_dim

        self.hidden_dim = 40
        self.recurrent_layers = 2
        self.dropout_p = 0.1

        self.input_embeded = nn.Linear(input_dim, self.hidden_dim//2)
        self.dropout = nn.Dropout(self.dropout_p)
        self.lstm = nn.LSTM(input_size=self.hidden_dim//2, hidden_size=self.hidden_dim, num_layers=self.recurrent_layers,
                            bidirectional=True)

        self.self_attention = nn.Sequential(
            nn.Linear(self.hidden_dim*2, self.hidden_dim*2),
            nn.ReLU(True),
            nn.Linear(self.hidden_dim*2, 1)
        )

        self.scale = 1.0/np.sqrt(self.hidden_dim)

        # initialize LSTM forget gate bias to be 1 as recommanded by http://proceedings.mlr.press/v37/jozefowicz15.pdf
        for names in self.lstm._all_weights:
            for name in filter(lambda n: "bias" in n, names):
                bias = getattr(self.lstm, name)
                n = bias.size(0)
                start, end = n // 4, n // 2
                bias.data[start:end].fill_(1.)

        self.output_linear = nn.Linear(self.hidden_dim * 2, self.hidden_dim)
        self.label = nn.Linear(self.hidden_dim*4, output_dim)

    def forward(self, input_sentences, batch_size=None):
        input = self.dropout(torch.tanh(self.input_embeded(input_sentences)))
        input = input.permute(1, 0, 2)
        if batch_size is None:
            h_0 = Variable(torch.zeros(2*self.recurrent_layers,
                                       self.batch_size, self.hidden_dim).to(device))
            c_0 = Variable(torch.zeros(2*self.recurrent_layers,
                                       self.batch_size, self.hidden_dim).to(device))
        else:
            h_0 = Variable(torch.zeros(2*self.recurrent_layers,
                                       batch_size, self.hidden_dim).to(device))
            c_0 = Variable(torch.zeros(2*self.recurrent_layers,
                                       batch_size, self.hidden_dim).to(device))

        self.lstm.flatten_parameters()

        output, (final_hidden_state, final_cell_state) = self.lstm(
            input, (h_0, c_0))
        output = output.permute(1, 0, 2)

        attn_ene = self.self_attention(output)
        attn_ene = attn_ene.reshape(
            self.batch_size, -1)

        # scale
        attn_ene.mul_(self.scale)
        attns = F.softmax(attn_ene, dim=1).unsqueeze(2)

        final_inputs = (output * attns).sum(dim=1)
        final_inputs2 = output.sum(dim=1)

        combined_inputs = torch.cat([final_inputs, final_inputs2], dim=1)
        logits = self.label(combined_inputs)

        return logits

    def training_step(self, batch, batch_idx):
        x, y = batch
        logits = self(x)
        loss = F.nll_loss(logits, y)
        return loss

    def validation_step(self, batch, batch_idx):
        x, y = batch
        logits = self(x)
        loss = F.nll_loss(logits, y)
        preds = torch.argmax(logits, dim=1)
        acc = accuracy(preds, y)

        # Calling self.log will surface up scalars for you in TensorBoard
        self.log('val_loss', loss, prog_bar=True)
        self.log('val_acc', acc, prog_bar=True)
        return loss

    def test_step(self, batch, batch_idx):
        # Here we just reuse the validation_step for testing
        return self.validation_step(batch, batch_idx)

    def configure_optimizers(self):
        optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)
        return optimizer

    ####################
    # DATA RELATED HOOKS
    ####################

    def prepare_data(self):
        # download
        MNIST(self.data_dir, train=True, download=True)
        MNIST(self.data_dir, train=False, download=True)

    def setup(self, stage=None):

        # Assign train/val datasets for use in dataloaders
        if stage == 'fit' or stage is None:
            mnist_full = MNIST(self.data_dir, train=True, transform=self.transform)
            self.mnist_train, self.mnist_val = random_split(mnist_full, [55000, 5000])

        # Assign test dataset for use in dataloader(s)
        if stage == 'test' or stage is None:
            self.mnist_test = MNIST(self.data_dir, train=False, transform=self.transform)

    def train_dataloader(self):
        return DataLoader(self.mnist_train, batch_size=32)

    def val_dataloader(self):
        return DataLoader(self.mnist_val, batch_size=32)

    def test_dataloader(self):
        return DataLoader(self.mnist_test, batch_size=32)

from LSTM_Classification_V3 import AttentionModel, clip_gradient
# from LSTM_Classification import AttentionModel, clip_gradient
from loss import LabelSmoothingCrossEntropy
from imblearn.over_sampling import RandomOverSampler
from sklearn.metrics import accuracy_score, cohen_kappa_score
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from catalyst.utils import set_global_seed
from catalyst.dl import SupervisedRunner, Runner
from catalyst.utils import metrics
from catalyst.dl.callbacks import AccuracyCallback, AUCCallback, F1ScoreCallback, EarlyStoppingCallback, CriterionCallback

import torch
import torch.nn as nn
from torch.utils.data import Dataset as BaseDataset
from torch.utils.data import DataLoader, TensorDataset
from torch.nn import functional as F

from collections import OrderedDict
import matplotlib.pyplot as plt
import os
import time
import numpy as np
import pandas as pd
np.set_printoptions(threshold=np.inf)
pd.options.display.width = 0

# reproduce
SEED = 15
set_global_seed(SEED)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# read csv files
def data_together(filepath):
    csvs = []
    dfs = []

    for subdir, dirs, files in os.walk(filepath):
        for file in files:
            # print os.path.join(subdir, file)
            filepath = subdir + os.sep + file
            if filepath.endswith(".csv"):
                csvs.append(filepath)

    for f in csvs:
        dfs.append(pd.read_csv(f))

    return dfs, csvs

# determine the supported device

def get_device():
    if torch.cuda.is_available():
        device = torch.device('cuda:0')
    else:
        device = torch.device('cpu')  # don't have GPU
    return device

# convert a df to tensor to be used in pytorch

def numpy_to_tensor(ay, tp):
    device = get_device()
    return torch.from_numpy(ay).type(tp).to(device)

# count model parameters

def count_parameters(model):
    return sum(p.numel() for p in model.parameters() if p.requires_grad)

class CustomRunner(Runner):

    def _handle_batch(self, batch):
        x, y = batch
        # y_hat, attention = self.model(x)
        outputs = self.model(x)

        loss = F.cross_entropy(outputs['logits'], y)
        accuracy01, accuracy02 = metrics.accuracy(
            outputs['logits'], y, topk=(1, 2))
        self.batch_metrics = {
            "loss": loss,
            "accuracy01": accuracy01,
            "accuracy02": accuracy02,
        }

        if self.is_train_loader:
            loss.backward()
            self.optimizer.step()
            self.optimizer.zero_grad()

# classification label smoothing
class LabelSmoothingCrossEntropy(nn.Module):
    def __init__(self):
        super(LabelSmoothingCrossEntropy, self).__init__()

    def forward(self, x, target, smoothing=0.1):
        confidence = 1. - smoothing
        logprobs = F.log_softmax(x, dim=-1)
        nll_loss = -logprobs.gather(dim=-1, index=target.unsqueeze(1))
        nll_loss = nll_loss.squeeze(1)
        smooth_loss = -logprobs.mean(dim=-1)
        loss = confidence * nll_loss + smoothing * smooth_loss
        return loss.mean()

# self attention with lstm
class AttentionModel(torch.nn.Module):
    def __init__(self, trial,batch_size,input_dim, output_dim):
        super(AttentionModel, self).__init__()

        self.batch_size = batch_size
        self.input_dim = input_dim
        self.output_dim = output_dim

        self.hidden_dim = trial.suggest_int("hidden_dim", 40, 80,step=4,log=False)
        self.recurrent_layers = trial.suggest_int("recurrent_layers", 1, 3)
        self.dropout_p = trial.suggest_float("dropout_rate", 0, 0.3,step=0.1,log=False)

        self.input_embeded = nn.Linear(input_dim, self.hidden_dim//2)
        self.dropout = nn.Dropout(self.dropout_p)
        self.lstm = nn.LSTM(input_size=self.hidden_dim//2, hidden_size=self.hidden_dim, num_layers=self.recurrent_layers,
                            bidirectional=True)

        self.self_attention = nn.Sequential(
            nn.Linear(self.hidden_dim*2, self.hidden_dim*2),
            nn.ReLU(True),
            nn.Linear(self.hidden_dim*2, 1)
        )

        self.scale = 1.0/np.sqrt(self.hidden_dim)

        # initialize LSTM forget gate bias to be 1 as recommanded by http://proceedings.mlr.press/v37/jozefowicz15.pdf
        for names in self.lstm._all_weights:
            for name in filter(lambda n: "bias" in n, names):
                bias = getattr(self.lstm, name)
                n = bias.size(0)
                start, end = n // 4, n // 2
                bias.data[start:end].fill_(1.)

        self.output_linear = nn.Linear(self.hidden_dim * 2, self.hidden_dim)
        self.label = nn.Linear(self.hidden_dim*4, output_dim)

    def forward(self, input_sentences, batch_size=None):

        input = self.dropout(torch.tanh(self.input_embeded(input_sentences)))
        input = input.permute(1, 0, 2)
        if batch_size is None:
            h_0 = Variable(torch.zeros(2*self.recurrent_layers,
                                       self.batch_size, self.hidden_dim).to(device))
            c_0 = Variable(torch.zeros(2*self.recurrent_layers,
                                       self.batch_size, self.hidden_dim).to(device))
        else:
            h_0 = Variable(torch.zeros(2*self.recurrent_layers,
                                       batch_size, self.hidden_dim).to(device))
            c_0 = Variable(torch.zeros(2*self.recurrent_layers,
                                       batch_size, self.hidden_dim).to(device))

        self.lstm.flatten_parameters()

        output, (final_hidden_state, final_cell_state) = self.lstm(
            input, (h_0, c_0))
        output = output.permute(1, 0, 2)

        attn_ene = self.self_attention(output)
        attn_ene = attn_ene.reshape(
            self.batch_size, -1)

        # scale
        attn_ene.mul_(self.scale)
        attns = F.softmax(attn_ene, dim=1).unsqueeze(2)

        final_inputs = (output * attns).sum(dim=1)
        final_inputs2 = output.sum(dim=1)

        combined_inputs = torch.cat([final_inputs, final_inputs2], dim=1)
        logits = self.label(combined_inputs)

        return logits

if __name__ == "__main__":
    # sample data

    data_path = 'R:/CROPPHEN-Q2067/MoDS/Dabang_Sheng/Data/cleaned_data_25753.csv'
    df_all = pd.read_csv(data_path)
    labels = df_all.iloc[:, 2]
    df_all = df_all.iloc[:, 12:210].copy()
    # remove bigger than 1
    mask = (df_all <= 1).all(axis=1)
    df_all = df_all[mask]
    labels = labels[mask]

    # print((df_all.values >1).any())
    X = df_all
    y = labels

    le = LabelEncoder()
    le.fit(y)
    print(le.classes_)
    class_names = le.classes_
    y = le.transform(y)

    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.3, random_state=SEED, stratify=y)

    # # normalizeation
    # scaler = StandardScaler()
    # scaler.fit(X_train)

    # scaler_data_ = np.array([scaler.scale_, scaler.mean_, scaler.var_])
    # np.save("standard_scaler_2019filtered.npy", scaler_data_)

    # Using the standard deviation, mean and variance results from above.

    # X_train = scaler.transform(X_train)
    # X_test = scaler.transform(X_test)

    X_train_resampled, y_train_resampled = X_train, y_train

    # DataLoader definition
    # model hyperparameters
    INPUT_DIM = 1
    OUTPUT_DIM = 5
    HID_DIM = 64
    DROPOUT = 0.1
    RECURRENT_Layers = 2
    # LR = 0.001  # learning rate
    EPOCHS = 400
    BATCH_SIZE = 32
    num_classes = 5

    unique_elements, counts_elements = np.unique(y_train, return_counts=True)
    weights = [1/i for i in counts_elements]
    weights[2] = weights[2]/10
    print(np.asarray((unique_elements, counts_elements)))
    print(weights)
    samples_weight = np.array([weights[t] for t in y_train])
    samples_weights = torch.FloatTensor(samples_weight).to(device)
    class_weights = torch.FloatTensor(weights).to(device)
    sampler = torch.utils.data.sampler.WeightedRandomSampler(
        samples_weights, len(X_train_resampled), replacement=True)

    # # check sample no.
    # unique_elements, counts_elements = np.unique(y, return_counts=True)
    # print("Frequency of unique values of the said array:")
    # print(np.asarray((unique_elements, counts_elements)))

    # prepare PyTorch Datasets

    X_train_tensor = numpy_to_tensor(
        X_train_resampled.to_numpy(), torch.FloatTensor)
    y_train_tensor = numpy_to_tensor(y_train_resampled, torch.long)
    X_test_tensor = numpy_to_tensor(X_test.to_numpy(), torch.FloatTensor)
    y_test_tensor = numpy_to_tensor(y_test, torch.long)

    X_train_tensor = torch.unsqueeze(X_train_tensor, 2)
    X_test_tensor = torch.unsqueeze(X_test_tensor, 2)

    train_ds = TensorDataset(X_train_tensor, y_train_tensor)
    valid_ds = TensorDataset(X_test_tensor, y_test_tensor)

    train_dl = DataLoader(train_ds, batch_size=BATCH_SIZE,
                          sampler=sampler, drop_last=True, num_workers=0)
    valid_dl = DataLoader(valid_ds, batch_size=BATCH_SIZE,
                          shuffle=False, drop_last=True, num_workers=0)

    ground_truth = []
    for i in valid_dl:
        ground_truth.append(i[1].cpu().numpy().tolist())

    # print(ground_truth.flatten())
    ground_truth = [item for sublist in ground_truth for item in sublist]

    # Catalyst loader:

    loaders = OrderedDict()
    loaders["train"] = train_dl
    loaders["valid"] = valid_dl

    # model, criterion, optimizer, scheduler

    model = AttentionModel(BATCH_SIZE, INPUT_DIM, HID_DIM,
                           OUTPUT_DIM, RECURRENT_Layers, DROPOUT).to(device)

    print(model)
    count = count_parameters(model)
    print(count)

    criterion = torch.nn.CrossEntropyLoss()
    # criterion = LabelSmoothingCrossEntropy()
    optimizer = torch.optim.Adam(model.parameters(), lr=0.006)
    scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, [10, 40, 60])


import sys
import matplotlib.pyplot as plt
import networkx as nx
import random

def connect_dots_with_minimal_distance(matrix_with_distances, graph):
    minimal_distance = 0
    i_min, j_min = 0, 0

    for i in range(n):
        for j in range(i + 1, n):
            if minimal_distance > matrix_with_distances[i][j]:
                minimal_distance = matrix_with_distances[i][j]
                i_min, j_min = i, j

    graph[i_min][j_min] = minimal_distance
    graph[j_min][i_min] = minimal_distance

    matrix_with_distances[i_min][j_min] = sys.maxsize
    matrix_with_distances[j_min][i_min] = sys.maxsize

    selection = [0] * n
    selection[i_min] = 1
    selection[j_min] = 1
    return selection

def connect_all_in_selection(matrix_with_distances, graph, selection):
    minimal_distance = sys.maxsize
    i_min, j_min = 0, 0

    for i in range(n):
        if selection[i] == 1:
            for j in range(n):
                if selection[j] == 0:
                    if minimal_distance > matrix_with_distances[i][j]:
                        minimal_distance = matrix_with_distances[i][j]
                        i_min, j_min = i, j

    graph[i_min][j_min] = minimal_distance
    graph[j_min][i_min] = minimal_distance

    matrix_with_distances[i_min][j_min] = matrix_with_distances[j_min][i_min] = sys.maxsize
    selection[i_min] = selection[j_min] = 1

def delete_connection(graph):
    maximal_distance = 0
    i_max = j_max = 0
    for i in range(n):
        for j in range(i + 1, n):
            if graph[i][j] > maximal_distance:
                maximal_distance = graph[i][j]
                i_max, j_max = i, j
    graph[i_max][j_max] = graph[j_max][i_max] = 0

n, k = 15, 3

matrix_with_distances = [[0 for i in range(n)] for i in range(n)]
for i in range(0, n):
    for j in range(i + 1, n):
        matrix_with_distances[i][j] = random.randint(1, 100)
        matrix_with_distances[j][i] = matrix_with_distances[i][j]

graph = [[0 for i in range(n)] for j in range(n)]

selection = connect_dots_with_minimal_distance(matrix_with_distances, graph)

while 0 in selection:
    connect_all_in_selection(matrix_with_distances, graph, selection)

for i in range(k - 1):
    delete_connection(graph)

graph_for_drawing = nx.Graph(strict=False)
for i in range(0, n):
    graph_for_drawing.add_node(i)
for i in range(0, n):
    for j in range(0, n):
        if graph[i][j] == 0:
            continue
        graph_for_drawing.add_edge(i, j, weight=graph[i][j])
        graph[i][j] = graph[j][i] = 0
nx.draw_circular(graph_for_drawing, with_labels=True)
pos = nx.circular_layout(graph_for_drawing)
edge_labels = nx.get_edge_attributes(graph_for_drawing, 'weight')
nx.draw_networkx_edge_labels(graph_for_drawing, pos=pos, edge_labels=edge_labels)
plt.show()


#!/usr/bin/python3

import os
import tempfile
import shutil
from org.zerlegen.pyXmind.xmind_manifest import XMindManifest
import org.zerlegen.pyXmind.xmind_utils as xmind_utils

import zipfile

################################################################################
# Wrapper over an Xmind workbook with methods for assembly
################################################################################

class XMindWorkbookBuilder:
    _in_content_xml = ""
    _in_meta_xml = ""

    _in_attachments = []

    _in_thumbnails = []
    _in_revisions = []
    #_build_dir = None

    ###########################################################################
    # Constructor
    #
    #

    def _init(in_template = None):
        pass

    ###########################################################################
    # set_from_template
    #
    # Prepopulate a builder object using an existing workbook
    #
    # in_book - workbook to use for prepopulating this object
    #
    #

    def set_from_template(self, in_template):

        extract_dir = tempfile.mkdtemp()
        test_wkbk = zipfile.ZipFile(in_template, 'r')
        test_wkbk.extractall(extract_dir)

        self.set_content_xml(os.path.join(extract_dir, 'content.xml'))
        self.set_meta_xml(os.path.join(extract_dir, 'meta.xml'))

        for file in os.listdir(os.path.join(extract_dir, 'attachments')):
            self.add_attachment(os.path.join(extract_dir, 'attachments', file))

    def set_content_xml(self, content_xml):
        self._in_content_xml = content_xml

    def set_meta_xml(self, meta_xml):
        self._in_meta_xml = meta_xml

    ###########################################################################
    # Add an attachement to this workbook
    #
    # attachment_file - path to the file to add
    # id - id string to use in manifest
    #

    def add_attachment(self, attachment_file, id):
        #
        #print("adding attachment: " + attachment_file )
        #
        self._in_attachments.append((attachment_file, id))

    ###########################################################################
    def add_thumbnail(self, thumbnail_file):
        self._in_thumbnails.append(thumbnail_file)

    ###########################################################################
    def add_revision(self, revision_file):
        self._in_revisions.append(revision_file)

    ###########################################################################
    #
    # out_file - output Xmind workbook
    #
    #

    def build(self, out_file):

        #######################################################################
        # Populate attachments in a subdirectory for this workbook
        #
        #
        # sub_name - name of workbook sub directory
        # abs_file_list - list of files to copy to sub_name (absolute paths)
        #
        #

        def populate_attachments():
            #
            print("creating attachments dir")
            #
            att_path = os.path.join(build_path, "attachments")
            os.mkdir(att_path)
            for (abs_file, id) in self._in_attachments:
                split_filename = os.path.basename(abs_file).split('.')
                if (len(split_filename) > 1):
                    dest = os.path.join(att_path, (id + '.' + split_filename[-1]))
                else:
                    dest = os.path.join(att_path, id)
                #
                print("copying file: " + abs_file + " to: " + dest)
                #
                shutil.copyfile(abs_file, dest)

        #######################################################################
        def zipup_workbook(build_path, out_file):
            print("zipping up workbook...")

            tmp_dir = os.getcwd()
            os.chdir(build_path)

            print("zip out: " + os.path.join(tmp_dir, out_file))
            outzip = zipfile.ZipFile(os.path.join(tmp_dir, out_file), 'w')
            for (dir, subs, files) in os.walk('.'):
                for file in files:
                    print("adding file: " + file + " to zip...")
                    outzip.write(os.path.join(dir, file))

            outzip.close()
            os.chdir(tmp_dir)

        #######################################################################
        def build_manifest(build_path):
            #
            print("building manifest...")
            #
            meta_inf = os.path.join(build_path, "META-INF")
            os.mkdir(meta_inf)

            manifest = XMindManifest()
            for (dir, subs, files) in os.walk(build_path):
                print("subs: " + str(subs))
                for file in files:

                    # determine path relative to workbook root
                    rel_path = dir.replace(build_path, "")
                    rel_path = rel_path.replace(os.path.sep, "", 1)
                    #
                    #print("rel path: " + str(rel_path.startswith(os.path.sep)))
                    #
                    if len(rel_path) == 0:
                        print("adding manifest file: " + file )
                        manifest.add_file_entry(file)
                    else:
                        print("adding manifest file: " +
                              os.path.join(rel_path, file))
                        manifest.add_file_entry(os.path.join(rel_path, file))

                for sub in subs:
                    print("adding manifest dir: " + sub)
                    manifest.add_file_entry(sub + os.path.sep)

            manifest.add_file_entry(os.path.join("META-INF", "manifest.xml"))
            print("writing manifest file to: " + os.path.join(meta_inf, "manifest.xml"))
            #
            manifest.write_manifest_file(os.path.join(meta_inf, "manifest.xml"))

        #
        #print("creating build dir: " + self._build_dir)
        #
        #build_path = tempfile.mkdtemp(self._build_dir)
        build_path = tempfile.mkdtemp()

        # populate build dir
        #
        print("copying content xml: " + self._in_content_xml + " to: " + build_path)
        #
        shutil.copyfile(self._in_content_xml, os.path.join(build_path, "content.xml"))
        ###
        #shutil.copyfile('./content.xml', os.path.join(build_path, "content.xml"))

        if len(self._in_meta_xml) > 0:
            print("copying meta xml: " + self._in_meta_xml + " to: " + build_path)
            #
            shutil.copyfile(self._in_meta_xml, os.path.join(build_path, "meta.xml"))

        #
        print("populating attachments...")
        #
        #print(self._in_attachments)
        populate_attachments()

        #
        #print("populating thumbnails...")
        #
        #populate_sub_dir("Thumbnails", self._in_thumbnails, preserve_filenames)
        #
        #print("populating revisions...")
        #
        #populate_sub_dir("Revisions", self._in_revisions, True)

        build_manifest(build_path)
        zipup_workbook(build_path, out_file)

        # cleanup build dir
        try:
            shutil.rmtree(build_path)
        except OSError:
            print("Error deleting build dir: " + str(OSError))

################################################################################
# BEGIN UNIT TESTS
################################################################################

################################################################################
# Basic unit test - manually duplicate a specified workbook

def test_build_workbook(in_book, out_book):
    extract_dir = tempfile.mkdtemp()
    test_wkbk = zipfile.ZipFile(in_book, 'r')
    test_wkbk.extractall(extract_dir)

    builder = XMindWorkbookBuilder()
    builder.set_content_xml(os.path.join(extract_dir, 'content.xml'))
    builder.set_meta_xml(os.path.join(extract_dir, 'meta.xml'))
    builder.add_attachment(os.path.join(extract_dir, 'attachments',
                                        "0g9t76dl47617sja3r840po3ug.txt"),
                                        "0g9t76dl47617sja3r840po3ug")
    builder.add_attachment(os.path.join(extract_dir, "attachments",
                                        "5s33aeq5fc2bmb1qcnlhbqh69t.jpg"),
                                        "5s33aeq5fc2bmb1qcnlhbqh69t")

    #    builder.add_attachment_dir(os.path.join(extract_dir, 'attachments'))
    builder.build(os.path.join(os.curdir, out_book))

if (__name__ == "__main__"):
    test_build_workbook('test-build.xmind', 'test-out.xmind')


#!/usr/bin/env python
# -*- coding: utf-8 -*-

numeros = [54, 82, 65, 98212, 51212, 413, 43, 67, 94]

pares = []
impares = []

for x in numeros:
	if x % 2 == 0:
		pares.append(x)
	else:
		impares.append(x)

print("Pares: {0}" .format(pares))
print("Ímpares: {0}" .format(impares))