from PyQt5.uic import loadUi
from enum import unique
from operator import index
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtCore import Qt, QAbstractTableModel
from PyQt5.QtGui import QPixmap, QImage, QFont
from PyQt5.QtWidgets import QFileDialog , QTableWidget,QMessageBox, QAction, QMainWindow, QTableView, QPushButton, QToolTip, QApplication, QTableWidgetItem

import os
import nltk # Library nltk
# nltk.download('stopwords')
import string # Library string
import re # Library regex
import pandas as pd # Library Pandax

#boolean
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer, PorterStemmer
import glob
import numpy as np
import sys
import math
Stopwords = set(stopwords.words('english'))

#indexing data
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

# ======== Tokenize ========
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.tokenize import word_tokenize

# ======== StopWord & Stemmer ========
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
from Sastrawi.Dictionary.ArrayDictionary import ArrayDictionary
from Sastrawi.StopWordRemover.StopWordRemover import StopWordRemover
from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory

# ========= Boolean ===========
from contextlib import redirect_stdout

class Ui_MainWindow(QMainWindow):

    def __init__(self):
        QMainWindow.__init__(self)
        loadUi('stkifix.ui', self)

        #Deklarasi
        self.savefile = []
        self.filename = []
        self.allWord = []
        self.token_word = []

        self.stopWord_word = []
        self.stemming_word = []

        #Function Button
        self.pushButton.clicked.connect(self.addFile)
        self.pushButton_2.clicked.connect(self.resetFile)
        self.pushButton_3.clicked.connect(self.booleanSearch)
        self.pushButton_7.clicked.connect(self.preprocessingFile)
        self.pushButton_4.clicked.connect(self.tf_idf)
        self.pushButton_5.clicked.connect(self.jaccardFunction)
        self.pushButton_6.clicked.connect(self.ngramFunction)
        self.pushButton_8.clicked.connect(self.cosineSimilarity)

    def addFile(self):
        self.fileName, fileType = QFileDialog.getOpenFileName(self.centralwidget, "Open File", "",
                                                              "*.txt;;All Files(*)")
        self.filePathName = os.path.basename(self.fileName)
        self.split = os.path.splitext(self.filePathName)[0]
        self.savefile.append(self.fileName)
        self.filename.append(self.split)
        self.listWidget.addItem(self.fileName)

        self.casefoldingWord(self.fileName)

        self.processingFile()
        self.token_word.extend(self.jadi)
        self.stopWord_word.extend(self.tokeStop)

        self.stemming_word.extend(self.tokenStem)
        self.stemming_word = list(dict.fromkeys(self.stemming_word))

        self.invertedIndex()
        self.printIncidence()

    def resetFile(self):
        self.stopWord_word.clear()
        self.stemming_word.clear()
        self.items_clear()

        self.tableWidget.clear()
        self.listWidget.clear()
        self.listWidget_2.clear()
        self.listWidget_3.clear()
        self.listWidget_4.clear()
        self.savefile.clear()

    def getFileNameOnly(self,filenameonly):
        filePathName = os.path.basename(filenameonly)
        splitName = os.path.splitext(filePathName)[0]
        return splitName

    def casefoldingWord(self,fileName):

        self.kalimatOpen = open(fileName,'r').read()
        self.tokenize = self.kalimatOpen.lower()
        # Menghapus angka
        self.angka = re.sub(r"\d+", "", self.tokenize)
        # Menghapus tanda baca
        self.tandabaca = self.angka.translate(str.maketrans("","",string.punctuation))
        # Menghapus whitespace
        self.whitespace = self.tandabaca.strip()
        # Menghapus beberapa whitespace menjadi whitespace tunggal
        self.whitespacetunggal = re.sub('\s+',' ',self.whitespace)

    def processingFile(self):
        # Case Folding -> Tokenizing -> Filtering (Stopword) -> Stemming
        # Tokenizing
        self.jadi = nltk.tokenize.word_tokenize(self.whitespacetunggal)
        self.frekuensi_tokens = nltk.FreqDist(self.jadi).most_common()

        #Filtering (StopWord)
        factory = StopWordRemoverFactory()
        stopword = factory.create_stop_word_remover()
        self.stop = stopword.remove(self.whitespacetunggal)
        self.tokeStop = nltk.tokenize.word_tokenize(self.stop)

        # Stemming
        factory = StemmerFactory()
        self.stemmer = factory.create_stemmer()
        self.stemming   = self.stemmer.stem(self.stop)
        self.tokenStem = nltk.tokenize.word_tokenize(self.stemming)
        # self.kalimatOpen = nltk.tokenize.word_tokenize(self.stop)

    def preprocessingFile(self):
        self.listWidget_2.clear()
        self.listWidget_3.clear()
        self.listWidget_4.clear()
        self.fileNamePre = self.listWidget.currentItem().text()
        self.splitPre = self.getFileNameOnly(self.fileNamePre)
        self.casefoldingWord(self.fileNamePre)

        self.processingFile()

        #Add listWidget
        self.label_2.setText('Kalimat Text : {}'.format(self.kalimatOpen))
        self.listWidget_2.addItem('Token Kata : \n{}'.format(self.jadi))
        self.listWidget_2.addItem('\nFrekuensi Kata : \n{}'.format(self.frekuensi_tokens))
        self.listWidget_3.addItem('Stopword : \n{}'.format(self.stop))
        self.listWidget_4.addItem('Stemming : \n{}'.format(self.stemming))

        # self.counter += 1
        # self.increase += 1

    def invertedIndex(self):
        self.listWidget_7.clear()

        for item in range(len(self.stemming_word)):
            exist_file = list()
            for file in self.savefile:
                with open(file, 'r') as namaFile:
                    for isi in namaFile:
                        isi = isi.lower()
                        if self.stemming_word[item] in isi:
                            exist_file.append(os.path.basename(file))
                            exist_file = list(dict.fromkeys(exist_file))

            self.listWidget_7.addItem('{}\t: <{}>'.format(self.stemming_word[item], exist_file))

    def printIncidence(self):

        self.items_clear()

        panjang_col = len(self.savefile)
        panjang_row = len(self.stemming_word)

        self.tableWidget_2.setColumnCount(panjang_col)
        self.tableWidget_2.setRowCount(panjang_row)

        self.tableWidget_2.setHorizontalHeaderLabels(self.filename)
        self.tableWidget_2.setVerticalHeaderLabels(self.stemming_word)

        for x in range(len(self.stemming_word)):
            for y in range(len(self.filename)):
                with open(self.savefile[y], 'r') as openFile:
                    for content in openFile:
                        if self.stemming_word[x] in content.lower():
                            self.tableWidget_2.setItem(x, y, QTableWidgetItem('1'))
                        else:
                            self.tableWidget_2.setItem(x, y, QTableWidgetItem('0'))

    # =========== Boolean ==============
    def finding_all_unique_words_and_freq(self,words):
        words_unique = []
        word_freq = {}
        for word in words:
            if word not in words_unique:
                words_unique.append(word)
        for word in words_unique:
            word_freq[word] = words.count(word)
        return word_freq

    def finding_freq_of_word_in_doc(self,word,words):
        freq = words.count(word)

    def remove_special_characters(self,text):
        regex = re.compile('[^a-zA-Z0-9\s]')
        text_returned = re.sub(regex,'',text)
        return text_returned

    def booleanSearch(self):

        self.listWidget_12.clear()
        self.listWidget_5.clear()
        totalBoolean = []
        for count in self.savefile:
            totalBoolean.append(count)

        all_words = []
        dict_global = {}
        idx = 1
        files_with_index = {}
        for file in self.savefile:
            fname = file
            file = open(file , "r")
            text = file.read()
            text = self.remove_special_characters(text)
            text = re.sub(re.compile('\d'),'',text)
            sentences = sent_tokenize(text)
            words = word_tokenize(text)
            words = [word for word in words if len(words)>1]
            words = [word.lower() for word in words]
            words = [word for word in words if word not in Stopwords]
            dict_global.update(self.finding_all_unique_words_and_freq(words))
            files_with_index[idx] = os.path.basename(fname)
            idx = idx + 1

        unique_words_all = set(dict_global.keys())

        linked_list_data = {}
        for word in unique_words_all:
            linked_list_data[word] = SlinkedList()
            linked_list_data[word].head = Node(1,Node)
        word_freq_in_doc = {}
        idx = 1
        for file in self.savefile:
            file = open(file, "r")
            text = file.read()
            text = self.remove_special_characters(text)
            text = re.sub(re.compile('\d'),'',text)
            sentences = sent_tokenize(text)
            words = word_tokenize(text)
            words = [word for word in words if len(words)>1]
            words = [word.lower() for word in words]
            words = [word for word in words if word not in Stopwords]
            word_freq_in_doc = self.finding_all_unique_words_and_freq(words)
            for word in word_freq_in_doc.keys():
                linked_list = linked_list_data[word].head
                while linked_list.nextval is not None:
                    linked_list = linked_list.nextval
                linked_list.nextval = Node(idx ,word_freq_in_doc[word])
            idx = idx + 1

            #Step -6 Query processing and output generation
            userInput = self.textEdit.toPlainText().lower()
            query = word_tokenize(userInput)
            connecting_words = []
            cnt = 1
            different_words = []
            for word in query:
                if word.lower() != "and" and word.lower() != "or" and word.lower() != "not":
                    different_words.append(word.lower())
                else:
                    connecting_words.append(word.lower())

            total_files = len(files_with_index)
            zeroes_and_ones = []
            zeroes_and_ones_of_all_words = []
            for word in (different_words):
                if word.lower() in unique_words_all:
                    zeroes_and_ones = [0] * total_files
                    linkedlist = linked_list_data[word].head
                    # self.listWidget_5.addItem(word)
                    while linkedlist.nextval is not None:
                        zeroes_and_ones[linkedlist.nextval.doc - 1] = 1
                        linkedlist = linkedlist.nextval
                    zeroes_and_ones_of_all_words.append(zeroes_and_ones)
                else:
                    self.listWidget_5.addItem("{} not found".format(word))
            # print(zeroes_and_ones_of_all_words)
            # for zeroes_word in zeroes_and_ones_of_all_words:
            #     self.listWidget_5.addItem(str(zeroes_word))
            for word in connecting_words:
                word_list1 = zeroes_and_ones_of_all_words[0]
                word_list2 = zeroes_and_ones_of_all_words[1]
                if word == "and":
                    bitwise_op = [w1 & w2 for (w1,w2) in zip(word_list1,word_list2)]
                    zeroes_and_ones_of_all_words.remove(word_list1)
                    zeroes_and_ones_of_all_words.remove(word_list2)
                    zeroes_and_ones_of_all_words.insert(0, bitwise_op);
                elif word == "or":
                    bitwise_op = [w1 | w2 for (w1,w2) in zip(word_list1,word_list2)]
                    zeroes_and_ones_of_all_words.remove(word_list1)
                    zeroes_and_ones_of_all_words.remove(word_list2)
                    zeroes_and_ones_of_all_words.insert(0, bitwise_op);
                elif word == "not":
                    bitwise_op = [not w1 for w1 in word_list2]
                    bitwise_op = [int(b == True) for b in bitwise_op]
                    zeroes_and_ones_of_all_words.remove(word_list2)
                    zeroes_and_ones_of_all_words.remove(word_list1)
                    bitwise_op = [w1 & w2 for (w1,w2) in zip(word_list1,bitwise_op)]
                    zeroes_and_ones_of_all_words.insert(0, bitwise_op);

            files = []

            print(zeroes_and_ones_of_all_words)

            lis = zeroes_and_ones_of_all_words[0]
            cnt = 1
            for index in lis:
                if index == 1:
                    files.append(files_with_index[cnt])
                cnt = cnt+1

        self.listWidget_12.addItem(str(zeroes_and_ones_of_all_words))
        self.listWidget_5.addItem(str(files))

    # ========= TF/IDF ===========

    def tf_idf(self):

        total = []
        for count in self.savefile:
            total.append(0)

        # ========================= Buat Kolom ===================================
        font = QFont()
        font.setBold(True)

        userInput = self.textEdit_2.toPlainText().lower()
        # ============== PREPROCESSING KEYWORD ======================
        factory = StopWordRemoverFactory()
        factoryStem = StemmerFactory()

        angka = re.sub(r"\d+", "", userInput)
        tandabaca = angka.translate(str.maketrans("","",string.punctuation))
        whitespace = tandabaca.strip()
        whitespacetunggal = re.sub('\s+',' ',whitespace)

        # Stop Word
        stopword = factory.create_stop_word_remover()
        stop = stopword.remove(whitespacetunggal)

        # Stemming
        stemmer = factoryStem.create_stemmer()
        stemming   = stemmer.stem(stop)
        tokenStem = nltk.tokenize.word_tokenize(stemming)
        sentenceIsi = ' '.join(tokenStem)
        # ============== END PREPROCESSING KEYWORD ===================
        userInput = sentenceIsi.split()

        kolom_tf = ['df', 'D/df', 'IDF', 'IDF+1']

        jarak_W = len(self.savefile) + len(kolom_tf)
        panjang_kolom = len(self.savefile)*2 + len(kolom_tf) + 1

        self.tableWidget.setColumnCount(panjang_kolom)
        self.tableWidget.setRowCount(len(userInput)+3)

        self.tableWidget.horizontalHeader().setVisible(False)
        self.tableWidget.verticalHeader().setVisible(False)

        # ========== Span Tf ==========

        ''' Format bikin span : tableTf.setSpan(row, column, rowSpan, columnSpan) '''

        self.tableWidget.setSpan(0, 1, 1, len(self.savefile))
        newItem = QTableWidgetItem("tf")
        newItem.setTextAlignment(QtCore.Qt.AlignCenter)
        self.tableWidget.setItem(0, 1, newItem)
        self.tableWidget.item(0, 1).setFont(font)

        # ========== Span df ==========

        self.tableWidget.setSpan(0, len(self.savefile)+1, 2, 1)
        newItem = QTableWidgetItem("df")
        newItem.setTextAlignment(QtCore.Qt.AlignCenter)
        self.tableWidget.setItem(0, len(self.savefile)+1, newItem)
        self.tableWidget.item(0, len(self.savefile)+1).setFont(font)

        # ========== Span D/df ==========

        self.tableWidget.setSpan(0, len(self.savefile)+2, 2, 1)
        newItem = QTableWidgetItem("D / df")
        newItem.setTextAlignment(QtCore.Qt.AlignCenter)
        self.tableWidget.setItem(0, len(self.savefile)+2, newItem)
        self.tableWidget.item(0, len(self.savefile)+2).setFont(font)

        # ========== Span IDF ==========

        self.tableWidget.setSpan(0, len(self.savefile)+3, 2, 1)
        newItem = QTableWidgetItem("IDF")
        newItem.setTextAlignment(QtCore.Qt.AlignCenter)
        self.tableWidget.setItem(0, len(self.savefile)+3, newItem)
        self.tableWidget.item(0, len(self.savefile)+3).setFont(font)

        # ========== Span IDF+1 ==========

        self.tableWidget.setSpan(0, len(self.savefile)+4, 2, 1)
        newItem = QTableWidgetItem("IDF + 1")
        newItem.setTextAlignment(QtCore.Qt.AlignCenter)
        self.tableWidget.setItem(0, len(self.savefile)+4, newItem)
        self.tableWidget.item(0, len(self.savefile)+4).setFont(font)

        # ========== Span W ==========

        self.tableWidget.setSpan(0, len(self.savefile)+5, 1, len(self.savefile))
        newItem = QTableWidgetItem("W = tf*(IDF+1)")
        newItem.setTextAlignment(QtCore.Qt.AlignCenter)
        self.tableWidget.setItem(0, len(self.savefile)+5, newItem)
        self.tableWidget.item(0, len(self.savefile)+5).setFont(font)

        # ========== MAKE TABLE ==========

        # __Print Document di kolom tf (kiri)__
        for y in range(len(self.savefile)):
            cell_item = QTableWidgetItem(str(self.filename[y]))
            cell_item.setTextAlignment(QtCore.Qt.AlignCenter)
            self.tableWidget.setItem(1, y+1, cell_item)
            self.tableWidget.item(1, y+1).setFont(font)

        # __Print Document di kolom tf (kanan)__
        for y in range(len(self.savefile)):
            cell_item = QTableWidgetItem(str(self.filename[y]))
            cell_item.setTextAlignment(QtCore.Qt.AlignCenter)
            self.tableWidget.setItem(1, y+1+jarak_W, cell_item)
            self.tableWidget.item(1, y+1+jarak_W).setFont(font)

        # ====================== END BUAT KOLOM ===============================

        # # __Isi Tf-Idf__
        for x in range(len(userInput)): # 0 < x < 3 (x = 0)

            exist_in = []

            for y in range((panjang_kolom)): # 0 < y < 11 (y = 0)

                # __Print userInput di row__
                self.tableWidget.setItem(x+2, 0, QTableWidgetItem(str(userInput[x])))

                # __Print nilai tf per document__
                if y < len(self.savefile):
                    with open(self.savefile[y], 'r') as openFile:
                        for content in openFile:
                            if userInput[x] in content.lower():
                                exist_in.append(1)
                            else:
                                exist_in.append(0)

                    self.tableWidget.setItem(x+2, y+1, QTableWidgetItem(str(exist_in[y])))

                # __df__
                df = 0
                for count in exist_in:
                    if count > 0:
                        df = df + 1 # df = 1

                # __D/df__
                if df != 0:
                    D = round(len(self.savefile) / df, 3)
                else:
                    D = 1

                # __idf__
                idf = round(math.log(D), 3)

                # __idf+1__
                idf_1 = round(idf+1,3)

                # __W__
                W = []
                for freq in range(len(exist_in)):
                    W.append(round(exist_in[freq]*(idf+1), 3))

                if len(W) == len(self.savefile) and y == len(self.savefile):
                    # __SUM__
                    zipped_list = zip(total, W)
                    total = [x+y for (x, y) in zipped_list]

                # __Print nilai setelah tf dan sebelum W__
                if y == (len(self.savefile)+1) and y <= jarak_W:
                    self.tableWidget.setItem(x+2, y, QTableWidgetItem(str(df)))
                    self.tableWidget.setItem(x+2, y+1, QTableWidgetItem(str(D)))
                    self.tableWidget.setItem(x+2, y+2, QTableWidgetItem(str(idf)))
                    self.tableWidget.setItem(x+2, y+3, QTableWidgetItem(str(idf_1)))

                # __Print nilai W per dokumen__
                if y > jarak_W:
                    self.tableWidget.setItem(x+2, y, QTableWidgetItem(str(W[y-(jarak_W+1)])))

                if x == len(userInput)-1: # x = 2. userInput = 2

                    if y == jarak_W: # y = 7
                        item_sum = QTableWidgetItem('Sum :')
                        self.tableWidget.setItem(x+3, y, item_sum)
                        self.tableWidget.item(x+3, y).setFont(font)
                    elif y > jarak_W:
                        total_sum = QTableWidgetItem(str(total[y-(jarak_W+1)]))
                        self.tableWidget.setItem(x+3, y, total_sum)
                        self.tableWidget.item(x+3, y).setFont(font)

        file_rank = self.filename.copy()

        self.listWidget_8.addItem('Ranking:')

        for i in range(len(self.savefile)-1):
            for j in range(len(self.savefile)-i-1):
                if total[j] < total[j+1]:
                    temp_total = total[j]
                    total[j] = total[j+1]
                    total[j+1] = temp_total

                    temp = file_rank[j]
                    file_rank[j] = file_rank[j+1]
                    file_rank[j+1] = temp

        print(type(file_rank))
        for i in range(len(file_rank)):
            print(i)
            self.listWidget_8.addItem('{}. {}'.format(i,file_rank[i]))

        # print(total)
        # print(file_rank)

    # ============= Jaccard =============
    def jaccardFunction(self):
        self.listWidget_6.clear()
        self.listWidget_11.clear()
        total = []
        for count in self.savefile:
            total.append(count)

        userInput = self.textEdit_3.toPlainText().lower()
        # print(userInput)

        # ============== PREPROCESSING KEYWORD ======================
        factory = StopWordRemoverFactory()
        factoryStem = StemmerFactory()

        angka = re.sub(r"\d+", "", userInput)
        tandabaca = angka.translate(str.maketrans("","",string.punctuation))
        whitespace = tandabaca.strip()
        whitespacetunggal = re.sub('\s+',' ',whitespace)

        # Stop Word
        stopword = factory.create_stop_word_remover()
        stop = stopword.remove(whitespacetunggal)

        # Stemming
        stemmer = factoryStem.create_stemmer()
        stemming   = stemmer.stem(stop)
        tokenStem = nltk.tokenize.word_tokenize(stemming)
        sentenceUser = ' '.join(tokenStem)
        # ============== END PREPROCESSING KEYWORD ===================

        userInput = re.split(r'\W+', sentenceUser)

        a = userInput
        self.listWidget_6.addItem('Orignal Text : {}'.format(a))
        increaseNo = 0
        hasil_urutJaccard = list()
        for folder in total:
            self.listWidget_6.addItem('==========================================')
            with open(folder,'r') as namafile:
                for isi in namafile:
                    isi = isi.lower()
                    # ============== PREPROCESSING KEYWORD ======================
                    factory = StopWordRemoverFactory()
                    factoryStem = StemmerFactory()

                    angka = re.sub(r"\d+", "", isi)
                    tandabaca = angka.translate(str.maketrans("","",string.punctuation))
                    whitespace = tandabaca.strip()
                    whitespacetunggal = re.sub('\s+',' ',whitespace)

                    # Stop Word
                    stopword = factory.create_stop_word_remover()
                    stop = stopword.remove(whitespacetunggal)

                    # Stemming
                    stemmer = factoryStem.create_stemmer()
                    stemming   = stemmer.stem(stop)
                    tokenStem = nltk.tokenize.word_tokenize(stemming)
                    sentenceIsi = ' '.join(tokenStem)
                    # ============== END PREPROCESSING KEYWORD ===================
                    isi = sentenceIsi.split()

                    self.listWidget_6.addItem('{} :'.format(self.filename[increaseNo]))
                    self.listWidget_6.addItem('Original Text : [{}]'.format(isi))
                    c = set(a).intersection(isi)
                    d = round(float(len(c)) / (len(a)+len(isi)-len(c)),3)
                    self.listWidget_6.addItem('{} / {} : [{}]'.format(len(c),(len(a)+len(isi)-len(c)),d))
                    hasil_urutJaccard.append([self.filename[increaseNo],d])
                    increaseNo +=1

        # print(d)

        hasil_urutJaccard.sort(key=lambda row: (row[1]),reverse=True)

        increaseNo = 0
        self.listWidget_11.addItem('Ranking')
        for folder in total:
            self.listWidget_11.addItem('{} : [{}]'.format(hasil_urutJaccard[increaseNo][0],hasil_urutJaccard[increaseNo][1]))
            increaseNo+=1

    # ================= N Gram ===============
    def ngramFunction(self):
        self.listWidget_9.clear()
        self.listWidget_10.clear()
        total = []
        for count in self.savefile:
            total.append(count)

        userInput = self.textEdit_4.toPlainText().lower()
        # print(userInput)
        # Casefolding
        # ============== PREPROCESSING KEYWORD ======================
        factory = StopWordRemoverFactory()
        factoryStem = StemmerFactory()

        angka = re.sub(r"\d+", "", userInput)
        tandabaca = angka.translate(str.maketrans("","",string.punctuation))
        whitespace = tandabaca.strip()
        whitespacetunggal = re.sub('\s+',' ',whitespace)

        # Stop Word
        stopword = factory.create_stop_word_remover()
        stop = stopword.remove(whitespacetunggal)

        # Stemming
        stemmer = factoryStem.create_stemmer()
        stemming   = stemmer.stem(stop)
        tokenStem = nltk.tokenize.word_tokenize(stemming)
        # ============== END PREPROCESSING KEYWORD ===================

        sentenceUser = ' '.join(tokenStem)
        userInput = re.split(r'\W+', sentenceUser)

        m = 2 # nilai untuk jumlah grams
        nGramSearchList = self.getNGrams(userInput,m)

        print(nGramSearchList)
        self.listWidget_9.addItem('Orignal Text : {}'.format(nGramSearchList))

        increaseNo = 0
        hasil_urutnGram = list()
        for folder in total:
            self.listWidget_9.addItem('==========================================')
            for i in range(len(nGramSearchList)):
                with open(folder,'r') as namafile:
                    a = 0
                    for isi in namafile:
                        isi = isi.lower()
                        # ========== PREPROCESSING ISI ==================

                        angka = re.sub(r"\d+", "", isi)
                        tandabaca = angka.translate(str.maketrans("","",string.punctuation))
                        whitespace = tandabaca.strip()
                        whitespacetunggal = re.sub('\s+',' ',whitespace)

                        # Stop Word
                        stopword = factory.create_stop_word_remover()
                        stop = stopword.remove(whitespacetunggal)

                        # Stemming
                        stemmer = factoryStem.create_stemmer()
                        stemming   = stemmer.stem(stop)
                        tokenStem = nltk.tokenize.word_tokenize(stemming)
                        sentenceIsi = ' '.join(tokenStem)

                        # ========= END PREPROCESSING ISI ==============
                        isi = sentenceIsi.split()
                        fileGetNGrams = self.getNGrams(isi,m)
                        self.listWidget_9.addItem('{} : \n{}'.format(self.filename[increaseNo],fileGetNGrams))

                        # print(len(fileGetNGrams)) // Jumlah panjang listnya
                        for j in range(len(fileGetNGrams)):
                            if nGramSearchList[i] == fileGetNGrams[j]:
                                a +=1
            d = round(float(a)/(len(isi)),3)
            hasil_urutnGram.append([self.filename[increaseNo],d])
            self.listWidget_9.addItem('{} / {}: {}'.format(a,len(isi),d))
            increaseNo +=1

        hasil_urutnGram.sort(key=lambda row: (row[1]),reverse=True)

        increaseNo = 0
        self.listWidget_10.addItem('Ranking')
        for folder in total:
            self.listWidget_10.addItem('{} : [{}]'.format(hasil_urutnGram[increaseNo][0],hasil_urutnGram[increaseNo][1]))
            increaseNo+=1

    def getNGrams(self,wordlist,n):
        ngrams = []
        for i in range(len(wordlist)-(n-1)):
            ngrams.append(wordlist[i:i+n])
        return ngrams

    def items_clear(self):
        for item in self.tableWidget_2.selectedItems():
            newitem = QTableWidgetItem()
            self.tableWidget_2.setItem(item.row(), item.column(), newitem)

    # =================== Cosine Similarity =============
    def cosineSimilarity(self):
        self.listWidget_13.clear()

        header_file = []
        documents = []
        total = []

        for count in self.savefile:
            total.append(count)

        userInput = self.textEdit_5.toPlainText().lower()

        # ======== Preprocessing ==========
        # Casefolding
        factory = StopWordRemoverFactory()
        factoryStem = StemmerFactory()

        angka = re.sub(r"\d+", "", userInput)
        tandabaca = angka.translate(str.maketrans("","",string.punctuation))
        whitespace = tandabaca.strip()
        whitespacetunggal = re.sub('\s+',' ',whitespace)

        # Stop Word
        stopword = factory.create_stop_word_remover()
        stop = stopword.remove(whitespacetunggal)

        # Stemming
        stemmer = factoryStem.create_stemmer()
        stemming   = stemmer.stem(stop)
        tokenStem = nltk.tokenize.word_tokenize(stemming)
        # ======== Stem Keyword End ======

        sentenceUser = ' '.join(tokenStem)
        documents.append(sentenceUser)
        print(documents)

        header_file.append("User Input")
        header_file.extend(self.filename)
        print(header_file)

        for i in range(len(total)):
            document_word = []
            with open(total[i],'r') as file:
                for isi in file:
                    tokenize = isi.lower()
                    angka = re.sub(r"\d+", "", tokenize)
                    tandabaca = angka.translate(str.maketrans("","",string.punctuation))
                    whitespace = tandabaca.strip()
                    whitespacetunggal = re.sub('\s+',' ',whitespace)

                    stopword = factory.create_stop_word_remover()
                    stop = stopword.remove(whitespacetunggal)

                    # Stemming
                    stemmer = factoryStem.create_stemmer()
                    stemming   = stemmer.stem(stop)
                    tokenStem = nltk.tokenize.word_tokenize(stemming)

                    document_word.extend(tokenStem)
            sentence = ' '.join(document_word)
            documents.append(sentence)

        print(documents)

        count_vectorizer = CountVectorizer(stop_words=stopword)
        count_vectorizer = CountVectorizer()
        sparse_matrix = count_vectorizer.fit_transform(documents)

        doc_term_matrix = sparse_matrix.todense()
        df = pd.DataFrame(doc_term_matrix,
                        columns=count_vectorizer.get_feature_names_out(),
                        index=[header_file])

        self.tableWidget_3.setColumnCount(len(df.columns))
        self.tableWidget_3.setRowCount(len(df.index))

        self.tableWidget_3.setHorizontalHeaderLabels(df.columns)
        self.tableWidget_3.setVerticalHeaderLabels(header_file)

        for i in range(len(df.index)):
            for j in range(len(df.columns)):
                self.tableWidget_3.setItem(i,j,QTableWidgetItem(str(df.iloc[i,j])))

        hasil = cosine_similarity(df, df)

        # Cosine_similarity fungsi dari sklearn
        # K(x,y) = <X,y> / (||X||*||Y||)
        print(hasil)

        hasil_urutConsaint = list()
        jmlh_file = 1
        for i in range(len(header_file)-1):
            self.listWidget_13.addItem("{} : {}".format(header_file[jmlh_file],str(round(hasil[0][jmlh_file],4))))
            hasil_urutConsaint.append([header_file[jmlh_file],str(round(hasil[0][jmlh_file],4))])
            jmlh_file +=1

        self.listWidget_13.addItem('=================')

        # print(d)

        hasil_urutConsaint.sort(key=lambda row: (row[1]),reverse=True)

        increaseNo = 0
        self.listWidget_13.addItem('Ranking')
        for folder in total:
            self.listWidget_13.addItem('{} : {}'.format(hasil_urutConsaint[increaseNo][0],hasil_urutConsaint[increaseNo][1]))
            increaseNo+=1

        total.clear()
        header_file.clear()

class Node:
    def __init__(self ,docId, freq = None):
        self.freq = freq
        self.doc = docId
        self.nextval = None

class SlinkedList:
    def __init__(self ,head = None):
        self.head = head

app = QApplication([])
window = Ui_MainWindow()
window.show()
app.exec_()


#!/bin/bash
# -*- coding: utf-8 -*-
# written by Tong
import numpy as np
import matplotlib.pyplot as plt
from astropy.io import fits
import sys
import os
import string
from datetime import datetime
from scipy.interpolate import lagrange
from scipy import interpolate
from scipy.optimize import curve_fit
import pandas as pd
from astropy.timeseries import LombScargle
import stingray
from stingray import Lightcurve, Powerspectrum, AveragedPowerspectrum

font1 = {'family': 'Normal',
         'weight': 'normal',
         'size': 16, }

def phase_fold(time,rate,period,binnumber=10):
    ## 周期折叠，period为这个源QPO的周期 ##
    turns=time*1/period-(time*1/period).astype('int')
    fluxmean=np.zeros(binnumber)
    for i in range(len(fluxmean)):
        fluxmean[i]=np.mean(rate[np.where((turns<(i+1)/binnumber)&(turns>i/binnumber))])
    x=np.linspace(0,1,binnumber+1)
    x=x[:-1]

    x2=np.concatenate((x,x+1));y2=np.concatenate((fluxmean,fluxmean))
    ## 为了画出来漂亮一些，仅此而已
    plt.step(x2,y2)
    plt.xlabel('Phase',font1)
    plt.ylabel('Count rate (counts s-1)',font1)
    plt.tick_params(labelsize=16)
    plt.show()

def filter_energy(time,energy,band):
    T=time
    E=energy
    i=0
    if len(time)!=len(energy):
        print('error')
        return None
    else:
        while i <len(E):
            if E[i]<band[0] or E[i]>band[1]:
                E=np.delete(E,i)
                T=np.delete(T,i)
            else:
                i+=1
    return T
def get_LS(time, flux,freq):
    ## Lomb-Scagrle周期图算法，参考VanderPlas J. T., 2018, ApJS, 236, 16 ##
    x = time
    y = flux
    plt.figure(1,(9,6))

    # LS = LombScargle(x, y, dy = 1, normalization = 'standard', fit_mean = True,
    #                  center_data = True).power(freq, method = 'cython')
    LS = LombScargle(x, y,normalization = 'standard')
    power = LS.power(freq)
    FP=LS.false_alarm_probability(power.max(),minimum_frequency = freq[0], maximum_frequency = freq[-1],method='baluev')
    FP_99 = LS.false_alarm_level(0.0027, minimum_frequency = freq[0], maximum_frequency = freq[-1],method='baluev')
    FP_90 = LS.false_alarm_level(0.05,  minimum_frequency=freq[0],
                                 maximum_frequency=freq[-1], method='baluev')
    FP_68 = LS.false_alarm_level(0.32, minimum_frequency=freq[0],
                                 maximum_frequency=freq[-1], method='baluev')
    plt.plot([freq[0], freq[-1]], [FP_99, FP_99], '--')
    plt.plot([freq[0], freq[-1]], [FP_90, FP_90], '--')
    plt.plot([freq[0], freq[-1]], [FP_68, FP_68], '--')

    plt.title('FP={0}'.format(FP),font1)
    plt.semilogx()
    plt.xlabel('frequency (Hz)')
    plt.ylabel('Normalized Lomb-Scargle power')
    plt.plot(freq, power)
    print(1./freq[np.where(power==np.max(power))])
    plt.xlabel('frequency',font1)
    plt.ylabel('normalized LSP',font1)
    plt.tick_params(labelsize=16)
    plt.show()
    res=1e5*power
    res=np.round(res,2)
    return [FP, 1. / freq[np.where(power == np.max(power))],np.max(power),res]

def plot_pds(time,flux):
    ## 画光变曲线，即光子流量随时间的变化 ##
    lc = Lightcurve(time, flux)
    fig, ax = plt.subplots(1, 1, figsize=(10, 6))
    ax.plot(lc.time, lc.counts, lw=2, color='blue')
    ax.set_xlabel("Time (s)", fontproperties=font1)
    ax.set_ylabel("Counts (cts)/bin", fontproperties=font1)
    ax.tick_params(axis='x', labelsize=16)
    ax.tick_params(axis='y', labelsize=16)
    ax.tick_params(which='major', width=1.5, length=7)
    ax.tick_params(which='minor', width=1.5, length=4)
    plt.show()

    ## 画功率谱图 ##
    ps = Powerspectrum(lc,norm='leahy')
    ps=ps.rebin(df=4.7e-3,f=np.arange(1./100,0.5/1,1./100))

    fig, ax1 = plt.subplots(1, 1, figsize=(9, 6), sharex=True)
    ax1.loglog()
    ax1.step(ps.freq, ps.power, lw=2, color='blue')
    ax1.set_xlabel("Frequency (Hz)", fontproperties=font1)
    ax1.set_ylabel("Power ", fontproperties=font1)
    # ax1.set_yscale('log')
    ax1.tick_params(axis='x', labelsize=16)
    ax1.tick_params(axis='y', labelsize=16)
    ax1.tick_params(which='major', width=1.5, length=7)
    ax1.tick_params(which='minor', width=1.5, length=4)
    for axis in ['top', 'bottom', 'left', 'right']:
        ax1.spines[axis].set_linewidth(1.5)
    plt.show()

    ## 画average的功率谱，对于绝大多数的源不需要做这张图，除非所给的参考文献中有做average power density spectrum ###
    avg_ps = AveragedPowerspectrum(lc, 500,dt=lc.time[1]-lc.time[0],norm='leahy')
    print("Number of segments: %d" % avg_ps.m)
    fig, ax1 = plt.subplots(1, 1, figsize=(9, 6))
    ax1.loglog()
    ax1.step(avg_ps.freq, avg_ps.power, lw=2, color='blue')
    ax1.set_xlabel("Frequency (Hz)", fontproperties=font1)
    ax1.set_ylabel("Power ", fontproperties=font1)
    ax1.set_yscale('log')
    ax1.tick_params(axis='x', labelsize=16)
    ax1.tick_params(axis='y', labelsize=16)
    ax1.tick_params(which='major', width=1.5, length=7)
    ax1.tick_params(which='minor', width=1.5, length=4)
    for axis in ['top', 'bottom', 'left', 'right']:
        ax1.spines[axis].set_linewidth(1.5)
    plt.show()

def read_SAS_lc():
    # 1,2,3分别代表mos1,mos2,pn的light curve，也可以加起来用，记为_all;
    # 根据实际情况来决定lomb-scargle的输入

    # 要注意的是，如果在run_XMMproducts_spectra.py中没有自行指定tmin和tmax，这里就不能直接对三个detector的lc进行加减
    dt = 10
    path = '/Volumes/pulsar/HLX1/0655510201/cal/'
    os.chdir(path)
    mode=['mos1','mos2','pn']
    # filename1=mode[0]+'_lccorr_bin{0}.lc'.format(int(dt))
    # filename2=mode[1]+'_lccorr_bin{0}.lc'.format(int(dt))
    # filename3=mode[2]+'_src_lc_bin{0}.lc'.format(dt)
    filename3=mode[2]+'_lccorr_bin{0}.lc'.format(dt)
    # lc1=fits.open(filename1);
    # lc2=fits.open(filename2);
    lc3=fits.open(path+filename3)
    # time1=lc1[1].data['TIME'][0:-100];rate1=lc1[1].data['RATE'][0:-100]
    # time2=lc2[1].data['TIME'][0:-100];rate2=lc2[1].data['RATE'][0:-100]
    time3=lc3[1].data['TIME'];rate3=lc3[1].data['RATE']
    rate3=np.nan_to_num(rate3)
    rate3[np.where(rate3<0)]=0
    freq=np.arange(1./100,0.5/dt,1./100)

    time_all=time3;rate_all=rate3
    index_gti=np.where(rate_all>0)
    time_all=time_all[index_gti];rate_all=rate_all[index_gti]

    phase_fold(time_all,rate_all,1/1.5e-4,binnumber=20)
    get_LS(time_all,rate_all,freq)
    plot_pds(time_all,rate_all)
read_SAS_lc()


class NumMatrix:

    def __init__(self, matrix: List[List[int]]):
        if not matrix or not matrix[0]:
            return
        self.dp = [[0] * len(matrix[0]) for _ in range(len(matrix))]
        self.dp[0][0] = matrix[0][0]
        for i in range(1, len(matrix)):
            self.dp[i][0] = self.dp[i - 1][0] + matrix[i][0]
        for j in range(1, len(matrix[0])):
            self.dp[0][j] = self.dp[0][j - 1] + matrix[0][j]
        for i in range(1, len(matrix)):
            for j in range(1, len(matrix[0])):
                self.dp[i][j] = self.dp[i - 1][j] + self.dp[i][j - 1] - \
                    self.dp[i - 1][j - 1] + matrix[i][j]

    def sumRegion(self, row1: int, col1: int, row2: int, col2: int) -> int:
        res = self.dp[row2][col2]
        if col1 > 0:
            res -= self.dp[row2][col1 - 1]
        if row1 > 0:
            res -= self.dp[row1 - 1][col2]
        if col1 > 0 and row1 > 0:
            res += self.dp[row1 - 1][col1 - 1]
        return res

        # Your NumMatrix object will be instantiated and called as such:
        # obj = NumMatrix(matrix)
        # param_1 = obj.sumRegion(row1,col1,row2,col2)


# import serial
# ser = serial.Serial('/dev/ttyACM3')
import serial
import datetime
import os
import csv
dataFolder = "/home/lakitha/Documents/mintsData/"

def main():

    ser = serial.Serial(
        port='/dev/ttyACM0',\
        baudrate=9600,\
        parity  =serial.PARITY_NONE,\
        stopbits=serial.STOPBITS_ONE,\
        bytesize=serial.EIGHTBITS,\
            timeout=0)

    print("connected to: " + ser.portstr)

    #this will store the line
    line = []

    while True:
        for c in ser.read():
            line.append(chr(c))
            if chr(c) == '-':
                dataString     = (''.join(line))
                dataStringPost = dataString.replace('-', '')
                dataSplit(dataStringPost,datetime.datetime.now())
                line = []
                break

    ser.close()
        # if chr(c) == '-':
        #     print(''.join(line))

def dataSplit(dataString,dateTime):
    dataOut   = dataString.split('!')
    if(len(dataOut) == 2):
        tag       = dataOut[0]
        dataQuota = dataOut[1]
        if(tag.find("#mintsO")==0):
            sensorSplit(dataQuota,dateTime)

def sensorSplit(dataQuota,dateTime):
    dataOut    = dataQuota.split('>')
    if(len(dataOut) == 2):
        sensorID   = dataOut[0]
        sensorData = dataOut[1]
        sensorSend(sensorID,sensorData,dateTime)
        # print("Sensor ID  : "+sensorID)
        # print("Sensor Data: "+sensorData)
        # print("Date Time : " +str(dateTime))

def sensorSend(sensorID,sensorData,dateTime):
    if(sensorID=="HTU21D"):
        HTU21DWrite(sensorData,dateTime)
    if(sensorID=="BMP280"):
        BMP280Write(sensorData,dateTime)
    if(sensorID=="GAS1"):
        GAS1Write(sensorData,dateTime)

def HTU21DWrite(sensorData,dateTime):
    dataOut    = sensorData.split(':')
    sensorName = "HTU21D"
    if(len(dataOut) == 2):
        sensorDictionary = {
                "DateTime"           : str(dateTime),
        		"Temperature" :dataOut[0],
            	"Humdity"     :dataOut[1],
        	     }

    #Getting Write Path
    writePath = getWritePath(sensorName,dateTime)
    exists = directoryCheck(writePath)
    writeCSV2(writePath,sensorDictionary,exists)
    # print(writePath)
    print(sensorDictionary)
#
def BMP280Write(sensorData,dateTime):
    dataOut    = sensorData.split(':')
    sensorName = "BMP280"
    if(len(dataOut) == 2):
        sensorDictionary = {
                "DateTime"           : str(dateTime),
        		"Temperature" :dataOut[0],
            	"Pressure"     :dataOut[1],
        	     }

    #Getting Write Path
    writePath = getWritePath(sensorName,dateTime)
    exists = directoryCheck(writePath)
    writeCSV2(writePath,sensorDictionary,exists)
    # print(writePath)
    print(sensorDictionary)

def GAS1Write(sensorData,dateTime):
    dataOut    = sensorData.split(':')
    sensorName = "GAS1"
    if(len(dataOut) == 4):
        sensorDictionary = {
                "DateTime"   : str(dateTime),
        		"NO2"  :dataOut[0],
            	"SO2"  :dataOut[1],
                "O3"   :dataOut[2],
                "C02"  :dataOut[3],
        	     }
    #Getting Write Path
    writePath = getWritePath(sensorName,dateTime)
    exists = directoryCheck(writePath)
    writeCSV2(writePath,sensorDictionary,exists)
    # print(writePath)
    print(sensorDictionary)

def writeCSV2(writePath,sensorDictionary,exists):
    keys =  list(sensorDictionary.keys())
    with open(writePath, 'a') as csv_file:
        writer = csv.DictWriter(csv_file, fieldnames=keys)
        print(exists)
        if(not(exists)):
            writer.writeheader()
        writer.writerow(sensorDictionary)

def getWritePath(labelIn,dateTime):
    writePath = dataFolder +str(dateTime.year) + "/" + str(dateTime.month) +"/mintsO"+ labelIn + str(dateTime.year) + str(dateTime.month) + str(dateTime.day) + ".csv"
    return writePath;

def getListDictionaryFromPath(dirPath):
    print("Reading : "+ dirPath)
    reader = csv.DictReader(open(dirPath))
    reader = list(reader)

def fixCSV(keyIn,valueIn,currentDictionary):
    editedList       = editDictionaryList(currentDictionary,keyIn,valueIn)
    return editedList

def editDictionaryList(dictionaryListIn,keyIn,valueIn):
    for dictionaryIn in dictionaryListIn:
        dictionaryIn[keyIn] = valueIn

    return dictionaryListIn

def getDateDataOrganized(currentCSV,nodeID):
    currentCSVName = os.path.basename(currentCSV)
    nameOnly = currentCSVName.split('-Organized.')
    dateOnly = nameOnly[0].split(nodeID+'-')
    print(dateOnly)
    dateInfo = dateOnly[1].split('-')
    print(dateInfo)
    return dateInfo

def getFilePathsforOrganizedNodes(nodeID,subFolder):
    nodeFolder = subFolder+ nodeID+'/';
    pattern = "*Organized.csv"
    fileList = [];
    for path, subdirs, files in os.walk(nodeFolder):
        for name in files:
            if fnmatch(name, pattern):
                fileList.append(os.path.join(path, name))
    return sorted(fileList)

def getLocationList(directory, suffix=".csv"):
    filenames = listdir(directory)
    dateList = [ filename for filename in filenames if filename.endswith( suffix ) ]
    return sorted(dateList)

def getListDictionaryCSV(inputPath):
    # the path will depend on the node ID
    reader = csv.DictReader(open(inputPath))
    reader = list(reader)
    return reader

def writeCSV(reader,keys,outputPath):
    directoryCheck(outputPath)
    csvWriter(outputPath,reader,keys)

def directoryCheck(outputPath):
    exists = os.path.isfile(outputPath)
    directoryIn = os.path.dirname(outputPath)
    if not os.path.exists(directoryIn):
        os.makedirs(directoryIn)
    return exists

def csvWriter(writePath,organizedData,keys):
    with open(writePath,'w') as output_file:
        writer = csv.DictWriter(output_file, fieldnames=keys)
        writer.writeheader()
        writer.writerows(organizedData)

def gainDirectoryInfo(dailyDownloadLocation):
    directoryPaths = []
    directoryNames = []
    directoryFiles = []
    for (dirpath, dirnames, filenames) in walk(dailyDownloadLocation):
        directoryPaths.extend(dirpath)
        directoryNames.extend(dirnames)
        directoryFiles.extend(filenames)

    return directoryPaths,directoryNames,directoryFiles;

if __name__ == "__main__":
   main()


"""
URL routing for the geohashing app
"""

from django.conf.urls import url

from rest_framework.urlpatterns import format_suffix_patterns

from . import views

urlpatterns = [
    url(
        r'^(?P<year>\d{4})-(?P<month>\d{1,2})-(?P<day>\d{1,2})$',
        views.get_geohash,
        name='geohash'
    ),
]

urlpatterns = format_suffix_patterns(urlpatterns)


# pip install flask-restful
# reqparse => para indicar las caracterisiticas que deben de cumplirse para poder continuar con la logica de la api

from flask import Flask, request
from flask_restful import Resource, Api, reqparse

app = Flask(__name__)
api = Api(app)

items = [
    {
        'prodNom': 'Sapolio',
        'prodPrec': 5.40,
        'prodCar': ['LIMPIA TODO', 'DESENGRASANTE', 'LAVAVAJILLAS']
    },
    {
        'prodNom': 'Ayudin',
        'prodPrec': 3.8,
        'prodCar': ['ESPONJA GRATIS', 'LIQUIDO', 'DESENGRASANTE', 'PULIDOR']
    },
    {
        'prodNom': 'Pepsi 3L',
        'prodPrec': 5.8,
        'prodCar': ['GASEOSA', 'CARBONATADA', 'LIQUIDO', 'ALTO EN AZUCAR']
    }
]

# tener un CRUD y que ademas, de acuerdo a un buscador, que me indique todos los productos que tengan esa caracteristica
# 127.0.0.1/buscar {"palabra": "DESENGRASANTE"}

@app.route('/')
def inicio():
    return 'La api funciona'

#palabra, es lo que le pasamos por el POST
@app.route('/buscar', methods=['POST'])
def buscar():
    data = request.get_json()
    palabra = data['palabra'].lower()
    # print(palabra)
    resultado = []
    for item in items:
        # print(item['prodCar'])
        for caracteristica in item['prodCar']:
            if caracteristica.lower() == palabra:
                resultado.append(item)
    # print(caracteristica)
    return {
        'ok': True,
        'content': resultado
    }

# Clase Item que hereda de la clase Resource
class Item(Resource):
    parser = reqparse.RequestParser()
    parser.add_argument(
        'prodNom',
        type=str,
        required=True,
        help='Falta el nombre del producto'
    )

    parser.add_argument(
        'prodPrec',
        type=float,
        required=True,
        help='Falta el precio del producto'
    )

    parser.add_argument(
        'prodCar',
        type=list,
        required=False,
        location='json',
        help='Falta las caracteristicas del producto'
    )

    # METODOS
    def get(self, id):
        if len(items) > id:
            return {
                'ok': True,
                'content': None,
                'message': items[id],
            }
        else:
            return {
                'ok': False,
                'content': 'No se encontro el item a buscar',
                'message': None,
            }

    def post(self):
        # realiza la validacion que configuramos previamente. Si todo esta correcto, lo asigna a la variable data
        data = self.parser.parse_args()
        items.append(data)
        return {
            'ok': True,
            'content': data,
            'message': 'Se agrego exitosamente el item',
        }

    def put(self, id):
        if len(items) > id:
            data = self.parser.parse_args()
            items[id] = data
            return {
                'ok': True,
                'content': data,
                'message': 'Se actualizo el item',
            }, 201
        else:
            return {
                'ok': False,
                'content': None,
                'message': 'No se encontro el item a actualizar',
            }

    def delete(self, id):
        if len(items) > id:
            items.pop(id)
            return {
                'ok': True,
                'content': None,
                'message': 'Se elimino el item',
            }
        else:
            return {
                'ok': False,
                'content': None,
                'message': 'No se encontro el item a actualizar',
            }

# Con el uso de flask_restfull ya no se necesitan decoradores.
# Solamente se pasas un parametro para agregar un recurso a la api.
# Le pasamos la clase, en este caso Item

api.add_resource(Item, '/item', '/item/<int:id>')

# ejecutamos el programa
if __name__ == '__main__':
    app.run(debug=True, port=5002)


#!/usr/bin/env python
# -*- python-indent-offset: 4; -*-

# -*- mode: python-mode; python-indent-offset: 4 -*-
from __future__ import print_function
from builtins import input
import sys
import os
import matplotlib
if matplotlib.get_backend() != "TKAgg":
    matplotlib.use("TKAgg")

import pmagpy.pmag as pmag
import pmagpy.pmagplotlib as pmagplotlib
import pmagpy.new_builder as nb

def main():
    """
    NAME
        dmag_magic.py

    DESCRIPTION
       plots intensity decay curves for demagnetization experiments

    SYNTAX
        dmag_magic -h [command line options]

    INPUT
       takes magic formatted magic_measurements.txt files

    OPTIONS
        -h prints help message and quits
        -f FILE: specify input file, default is: magic_measurements.txt
        -obj OBJ: specify  object  [loc, sit, sam, spc] for plot,
               default is by location
        -LT [AF,T,M]: specify lab treatment type, default AF
        -XLP [PI]: exclude specific  lab protocols,
               (for example, method codes like LP-PI)
        -N do not normalize by NRM magnetization
        -sav save plots silently and quit
        -fmt [svg,jpg,png,pdf] set figure format [default is svg]
    NOTE
        loc: location (study); sit: site; sam: sample; spc: specimen
    """
    if '-h' in sys.argv:
        print(main.__doc__)
        sys.exit()
    # initialize variables from command line + defaults
    FIG = {}  # plot dictionary
    FIG['demag'] = 1  # demag is figure 1
    in_file = pmag.get_named_arg_from_sys("-f", default_val="measurements.txt")
    plot_by = pmag.get_named_arg_from_sys("-obj", default_val="loc")
    name_dict = {'loc': 'location', 'sit': 'site',
                 'sam': 'sample', 'spc': 'specimen'}
    plot_key = name_dict[plot_by]
    LT = "LT-" + pmag.get_named_arg_from_sys("-LT", "AF") + "-Z"
    if LT == "LT-T-Z":
        units, dmag_key = 'K', 'treat_temp'
    elif LT == "LT-AF-Z":
        units, dmag_key = 'T', 'treat_ac_field'
    elif LT == 'LT-M-Z':
        units, dmag_key = 'J', 'treat_mw_energy'
    else:
        units = 'U'
    no_norm = pmag.get_flag_arg_from_sys("-N")
    norm = 0 if no_norm else 1
    no_plot = pmag.get_flag_arg_from_sys("-sav")
    plot = 0 if no_plot else 1
    fmt = pmag.get_named_arg_from_sys("-fmt", "svg")
    XLP = pmag.get_named_arg_from_sys("-XLP", "")
    dir_path = pmag.get_named_arg_from_sys("-WD", os.getcwd())
    spec_file = pmag.get_named_arg_from_sys("-fsp", default_val="specimens.txt")
    samp_file = pmag.get_named_arg_from_sys("-fsa", default_val="samples.txt")
    site_file = pmag.get_named_arg_from_sys("-fsi", default_val="sites.txt")

    # create contribution and add required headers
    fnames = {"specimens": spec_file, "samples": samp_file, 'sites': site_file}
    contribution = nb.Contribution(dir_path, single_file=in_file,
                                   custom_filenames=fnames)
    file_type = list(contribution.tables.keys())[0]
    print(len(contribution.tables['measurements'].df), ' records read from ', in_file)
    # add plot_key into measurements table
    if plot_key not in contribution.tables['measurements'].df.columns:
        contribution.propagate_name_down(plot_key, 'measurements')
    data_container = contribution.tables[file_type]
    # pare down to only records with useful data
    # grab records that have the requested code
    data_slice = data_container.get_records_for_code(LT)
    # and don't have the offending code
    data = data_container.get_records_for_code(XLP, incl=False, use_slice=True,
                                               sli=data_slice, strict_match=False)

    # make sure quality is in the dataframe
    if 'quality' not in data.columns:
        data['quality'] = 'g'
    # get intensity key and make sure intensity data is not blank
    intlist = ['magn_moment', 'magn_volume', 'magn_mass']
    IntMeths = [col_name for col_name in data.columns if col_name in intlist]
    # get rid of any entirely blank intensity columns
    for col_name in IntMeths:
        if not data[col_name].any():
            data.drop(col_name, axis=1, inplace=True)
    IntMeths = [col_name for col_name in data.columns if col_name in intlist]
    if len(IntMeths) == 0:
        print('No intensity headers found')
        sys.exit()

    int_key = IntMeths[0] # plot first intensity method found - normalized to initial value anyway - doesn't matter which used
    data = data[data[int_key].notnull()]
    # make list of individual plots
    # by default, will be by location_name
    plotlist = data[plot_key].unique()
    plotlist.sort()
    pmagplotlib.plot_init(FIG['demag'], 5, 5)
    # iterate through and plot the data
    for plt in plotlist:
        plot_data = data[data[plot_key] == plt].copy()
        if plot:
            print(plt, 'plotting by: ', plot_key)
        if len(plot_data) > 2:
            title = plt
            spcs = []
            spcs = plot_data['specimen'].unique()
            for spc in spcs:
                INTblock = []
                spec_data = plot_data[plot_data['specimen'] == spc]
                for ind, rec in spec_data.iterrows():
                    INTblock.append([float(rec[dmag_key]), 0, 0, float(rec[int_key]), 1, rec['quality']])
                if len(INTblock) > 2:
                    pmagplotlib.plotMT(FIG['demag'], INTblock,
                                       title, 0, units, norm)

            if not plot:
                files = {}
                for key in list(FIG.keys()):
                    files[key] = title + '_' + LT + '.' + fmt
                pmagplotlib.saveP(FIG, files)
                #sys.exit()
            else:
                pmagplotlib.drawFIGS(FIG)
                prompt = " S[a]ve to save plot, [q]uit,  Return to continue:  "
                ans = input(prompt)
                if ans == 'q':
                    sys.exit()
                if ans == "a":
                    files = {}
                    for key in list(FIG.keys()):
                        files[key] = title + '_' + LT + '.' + fmt
                    pmagplotlib.saveP(FIG, files)
            pmagplotlib.clearFIG(FIG['demag'])

if __name__ == "__main__":
    main()


import logging
import os
import re
from datetime import datetime, timezone
import pymongo
from feedgen.feed import FeedGenerator
import azure.functions as func
import xml.dom.minidom

def xmlPretty(string):
    #string=string.encode("utf-8").decode('utf-8','strict')
    xml_s = xml.dom.minidom.parseString(string)
    xml_pretty_str = xml_s.toprettyxml()
    return xml_pretty_str

def get_connstring():
    return os.getenv('CONNSTRING_MONGO', "mongodb://localhost:27017/")

def mongo_collection(col="anforanden"):
    myclient = pymongo.MongoClient(get_connstring())
    mydb = myclient["rod"]
    mycol = mydb[col]
    return mycol

def formatera_text_n(txt):
    """ Fixa texten till mer läsbart format men byter p-taggar till radbryt och behåller andra radbryt"""
    if txt is not None:
        txt=txt.replace('
','\n').replace('
','').replace('','').replace('','').replace('STYLEREF Kantrubrik \* MERGEFORMAT','').replace('-\n','')\
            .replace("&", "&").replace("<","<").replace(">", ">").replace("\"", """).replace("'", "'")
    return txt.encode("utf-8").decode('utf-8','strict')

def get_cursor(search_string, authority, maxhits):
    # Fetch search string for abreviated authorities with complex search strings
    if authority:
        if len(authority)<5:
            authority=authority.upper()
            auth=mongo_collection('authorities').find({'Entitet':authority})
            authority_search=auth[0]['Söksträng']
        else:
            authority=authority.title()
            auth=mongo_collection('authorities').find({'Entitet':authority})
            authority_search=auth[0]['Söksträng']
    else:
        authority_search=''

    # Depending on parameter input in the function this if-statement chooses how the Mongo query should be specified
    if search_string:
        search_string = search_string.replace("%20"," ")
        if len(search_string) > 3:
            if not authority:
                        filter=   {
                        "$text" : { "$search": search_string }
                    }

            else:
                filter=   {
                            "$text" : { "$search": search_string },
                            "authorities": authority
                        }
        else:
            if not authority:
                        filter=   {}
            else:
                filter=   {
                            "authorities": authority
                        }
    else:
        if not authority:
                    filter=   {}
        else:
            filter=   {
                        "authorities": authority
                    }

    # Pass the query to Mongo
    cursor=mongo_collection().find(sort=[("dok_datum", pymongo.DESCENDING),("anforande_nummer",pymongo.DESCENDING)],limit=maxhits, filter=filter)

    return cursor, authority_search

# Start looping in items to the rss
def rss_feedgen(search_string, authority, maxhits):
    cursor, authority_search = get_cursor(search_string, authority, maxhits)
    # Create the rss template
    rss_head="""<?xml version="1.0" encoding="UTF-8"?><rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0"><channel><title>RiR Datalab RSS feed</title><link>http://rod4sai.azurewebsites.net/</link><description>Anföranden i Riksdagen</description><language>sv</language><copyright>RiR Datalab 2020</copyright><atom:link href="http://rod4sai.azurewebsites.net" rel="self" type="application/rss+xml"/>"""

    reg_fallback=re.compile(f'((?<=
)(?:(?!<\/p>).)*.*(?:(?!<\/p>).)*(?=<\/p>))')
    reg_str=re.compile(f'((?<=
)(?:(?!<\/p>).)*{authority_search}(?:(?!<\/p>).)*(?=<\/p>))')
    item=''
    for row in cursor:
        if authority:
            try:
                summary = reg_str.search(row['anforandetext']).group(1)
            except:
                if len(row['anforandetext'])>400:
                    summary = f"{row['anforandetext'][:400]}..."
                else:
                    summary = row['anforandetext']
        else:
            try:
                summary = reg_fallback.search(row['anforandetext']).group(1)
            except Exception as ex:
                if len(row['anforandetext'])>400:
                    summary = f"{row['anforandetext'][:400]}..."
                else:
                    summary = row['anforandetext']

        link=f"<link>{row['protokoll_url_www']}</link>"
        title=f"<title>{row['avsnittsrubrik']}</title>"
        description=f"<description>{formatera_text_n(summary)}</description>"
        date=row['dok_datum'].replace(tzinfo=timezone.utc).strftime("%a, %d %b %Y %H:%M:%S %z")
        pubDate=f"<pubDate>{date}</pubDate>"
        guid=f"""<guid isPermaLink="false">{row['anforande_url_html']} </guid>"""
        category=f"<category>{row['topic']}</category>"
        source_url=f"""<source url="{row['protokoll_url_www']}"> {row['talare']} </source>"""

        item+=f"<item>{link}{title}{description}{pubDate}{guid}{category}{source_url}</item>"
    return rss_head+item+"</channel></rss>"

def main(req: func.HttpRequest) -> func.HttpResponse:
    #logging.info('Python HTTP trigger function processed a request.')

    searchstring = req.params.get("searchstring")
    authority = req.params.get("authority")
    maxhits = req.params.get("maxhits")

    if not searchstring:
        searchstring=""

    if maxhits:
        maxhits=int(maxhits)
    else:
        maxhits=25
    try:
        rss_string = rss_feedgen(searchstring, authority, maxhits)
        pass
    except Exception as ex:
        return func.HttpResponse(ex,status_code=200)

    return func.HttpResponse(rss_string,status_code=200)


# author = 'Erin Dugan'
# modified by 'Rui Liu'
# 2019-05-06

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import csv
import re
import sys
from selenium.common.exceptions import NoSuchElementException

argvalue = 1
endpage = 1000

index = argvalue

driver = webdriver.Chrome()
driver2 = webdriver.Chrome()

driver.get("https://www.petfinder.com/search/cats-for-adoption/us/ny/11790/?page=" + str(index))

csv_file = open('pets_Denver10mi_' + str(index) + '.csv', 'w')
writer = csv.writer(csv_file)

#STEPS:
# 1 - From search result page - Click each dog's pic (pet_tile) to go to its page
# 2 - Scrape individual page
# 3 - Return to result page, select next pet until no more on page
# 4 - Next page, then repeat until no more pages

while index <= endpage:  #While Next button is at bottom of page
	try:
		print("Scraping Page #: " + str(index))
		index = index + 1

		# Find all the pets on the page
		pet_tiles = driver.find_elements_by_xpath('//a[@class="petCard-overlay-link"]')
		pets = [x.get_attribute("href") for x in pet_tiles]

		for pet in pets:   # Find all the info on each pet's page
			driver2.get(pet)
			pet_dict = {}  #initialize dictionary

			try:
				name = driver2.find_element_by_xpath('.//h1[@id="Detail_Main"]').text
			except NoSuchElementException:
				name = 0

			try:
			    organization = driver2.find_element_by_xpath('.//pf-truncate[@line-count="3"]').text
			except NoSuchElementException:
			    organization = 0

			try:
			    story = driver2.find_element_by_xpath('(.//div[@class="u-vr4x"])[2]').text
			except NoSuchElementException:
			    story = 0

			try:
				info1 = driver2.find_element_by_xpath('.//div[@class="card-section-inner"]').text
			except NoSuchElementException:
			    info1 = 0

			try:
				info2 = driver2.find_element_by_xpath('//div[@class="grid grid_gutterLg u-vr4x"]').text
			except NoSuchElementException:
			    info2 = 0

			try:
				script_info = driver2.find_elements_by_tag_name("script")
				info_all = 0
				for elem in script_info:
					if "published_at" in elem.get_attribute("innerHTML"):
						info_all = elem.get_attribute("innerHTML")
			except NoSuchElementException:
			    info_all = 0

			pet_dict['name'] = name
			pet_dict['organization'] = organization
			pet_dict['info1'] = info1
			pet_dict['info2'] = info2
			pet_dict['story'] = story
			pet_dict['info_all'] = info_all

			writer.writerow(pet_dict.values())  #writing to csv file

		# Locate the next button on the page.
		wait_button = WebDriverWait(driver, 10)
		next_button = wait_button.until(EC.element_to_be_clickable((By.XPATH,'.//button[@class="fieldBtn fieldBtn_altHover m-fieldBtn_iconRt m-fieldBtn_tight m-fieldBtn_full"]')))
		next_button.click()
		time.sleep(1)   #1-second wait while page loads

	except Exception as e:
		print(e)
		csv_file.close()
		driver.close()
		driver2.close()
		break


import numpy as np
import matplotlib.pyplot as plt
import imageio as imio
import os
plt.rcParams['image.cmap'] = 'gray'

imsize = (192,168)
nums = ['B' + (n < 10)*'0' + str(n) for n in range(1,40)]
nums.remove('B14') # for some reason the dataset is missing 'B14'
path = 'yalefaces_cropped/CroppedYale/yale{}/'

# load face matrix
X = []
for n in nums:
    for d in os.listdir(path.format(n)):
        im = imio.imread(path.format(n)+d)
        X.append(im.flatten().astype('float64'))

# each face flattened into column
X = np.array(X).T

def imshow(ax,im):
    ax.imshow(im)
    ax.set_xticks([]), ax.set_yticks([])

X_ave = X.mean(axis=1).reshape((X.shape[0],1))
X0 = X - X_ave # remove mean from each row

u,s,v = np.linalg.svd(X,full_matrices=False)
u0,s0,v0 = np.linalg.svd(X0,full_matrices=False)

fig,ax = plt.subplots()
imshow(ax,X_ave.reshape(imsize))
ax.set_title('The "average" face')

# plot singular values
def plot_singvals(s,title):
    cumu = []
    tot = np.sum(s)
    for i in range(len(s)):
        cumu.append(np.sum(s[:i])/tot)

    fig,axes = plt.subplots(2,1,sharex=True)
    axes[0].plot(s/tot,'ro',markersize=2)
    axes[0].set_ylabel('$\sigma_j$ normalized')
    axes[1].plot(cumu,'r',linewidth=3)
    axes[1].axhline(0.95,label='95%')
    axes[1].legend()
    axes[1].set_ylabel('cumulative variance')
    axes[0].set_title(title)

plot_singvals(s0,'Singular values of cropped faces')

# plot principle components of the face matrix
def plot_PCs(u,s,title,rows=3,cols=5):
    fig,axes = plt.subplots(rows,cols)

    for r in range(rows):
        for c in range(cols):
            n = r*cols+c
            im = u[:,n].reshape(imsize)
            imshow(axes[r,c],im)
            axes[r,c].set_xlabel('$\sigma_{' + str(n+1) +
                '}$ = ' + str(int(s[n])))

    fig.text(0.5,0.95,title,ha='center',fontsize=12)

plot_PCs(u0,s0,'The first 15 principle components of the cropped faces')

# projection of face matrix
def plot_proj(X,u,n,seed,title):
    fig,axes = plt.subplots(3,4)
    np.random.seed(seed)
    for i in range(3):
        face = np.random.randint(X.shape[1])
        print(face)
        for j in range(axes.shape[1]-1):
            im = X[:,face+j].reshape(imsize)
            proj = u[:,:n].T @ X[:,face+j]
            imshow(axes[i,j],im)
            axes[i,j].set_xlabel('shot ' + str(j+1))
            axes[i,axes.shape[1]-1].plot(proj)
            axes[i,axes.shape[1]-1].set_xticks([])
            axes[i,axes.shape[1]-1].set_yticks([])
    fig.text(0.5,0.95,title,ha='center',fontsize=12)

plot_proj(X0,u0,15,10,
          'Projection of 3 faces onto the first 15 principle componenets')

# low rank approximations of faces
fig,axes = plt.subplots(2,5)
for row in range(len(axes)):
    face = np.random.randint(X.shape[1])
    face_full = X[:,face].reshape(imsize)
    axes[row,0].imshow(face_full)
    axes[row,0].set_xlabel('full rank')
    face_r = np.zeros(face_full.size)
    rank = [0,10,100,250,500]
    for i in range(1,len(rank)):
        for r in range(rank[i-1],rank[i]):
            face_r += s[r] * u[:,r] * v[r,face]
        axes[row,i].imshow(face_r.reshape(imsize))
        axes[row,i].set_xlabel('rank {}'.format(rank[i]))
    for ax in axes[row]:
        ax.set_xticks([]), ax.set_yticks([])

fig.text(0.5,0.95,'Low rank reconstructions of faces',
         ha='center',fontsize=12)

# uncropped images
path = 'yalefaces/'
imsize = (243,320)
subjs = ['subject' + (n < 10)*'0' + str(n) for n in range(1,16)]
X = []
for pic in os.listdir(path):
    im = imio.imread(os.path.join(path,pic))
    X.append(im.flatten())
X = np.array(X).T

X = X - np.mean(X,axis=1).reshape((X.shape[0],1))
u,s,v = np.linalg.svd(X,full_matrices=False)

# plot singular values
plot_singvals(s,'Singular Values of uncropped faces')

# plot PCs of uncropped faces
plot_PCs(u,s,'The first 15 principle components of the cropped faces')

plot_proj(X,u,25,70,
          'Projection of 3 faces onto the first 15 priciple components')

SVD and ML - Facial images and music classification/faces.py


from functools import partial
import numpy as np
from . import augmentor_utils, database_sampler
from ...utils import common_utils

class DataAugmentor(object):
    def __init__(self, root_path, augmentor_configs, class_names, logger=None):
        self.root_path = root_path
        self.class_names = class_names
        self.logger = logger

        self.data_augmentor_queue = []
        aug_config_list = augmentor_configs if isinstance(augmentor_configs, list) \
            else augmentor_configs.AUG_CONFIG_LIST

        for cur_cfg in aug_config_list:
            if not isinstance(augmentor_configs, list):
                if cur_cfg.NAME in augmentor_configs.DISABLE_AUG_LIST:
                    continue
            cur_augmentor = getattr(self, cur_cfg.NAME)(config=cur_cfg)
            self.data_augmentor_queue.append(cur_augmentor)

    def gt_sampling(self, config=None):
        db_sampler = database_sampler.DataBaseSampler(
            root_path=self.root_path,
            sampler_cfg=config,
            class_names=self.class_names,
            logger=self.logger
        )
        return db_sampler

    def __getstate__(self):
        d = dict(self.__dict__)
        del d['logger']
        return d

    def __setstate__(self, d):
        self.__dict__.update(d)

    def random_world_flip(self, data_dict=None, config=None):
        if data_dict is None:
            return partial(self.random_world_flip, config=config)
        gt_boxes, points = data_dict['gt_boxes'], data_dict['points']
        for cur_axis in config['ALONG_AXIS_LIST']:
            assert cur_axis in ['x', 'y']
            gt_boxes, points = getattr(augmentor_utils, 'random_flip_along_%s' % cur_axis)(
                gt_boxes, points,
            )

        data_dict['gt_boxes'] = gt_boxes
        data_dict['points'] = points
        return data_dict

    def random_world_rotation(self, data_dict=None, config=None):
        if data_dict is None:
            return partial(self.random_world_rotation, config=config)
        rot_range = config['WORLD_ROT_ANGLE']
        if not isinstance(rot_range, list):
            rot_range = [-rot_range, rot_range]
        gt_boxes, points = augmentor_utils.global_rotation(
            data_dict['gt_boxes'], data_dict['points'], rot_range=rot_range
        )

        data_dict['gt_boxes'] = gt_boxes
        data_dict['points'] = points
        return data_dict

    def random_world_scaling(self, data_dict=None, config=None):
        if data_dict is None:
            return partial(self.random_world_scaling, config=config)
        gt_boxes, points = augmentor_utils.global_scaling(
            data_dict['gt_boxes'], data_dict['points'], config['WORLD_SCALE_RANGE']
        )
        data_dict['gt_boxes'] = gt_boxes
        data_dict['points'] = points
        return data_dict

    def filter_by_min_points(self, data_dict=None, config=None):
        if data_dict is None:
            return partial(self.filter_by_min_points, config=config)
        gt_boxes = data_dict['gt_boxes']
        if gt_boxes.size == 0:  # empty
            return data_dict

        points_num = config['POINTS_NUM']
        assert isinstance(points_num, list)

        points_num_dict = {}
        for name_num in points_num:
            name, min_num = name_num.split(':')
            min_num = int(min_num)
            points_num_dict[name] = min_num
        if 'gt_boxes_points_num' in data_dict:
            gt_boxes = data_dict['gt_boxes']
            gt_names = data_dict['gt_names']
            gt_boxes_points_num = data_dict['gt_boxes_points_num']
            keeped = np.zeros_like(gt_boxes_points_num, dtype=np.bool)
            for index, (name, num) in enumerate(zip(gt_names, gt_boxes_points_num)):
                if num > points_num_dict[name]:
                    keeped[index] = True
            data_dict['gt_boxes'] = gt_boxes[keeped]
            data_dict['gt_names'] = gt_names[keeped]
            data_dict['gt_boxes_points_num'] = gt_boxes_points_num[keeped]
            if 'gt_boxes_mask' in data_dict:
                gt_boxes_mask = data_dict['gt_boxes_mask']
                data_dict['gt_boxes_mask'] = gt_boxes_mask[keeped]

        else:
            raise Exception
        return data_dict

    def filter_by_camera_fov(self, data_dict=None, config=None):
        if data_dict is None:
            return partial(self.filter_by_camera_fov, config=config)

        fov = config['FOV']
        assert isinstance(fov, list)
        fov_neg_angle, fov_pos_angle = fov[0], fov[1]
        fov_neg_cos, fov_pos_cos = np.cos(fov_neg_angle / 180 * np.pi), np.cos(fov_pos_angle / 180 * np.pi)
        # filter points
        points = data_dict['points']
        x, y = points[:, 0], points[:, 1]
        fov_flag = np.logical_or(
            (x / np.linalg.norm((x, y), ord=2, axis=0) > fov_neg_cos) & (y < 0),
            (x / np.linalg.norm((x, y), ord=2, axis=0) > fov_pos_cos) & (y > 0)
        )
        data_dict['points'] = points[fov_flag]
        # filter box
        if 'gt_boxes' in data_dict:
            # filter box which center is not in fov
            gt_boxes = data_dict['gt_boxes']
            if gt_boxes.size == 0:  # empty
                return data_dict

            center_x, center_y = gt_boxes[:, 0], gt_boxes[:, 1]
            x, y = center_x, center_y
            fov_flag = np.logical_or(
                (x / np.linalg.norm((x, y), ord=2, axis=0) > fov_neg_cos) & (y < 0),
                (x / np.linalg.norm((x, y), ord=2, axis=0) > fov_pos_cos) & (y > 0)
            )
            data_dict['gt_boxes'] = gt_boxes[fov_flag]
            if 'gt_names' in data_dict:
                gt_names = data_dict['gt_names']
                data_dict['gt_names'] = gt_names[fov_flag]
            if 'gt_boxes_points_num' in data_dict:
                gt_boxes_points_num = data_dict['gt_boxes_points_num']
                data_dict['gt_boxes_points_num'] = gt_boxes_points_num[fov_flag]
            if 'gt_boxes_mask' in data_dict:
                gt_boxes_mask = data_dict['gt_boxes_mask']
                data_dict['gt_boxes_mask'] = gt_boxes_mask[fov_flag]
        return data_dict

    def forward(self, data_dict):
        """
        Args:
            data_dict:
                points: (N, 3 + C_in)
                gt_boxes: optional, (N, 7) [x, y, z, dx, dy, dz, heading]
                gt_names: optional, (N), string
                ...

        Returns:
        """
        for cur_augmentor in self.data_augmentor_queue:
            data_dict = cur_augmentor(data_dict=data_dict)
        if data_dict['gt_boxes'].size > 0:
            data_dict['gt_boxes'][:, 6] = common_utils.limit_period(
                data_dict['gt_boxes'][:, 6], offset=0.5, period=2 * np.pi
            )
        if 'calib' in data_dict:
            data_dict.pop('calib')
        if 'road_plane' in data_dict:
            data_dict.pop('road_plane')
        if 'gt_boxes_mask' in data_dict:
            gt_boxes_mask = data_dict['gt_boxes_mask']
            data_dict['gt_boxes'] = data_dict['gt_boxes'][gt_boxes_mask]
            data_dict['gt_names'] = data_dict['gt_names'][gt_boxes_mask]
            data_dict.pop('gt_boxes_mask')
        return data_dict


# -*- coding: utf-8 -*-
"""
Created on Tue Jan  8 19:06:21 2019

@author: hjjiang
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import cross_val_score
from sklearn.cluster import KMeans
from sklearn.cluster import AgglomerativeClustering
from sklearn.manifold import TSNE
from sklearn.decomposition import PCA
from pyclustering.cluster.kmedoids import kmedoids
from sklearn.preprocessing import normalize
from sklearn.mixture import GaussianMixture
from sklearn.feature_selection import RFECV
from sklearn.metrics import accuracy_score

##############################################################################
#------------------调参区域-------------------
#无监督学习
plot_E5_1 = 0 #E3-E7的TSNE图
plot_E5_2 = 0 #E5的ICM,TE的TSNE图
plot_E5_3 = 0 #E5的EPI,PE的TSNE图
method_E5_1 = 3 #分离E5的TE,ICM的聚类方法 见下方聚类方法
method_E5_2 = 3 #分离E5的EPI,PE的聚类方法 见下方聚类方法

plot_E7_1 = 0 #E7的ICM,TE的TSNE图
plot_E7_2 = 0 #E7的EPI,PE的TSNE图
method_E7_1 = 3 #分离E7的ICM,TE的聚类方法 见下方聚类方法
method_E7_2 = 3 #分离E7的EPI,PE的聚类方法 见下方聚类方法
'''
聚类方法
1:KMeans
2:GMM
3:AgglomerativeClustering
'''
#监督学习
method_LDA = 0
method_LinearSVC = 1
method_SVM = 0
method_RandomForest = 0
method_DecisionTree = 0
method_MultilayerPerceptron = 0
method_NearestNeighbor = 0

#画出E3,E4,E6,E7谱系分类情况 直方图
plot_LDA = 0
plot_LinearSVC = 1
plot_SVM = 0
plot_RandomForest = 0
plot_DecisionTree = 0
plot_MultilayerPerceptron = 0
plot_NearestNeighbor = 0

#特征选择 只有LDA LinearSVC可以设置feature_select_method_RFECV=1
feature_select_method_RFECV = 0 #recursive feature elimination
##############################################################################
#------数据读取，用我给的csv，已经把label放进去了-------
dat = pd.read_csv("./embryonic_data_490genes.csv", index_col = 0)
allX, ally = dat.iloc[:,1:], dat['label'] #提取数据和标签

#--------数据预处理---------------------------
allX = np.log10(allX + 1) #对数据取log

##############################################################################
#------------------无监督学习-------------------
#-----------TSNE，目的是比较每天的差异---------------------
X_2d = TSNE(n_components=2,random_state=0).fit_transform(allX)
if plot_E5_1:
    plt.figure(figsize=(6, 5))
    colors = 'r', 'g', 'b', 'c', 'm'
    la=np.array([3,4,5,6,7])
    for i, c, label in zip(la, colors, la):
        plt.scatter(X_2d[ally == i, 0], X_2d[ally == i, 1], c=c, label=label)
    plt.legend()
    plt.show()

#------E5分离pre-lineage-----------------------------------
threshold = 15 #根据上面的图人为设定的pre-lineage的界限

E5_without_pre = allX[(X_2d[:,0] < threshold) & (ally == 5)]  #E5_without_pre
allX['lineage'] = 0 #标签初始化为0   TE->1;ICM ->2(进一步分裂):EPI->2, PE->3; pre-lineage->4
allX['lineage'][(X_2d[:,0] > threshold) & (ally == 5)] = 4 # 给E5的pre-lineage贴标签

pca = PCA(n_components=3) #特征提取
E5_without_pre_processed = pca.fit_transform(E5_without_pre)

#------聚类：E5_without_pre，分成2类ICM和TE----------------
if method_E5_1 == 1:
    estimator = KMeans(n_clusters=2).fit(E5_without_pre_processed)
    label_pred = estimator.labels_  # 获取聚类标签 根据论文，多的是TE，少的是ICM(EPI+PE)

if method_E5_1 == 2:
    estimator = GaussianMixture(n_components=2, covariance_type='diag', random_state=0).fit(E5_without_pre_processed)
    label_pred = estimator.predict(E5_without_pre_processed)

if method_E5_1 == 3:
    estimator=AgglomerativeClustering(affinity='euclidean', compute_full_tree='auto',
            connectivity=None, linkage='ward', memory=None, n_clusters=2).fit(E5_without_pre_processed)
    label_pred = estimator.labels_

ind = int(sum(label_pred==0) < sum(label_pred==1)) #指示因子
allX['lineage'][(X_2d[:,0] < threshold) & (ally == 5)] = label_pred*(1-2*ind) +ind + 1

'''
这里有一个逻辑判断,由于少的那一类的标签，要定义为ICM，即贴上“2”的标签
1.label_pred==0 说明0少一些，0表示ICM
    ind = 1
    label_pred(1-2ind)+ind+1 = -label_pred+2   0(ICM)->2, 1(TE)->1
2.label_pred==1 说明1少一些，1表示ICM
    ind = 0
    label_pred(1-2ind)+ind+1 = label_pred+1    1(ICM)->2, 0(TE)->1
TE: 1  ICM: 2
'''

if plot_E5_2:
    X_2d = TSNE(n_components=2,random_state=0).fit_transform(E5_without_pre)
    plt.figure(figsize=(6, 5))
    colors = 'r', 'g'
    la=np.array([0,1])
    for i, c, label in zip(la, colors, la):
        plt.scatter(X_2d[label_pred == i, 0], X_2d[label_pred == i, 1], c=c, label=label)
    plt.legend()
    plt.title('ICM-TE t-SNE plot')
    plt.xlabel('t-SNE 1')
    plt.ylabel('t-SNE 2')
    plt.show()

#------聚类：E5_ICM，分成2类EPI和PE----------------
E5_ICM = allX[allX['lineage']== 2]  #选出E5中所有ICM
E5_ICM = E5_ICM.iloc[:,:-1]

if method_E5_2 == 1: #KNN
    estimator = KMeans(n_clusters=2).fit(E5_ICM)  #获取聚类标签 根据论文，多的是EPI，少的是PE
    label_pred_EPI_PE = estimator.labels_  # 获取聚类标签 根据论文，多的是TE，少的是ICM(EPI+PE)

if method_E5_2 == 2: #GMM
    estimator = GaussianMixture(n_components=2, covariance_type='diag', random_state=0).fit(E5_ICM)
    label_pred_EPI_PE = estimator.predict(E5_ICM)

if method_E5_2 == 3: #AgglomerativeClustering
    estimator=AgglomerativeClustering(affinity='euclidean', compute_full_tree='auto',
            connectivity=None, linkage='ward', memory=None, n_clusters=2).fit(E5_ICM)
    label_pred_EPI_PE = estimator.labels_

ind = int(sum(label_pred_EPI_PE==0) < sum(label_pred_EPI_PE==1)) #指示因子
allX['lineage'][allX['lineage']== 2] = label_pred_EPI_PE*(1-2*ind) +ind + 2 #重新贴标签，EPI=2,PE=3

if plot_E5_3:
    X_2d_EPI_PE = TSNE(n_components=2,random_state=0).fit_transform(E5_ICM)
    plt.figure(figsize=(6, 5))
    colors = 'r', 'g'
    la=np.array([0,1])
    for i, c, label in zip(la, colors, la):
        plt.scatter(X_2d_EPI_PE[label_pred_EPI_PE == i, 0], X_2d_EPI_PE[label_pred_EPI_PE == i, 1], c=c, label=label)
    plt.legend()
    plt.title('EPI-PE t-SNE plot')
    plt.xlabel('t-SNE 1')
    plt.ylabel('t-SNE 2')
    plt.show()

#E5结果统计
print('E5 TE NUM:', sum(allX['lineage']==1))
print('E5 EPI NUM:', sum(allX['lineage']==2))
print('E5 PE NUM:', sum(allX['lineage']==3))
print('E5 PRE NUM:', sum(allX['lineage']==4), '\n')

#E5聚类结果提取
new5= allX[allX['lineage'] != 0]
new_data5, new_label5 = new5.iloc[:,:490], new5['lineage']

##############################################################################
#----------------------------E7聚类-----------------------------------
allX['lineage'] = 0 #标签初始化为0   TE->1;ICM ->2(进一步分裂):EPI->2, PE->3;
E7_data = allX[ally == 7]
E7_data = E7_data.iloc[:,:-1]
E7_data = normalize(E7_data)
pca = PCA(n_components=3) #特征提取
E7_data = pca.fit_transform(E7_data)

#------聚类:分成2类ICM和TE----------------
if method_E7_1 == 1:
    estimator = KMeans(n_clusters=2).fit(E7_data)
    label_pred = estimator.labels_  # 获取聚类标签 根据论文，多的是TE，少的是ICM(EPI+PE)

if method_E7_1 == 2:
    estimator = GaussianMixture(n_components=2, covariance_type='diag', random_state=0).fit(E7_data)
    label_pred = estimator.predict(E7_data)

if method_E7_1 == 3:
    estimator = AgglomerativeClustering(affinity='euclidean', compute_full_tree='auto',
            connectivity=None, linkage='ward', memory=None, n_clusters=2).fit(E7_data)
    label_pred = estimator.labels_

ind = int(sum(label_pred == 0) < sum(label_pred == 1)) #指示因子
allX['lineage'][ally == 7] = label_pred*(1-2*ind) +ind + 1

if plot_E7_1:
    X_2d = TSNE(n_components=2,random_state=0).fit_transform(allX[ally == 7].iloc[:,:-1])
    plt.figure(figsize=(6, 5))
    colors = 'r', 'g'
    la=np.array([0,1])
    for i, c, label in zip(la, colors, la):
        plt.scatter(X_2d[label_pred == i, 0], X_2d[label_pred == i, 1], c=c, label=label)
    plt.legend()
    plt.title('ICM-TE t-SNE plot')
    plt.xlabel('t-SNE 1')
    plt.ylabel('t-SNE 2')
    plt.show()

#------聚类：E7_ICM，分成2类EPI和PE----------------
E7_ICM = allX[allX['lineage']== 2]  #选出E7中所有ICM
E7_ICM = E7_ICM.iloc[:,:-1]

if method_E7_2 == 1: #KNN
    estimator = KMeans(n_clusters=2).fit(E7_ICM)  #获取聚类标签 根据论文，多的是EPI，少的是PE
    label_pred_EPI_PE = estimator.labels_  # 获取聚类标签 根据论文，多的是TE，少的是ICM(EPI+PE)

if method_E7_2 == 2: #GMM
    estimator = GaussianMixture(n_components=2, covariance_type='diag', random_state=0).fit(E7_ICM)
    label_pred_EPI_PE = estimator.predict(E7_ICM)

if method_E7_2 == 3: #AgglomerativeClustering
    estimator = AgglomerativeClustering(affinity='euclidean', compute_full_tree='auto',
            connectivity=None, linkage='ward', memory=None, n_clusters=2).fit(E7_ICM)
    label_pred_EPI_PE = estimator.labels_

ind = int(sum(label_pred_EPI_PE==0) < sum(label_pred_EPI_PE==1)) #指示因子
allX['lineage'][allX['lineage']== 2] = label_pred_EPI_PE*(1-2*ind) +ind + 2 #重新贴标签，EPI=2,PE=3

if plot_E7_2:
    X_2d_EPI_PE = TSNE(n_components=2,random_state=0).fit_transform(E7_ICM)
    plt.figure(figsize=(6, 5))
    colors = 'r', 'g'
    la=np.array([0,1])
    for i, c, label in zip(la, colors, la):
        plt.scatter(X_2d_EPI_PE[label_pred_EPI_PE == i, 0], X_2d_EPI_PE[label_pred_EPI_PE == i, 1], c=c, label=label)
    plt.legend()
    plt.title('EPI-PE t-SNE plot')
    plt.xlabel('t-SNE 1')
    plt.ylabel('t-SNE 2')
    plt.show()

#E7结果统计
print('E7 TE NUM:', sum(allX['lineage']==1))
print('E7 EPI NUM:', sum(allX['lineage']==2))
print('E7 PE NUM:', sum(allX['lineage']==3))

#E7聚类结果提取
new7= allX[allX['lineage'] != 0]
new_data7, new_label7 = new7.iloc[:,:490], new7['lineage']

##############################################################################
#------------------监督学习-------------------
dat_3 = allX[ally == 3].iloc[:,:-1]
dat_4 = allX[ally == 4].iloc[:,:-1]
dat_6 = allX[ally == 6].iloc[:,:-1]
dat_7 = allX[ally == 7].iloc[:,:-1]

#Linear Discriminant Analysis with feature selection
if method_LDA:
    from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
    if feature_select_method_RFECV:
        LDA = LinearDiscriminantAnalysis()
        rfe = RFECV(estimator=LDA, step=10, cv=8)
        rfe = rfe.fit(new_data5, new_label5)
        index = np.where(rfe.support_ == True)[0]
        new_data5_selected = new_data5.iloc[:,index]
        LDA.fit(new_data5_selected, new_label5)
        LDA_prediction3=LDA.predict(dat_3.iloc[:,index])
        LDA_prediction4=LDA.predict(dat_4.iloc[:,index])
        LDA_prediction6=LDA.predict(dat_6.iloc[:,index])
        LDA_prediction7=LDA.predict(dat_7.iloc[:,index])
        print('----------------------RFECV----------------------')
    else:
        LDA = LinearDiscriminantAnalysis()
        new_data5_selected = new_data5
        LDA.fit(new_data5_selected, new_label5)
        LDA_prediction3=LDA.predict(dat_3)
        LDA_prediction4=LDA.predict(dat_4)
        LDA_prediction6=LDA.predict(dat_6)
        LDA_prediction7=LDA.predict(dat_7)

    if plot_LDA:
        for i in range(4):
            plt.figure(figsize=(6, 5))
            name_list = ['TE','EPI','PE','PRE']
            if i < 2:
                num_list = [sum(eval('LDA_prediction' + str(i+3)) == 1),sum(eval('LDA_prediction' + str(i+3)) == 2),sum(eval('LDA_prediction' + str(i+3)) == 3),sum(eval('LDA_prediction' + str(i+3)) == 4)]
            else:
                num_list = [sum(eval('LDA_prediction' + str(i+4)) == 1),sum(eval('LDA_prediction' + str(i+4)) == 2),sum(eval('LDA_prediction' + str(i+4)) == 3),sum(eval('LDA_prediction' + str(i+4)) == 4)]
            plt.bar(range(len(num_list)), num_list,color='rgbc',tick_label=name_list)
            plt.title('LDA-'+'E'+str(i+3)) if i <2 else plt.title('LDA-'+'E'+str(i+4))
            for a,b in zip(range(len(num_list)), num_list):
                plt.text(a, b+0.05, '%.0f' % b, ha='center', va='bottom', fontsize=10)
            plt.show()

    scores_LDA = cross_val_score(LDA, new_data5_selected, new_label5, cv=8)
    scores_LDA_mean=scores_LDA.mean()
    print('---------scores_LDA_mean---------:', scores_LDA_mean)
    scores_LDA_E7_test = accuracy_score(new_label7, LDA_prediction7)
    print('---------scores_LDA_E7_test---------:', scores_LDA_E7_test)

#SVM
if method_SVM:
    from sklearn.svm import SVC
    #if feature_select_method_REFCV: svm does not expose "coef_" or "feature_importances_" attributes
    SVM = SVC(kernel='rbf', gamma='auto')
    SVM.fit(new_data5, new_label5)
    SVM_prediction3=SVM.predict(dat_3)
    SVM_prediction4=SVM.predict(dat_4)
    SVM_prediction6=SVM.predict(dat_6)
    SVM_prediction7=SVM.predict(dat_7)
    scores_SVM = cross_val_score(SVM, new_data5, new_label5, cv=8)
    scores_SVM_mean=scores_SVM.mean()
    print('---------scores_SVM_mean---------:', scores_SVM_mean)
    scores_SVM_E7_test = accuracy_score(new_label7, SVM_prediction7)
    print('---------scores_SVM_E7_test---------:', scores_SVM_E7_test)

    if plot_SVM:
        for i in range(4):
            plt.figure(figsize=(6, 5))
            name_list = ['TE','EPI','PE','PRE']
            if i < 2:
                num_list = [sum(eval('SVM_prediction' + str(i+3)) == 1),sum(eval('SVM_prediction' + str(i+3)) == 2),sum(eval('SVM_prediction' + str(i+3)) == 3),sum(eval('SVM_prediction' + str(i+3)) == 4)]
            else:
                num_list = [sum(eval('SVM_prediction' + str(i+4)) == 1),sum(eval('SVM_prediction' + str(i+4)) == 2),sum(eval('SVM_prediction' + str(i+4)) == 3),sum(eval('SVM_prediction' + str(i+4)) == 4)]
            plt.bar(range(len(num_list)), num_list,color='rgbc',tick_label=name_list)
            plt.title('SVM-'+'E'+str(i+3)) if i <2 else plt.title('SVM-'+'E'+str(i+4))
            for a,b in zip(range(len(num_list)), num_list):
                plt.text(a, b+0.05, '%.0f' % b, ha='center', va='bottom', fontsize=10)
            plt.show()

#LinearSVC
if method_LinearSVC:
    from sklearn.svm import LinearSVC
    if feature_select_method_RFECV:
        LSVC = LinearSVC(C=0.01, penalty='l1', dual=False, max_iter=5000)
        rfe = RFECV(estimator=LSVC, step=100, cv=8)
        rfe = rfe.fit(new_data5, new_label5)
        index = np.where(rfe.support_ == True)[0]
        new_data5_selected = new_data5.iloc[:,index]
        LSVC.fit(new_data5_selected, new_label5)
        LSVC_prediction3=LSVC.predict(dat_3.iloc[:,index])
        LSVC_prediction4=LSVC.predict(dat_4.iloc[:,index])
        LSVC_prediction6=LSVC.predict(dat_6.iloc[:,index])
        LSVC_prediction7=LSVC.predict(dat_7.iloc[:,index])
        print('----------------------RFECV----------------------')

    else:
        LSVC = LinearSVC(C=0.01, penalty='l1', dual=False, max_iter=5000)
        new_data5_selected = new_data5
        LSVC.fit(new_data5_selected, new_label5)
        LSVC_prediction3=LSVC.predict(dat_3)
        LSVC_prediction4=LSVC.predict(dat_4)
        LSVC_prediction6=LSVC.predict(dat_6)
        LSVC_prediction7=LSVC.predict(dat_7)

    if plot_LinearSVC:
        for i in range(4):
            plt.figure(figsize=(6, 5))
            name_list = ['TE','EPI','PE','PRE']
            if i < 2:
                num_list = [sum(eval('LSVC_prediction' + str(i+3)) == 1),sum(eval('LSVC_prediction' + str(i+3)) == 2),sum(eval('LSVC_prediction' + str(i+3)) == 3),sum(eval('LSVC_prediction' + str(i+3)) == 4)]
            else:
                num_list = [sum(eval('LSVC_prediction' + str(i+4)) == 1),sum(eval('LSVC_prediction' + str(i+4)) == 2),sum(eval('LSVC_prediction' + str(i+4)) == 3),sum(eval('LSVC_prediction' + str(i+4)) == 4)]
            plt.bar(range(len(num_list)), num_list,color='rgbc',tick_label=name_list)
            plt.title('LinearSVC-'+'E'+str(i+3)) if i <2 else plt.title('LinearSVC-'+'E'+str(i+4))
            for a,b in zip(range(len(num_list)), num_list):
                plt.text(a, b+0.05, '%.0f' % b, ha='center', va='bottom', fontsize=10)
            plt.show()

    scores_LSVC = cross_val_score(LSVC, new_data5_selected, new_label5, cv=8)
    scores_LSVC_mean=scores_LSVC.mean()
    print('---------scores_LSVC_mean---------:', scores_LSVC_mean)
    scores_LSVC_E7_test = accuracy_score(new_label7, LSVC_prediction7)
    print('---------scores_LSVC_E7_test---------:', scores_LSVC_E7_test)

#Random Forest
if method_RandomForest:
    from sklearn.ensemble import RandomForestClassifier
    RF=RandomForestClassifier(n_estimators=100, max_depth=10,random_state=0)
    scores_RF = cross_val_score(RF, new_data5, new_label5, cv=8)
    scores_RF_mean=scores_RF.mean()
    print('---------scores_RF_mean---------:', scores_RF_mean)
    RF.fit(new_data5,new_label5)
    RF_prediction3=RF.predict(dat_3)
    RF_prediction4=RF.predict(dat_4)
    RF_prediction6=RF.predict(dat_6)
    RF_prediction7=RF.predict(dat_7)
    scores_RF_E7_test = accuracy_score(new_label7, RF_prediction7)
    print('---------scores_RF_E7_test---------:', scores_RF_E7_test)

    if plot_RandomForest:
        for i in range(4):
            plt.figure(figsize=(6, 5))
            name_list = ['TE','EPI','PE','PRE']
            if i < 2:
                num_list = [sum(eval('RF_prediction' + str(i+3)) == 1),sum(eval('RF_prediction' + str(i+3)) == 2),sum(eval('RF_prediction' + str(i+3)) == 3),sum(eval('RF_prediction' + str(i+3)) == 4)]
            else:
                num_list = [sum(eval('RF_prediction' + str(i+4)) == 1),sum(eval('RF_prediction' + str(i+4)) == 2),sum(eval('RF_prediction' + str(i+4)) == 3),sum(eval('RF_prediction' + str(i+4)) == 4)]
            plt.bar(range(len(num_list)), num_list,color='rgbc',tick_label=name_list)
            plt.title('RandomForest-'+'E'+str(i+3)) if i <2 else plt.title('RandomForest-'+'E'+str(i+4))
            for a,b in zip(range(len(num_list)), num_list):
                plt.text(a, b+0.05, '%.0f' % b, ha='center', va='bottom', fontsize=10)
            plt.show()

#Decision Tree
if method_DecisionTree:
    from sklearn import tree
    DT= tree.DecisionTreeClassifier()
    scores_DT = cross_val_score(DT, new_data5, new_label5, cv=8)
    scores_DT_mean=scores_DT.mean()
    print('---------scores_DT_mean---------:', scores_DT_mean)
    DT.fit(new_data5,new_label5)
    DT_prediction3=DT.predict(dat_3)
    DT_prediction4=DT.predict(dat_4)
    DT_prediction6=DT.predict(dat_6)
    DT_prediction7=DT.predict(dat_7)
    scores_DT_E7_test = accuracy_score(new_label7, DT_prediction7)
    print('---------scores_DT_E7_test---------:', scores_DT_E7_test)

    if plot_DecisionTree:
        for i in range(4):
            plt.figure(figsize=(6, 5))
            name_list = ['TE','EPI','PE','PRE']
            if i < 2:
                num_list = [sum(eval('DT_prediction' + str(i+3)) == 1),sum(eval('DT_prediction' + str(i+3)) == 2),sum(eval('DT_prediction' + str(i+3)) == 3),sum(eval('DT_prediction' + str(i+3)) == 4)]
            else:
                num_list = [sum(eval('DT_prediction' + str(i+4)) == 1),sum(eval('DT_prediction' + str(i+4)) == 2),sum(eval('DT_prediction' + str(i+4)) == 3),sum(eval('DT_prediction' + str(i+4)) == 4)]
            plt.bar(range(len(num_list)), num_list,color='rgbc',tick_label=name_list)
            plt.title('DecisionTree-'+'E'+str(i+3)) if i <2 else plt.title('DecisionTree-'+'E'+str(i+4))
            for a,b in zip(range(len(num_list)), num_list):
                plt.text(a, b+0.05, '%.0f' % b, ha='center', va='bottom', fontsize=10)
            plt.show()

#MLP
if method_MultilayerPerceptron:
    from sklearn.neural_network import MLPClassifier
    MLP= MLPClassifier(solver='lbfgs', alpha=1e-5, hidden_layer_sizes=9, random_state=1)
    scores_MLP = cross_val_score(MLP, new_data5, new_label5, cv=8)
    scores_MLP_mean=scores_MLP.mean()
    print('---------scores_MLP_mean---------:', scores_MLP_mean)
    MLP.fit(new_data5,new_label5)
    MLP_prediction3=MLP.predict(dat_3)
    MLP_prediction4=MLP.predict(dat_4)
    MLP_prediction6=MLP.predict(dat_6)
    MLP_prediction7=MLP.predict(dat_7)
    scores_MLP_E7_test = accuracy_score(new_label7, MLP_prediction7)
    print('---------scores_MLP_E7_test---------:', scores_MLP_E7_test)

    if plot_MultilayerPerceptron:
        for i in range(4):
            plt.figure(figsize=(6, 5))
            name_list = ['TE','EPI','PE','PRE']
            if i < 2:
                num_list = [sum(eval('MLP_prediction' + str(i+3)) == 1),sum(eval('MLP_prediction' + str(i+3)) == 2),sum(eval('MLP_prediction' + str(i+3)) == 3),sum(eval('MLP_prediction' + str(i+3)) == 4)]
            else:
                num_list = [sum(eval('MLP_prediction' + str(i+4)) == 1),sum(eval('MLP_prediction' + str(i+4)) == 2),sum(eval('MLP_prediction' + str(i+4)) == 3),sum(eval('MLP_prediction' + str(i+4)) == 4)]
            plt.bar(range(len(num_list)), num_list,color='rgbc',tick_label=name_list)
            plt.title('MultilayerPerceptron-'+'E'+str(i+3)) if i <2 else plt.title('MultilayerPerceptron-'+'E'+str(i+4))
            for a,b in zip(range(len(num_list)), num_list):
                plt.text(a, b+0.05, '%.0f' % b, ha='center', va='bottom', fontsize=10)
            plt.show()

#Nearest Neighbour Classification
if method_NearestNeighbor:
    from sklearn.neighbors import KNeighborsClassifier
    KNN = KNeighborsClassifier(n_neighbors=6)
    scores_KNN = cross_val_score(KNN, new_data5, new_label5, cv=8)
    scores_KNN_mean=scores_KNN.mean()
    print('---------scores_KNN_mean---------:', scores_KNN_mean)
    KNN.fit(new_data5,new_label5)
    KNN_prediction3=KNN.predict(dat_3)
    KNN_prediction4=KNN.predict(dat_4)
    KNN_prediction6=KNN.predict(dat_6)
    KNN_prediction7=KNN.predict(dat_7)
    scores_KNN_E7_test = accuracy_score(new_label7, KNN_prediction7)
    print('---------scores_KNN_E7_test---------:', scores_KNN_E7_test)

    if plot_NearestNeighbor:
        for i in range(4):
            plt.figure(figsize=(6, 5))
            name_list = ['TE','EPI','PE','PRE']
            if i < 2:
                num_list = [sum(eval('KNN_prediction' + str(i+3)) == 1),sum(eval('KNN_prediction' + str(i+3)) == 2),sum(eval('KNN_prediction' + str(i+3)) == 3),sum(eval('KNN_prediction' + str(i+3)) == 4)]
            else:
                num_list = [sum(eval('KNN_prediction' + str(i+4)) == 1),sum(eval('KNN_prediction' + str(i+4)) == 2),sum(eval('KNN_prediction' + str(i+4)) == 3),sum(eval('KNN_prediction' + str(i+4)) == 4)]
            plt.bar(range(len(num_list)), num_list,color='rgbc',tick_label=name_list)
            plt.title('NearestNeighbor-'+'E'+str(i+3)) if i <2 else plt.title('NearestNeighbor-'+'E'+str(i+4))
            for a,b in zip(range(len(num_list)), num_list):
                plt.text(a, b+0.05, '%.0f' % b, ha='center', va='bottom', fontsize=10)
            plt.show()