import os

try:
	import openpyxl
except:
	print("could not find openpyxl module")

def getFilesInDirectory(directory):
	fileNames = []
	excelFile_Paths = []

	current_path = os.getcwd()
	for file_in_directory in os.listdir(os.path.join(current_path, directory)):
		if file_in_directory.endswith(".xlsx"):
			excelFile_Paths.append(os.path.join(current_path, file_in_directory))
			fileNames.append(file_in_directory)

	return fileNames, excelFile_Paths

def convert_color_code(color_index):
    if isinstance(color_index, str):
        return color_index
    if isinstance(color_index, int):
        if color_index <= 63:
            return COLOR_INDEX[color_index]
        else:
            return "System background"

def titleRowsOfSheet(sheet, sheetname):
	# row_count = sheet.max_row
	# col_count = sheet.max_column
	mergedRanges = sheet.merged_cells.ranges
	title_rows = []

	# values update in loop
	column_max = 1
	column_min = sheet.max_column

	for row in sheet.iter_rows():
		all_filled = True
		all_empty = True

		for cell in row:
			if type(cell).__name__ == "MergedCell":
				first_cell = [cell_in_merge for cell_in_merge in sheet.merged_cells.ranges if cell.coordinate in cell_in_merge][0].start_cell
				merged_cell_color_index = first_cell.fill.end_color.index
				color_value = convert_color_code(merged_cell_color_index)
			else:
				color_index = cell.fill.end_color.index
				color_value = convert_color_code(color_index)

			if color_value != "00000000":
				# print("PASS "+ cell.coordinate)
				if cell.column > column_max:
					column_max = cell.column
				if cell.column < column_min:
					column_min = cell.column
				all_empty = False

			else:
				# print("FAIL "+ cell.coordinate)

				if cell.value != None:
					all_filled = False
					break;

		if all_filled and not all_empty:
			title_rows.append(row[0].row)  # add row numbers

	print(f"sheet: {sheetname}, min: {column_min}, max: {column_max}, title rows: {title_rows}")

	return column_min, column_max, title_rows

def columnNamesOfSheet(sheet, title_rows, min_column, max_column, sheetname):
	title_values = []
	column_count = sheet.max_column
	for col in range(min_column, max_column+1):
		column_values = []

		for row in range(title_rows[0], title_rows[-1] + 1):
			cell = sheet.cell(row=row, column=col)

			if type(cell).__name__ == "MergedCell":
				cell_value = [cell_in_merge for cell_in_merge in sheet.merged_cells.ranges if cell.coordinate in cell_in_merge][0].start_cell.value
			else:
				cell_value = cell.value

			if cell_value == None:
				cell_value = ""

			cell_value = str(cell_value)

			if (cell_value not in column_values):
				column_values.append(cell_value)

		title_values.append(" ".join(column_values))

	print(f"{sheetname}: {title_values}")
	return title_values

def main():
	fileNames, excelFile_Paths = getFilesInDirectory(directory)
	current_path = os.getcwd()

	for fileIndex in range(0, len(excelFile_Paths)):
		excelFile = openpyxl.load_workbook(excelFile_Paths[fileIndex])

		folder_name = fileNames[fileIndex].replace(".xlsx", "")
		folder_path = os.path.join(current_path, folder_name)

		try:
			os.mkdir(folder_path)
		except (FileExistsError):
			print(f"{folder_name} already exists")

		if set_sheetname_part:
			for sheetname in excelFile.sheetnames:
				if set_sheetname_part not in sheetname:
					continue

				sheet = excelFile[sheetname]
				column_min, column_max, title_rows = titleRowsOfSheet(sheet, sheetname)
				columnNames = columnNamesOfSheet(sheet, title_rows, column_min, column_max, sheetname)

				outputFile_path = os.path.join(folder_path, sheetname + ".csv")

				with open(outputFile_path, "w") as outputFile:
					for i in range(0, len(columnNames) - 1):
						outputFile.write(columnNames[i] + ',')
					outputFile.write(columnNames[len(columnNames) - 1] + '\n')

					for row in sheet.iter_rows(min_row=title_rows[-1]+1):
						for cell in row:
							if cell.value != None:
								cell_value = str(cell.value)
								cell_value = cell_value.replace("\n", " ")
								cell_value = cell_value.replace(",", ";")

							else:
								cell_value = ""

							outputFile.write(cell_value + ',')

						outputFile.write('\n')

		else:
			for sheetname in excelFile.sheetnames:
				sheet = excelFile[sheetname]
				column_min, column_max, title_rows = titleRowsOfSheet(sheet, sheetname)
				columnNames = columnNamesOfSheet(sheet, title_rows, column_min, column_max, sheetname)
				outputFile_path = os.path.join(folder_path, sheetname + ".csv")

				with open(outputFile_path, "w") as outputFile:
					for i in range(0, len(columnNames) - 1):
						outputFile.write(columnNames[i] + ',')

					outputFile.write(columnNames[len(columnNames) - 1] + '\n')

					for row in sheet.iter_rows(min_row=title_rows[-1]+1):
						for cell in row:
							if cell.value != None:
								cell_value = str(cell.value)
								cell_value = cell_value.replace("\n", " ")
								cell_value = cell_value.replace(",", ";")

							else:
								cell_value = ""
							outputFile.write(cell_value + ',')

						outputFile.write('\n')

if __name__ == "__main__":
	set_sheetname_part = "IPX"
	directory = "" # input("Folder: ")
	main()


#!/usr/bin/env python
# coding=utf-8
# Python3
"""
@Author: liuhaojie
@File  : webshellDc.py
@Idea  : IntelliJ IDEA
@Date  : 2020/7/15
@Desc  : Descriptions
"""
import logging
import os
import numpy as np

from sklearn.externals import joblib

logging.basicConfig(level=logging.DEBUG,
                    filename='log/checklog.log',
                    filemode='a',
                    format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s')
logger = logging.getLogger(__name__)
logger.addHandler(logging.StreamHandler())

class WebshellDec(object):
    def __init__(self, vs):
        super(WebshellDec, self).__init__()
        self.cv = joblib.load("model/countvectorizer_" + vs + ".pkl")
        self.transformer = joblib.load("model/tfidftransformer_" + vs + ".pkl")
        self.mlp = joblib.load("model/mlp_" + vs + ".pkl")

    @staticmethod
    def load_file(file_path):
        t = b''
        with open(file_path, "rb") as f:
            for line in f:
                line = line.strip(b'\r\t')
                t += line
        return t

    def checkdir(self, path):
        counter, webshell_number, normal_number = 0, 0, 0
        for r, d, files in os.walk(path):
            for file in files:
                file_path = r + '/' + file
                if os.path.splitext(file)[-1].lower() in ['.php', '.jsp', '.jspx', '.java', '.asp', '.aspx']:
                    t = self.load_file(file_path)
                    t_list = list()
                    t_list.append(t)
                    x = self.cv.transform(t_list).toarray()
                    x = self.transformer.transform(x).toarray()
                    y_pred = self.mlp.predict(x)
                    counter += 1
                    if y_pred[0] == 1:
                        logger.info("{} is webshell".format(file_path))
                        webshell_number += 1
                    else:
                        logger.info("{} is not webshell".format(file_path))
                        normal_number += 1
        logger.info("检测总量:%i, 检测出webshell:%i, 检测出正常文件:%i" % (counter, webshell_number, normal_number))

    def check(self, input_data):
        if os.path.isdir(input_data):
            return self.checkdir(input_data)
        elif os.path.isfile(input_data):
            t = [self.load_file(input_data)]
            x = self.cv.transform(t).toarray()
            x = self.transformer.transform(x).toarray()
            y_pred = self.mlp.predict(x)[0]
            logger.info("输入文件：%s" % input_data)
            logger.info("检测结果：%s" % "webshell" if y_pred == 1 else "正常文件")
        else:
            if type(input_data) is bytes or type(input_data) is str:
                try:
                    t = [input_data.decode()]
                except AttributeError:
                    t = [input_data]
                x = self.cv.transform(t).toarray()
                x = self.transformer.transform(x).toarray()
                y_pred = self.mlp.predict(x)
                logger.info("输入代码：%s" % input_data)
                logger.info("检测结果：%s" % "webshell" if y_pred == 1 else "正常代码")
            else:
                logger.info("输入无效")

if __name__ == "__main__":
    assert (np.__version__ >= "1.18.5")
    logger.info("输入模型版本号...")
    while True:
        try:
            version = input("version:")
            logger.info("初始化模型...")
            webshelldc = WebshellDec(version)
            logger.info("初始化模型完成，输入检测内容，支持脚本文件、文本代码、文件路径")
            break
        except FileNotFoundError:
            logger.info("对应版本模型不存在，请重新输入模型版本号...")

    while True:
        inputs = input("inputs:")
        if inputs == "exit":
            logger.info("退出检测！")
            break
        else:
            webshelldc.check(inputs)
            logger.info("继续输入：")
# text = '<?php@eval($_GET[\'p\'])\n<?php assert (    $_GET[\'p\']\n)\n$func="test";$b374k=$func(\'$x\', \'ev\'.\'al\')\n$b=$W(\'\',$S);$b();\n;$pouet($pif,$paf);\n${$pouet}\n\'pouet\'.\'pif\' . \'pouet\' . "lol" ."kwainkwain"\n'
# print(webshelldc.check(text))


import sys
import heapq
from collections import namedtuple

stdin = sys.stdin

Query = namedtuple('Query', 'id period')

if __name__ == '__main__':
    queries = []
    for row in stdin:
        row = row.strip()
        if '#' in row:
            n = int(stdin.readline().strip())
            break
        q = Query(*list(map(int, row.split()[1:])))
        queries.append(q)

    heap_smallest_to_largest = []
    heap_largest_to_smallest = []

    queries.sort(key=lambda x: x.period)
    n_queries = 0
    for i in range(1, n+1):
        for q in queries:
            if len(heap_smallest_to_largest) >= n:
                largest = abs(heap_largest_to_smallest[0][0])
                if q.period * i > largest:
                    break
            heapq.heappush(heap_largest_to_smallest, (-q.period * i, q.id))
            heapq.heappush(heap_smallest_to_largest, (q.period * i, q.id))

    while n_queries < n:
        print(heapq.heappop(heap_smallest_to_largest)[1])
        n_queries += 1


import requests
url = "http://api.etherscan.io/api?module=stats&action=tokensupply&contractaddress=0xc58c0fca06908e66540102356f2e91edcaeb8d81&apikey=YourApiKeyToken"
response = requests.get(url)

supply=response.json()
currentsupply=int(supply.get("result"))
roundedsupply= int(currentsupply/int(1000000000000000000))

burnedtokens= int(1000000)-roundedsupply

print("remaining tokens =" ,roundedsupply)
print("total burned tokens =" ,burnedtokens)
print("initial supply = 1000000")


import torch
import torch.nn as nn

class Word2Vec(nn.Module):
    def __init__(self, vocab_size, embed_size):
        super(Word2Vec, self).__init__()
        self.word_embed = nn.Embedding(vocab_size, embed_size)
        self.context_embed = nn.Embedding(vocab_size, embed_size)
        self.output = nn.LogSigmoid()

    def forward(self, target_word, context_word):
        # use `torch.bmm` for batch matrix multiplication
        target_embed = self.word_embed(target_word).unsqueeze(2)  # Shape: [batch_size, embed_size, 1]
        context_embed = self.context_embed(context_word).unsqueeze(1)  # Shape: [batch_size, 1, embed_size]
        scores = torch.bmm(context_embed, target_embed).squeeze()  # Shape: [batch_size]

        loglogits = self.output(scores)
        return loglogits


# to
# i can webscrape it without having to render it
# i just have to fish out the
# <div class="js-tweet-text-container"> </div>
# objects

# stages
# 1 download a searchpage every 4 seconds
# extract the tweet objects from the page into an array
# create a coherent timeline (elimiate doubles)
# save to disk in some format

import tweepy
import queue
import datetime
import pprint
import time
import multiprocessing
import threading
from main import ROOTDIR, dir_sep as dirsep
from utils import handle_ratelimit, escape_string, log
import csv

raw_tweet_queue = queue.Queue()
downloader_done = False

# TODO make it so that you download a set number of tweets rather
# TODO than a daterange cuz twitter just doesn't want me to have nice thing

def test(api):
    search_term = '@ISPuuuv'
    #search_term = '@Potus'
    dump_dir = ROOTDIR + dirsep + search_term + dirsep
    today = datetime.date.today()
    start_date = today.replace(month=today.month-1)
    end_date = start_date.replace(day=start_date.day +1)

    dlr = Tweet_Downloader()
    writer = Tweets_To_Csv()

    dl_process = threading.Thread(target=dlr.daterange_scrape, args=(api, search_term, start_date, end_date))
    dl_process.start()
    # join ends the thread once it's finished
    dl_process.join()

    writer_process = threading.Thread(target=writer.run, args=(dump_dir, search_term, start_date))
    writer_process.start()
    writer_process.join()

class Tweet_Downloader:
    def daterange_scrape(self, api, search_term, date_start, date_end):
        # the date arguments must be datetime.date objects
        log("starting datrange scrape")

        results = []
        since_id_search_term = self.__date_to_search_term(date_start, date_end, search_term)
        print(api.search(since_id_search_term))
        # this gets the since_id associated with the search term
        while not len(results) > 0:
            log("getting results")
            try:
                for page in tweepy.Cursor(api.search, q=since_id_search_term).pages():
                    results = page
                    print(page[0].created_at)
                    #break
            except Exception as e:
                log(str(e))
                handle_ratelimit("since id getter")

        since_id = results[0].id

        self.__download_tweets(api, search_term, since_id, date_end)

    def __date_to_search_term(self, date_start, date_end, search_term):
        # searchterm since:2017-09-18 until:2017-09-19
        # this string used as searchterm will search the daterange specified
        return str(search_term + " since:" + str(date_start) + " until:" + str(date_end))

    def __tweets_to_queue(self, tweets):
        # if in con't place the items in the queue it waits instead of throwing an exept (apparently)
        for tweet in tweets:
            global raw_tweet_queue
            raw_tweet_queue.put(tweet)

    def __check_stream_end(self, results, stream_end):
        # stream_end is a datetime.datetime object of where the stream ends.
        #stream_end = datetime(stream_end.year, stream_end.month, stream_end.day)

        if results[0].created_at > datetime.datetime(stream_end.year, stream_end.month, stream_end.day):
            return True
        return False

    def __truncate_stream_end(self, page, stream_end_date):
        # cuts of the tweets that fall outside of the time range
        return page#TODO

    def __download_tweets(self, api, search_term, since_id, stream_end_date):

        end_of_stream = False
        i = 1
        while not end_of_stream:
            print("trying page download")
            try:
                for page in tweepy.Cursor(api.search, q=search_term, since_id=since_id).pages():
                    print("downloaded page")
                    # page is a list of statuses

                    if self.__check_stream_end(page, stream_end_date):
                        end_of_stream = True

                        page = self.__truncate_stream_end(page, stream_end_date)
                        self.__tweets_to_queue(page)  # adds truncated page onto queue before exiting
                        break  # breaks out of try exept loop
                        # if break is not here it will continue until it hits the rate limit

                    #if i % 5 == 0:
                    log("got page " + str(i))
                    i += 1

                    self.__tweets_to_queue(page)

            except Exception as e:
                log(str(e))
                handle_ratelimit("dler")
        global downloader_done
        downloader_done = True

# methods for saving tweet data to a csv file
class Tweets_To_Csv:
    def run(self, dump_dir, searchterm, date):
        filename = searchterm + " " + str(date) + ".csv"
        with open(filename, 'w', newline='', encoding='utf-8') as csvfile:
            fieldnames = ["posted_at", "tweet_id", "in_reply_to_tweet_id", "in_reply_to_user_id", "user_id",
                          "user_name", "user_screen_name", "user_verified", "tweet_text", "retweets_no", "likes",
                          "application_used", "user_location", "coordinates", "user_timezone",
                          "follower_count", "following_count", "user_likes_no", "lang", "user_created"]

            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
            writer.writeheader()
            # while not downloader_done:
            i = 1

            while True:
                q = raw_tweet_queue
                if q.empty():
                    log("waiting 3 seconds for tweets to download")
                    time.sleep(3)
                else:
                    self.__mask_and_write(q.get(), writer)
                    if i % 30 == 0:
                        log(str(i) + " tweets encoded")
                    i+=1

                if q.empty() and downloader_done:
                    break

    def __mask_and_write(self, tweet, writer):
        posted_at = str(tweet.created_at)  # have to cast cause it's a datetime object
        tweet_id = tweet.id_str
        in_reply_to_tweet_id = tweet.in_reply_to_status_id_str
        in_reply_to_user_id = tweet.in_reply_to_user_id_str
        user_id = tweet.user.id_str
        user_name = tweet.user.name
        user_screen_name = tweet.user.screen_name
        user_verified = 0
        if tweet.user.verified:  # this is to save space
            user_verified = 1
        tweet_text = tweet.text
        tweet_text = escape_string(tweet_text)
        # replies_no = tweet.? #don't think it's in here
        retweets_no = tweet.retweet_count
        likes = tweet.favorite_count
        application_used = tweet.source_url
        # possibly_sensitive = 0
        # if tweet.retweeted_status.possibly_sensitive:
        #    possibly_sensitive = 1
        user_location = tweet.user.location
        coordinates = tweet.coordinates
        user_timezone = tweet.user.time_zone
        follower_count = tweet.user.followers_count
        following_count = tweet.user.friends_count
        user_likes_no = tweet.user.favourites_count
        lang = tweet.lang
        user_created = str(tweet.user.created_at)  # have to cast cause it's a datetime object

        writer.writerow({"posted_at": posted_at, "tweet_id": tweet_id, "in_reply_to_tweet_id": in_reply_to_tweet_id,
                     "in_reply_to_user_id": in_reply_to_user_id, "user_id": user_id, "user_name": user_name,
                     "user_screen_name": user_screen_name, "user_verified": user_verified, "tweet_text": tweet_text,
                     "retweets_no": retweets_no, "likes": likes, "application_used": application_used, "user_location":
                         user_location, "coordinates": coordinates, "user_timezone": user_timezone, "follower_count":
                         follower_count, "following_count": following_count, "user_likes_no": user_likes_no, "lang": lang,
                     "user_created": user_created})


# -*- coding: utf-8 -*-
"""
    Deveiate Colorscheme
    ~~~~~~~~~~~~~~~~~~~~

    Converted by Vim Colorscheme Converter
"""
from pygments.style import Style
from pygments.token import Token, Keyword, Number, Generic, String, Operator, Comment, Name

class DeveiateStyle(Style):

    background_color = '#00000f'
    styles = {
        Token:              'noinherit #f6dfb2 bg:#00000f',
        Name.Entity:        'noinherit #e9b96e',
        Number:             'noinherit #fce94f',
        Comment.Preproc:    '#00cbcd bold',
        Name.Constant:      'noinherit #76a3d7',
        Name.Attribute:     'noinherit #ad7fa8',
        Name.Tag:           'noinherit #ffffff bold',
        Comment:            'noinherit #a82419',
        Generic.Deleted:    'noinherit #f8f8f8 bg:#420e09',
        Generic.Inserted:   'noinherit #f8f8f8 bg:#253b22',
        Keyword:            'noinherit #ffffff bold',
        Name.Variable:      'noinherit #729fcf',
        Generic.Emph:       '#ad7fa8 underline',
        String:             'noinherit #da8d53',
        Name.Function:      'noinherit #ad7fa8',
        Generic.Output:     'noinherit #222222',
        Number.Float:       'noinherit #9bda8b',
    }


n = int(input())
result = 0

for i in range(1, n + 1):
    m = list(map(int, str(i))) # 숫자 자리별로 하나씩 배열 m에 넣어줌
    s = i + sum(m) # 분해합

    if(s == n): # 분해합 결과와 n이 같으면
        result = i
        break

print(result)


""" Very simple demonstration of how match_title_ents works.
"""
from match_title_ents import match_title_ents
import spacy

nlp = spacy.load('en_core_web_sm')

with open("tests/test_data/title1.txt", "r") as text_file:
    title1 = nlp(text_file.read())

with open("tests/test_data/article1.txt", "r") as text_file:
    article1 = nlp(text_file.read())

with open("tests/test_data/title2.txt", "r") as text_file:
    title2 = nlp(text_file.read())

with open("tests/test_data/article2.txt", "r") as text_file:
    article2 = nlp(text_file.read())

print(match_title_ents(title1, article1))

print(match_title_ents(title2, article2))


from matplotlib import pyplot as plt
from matplotlib.colors import ListedColormap

from mne.filter import filter_data

from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC, LinearSVC
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import make_moons, make_circles, make_classification
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis

import scipy.signal as scisig
import numpy as np
import pandas as pd
import csv
import pickle
import datetime

def ConvertEasyToDataFrame(filename):
    """ This function converts a single .easy file (from NIC2) to an easy-to-use dataframe.
    Uses both the .easy file and .info file (containing metadata)

    ---- Input ----
    filename: string containing the .easy filepath

    ---- Output ----
    df: dataframe containing all the EEG, accelerometer, and event marker data
    fs: sampling rate for the EEG data (Hz)
    fs_accel: sampling rate for the accelerometer data (Hz)

    """
    import pandas as pd

    # Read in the .easy file
    df = pd.read_csv(filename, delimiter='\t', header=None)

    # Get metadata from the .info file
    fname = filename[:-5] + '.info'
    with open(fname) as f:
        content = f.readlines()
    content = [x.strip() for x in content]

    # Get the channel names
    channel_info = [x for x in content if 'Channel ' in x]
    channel_names = []
    for ch in range(len(channel_info)):
        channel_names.append(channel_info[ch].split(': ')[1])

    channel_names.append('X')
    channel_names.append('Y')
    channel_names.append('Z')
    channel_names.append('STI 014')
    channel_names.append('DateTime')

    # Get sampling rates
    sampling_rates = [x for x in content if 'sampling rate: ' in x]
    fs_all = []
    for freq in range(len(sampling_rates)):
        fs_all.append(float(sampling_rates[freq].split(': ')[1].split(' ')[0]))

    # Store sampling rates
    fs = fs_all[0]
    fs_accel = fs_all[1]

    # Assign the column names
    df.columns = channel_names

    # Return dataframe and sampling rates
    return df, fs, fs_accel

def PlotAverageSpectrogram(epochs, trials, chanOfInt, fs, ds_factor):
    """
    Input
    epochs: your data epochs (raw)
    trials: list or array of trials you wish to include in the average (e.g., all trials of type A)
    chanOfInt: name of the channel of interest
    fs: sampling rate
    ds_factor: downsampling factor
    """

    Sxx_all = list()

    for trialOfInt in trials:
        f, t, Sxx = scisig.spectrogram(scisig.decimate(epochs[trialOfInt][chanOfInt], ds_factor), fs/ds_factor, nperseg=10)
        # nperseg determines size of time bins, number of time bins = fs/nperseg - 1
        # nperseg = number of segments per second, so 10 = 100 ms chunks
        Sxx_all.append(Sxx)

    Sxx_avg = np.mean(Sxx_all,0)

    return f, t, Sxx_avg

if __name__ == "__main__":
    subjID = input('Subject ID: ')
    EEGdevice = int(input('7 if DSI-7, 8 if Enobio: '))
    filepath_EEG = input('EEG data file path (.easy or .csv): ')
    filepath_behavioral = input('Behavioral data filepath (.csv): ')

    ## Load in data from the EEG file
    if EEGdevice == 7:
        EEGmeta = list()
        # Open and read ('r') csv file with raw data
        with open(filepath_EEG, 'r') as csvfile:
            for row in range(0,15):
                EEGmeta.append(csvfile.readline())

        # Save sampling frequency
        fs = int(EEGmeta[1].split(',')[1])

        # Skip the first 15 rows that contain the meta data
        EEGdata = pd.read_csv(filepath_EEG,',',skiprows=15)

    elif EEGdevice == 8:
        EEGdata, fs, fs_accel = ConvertEasyToDataFrame(filepath_EEG)

    else:
       raise ValueError('Invalid EEG device number')

    # Load behavioral data
    behavioralData = pd.read_csv(filepath_behavioral, ',')
    behavioralData = behavioralData.transpose()

    # Saving header information
    behavioralHeader = behavioralData.iloc[0]

    behavioralData = behavioralData.iloc[2:]
    behavioralData.columns = behavioralHeader

    ## Get event markers
    print('Get event markers')
    if EEGdevice == 7:
        # Uses audio pulses
        # Store where the values in trigger are equal to 8 (the audio trigger input channel number)
        index_trigger = np.where(EEGdata['Trigger']==8)

        # Initial peak/jump detection (note that it isn't clean all the time so just take initial)
        index_trigger = np.where(np.diff(EEGdata['Trigger'])>1) #use EEGdata['Trigger'] to make sure we get initial

        # Delete suspicious trigger pulses that are closer than 4 seconds to one another
        index_trigger = np.delete(index_trigger, np.where(np.diff(index_trigger[0]) < (4*fs)))

        # Remove initial sync pulse (there were two in this case)
        index_trigger = index_trigger[2:]

        # Check number of trials
        num_of_trials = behavioralData.shape[0]
        channels = EEGdata.columns[1:8]

        # If the number of trials is greater than number of pulses,
        # Likely last trial pulse was not sent, so estimate it
        if np.shape(index_trigger)[0] < num_of_trials:
            index_trigger = np.append(index_trigger, int(index_trigger[-1]+np.mean(np.diff(index_trigger))))

    elif EEGdevice == 8:
        # Use LSL event markers
        index_trigger = np.where(EEGdata['STI 014']!=0)
        index_trigger = index_trigger[0]
        num_of_trials = behavioralData.shape[0]
        channels = EEGdata.columns[0:7]

        if num_of_trials > len(index_trigger):
            # Task was run with first trial sending pulse of value 0
            # so remove the first trial since we don't know when it starts
            num_of_trials = num_of_trials - 1

    ## Separate into epochs
    print('Separate into epochs')
    trialLength = int(behavioralData['trialLength'][0])

    # Get trial timing
    t_trial_start = list()
    t_trial_end = list()

    # Creating lists of all trigger start and end locations
    for i in range(0,num_of_trials):
        t_trial_start.append(index_trigger[i])
        t_trial_end.append(index_trigger[i] + int(trialLength*fs))

    # Save rest period epochs as well as trials for comparison
    t_rest_start = list()
    t_rest_end = list()

    for i in range(num_of_trials-1):
        t_rest_start.append(t_trial_end[i])
        t_rest_end.append(t_trial_start[i+1])

    ## Cleaning EEG data
    print('Clean EEG data')
    EEGdata_filt = EEGdata.copy()
    eeg_data = EEGdata[channels].values * 1.0 # multiply by 1.0 to convert int to float
    filtered = filter_data(eeg_data.T, sfreq=fs, l_freq=1, h_freq=40)
    EEGdata_filt[channels] = filtered.T

    epochs_raw = []
    epochs_filt = []

    for trial in range(0,len(t_trial_start)):
        epochs_raw.append(EEGdata.loc[t_trial_start[trial]:t_trial_end[trial]][channels])
        epochs_filt.append(EEGdata_filt.loc[t_trial_start[trial]:t_trial_end[trial]][channels])

    ## Organize trials
    print('Organize trials')
    # Create lists for each trial type
    trialL = list()
    trialR = list()
    i = 0

    for letter in behavioralData['trialType'][1:]:
        if letter == 'L':
            trialL.append(i)
        elif letter == 'R':
            trialR.append(i)
        elif letter == 'B':
            trialR.append(i)
        elif letter == 'D':
            trialR.append(i)
        i += 1

    # Create a single list that includes which trial is which
    trial_type = list()
    i = 0

    for letter in behavioralData['trialType'][1:]:
        if letter == 'L':
            trial_type.append(0)
        elif letter == 'R':
            trial_type.append(1)
        elif letter == 'B':
            trial_type.append(0)
        elif letter == 'D':
            trial_type.append(1)
        i += 1

    ## Store the data
    print('Store the data')
    # Store spectrogram data for each trial, for each channel
    ds_factor = 4 # downsampling factor
    channels_to_use = ['C3', 'C4'] # to use all channels, use channels
    epochs_Sxx = [[]]*len(channels_to_use)
    i = 0

    for chanOfInt in channels_to_use:
        tmp = list()
        for trial in range(0,len(trial_type)):
            f, t, Sxx = scisig.spectrogram(scisig.decimate(epochs_raw[trial][chanOfInt].T, ds_factor), fs/ds_factor, nperseg=10)
            tmp.append(Sxx.flatten('C')) # if you want all frequency data
            #tmp.append(Sxx[1]) # if you just want second frequency bin (10-20 Hz if you do ds_factor = 3)
        epochs_Sxx[i] = tmp
        i += 1

    # Format for classification
    print('Format data for classification')
    epochs_collapsed = []

    # Concatenate all epochs such that shape is (num of trials x (flattened channels and spec data))
    # So this should be 100 x 2310 (which is 7 channels * 330 spectrogram data points)
    num_of_channels = np.shape(epochs_Sxx)[0]
    num_of_trials = np.shape(epochs_Sxx)[1]
    num_of_specData = np.shape(epochs_Sxx)[2]

    epochs_collapsed = np.asarray(epochs_Sxx).reshape(num_of_trials, num_of_channels * num_of_specData)

    h = .02  # step size in the mesh

    # preprocess dataset, split into training and test part
    print('Preprocess data for classification')
    X = [np.mean(epochs_collapsed,1),np.std(epochs_collapsed,1)]
    X = np.asarray(X).T
    y = trial_type
    args = np.arange(len(X))
    np.random.shuffle(args)
    X = [X[i] for i in args]
    y = [y[i] for i in args]
    X = StandardScaler().fit_transform(X)
    X_train, X_test, y_train, y_test = \
        train_test_split(X, y, test_size=.4, random_state=42)

    x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
    y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                        np.arange(y_min, y_max, h))

    ## Classify
    print('Classify')
    name = "Random Forest"
    clf = RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1)
    clf.fit(X_train, y_train)
    score = clf.score(X_test, y_test)
    print('Score: ' + str(score))

    # Save model
    print('Save model')
    model = clf
    time_to_save = datetime.datetime.now().isoformat()
    time_to_save = time_to_save.replace('T','-')
    time_to_save = time_to_save.replace(':','-')
    model_file = subjID + '_MI_classifier_' + time_to_save[:19] + '.sav'
    pickle.dump(model, open(model_file, 'wb'))

    # Save training/testing examples
    print('Save training/testing examples')
    filepath_export_data = 'data_for_training_' + time_to_save[:19] + '.npz'
    np.savez_compressed(filepath_export_data, X=X, y=y)


'''
    File: simple_autoencoder.py
    Author: Adam Good
    Description: An example of an autoencoder on the MNIST dataset
'''
import time
import argparse
import os
import torch
from torch import nn
from torch.nn import functional as F
import torch.optim as optim

from data_managment import get_dataloaders
from autoencoder_utils import train_model, print_epoch

SAE_LAYERS = [28*28, 512, 128, 32]

class SimpleAutoEncoder(nn.Module):
    '''
        A basic autoencoder
        input -> hidden -> bottleneck -> hidden -> output
    '''
    def __init__(self):
        super(SimpleAutoEncoder, self).__init__()
        self.input_size = SAE_LAYERS[0]
        self.bottleneck_size = SAE_LAYERS[-1]

        left = SAE_LAYERS[:-1]
        right = SAE_LAYERS[1:]
        encoding_layers = [nn.Linear(l, r) for l, r in zip(left, right)]
        self.bottleneck_layer = encoding_layers.pop()
        self.encoding_layers = nn.ModuleList(encoding_layers)

        left, right = right[::-1], left[::-1]
        decoding_layers = [nn.Linear(l, r) for l, r in zip(left, right)]
        self.output_layer = decoding_layers.pop()
        self.decoding_layers = nn.ModuleList(decoding_layers)

    def encode(self, original):
        '''
            Encode x to bottleneck
        '''
        encoding = original
        # encoding = original.view(-1, 1, self.input_size)
        for layer in self.encoding_layers:
            encoding = F.relu(layer(encoding))
        encoding = self.bottleneck_layer(encoding)
        return encoding

    def decode(self, encoding):
        '''
            Decode x back into the original data
        '''
        reconstruction = encoding
        for layer in self.decoding_layers:
            reconstruction = F.elu(layer(reconstruction))
        reconstruction = torch.sigmoid(self.output_layer(reconstruction))
        # reconstruction = reconstruction.view(-1, 1, 28, 28)
        return reconstruction

    def forward(self, x, *argv):
        '''
            Encode and reconstruct the input x
        '''
        return self.decode(self.encode(x))

    def run_epoch(self, optimizer, loss_fn, data_loader, epoch, log_interval=1, training=True):
        if training:
            self.train()
            epoch_type = "Training"
        else:
            self.eval()
            epoch_type = "Validation"

        dataset_size = len(data_loader.dataset)
        epoch_loss = 0.0
        num_items = 0
        batch_idx = 0
        start = time.time()

        for batch_idx, (data, _) in enumerate(data_loader):
            data = data.view(-1, 1, self.input_size)
            if training:
                optimizer.zero_grad()
            output = self(data)
            loss = loss_fn(data, output)
            epoch_loss += loss.item()
            num_items += len(data)
            if training:
                loss.backward()
                optimizer.step()

            if batch_idx % log_interval == 0:
                elapsed_time = time.time() - start
                print_epoch(epoch, epoch_loss, num_items, dataset_size, elapsed_time, epoch_type)

        elapsed_time = time.time() - start
        print_epoch(epoch, epoch_loss, num_items, dataset_size, elapsed_time, epoch_type, end='\n')

    def save(self, path):
        torch.save(self.state_dict(), path)

    def load(self, path):
        self.load_state_dict(torch.load(path))

def main():
    parser = argparse.ArgumentParser(description="Configure Training of Auto Encoder")
    parser.add_argument('-e', '--epochs', type=int, help='Number of Epochs for Training')
    parser.add_argument('-bs', '--batchsize', type=int, help='Training Batch Size')
    parser.add_argument('-lr', '--learningrate', type=float, default=1e-3, help='Learning Rate')
    parser.add_argument('-v', '--verify', action='store_true', help='Run Verification Epochs')
    parser.add_argument('-p', '--trainingpath', type=str, default='./models', help='Directory for Saving State During Training')
    parser.add_argument('-o', '--output', type=str, help='Name for Output File(s)')
    parser.add_argument('-C', '--clean', action='store_true', help='Cleans out Training Data')
    args = parser.parse_args()

    batch_size = args.batchsize
    num_epochs = args.epochs
    path = args.trainingpath
    output = args.output

    if not os.path.exists(path):
        os.makedirs(path)

    train_loader, test_loader = get_dataloaders(batch_size, batch_size)
    autoencoder = SimpleAutoEncoder()
    optimizer = optim.Adam(autoencoder.parameters(), lr=args.learningrate)

    # loss_fn = lambda x,y: F.binary_cross_entropy(y, x, reduction='sum')
    loss_fn = F.mse_loss
    train_model(autoencoder, optimizer, loss_fn, num_epochs, train_loader, test_loader, verify=args.verify, trainingpath=path, name=output)
    if args.clean:
        from shutil import rmtree
        rmtree(path)

if __name__ == '__main__':
    main()


from flask import request , make_response
from models.blog_models import blog
import json
import bson.json_util as json_util

def Blog():
    body = json.loads(request.data)
    title = body['title']
    author = body['author']
    content = body['content']
    user_ID = body['user_ID']

    blogs = blog(title = title , author=author , content= content , user_ID= user_ID)

    if not blogs.title and blogs.content:
        return make_response ({"message":"auhtor and content cannot be empty"} , 401)

    saved_blog = blogs.save_blog()
    json_version = json_util.dumps(saved_blog)

    return make_response({'message':"Your blog has been posted successfully" , "blog": json_version} , 201)

def GetAllBLogs():
    blogs = blog.get_blogs()
    json_Version = json_util.dumps(blogs)
    return make_response(json_Version , 201)


import spacy
import copy
import debug

class Word:
    """
    Stores a single word with ID and Type
    """
    def __init__(self, word, wtype, wid):
        self.word = word
        self.word_type = wtype
        self.id = wid

class Understood:
    """
    Stores what was understood by the parser as in words found in the wordlist
    stores the ids based in the wordfile
    """
    def __init__(self):
        self.verb_id = 0
        self.dir_id = 0
        self.obj1_id = 0
        self.obj2_id = 0
        self.is_question = False

    def clear(self):
        self.verb_id = 0
        self.dir_id = 0
        self.obj1_id = 0
        self.obj2_id = 0
        self.is_question = False

class Adv_Parser:
    """
    Class to handle the parsing.
    """

    def __init__(self):
        """
        Initializes the blank word list and the spacy parser
        """
        self.wordlist = []
        self.nlp = spacy.load("de_core_news_sm")

    def load_words(self):
        """
        Loads word list which will be understood by the adventure
        """
        with open(r".\res\words.csv", "r", encoding="utf-8") as wordfile:
            # csv_file = csv.reader(wordfile, delimiter=";")
            for row in wordfile:
                a,b,c = row.split(";")
                w = Word(a, int(b), int(c))
                self.wordlist.append(w)

    def parse(self, command):
        verb = direction = obj1 = obj2 = adj = out = 0

        for w in command.split(" "):
            for lookup in self.wordlist:
                if w.lower() == lookup.word:
                    if lookup.word_type == 1:
                        verb = lookup.id
                    elif lookup.word_type == 2:
                        direction = lookup.id
                    elif lookup.word_type == 3:
                        if obj1 == 0:
                            obj1 = lookup.id
                        else:
                            obj2 = lookup.id
                    elif lookup.word_type == 4:
                        adj = lookup.id
                    else:
                        out = lookup.id

        return [verb, direction, obj1, obj2, adj, out]

    def lookup_word(self, w):
        """
        Looks up w in the wordlist and returns the type and the id if found otherwise 0,0
        :param w: word to lookup
        :return: type and id if found
        """
        for lookup in self.wordlist:
            if w.lower() == lookup.word:
                return [int(lookup.word_type), int(lookup.id)]

        return [0, 0]

    def parse_spacy(self, command):
        """
        Parses command using spacy returns list of commands
        ;param command: user entry for command
        ;return list of Understood
        """
        doc = self.nlp(command)

        return_commands = []

        ud = Understood()

        for token in doc:
            if token.pos_ in ["NOUN",  "VERB", "PUNCT", "CCONJ", "ADJ", "X", "ADV", "PART", "PROPN"]:      #ignore everything which is not a noun or a verb or a conjucntion
                if token.pos_ == "PUNCT" or token.lemma_ == "und":   # command end and add in case of interpunction or the word "und"
                                                                     # TODO put "und" in variable
                    ud.is_question = (token.lemma_ == "?")
                    if ud.verb_id != 0 or ud.dir_id != 0: # return if at least a verb or a direction was found
                        return_commands.append(copy.deepcopy(ud))
                    verb_save = ud.verb_id
                    ud.clear() # reset
                    ud.verb_id = verb_save          # in case of multiple objects save verb
                else:
                    wordtype, id = self.lookup_word(token.lemma_)

                    if debug.DEBUG:
                        print(f"typ {wordtype}, ID {id} {token.lemma_}")
                    if wordtype == 0:
                        wordtype, id = self.lookup_word(token.text)

                    if wordtype == 1:
                        ud.verb_id = id
                    if wordtype == 2:
                        ud.dir_id = id
                    if wordtype == 3: # store a maximum of two objects per command
                        if ud.obj1_id == 0:
                            ud.obj1_id = id
                        elif ud.obj2_id == 0:
                            ud.obj2_id = id
            if debug.DEBUG:
                print(token.text, token.lemma_, token.pos_, token.tag_, token.dep_, token.shape_, token.is_alpha, token.is_stop)
        if ud.verb_id != 0 or ud.dir_id != 0:  # return if at least a verb or a direction was found
            return_commands.append(copy.deepcopy(ud))
        return return_commands


import csv

f = open("./covid19_articles.csv", "r", newline="")

rdr = csv.reader(f)
next(rdr)

# 속보 기사 개수
count = 0

for r in rdr:
    # 기사 제목에 [속보]가 포함되어 있다면 프린트 + 개수 추가
    if "[속보]" in r[2]:
        count += 1
        print(r[2])
print(f"속보 기사 개수 : {count}")

f.close()


# coding: utf-8
#Imports {{{1
from os import path
from datetime import date
import logging
# app engine
import webapp2
from google.appengine.api import users
from google.appengine.ext import db
# Шаблонизатор mako
from mako.template import Template
from mako.lookup import TemplateLookup
TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates')
lookup = TemplateLookup(
    directories=[TEMPLATES_DIR],
    input_encoding="utf-8",
    format_exceptions=True,
)
# проект
from models import *
import forms
from profile import build_residence
from lib.functions import get_i18n

# Global variables {{{1
empty_values = (None, '', []) # значения полей, при которых оно рассматривается как пустое

#class FillData(webapp2.RequestHandler) {{{1
class FillData(webapp2.RequestHandler):
    """/prof/fill_data"""
    #def __init__(self, *args, **kwargs) {{{2
    def __init__(self, *args, **kwargs):
        super(FillData, self).__init__(*args, **kwargs)
        self.WARNINGS = []

    def get(self, action=None, form=None, locationForm=None, Date=None): #{{{2
        """Предоставить пользователю форму для заполнения параметров.
        Отправляются только те данные, которые заполнены.
        form - форма с заполненными данными и ошибками
        Date - дата, за которую показывать значения
        """
        # locale
        i18n = get_i18n(self.request)
        _ = i18n.gettext
        lang = i18n.locale

        # animation
        animation = self.request.cookies.get('animation', True)
        if animation == 'false':
            animation = False
        else: # 'true' или что-то странное
            animation = True

        action_url = '/prof/fill_data'
        user = users.get_current_user()

        if user:    # Пользователь залогинен
            # Проверить, есть ли данный пользователь в нашем хранилище
            U = User.get_by_key_name(user.email())
            # ссылка для разлогинивания
            logout_url = users.create_logout_url(self.request.uri)

            # Если пользователь зарегистрирован
            if U:
                if not form:
                    form = forms.FillDataForm(U, Date=Date)
                residence = U.residence.name
                if locationForm is None:
                    locationForm = forms.LocationForm()

                template_values = {
                    '_': _,
                    'lang': lang,
                    'animation': animation,
                    'user': user, # login.mako
                    'logout_url': logout_url, # login.mako
                    'route': '/prof/fill_data',
                    'WARNINGS': self.WARNINGS,
                    'form': form,
                    'locationForm': locationForm,
                    'residence': residence,
                    'action_url': action_url,
                    'action': action
                }

                # Формируем и рендерим шаблон
                template = lookup.get_template("fill_data.mako")
                self.response.write(template.render(**template_values))
            # Если пользователь не зарегистрирован
            else:
                self.redirect("/prof")
        # Пользователь не залогинен
        else:
            self.redirect(users.create_login_url(self.request.uri))

    #def post(self) {{{2
    def post(self):
        """Заполнение тренда"""
        user = users.get_current_user()
        today = date.today()
        # два фиксированных параметра
        if user:
            U = User.get_by_key_name(user.email())
        else:
            return self.get()

        # Нажата кнопка "Сохранить" при выборе населённого пункта
        if self.request.get('submitLocationForm'):
            # получаем объект residence
            lang = self.request.get('lang')
            country = self.request.get('country')
            region = self.request.get('region')
            district = self.request.get('district')
            try:
                residence = build_residence(lang, country, region, district)
            except (IOError, KeyError): # Несовместимые пункты в select'ах
                return self.get()
            # строим форму
            form = forms.LocationForm(residence, formdata=self.request.POST)
            if form.validate():
                if form.locality.data: # если был запрос на изменение места жительства
                    name, id, url = form.locality.data.rsplit(';', 2)
                    keyname = "id_%s" % id
                    R = Residence.get_or_insert(keyname, id=int(id), name=name, url=url)
                    # Если м.ж. поменялось и у прежнего м.ж. это единственный пользователь, удаляем
                    if R.key() != U.residence.key() and U.residence.users.count() < 2:
                        U.residence.delete()
                    # Обязательные поля
                    U.residence = R
                    U.put()

                return self.get()
            else:
                return self.get(locationForm=form, action='editLocation')

        # Нажата кнопка "Обновить значения параметров"
        elif self.request.get('update_data'):
            form = forms.FillDataForm(U, formdata=self.request.POST)
            logging.warning(self.request.get('date'))

            if form.validate(): # Проверка пройдена (правда нужна только дата)
                Date = form.date.data
                return self.get(form=None, Date=Date)
            else:
                return self.get(form=form)

        # Нажата кнопка 'Занести значения'
        elif self.request.get('submit'):
            form = forms.FillDataForm(U, formdata=self.request.POST)
            logging.warning(self.request.get('date'))

            if form.validate(): # Проверка пройдена
                Date = form.date.data
                keyname = "%s_%s" % (U.nickname, Date.strftime("%Y%m%d"))
                UT = UserTrend.get_by_key_name(keyname, parent=U)
                if not UT:
                    ccd_key = "%s-%s" % (U.residence.key().name(), Date.strftime("%Y%m%d"))
                    UT = UserTrend(key_name=keyname, parent=U, date=today, ccd_key=ccd_key)

                all_values_are_empty = True # вначале все значения пустые

                # Подмена id не пройдёт, т.к. значения параметров и элементов берутся из валидированной формы
                for field in form:
                    if field.data not in empty_values:  # если значение есть
                        # 2 вида полей первого порядка (обычные и FormField)
                        if not hasattr(field, 'form'):  # обычный параметр
                            if field.name not in ('UTCdate', 'isToday'):
                                setattr(UT, field.name, field.data)
                                if field.name != 'date':
                                    all_values_are_empty = False
                        else:   # FormField (сложные параметры)
                            for subfield in field:
                                subfield_name = subfield.name.split('_')[1]
                                if subfield.data not in empty_values:   # если значение есть
                                    setattr(UT, subfield_name, subfield.data)
                                    all_values_are_empty = False
                                else:
                                    # если значение было, то свойство удаляется
                                    value = getattr(UT, subfield_name, None)
                                    if value not in empty_values:
                                        delattr(UT, subfield_name)
                    else:
                        # если значение было, то свойство удаляется
                        value = getattr(UT, field.name, None)
                        if value not in empty_values:
                            delattr(UT, field.name)

                if all_values_are_empty == True:
                    # пользователь не заполнил либо стёр данные, удаляем запись в хранилище
                    self.WARNINGS.append(u"Вы не заполнили данные")
                    db.delete(UT)
                else:
                    UT.put()
                    self.WARNINGS.append(u"Значения добавлены")

                return self.get(Date=Date)
            else:
                return self.get(form=form)


import os
import pickle
import numpy as np
from tqdm import trange
from glob import glob
import re
import csv
import math

class Imdb(object):

    def __init__(self, cfg, name=None, virtual=False, logger=None, **kwargs):
        self.cache_attr = [
            'imgpath', 'imgsize',
            'bbox_original', 'bbox_min', 'bbox_max',
            'minsize', 'maxsize',
            'angle', 'points', 'lab',
            'partname', 'level', 'bbox_mask'
        ]
        self.cfg = cfg
        self.name = name
        self.virtual = virtual
        self.logger = logger
        self.cache_filename = os.path.join(cfg.cache_path, '{}.pkl'.format(name))
        assert self.logger

        self._set(kwargs)

        if not virtual:
            try:
                assert not self.cfg.clear_cache
                cache = self._check_cache(cfg)
                for attr in self.cache_attr:
                    if attr in cache:
                        setattr(self, attr, cache[attr])
            except:
                self._parse_annotation()
                self._set_bbox_range()
                self._bbox_sanity()
                self._set_minmax()
                self._save_cache()

            self._filter()
            self._order()

        else:
            self.imgpath = np.empty(0)
            self.imgsize = np.empty((0, 2), dtype=np.int32)
            self.bbox_original = np.empty((0, 4), dtype=np.int32)
            self.bbox_min = np.empty((0, 4), dtype=np.int32)
            self.bbox_max = np.empty((0, 4), dtype=np.int32)
            self.minsize = np.empty(0, dtype=np.int32)
            self.maxsize = np.empty(0, dtype=np.int32)
            # For UMDFaces
            self.angle = np.empty((0, 3), dtype=np.float32) # yaw, pitch, roll
            self.points = np.empty((0, 21, 2), dtype=np.int32) # 21 points
            self.lab = np.empty((0, 3), dtype=np.float32) # mean LAB of face region
            # For Photoshop
            self.partname = np.empty(0)
            self.level = np.empty(0, dtype=np.int32)
            self.bbox_mask = np.empty((0, 4), dtype=np.int32)

    def _set(self, kwargs):
        self.ignore_cache = False

    def _parse_annotation(self):
        raise NotImplementedError()

    def _set_bbox_range(self):
        raise NotImplementedError()

    def _bbox_sanity(self):
        self.bbox_min[:,:2] = np.maximum(0, self.bbox_min[:,:2]) # top, left (minbox)
        self.bbox_max[:,:2] = np.maximum(0, self.bbox_max[:,:2]) # top, left (maxbox)

        if hasattr(self, 'imgsize'):
            self.bbox_min[:,2] = np.minimum(self.imgsize[:,0], self.bbox_min[:,2]) # bottom
            self.bbox_max[:,2] = np.minimum(self.imgsize[:,0], self.bbox_max[:,2]) # bottom
            self.bbox_min[:,3] = np.minimum(self.imgsize[:,1], self.bbox_min[:,3]) # right
            self.bbox_max[:,3] = np.minimum(self.imgsize[:,1], self.bbox_max[:,3]) # right

    def _set_minmax(self):
        min_width = self.bbox_min[:,3] - self.bbox_min[:,1]
        min_height = self.bbox_min[:,2] - self.bbox_min[:,0]
        max_width = self.bbox_max[:,3] - self.bbox_max[:,1]
        max_height = self.bbox_max[:,2] - self.bbox_max[:,0]

        self.minsize = np.maximum(min_width, (min_height / self.cfg.dataset.ar).astype(np.int32))
        self.maxsize = np.minimum(max_width, (max_height / self.cfg.dataset.ar).astype(np.int32))

    def _filter(self):
        reso_th = self.cfg.dataset.resolution_thres_factor * self.cfg.dataset.resolution
        reso_th = max(self.cfg.dataset.min_original_resolution, reso_th)
        mask_w = (self.bbox_max[:,3] - self.bbox_max[:,1]) > reso_th
        mask_h = (self.bbox_max[:,2] - self.bbox_max[:,0]) > reso_th * self.cfg.dataset.ar
        mask_minmax = self.minsize <= self.maxsize

        mask = mask_w & mask_h & mask_minmax
        if hasattr(self, 'angle'):
            angle_th = self.cfg.dataset.angle_th
            mask_angle = (-angle_th < self.angle) & (self.angle < angle_th)
            mask_angle = mask_angle.all(1)
            mask = mask & mask_angle

        self.imgpath = self.imgpath[mask]
        self.bbox_original = self.bbox_original[mask]
        self.bbox_min = self.bbox_min[mask]
        self.bbox_max = self.bbox_max[mask]
        self.minsize = self.minsize[mask]
        self.maxsize = self.maxsize[mask]
        if hasattr(self, 'imgsize'):
            self.imgsize = self.imgsize[mask]
        if hasattr(self, 'angle'):
            self.angle = self.angle[mask]
        if hasattr(self, 'points'):
            self.points = self.points[mask]
        if hasattr(self, 'lab'):
            self.lab = self.lab[mask]
        if hasattr(self, 'partname'):
            self.partname = self.partname[mask]
        if hasattr(self, 'level'):
            self.level = self.level[mask]
        if hasattr(self, 'bbox_mask'):
            self.bbox_mask = self.bbox_mask[mask]

    def _order(self):
        idx = np.argsort(self.imgpath, axis=0)
        self.imgpath = self.imgpath[idx]
        self.bbox_original = self.bbox_original[idx]
        self.bbox_min = self.bbox_min[idx]
        self.bbox_max = self.bbox_max[idx]
        self.minsize = self.minsize[idx]
        self.maxsize = self.maxsize[idx]
        if hasattr(self, 'imgsize'):
            self.imgsize = self.imgsize[idx]
        if hasattr(self, 'angle'):
            self.angle = self.angle[idx]
        if hasattr(self, 'points'):
            self.points = self.points[idx]
        if hasattr(self, 'lab'):
            self.lab = self.lab[idx]
        if hasattr(self, 'partname'):
            self.partname = self.partname[idx]
        if hasattr(self, 'level'):
            self.level = self.level[idx]
        if hasattr(self, 'bbox_mask'):
            self.bbox_mask = self.bbox_mask[idx]

    def split(self, num_2):
        imdb_1 = Imdb(self.cfg, self.name, virtual=True, logger=self.logger)
        imdb_2 = Imdb(self.cfg, self.name, virtual=True, logger=self.logger)
        # num_val = self.cfg.dataset.num_val[self.name]
        num_1 = len(self.imgpath) - num_2

        imdb_1.imgpath = self.imgpath[:num_1]
        imdb_1.bbox_original = self.bbox_original[:num_1]
        imdb_1.bbox_min = self.bbox_min[:num_1]
        imdb_1.bbox_max = self.bbox_max[:num_1]
        imdb_1.minsize = self.minsize[:num_1]
        imdb_1.maxsize = self.maxsize[:num_1]

        imdb_2.imgpath = self.imgpath[num_1:]
        imdb_2.bbox_original = self.bbox_original[num_1:]
        imdb_2.bbox_min = self.bbox_min[num_1:]
        imdb_2.bbox_max = self.bbox_max[num_1:]
        imdb_2.minsize = self.minsize[num_1:]
        imdb_2.maxsize = self.maxsize[num_1:]

        if hasattr(self, 'imgsize'):
            imdb_1.imgsize = self.imgsize[:num_1]
            imdb_2.imgsize = self.imgsize[num_1:]

        if hasattr(self, 'angle'):
            imdb_1.angle = self.angle[:num_1]
            imdb_2.angle = self.angle[num_1:]

        if hasattr(self, 'points'):
            imdb_1.points = self.points[:num_1]
            imdb_2.points = self.points[num_1:]

        if hasattr(self, 'lab'):
            imdb_1.lab = self.lab[:num_1]
            imdb_2.lab = self.lab[num_1:]

        if hasattr(self, 'partname'):
            imdb_1.partname = self.partname[:num_1]
            imdb_2.partname = self.partname[num_1:]

        if hasattr(self, 'level'):
            imdb_1.level = self.level[:num_1]
            imdb_2.level = self.level[num_1:]

        if hasattr(self, 'bbox_mask'):
            imdb_1.bbox_mask = self.bbox_mask[:num_1]
            imdb_2.bbox_mask = self.bbox_mask[num_1:]

        return imdb_1, imdb_2

    def _check_cache(self, cfg):
        assert not self.ignore_cache
        with open(self.cache_filename, 'rb') as f:
            cache = pickle.load(f)
            return cache

    def _save_cache(self):
        os.makedirs(self.cfg.cache_path, exist_ok=True)
        with open(self.cache_filename, 'wb') as f:
            cache_data = {}
            for attr in self.cache_attr:
                if hasattr(self, attr):
                    cache_data[attr] = getattr(self, attr)
            pickle.dump(cache_data, f)

class Union(Imdb):

    def _set(self, kwargs):
        self.train = kwargs['train']
        self.imdbs = dict()
        self.weights = dict()
        self.total = 0

    def merge(self, imdb, weight=0.0):
        assert self.virtual
        self.imdbs[imdb.name] = imdb
        self.weights[imdb.name] = weight

    def initialize(self):
        if self.train:
            total_candidates = [
                int(len(self.imdbs[imdb_name].imgpath) / self.weights[imdb_name]) \
                    for imdb_name in self.imdbs.keys()]
            self.total = np.array(total_candidates).min()
        else:
            self.total = 0

    def reset(self):
        self.imgpath = np.empty(0)
        self.imgsize = np.empty((0, 2), dtype=np.int32)
        self.bbox_original = np.empty((0, 4), dtype=np.int32)
        self.bbox_min = np.empty((0, 4), dtype=np.int32)
        self.bbox_max = np.empty((0, 4), dtype=np.int32)
        self.minsize = np.empty(0, dtype=np.int32)
        self.maxsize = np.empty(0, dtype=np.int32)
        # For UMDFaces
        self.angle = np.empty((0, 3), dtype=np.float32) # yaw, pitch, roll
        self.points = np.empty((0, 21, 2), dtype=np.int32) # 21 points
        self.lab = np.empty((0, 3), dtype=np.float32) # mean LAB of face region
        # For Photoshop
        self.partname = np.empty(0)
        self.level = np.empty(0, dtype=np.int32)
        self.bbox_mask = np.empty((0, 4), dtype=np.int32)

        for imdb_name in self.imdbs.keys():
            imdb = self.imdbs[imdb_name]
            length = len(imdb.imgpath)
            weight = self.weights[imdb_name]

            if self.train:
                num_select = int(self.total * weight)
                select = np.random.choice(length, size=num_select, replace=False)
            else:
                select = np.arange(length)
                num_select = len(select)

            self.imgpath = np.append(self.imgpath, imdb.imgpath[select], axis=0)
            self.bbox_original = np.append(self.bbox_original, imdb.bbox_original[select], axis=0)
            self.bbox_min = np.append(self.bbox_min, imdb.bbox_min[select], axis=0)
            self.bbox_max = np.append(self.bbox_max, imdb.bbox_max[select], axis=0)
            self.minsize = np.append(self.minsize, imdb.minsize[select], axis=0)
            self.maxsize = np.append(self.maxsize, imdb.maxsize[select], axis=0)

            if hasattr(imdb, 'imgsize') and len(imdb.imgsize) == length:
                self.imgsize = np.append(self.imgsize, imdb.imgsize[select], axis=0)
            else:
                self.imgsize = np.append(self.imgsize, np.zeros((num_select, 2)), axis=0)

            if hasattr(imdb, 'angle') and len(imdb.angle) == length:
                self.angle = np.append(self.angle, imdb.angle[select], axis=0)
            else:
                self.angle = np.append(self.angle, np.ones((num_select, 3)) * (-360), axis=0)

            if hasattr(imdb, 'points') and len(imdb.points) == length:
                self.points = np.append(self.points, imdb.points[select], axis=0)
            else:
                self.points = np.append(self.points, np.zeros((num_select, 21, 2)), axis=0)

            if hasattr(imdb, 'lab') and len(imdb.lab) == length:
                self.lab = np.append(self.lab, imdb.lab[select], axis=0)
            else:
                self.lab = np.append(self.lab, np.zeros((num_select, 3)), axis=0)

class AFLW(Imdb):

    def _set(self, kwargs):
        raise NotImplementedError()

    def _parse_annotation(self):
        root = os.path.join(self.cfg.data_path, self.name)

        import sqlite3 as sq
        aflw_sq_path = os.path.join(root, 'data/aflw.sqlite')
        aflw_sq = sq.connect(aflw_sq_path)
        aflw_cur = aflw_sq.cursor()
        aflw_cur.execute("SELECT name FROM sqlite_master WHERE type='table';")
        table_name = aflw_cur.fetchall()

        # fetch image_name, face_rect and feature coordinates from db# fetch i
        faces = aflw_cur.execute("SELECT * FROM Faces")
        face_ids = faces.fetchall()

        face_names = []
        face_rects = []
        face_features = []
        imgpath = list()
        bbox = list()
        for i in range(len(face_ids)):
            # get face_id and file_id
            face_id = face_ids[i][0]
            file_id_sqlite = "SELECT file_id FROM Faces WHERE face_id ='" + str(face_id) + "'"
            file_id = aflw_cur.execute(file_id_sqlite).fetchall()
            file_id = file_id[0][0] # 'img00035.jpg'
            if len(file_id) < 1:
                continue

            # get file_path
            face_name_query = "SELECT filepath FROM FaceImages WHERE file_id = '"+ file_id + "'"
            face_name = aflw_cur.execute(face_name_query).fetchall()
            face_name = face_name[0][0] # '3/image00035.jpg'

            # rect
            feature_rect_query = "SELECT FaceRect.x,FaceRect.y,FaceRect.w,FaceRect.h FROM FaceRect WHERE face_id ='" + str(face_id) + "'"
            feature_rect = aflw_cur.execute(feature_rect_query).fetchall() # [(62, 64, 348, 348)]
            if len(feature_rect) < 1:
                continue

            feature_rect = feature_rect[0]
            x = feature_rect[0]
            y = feature_rect[1]
            w = feature_rect[2]
            h = feature_rect[3]

            # coor (normalize to 0~1)
            feature_coor_query = "SELECT descr,FeatureCoords.x,FeatureCoords.y FROM FeatureCoords,FeatureCoordTypes WHERE face_id ='" + str(face_id) + "' AND FeatureCoords.feature_id = FeatureCoordTypes.feature_id"
            feature_coor = aflw_cur.execute(feature_coor_query).fetchall()
            coor_x = [-1 for k in range(5)]
            coor_y = [-1 for k in range(5)]
            for j in range(len(feature_coor)):
                if feature_coor[j][0] == 'LeftEyeCenter':
                    coor_x[0] = feature_coor[j][1]
                    coor_y[0] = feature_coor[j][2]
                elif feature_coor[j][0] == 'RightEyeCenter':
                    coor_x[1] = feature_coor[j][1]
                    coor_y[1] = feature_coor[j][2]
                elif feature_coor[j][0] == 'NoseCenter':
                    coor_x[2] = feature_coor[j][1]
                    coor_y[2] = feature_coor[j][2]
                elif feature_coor[j][0] == 'MouthLeftCorner':
                    coor_x[3] = feature_coor[j][1]
                    coor_y[3] = feature_coor[j][2]
                elif feature_coor[j][0] == 'MouthRightCorner':
                    coor_x[4] = feature_coor[j][1]
                    coor_y[4] = feature_coor[j][2]

            coor = []
            coor.append(coor_x)
            coor.append(coor_y)

            imgpath.append(os.path.join(self.name, 'aflw/data/flickr', face_name))
            bbox.append([y, x, y + h, x + w])
            # self.coor.append(coor)

        aflw_cur.close()
        aflw_sq.close()

        self.imgpath = np.array(imgpath)
        self.bbox_original = np.array(bbox, dtype=np.int32)

    def _set_bbox_range(self):
        top, left, bottom, right = np.split(self.bbox_original, np.arange(1, 4), axis=1)
        h = bottom - top
        w = right - left
        size_min = np.max((h, w), 0) * 1.2
        size_max = np.max((h, w), 0) * 1.4
        cx = np.mean((left, right), 0)
        cy = np.mean((top, bottom), 0)
        cx_min = cx
        cx_max = cx
        cy_min = cy - 0.15 * size_min
        cy_max = cy - 0.15 * size_max

        self.bbox_min = np.hstack((
            cy_min - 0.5 * size_min,
            cx_min - 0.4 * size_min,
            cy_min + 0.5 * size_min,
            cx_min + 0.4 * size_min)).astype(np.int32)

        self.bbox_max = np.hstack((
            cy_max - 0.5 * size_max,
            cx_max - 0.5 * size_max,
            cy_max + 0.5 * size_max,
            cx_max + 0.5 * size_max)).astype(np.int32)

class CelebA(Imdb):

    def _set(self, kwargs):
        raise NotImplementedError()

    def _parse_annotation(self):
        root = os.path.join(self.cfg.data_path, self.name)
        anno_file = open(os.path.join(root, 'Anno/list_bbox_celeba.txt'), 'r')
        lines = anno_file.readlines()

        for line in lines[2:]:
            assert '.jpg' in line
            elem = list(filter(lambda a: a != '', line.strip().split(' ')))
            imgname = elem[0]
            left, top, width, height = tuple(elem[1:])
            left = int(left)
            top = int(top)
            width = int(width)
            height = int(height)
            bbox = [top, left, top + height, left + width]

            relpath = os.path.join(self.name, 'Img/img_celeba')
            self.imgpath.append(os.path.join(relpath, imgname))
            self.bbox_original.append(bbox)

        anno_file.close()

    def _set_bbox_range(self):
        for bb_o in self.bbox_original:
            top, left, bottom, right = bb_o
            h = bottom - top
            w = right - left

            center_x = (left + right) / 2
            w_min = 0.8 * w
            top_min = int(top + 0.1 * h)
            left_min = int(center_x - w_min / 2)
            bottom_min = int(bottom - 0.25 * h)
            right_min = int(center_x + w_min / 2)
            center_y = (top_min + bottom_min) / 2

            h_new = 2 * (bottom_min - top_min)
            w_new = 2 * (right_min - left_min)

            top_pad = int(center_y - h_new / 2)
            left_pad = int(center_x - w_new / 2)
            bottom_pad = int(center_y + h_new / 2)
            right_pad = int(center_x + w_new / 2)

            bbox_min = [top_min, left_min, bottom_min, right_min]
            bbox_max = [top_pad, left_pad, bottom_pad, right_pad]
            self.bbox_min.append(bbox_min)
            self.bbox_max.append(bbox_max)

class CelebA_HQ(Imdb):

    def _set(self, kwargs):
        self.num_img = 30 * 1000

    def _parse_annotation(self):
        imgpath = list()
        imgsize = list()
        bbox = list()
        for i in range(self.num_img):
            imgname = '{:06d}.png'.format(i)
            imgpath.append(os.path.join(self.name, imgname))
            imgsize.append([1024, 1024])
            bbox.append([0, 0, 1024, 1024])

        self.imgpath = np.array(imgpath)
        self.imgsize = np.array(imgsize, dtype=np.int32)
        self.bbox_original = np.array(bbox, dtype=np.int32)

    def _set_bbox_range(self):
        self.bbox_min = np.copy(self.bbox_original)
        self.bbox_max = np.copy(self.bbox_original)
        # offset_x = 212
        # offset_y = 170
        # min_size_x = 600
        # min_size_y = 650

        # top = offset_y
        # bottom = top + min_size_y
        # left = offset_x
        # right = left + min_size_x

        # length = len(self.bbox_original)
        # self.bbox_min = np.array([[top, left, bottom, right]] * length, dtype=np.int32)
        # self.bbox_max = np.array([[0, 0, 1024, 1024]] * length, dtype=np.int32)

class IJB_C(Imdb):

    def _set(self, kwargs):
        raise NotImplementedError()

    def _parse_annotation(self):
        root = os.path.join(self.cfg.data_path, self.name)

        anno_file = open(os.path.join(root, 'protocols/ijbc_face_detection_ground_truth.csv'))
        lines = anno_file.readlines()

        for line in lines[1:]:
            imgpath, left, top, width, height, ignore = tuple(line.strip().split(','))
            if imgpath.split('/')[0] != 'nonfaces':
                left = int(left)
                top = int(top)
                width = int(width)
                height = int(height)
                ignore = int(ignore)
                imgpath = os.path.join(self.name, 'images', imgpath)
                bbox = [top, left, top + height, left + width]
                self.imgpath.append(imgpath)
                self.bbox_original.append(bbox)
                # self.ignore.append(ignore)

        anno_file.close()

    def _set_bbox_range(self):
        for bb_o in self.bbox_original:
            top, left, bottom, right = bb_o
            center_y = (top + bottom) / 2
            center_x = (left + right) / 2

            h = bottom - top
            w = right - left
            # top_min = int(top)
            # left_min = int(left + 0.1 * w)
            # bottom_min = int(bottom)
            # right_min = int(right - 0.1 * w)
            h_new = 2 * h
            w_new = 2 * w

            top_pad = int(center_y - h_new / 2)
            left_pad = int(center_x - w_new / 2)
            bottom_pad = int(center_y + h_new / 2)
            right_pad = int(center_x + w_new / 2)

            # bbox_min = [top_min, left_min, bottom_min, right_min]
            bbox_min = bb_o
            bbox_max = [top_pad, left_pad, bottom_pad, right_pad]
            self.bbox_min.append(bbox_min)
            self.bbox_max.append(bbox_max)

class UMDFaces(Imdb):

    def _parse_annotation(self):
        with open(os.path.join(self.cfg.cache_path, 'UMDFaces_img.csv')) as f:
            imgpath = list()
            imgsize = list()
            bbox = list()
            angle = list() # yaw, pitch, roll
            points = list() # 21 points
            lab = list()

            rdr = csv.DictReader(f)
            for row in rdr:
                imgpath.append(os.path.join(self.name, 'umdfaces_batch{}'.format(int(row['fold'])), row['FILE']))
                imgsize.append([int(row['img_h']), int(row['img_w'])])
                bbox.append([
                    float(row['FACE_Y']),
                    float(row['FACE_X']),
                    float(row['FACE_Y']) + float(row['FACE_HEIGHT']),
                    float(row['FACE_X']) + float(row['FACE_WIDTH'])])
                angle.append([
                    float(row['YAW']),
                    float(row['PITCH']),
                    float(row['ROLL'])])
                pts = []
                for i in range(21):
                    pts.append([
                        float(row['P{}X'.format(i + 1)]),
                        float(row['P{}Y'.format(i + 1)])
                        # int(float(row['VIS{}'.format(i + 1)]))
                    ])
                points.append(pts)
                lab.append([
                    float(row['meanL']) * 100 / 255,
                    float(row['meanA']) - 128,
                    float(row['meanB']) - 128])

        self.imgpath = np.array(imgpath)
        self.imgsize = np.array(imgsize, dtype=np.int32)
        self.bbox_original = np.array(bbox, dtype=np.int32)
        self.angle = np.array(angle, dtype=np.float32)
        self.points = np.array(points, dtype=np.int32)
        self.lab = np.array(lab, dtype=np.float32)

    def _set_bbox_range(self):
        top, left, bottom, right = np.split(self.bbox_original, np.arange(1, 4), axis=1)
        h = bottom - top
        w = right - left
        size_min = np.max((h, w), 0) * 1.3
        size_max = np.max((h, w), 0) * 1.7
        cx = np.mean((left, right), 0)
        cy = np.mean((top, bottom), 0)

        th_x, th_y = 0.2, 0.2
        yaw, pitch, roll = np.split(self.angle, 3, axis=1)
        # x
        cx_delta_min = 0.3 * np.sin(np.radians(yaw))
        cx_delta_max = 0.3 * np.sin(np.radians(yaw))
        cx_min = cx - w * np.clip(cx_delta_min, -th_x, th_x)
        cx_max = cx - w * np.clip(cx_delta_max, -th_x, th_x)
        # y
        cy_delta_min = 0.1 * h * np.sin(np.radians(pitch)) + 0.15 * size_min
        cy_delta_max = 0.1 * h * np.sin(np.radians(pitch)) + 0.15 * size_max
        cy_min = cy - np.clip(cy_delta_min, -th_y * size_min, th_y * size_min)
        cy_max = cy - np.clip(cy_delta_max, -th_y * size_max, th_y * size_max)

        self.bbox_min = np.hstack((
            cy_min - 0.5 * size_min,
            cx_min - 0.4 * size_min,
            cy_min + 0.5 * size_min,
            cx_min + 0.4 * size_min)).astype(np.int32)

        self.bbox_max = np.hstack((
            cy_max - 0.5 * size_max,
            cx_max - 0.5 * size_max,
            cy_max + 0.5 * size_max,
            cx_max + 0.5 * size_max)).astype(np.int32)

class PGGAN_published_100k(CelebA_HQ):

    def _set(self, kwargs):
        self.num_img = 100 * 1000

class PGGAN_published_nGPUs(Imdb):

    def _parse_annotation(self):
        root = os.path.join(self.cfg.data_path, self.name)

        imgpath = list()
        bbox_original = list()
        imgsize = list()
        for dirpath, dirnames, filenames in os.walk(root):
            if len(filenames) > 0 and len(dirnames) == 0:
                for fn in filenames:
                    if 'tick=' in fn:
                        p = re.compile('_tick=\d+_')
                        tick = int(p.search(fn).group()[6:-1])
                        # tick = int(fn.split('_')[-2].split('=')[-1])
                        resolution = 2 ** int((tick - 1 + 3000) / 1200)
                        bbox = [0, 0, resolution, resolution]
                        _dirpath = dirpath[len(self.cfg.data_path) + 1:]
                        imgpath.append(os.path.join(_dirpath, fn))
                        bbox_original.append(bbox)
                        imgsize.append([resolution, resolution])

        self.imgpath = np.array(imgpath)
        self.bbox_original = np.array(bbox_original, dtype=np.int32)
        self.imgsize = np.array(imgsize, dtype=np.int32)

    def _set_bbox_range(self):
        self.bbox_min = np.copy(self.bbox_original)
        self.bbox_max = np.copy(self.bbox_original)
        # ox_ratio = 212 / 1024
        # oy_ratio = 170 / 1024
        # minx_ratio = 600 / 1024
        # miny_ratio = 650 / 1024

        # for i in range(len(self.bbox_original)):
        #     h = self.bbox_original[i][2] - self.bbox_original[i][0]
        #     w = self.bbox_original[i][3] - self.bbox_original[i][1]
        #     if h >= 512 and w >= 512:
        #         top = int(h * ox_ratio)
        #         left = int(w * oy_ratio)
        #         height = int(h * minx_ratio)
        #         width = int(w * miny_ratio)
        #         self.bbox_min.append([top, left, top + height, left + width])
        #         self.bbox_max.append(self.bbox_original[i])
        #     else:
        #         self.bbox_min.append(self.bbox_original[i])
        #         self.bbox_max.append(self.bbox_original[i])

        # self.bbox_min = np.array(self.bbox_min, dtype=np.int32)
        # self.bbox_max = np.array(self.bbox_max, dtype=np.int32)

class PGGAN_trained(PGGAN_published_nGPUs):
    pass

class Glow(Imdb):

    def _set(self, kwargs):
        if 'test' in kwargs:
            self.test = kwargs['test']
        else:
            self.test = None

    def _parse_annotation(self):
        if self.test:
            imglist = glob(os.path.join(self.cfg.data_path, self.name, 'test', '*.jpg'))
        else:
            imglist = glob(os.path.join(self.cfg.data_path, self.name, 'train', '*.jpg'))

        imglist = ['/'.join(_.split('/')[-3:]) for _ in imglist]
        self.imgpath = np.array(imglist)
        self.bbox_original = np.array([[0, 0, 256, 256]] * len(imglist), dtype=np.int32)
        self.imgsize = np.array([[256, 256]] * len(imglist), dtype=np.int32)

    def _set_bbox_range(self):
        self.bbox_min = np.copy(self.bbox_original)
        self.bbox_max = np.copy(self.bbox_original)

class Glow_CelebA_HQ(Glow):
    pass

class Glow_UMDFaces(Glow):
    pass

class StarGAN_CelebA(Glow):
    pass

class BEGAN(Imdb):

    def _set(self, kwargs):
        raise NotImplementedError()

    def _parse_annotation(self):
        root = os.path.join(self.cfg.data_path, self.name)

        for dirpath, dirnames, filenames in os.walk(root):
            if len(filenames) > 0 and len(dirnames) == 0:
                for fn in filenames:
                    p = re.compile('_size=\d+_')
                    size = int(p.search(fn).group()[6:-1])
                    bbox = [0, 0, size, size]
                    _dirpath = dirpath[len(self.cfg.data_path) + 1:]
                    self.imgpath.append(os.path.join(_dirpath, fn))
                    self.bbox_original.append(bbox)

    def _set_bbox_range(self):
        self.bbox_min = np.copy(self.bbox_original)
        self.bbox_max = np.copy(self.bbox_original)

class DCGAN(Imdb):

    def _set(self, kwargs):
        raise NotImplementedError()

    def _parse_annotation(self):
        root = os.path.join(self.cfg.data_path, self.name)

        for dirpath, dirnames, filenames in os.walk(root):
            if len(filenames) > 0 and len(dirnames) == 0:
                for fn in filenames:
                    bbox = [0, 0, 64, 64]
                    _dirpath = dirpath[len(self.cfg.data_path) + 1:]
                    self.imgpath.append(os.path.join(_dirpath, fn))
                    self.bbox_original.append(bbox)

    def _set_bbox_range(self):
        self.bbox_min = np.copy(self.bbox_original)
        self.bbox_max = np.copy(self.bbox_original)

class LSGAN(DCGAN):
    pass

class DRAGAN(DCGAN):
    pass

class WGAN_GP(DCGAN):
    pass

class Photoshop(Imdb):

    def __init__(self, cfg, name=None, virtual=False, logger=None, **kwargs):
        super(Photoshop, self).__init__(cfg, name=name, virtual=virtual, logger=logger, **kwargs)

        weight = 0
        for k, v in self.cfg.dataset.swap_parts.items():
            weight += v

        assert math.isclose(weight, 1.0), 'Sum of swap_parts weight should be 1.0, but {}'.format(weight)

    def _parse_annotation(self):
        with open(os.path.join(self.cfg.data_path, 'Photoshop', 'Photoshop_result.csv')) as f:
            imgpath = list()
            imgsize = list()
            bbox = list()
            partname = list()
            level = list()
            bbox_mask = list()

            rdr = csv.DictReader(f)
            for row in rdr:
                imgpath.append(os.path.join(
                    'Photoshop',
                    'Photoshop_' + self.cfg.dataset.photoshop_ext,
                    row['filename'] + '.' + self.cfg.dataset.photoshop_ext)
                )
                imgsize.append([int(row['h']), int(row['w'])])
                bbox.append([
                    int(row['face_y']),
                    int(row['face_x']),
                    int(row['face_y']) + int(row['face_h']),
                    int(row['face_x']) + int(row['face_w'])])
                partname.append(row['partname'])
                level.append(int(row['level']))
                bbox_mask.append([
                    int(row['mask_y']),
                    int(row['mask_x']),
                    int(row['mask_y']) + int(row['mask_h']),
                    int(row['mask_x']) + int(row['mask_w'])
                ])

        self.imgpath = np.array(imgpath)
        self.imgsize = np.array(imgsize, dtype=np.int32)
        self.bbox_original = np.array(bbox, dtype=np.int32)

        self.partname = np.array(partname)
        self.level = np.array(level, dtype=np.int32)
        self.bbox_mask = np.array(bbox_mask, dtype=np.int32)

    def _filter(self):
        super(Photoshop, self)._filter()

        partname_valid = [key for key, value in self.cfg.dataset.swap_parts.items() if value > 0]
        mask_part = np.zeros(len(self.imgpath), dtype=np.bool)
        for pv in partname_valid:
            mask_part = mask_part | (self.partname == pv)

        mask = mask_part

        self.imgpath = self.imgpath[mask]
        self.imgsize = self.imgsize[mask]
        self.bbox_original = self.bbox_original[mask]
        self.bbox_min = self.bbox_min[mask]
        self.bbox_max = self.bbox_max[mask]
        self.minsize = self.minsize[mask]
        self.maxsize = self.maxsize[mask]
        if hasattr(self, 'angle'):
            self.angle = self.angle[mask]
        if hasattr(self, 'points'):
            self.points = self.points[mask]
        if hasattr(self, 'lab'):
            self.lab = self.lab[mask]
        if hasattr(self, 'partname'):
            self.partname = self.partname[mask]
        if hasattr(self, 'level'):
            self.level = self.level[mask]
        if hasattr(self, 'bbox_mask'):
            self.bbox_mask = self.bbox_mask[mask]

    def _set_bbox_range(self):
        top, left, bottom, right = np.split(self.bbox_original, np.arange(1, 4), axis=1)
        h = bottom - top
        w = right - left
        size_min = np.max((h, w), 0) * 1.1
        size_max = np.max((h, w), 0) * 1.5
        cx = np.mean((left, right), 0)
        cy = np.mean((top, bottom), 0) - 0.1 * h

        self.bbox_min = np.hstack((
            cy - 0.5 * size_min,
            cx - 0.4 * size_min,
            cy + 0.5 * size_min,
            cx + 0.4 * size_min)).astype(np.int32)

        self.bbox_max = np.hstack((
            cy - 0.5 * size_max,
            cx - 0.5 * size_max,
            cy + 0.5 * size_max,
            cx + 0.5 * size_max)).astype(np.int32)

class Sample_1_M1_Real(Imdb):
    ''' 201~400 real images from Sample_1, task 1 '''

    def _parse_annotation(self):

        with open(os.path.join(self.cfg.data_path, 'RnDChallenge', 'Sample_1_M1_imsize.csv'), 'r') as f:
            rdr = csv.DictReader(f)

            imgpath = list()
            bbox_original = list()
            imgsize = list()
            for row in rdr:
                idx = int(re.findall('\d{5}', row['filename'])[0])
                if idx > 200:
                    imgpath.append(os.path.join('RnDChallenge', 'Sample_1', row['filename']))
                    h = int(row['h'])
                    w = int(row['w'])
                    bbox_original.append([0, 0, h, w])
                    imgsize.append([h, w])

            self.imgpath = np.array(imgpath)
            self.bbox_original = np.array(bbox_original, dtype=np.int32)
            self.imgsize = np.array(imgsize, dtype=np.int32)

    def _set_bbox_range(self):
        self.bbox_min = np.copy(self.bbox_original)
        self.bbox_max = np.copy(self.bbox_original)

class Sample_1_M2_Real(Imdb):
    ''' Real images from Sample_1, task 2 '''

    def _set(self, kwargs):
        self.ignore_cache = True
        self.real_fake = 'real'
        self.subdir = 'Sample_1'
        self.anno_file = 'Sample_1_M2_bbox.csv'

    def _parse_annotation(self):
        root = os.path.join(self.cfg.data_path, 'RnDChallenge')

        with open(os.path.join(root, self.anno_file)) as f:
            imgpath = list()
            imgsize = list()
            bbox = list()

            rdr = csv.DictReader(f)
            for row in rdr:
                if row['real_fake'] != self.real_fake:
                    continue

                imgpath.append(os.path.join('RnDChallenge', self.subdir, row['filename']))
                imgsize.append([int(row['h']), int(row['w'])])
                bbox.append([
                    int(row['face_y']),
                    int(row['face_x']),
                    int(row['face_y']) + int(row['face_h']),
                    int(row['face_x']) + int(row['face_w'])])

        self.imgpath = np.array(imgpath)
        self.imgsize = np.array(imgsize, dtype=np.int32)
        self.bbox_original = np.array(bbox, dtype=np.int32)

    def _set_bbox_range(self):
        top, left, bottom, right = np.split(self.bbox_original, np.arange(1, 4), axis=1)
        h = bottom - top
        w = right - left
        size_min = np.max((h, w), 0) * 1.1
        size_max = np.max((h, w), 0) * 1.5
        cx = np.mean((left, right), 0)
        cy = np.mean((top, bottom), 0) - 0.1 * h

        self.bbox_min = np.hstack((
            cy - 0.5 * size_min,
            cx - 0.4 * size_min,
            cy + 0.5 * size_min,
            cx + 0.4 * size_min)).astype(np.int32)

        self.bbox_max = np.hstack((
            cy - 0.5 * size_max,
            cx - 0.5 * size_max,
            cy + 0.5 * size_max,
            cx + 0.5 * size_max)).astype(np.int32)

class Sample_1_M2_Syn(Sample_1_M2_Real):
    ''' Fake images from Sample_1, task 2 '''

    def _set(self, kwargs):
        self.ignore_cache = True
        self.real_fake = 'fake'
        self.subdir='Sample_1'
        self.anno_file = 'Sample_1_M2_bbox.csv'

class Sample_1_M1_GAN(Imdb):
    ''' 1~200 fake images from Sample_1, task 1 '''

    def _parse_annotation(self):

        with open(os.path.join(self.cfg.data_path, 'RnDChallenge', 'Sample_1_M1_imsize.csv'), 'r') as f:
            rdr = csv.DictReader(f)

            imgpath = list()
            bbox_original = list()
            imgsize = list()
            for row in rdr:
                idx = int(re.findall('\d{5}', row['filename'])[0])
                if idx <= 200:
                    imgpath.append(os.path.join('RnDChallenge', 'Sample_1', row['filename']))
                    h = int(row['h'])
                    w = int(row['w'])
                    bbox_original.append([0, 0, h, w])
                    imgsize.append([h, w])

            self.imgpath = np.array(imgpath)
            self.bbox_original = np.array(bbox_original, dtype=np.int32)
            self.imgsize = np.array(imgsize, dtype=np.int32)

    def _set_bbox_range(self):
        self.bbox_min = np.copy(self.bbox_original)
        self.bbox_max = np.copy(self.bbox_original)

class Sample_2_GAN(Imdb):
    ''' 60 gan images from Sample_2 '''

    def _parse_annotation(self):
        root = os.path.join(self.cfg.data_path, 'RnDChallenge', 'Sample_2', 'gan_jpg')

        imglist = glob(os.path.join(root, 'gan_*.jpg'))
        imgpath = [os.path.join('RnDChallenge', 'Sample_2', 'gan_jpg', os.path.basename(_)) for _ in imglist]
        bbox_original = [[0, 0, 128, 128]] * len(imgpath)
        imgsize = [[0, 0, 128, 128]] * len(imgpath)

        assert len(imgpath) > 0

        self.imgpath = np.array(imgpath)
        self.bbox_original = np.array(bbox_original, dtype=np.int32)
        self.imgsize = np.ones((self.imgpath.shape[0], 2), dtype=np.int32) * 128

    def _set_bbox_range(self):
        self.bbox_min = np.copy(self.bbox_original)
        self.bbox_max = np.copy(self.bbox_original)

class Sample_2_Syn(Imdb):
    ''' Fake images from Sample_2 '''

    def _set(self, kwargs):
        self.ignore_cache = True

    def _parse_annotation(self):
        root = os.path.join(self.cfg.data_path, 'RnDChallenge', 'Sample_2', 'syn')

        imgpath = glob(os.path.join(root, '*.jpg'))
        imgpath = ['/'.join(_.split('/')[-4:]) for _ in imgpath]
        bbox_original = [[0, 0, 128, 128]] * len(imgpath)

        assert len(imgpath) > 0

        self.imgpath = np.array(imgpath)
        self.bbox_original = np.array(bbox_original, dtype=np.int32)
        self.imgsize = np.ones((self.imgpath.shape[0], 2), dtype=np.int32) * 128

    def _set_bbox_range(self):
        self.bbox_min = np.copy(self.bbox_original)
        self.bbox_max = np.copy(self.bbox_original)


import json
import urllib
import pandas as pd
import dash_html_components as html
import dash_bootstrap_components as dbc
import dash_core_components as dcc

from dash.dependencies import Input, Output, State
from app import app

def getInfo():
    urlData = 'https://data.bmkg.go.id/DataMKG/TEWS/gempaterkini.json'
    web = urllib.request.urlopen(urlData)
    if web.getcode() == 200:
        data = web.read()
        JSON = json.loads(data)

        dicts = []
        df_gempa = pd.DataFrame()

        for i in JSON['Infogempa']['gempa']:
            dicts.append(i)

        df_gempa = df_gempa.append(dicts, ignore_index=True)

        waktu_gempa = df_gempa['Tanggal'] + '\n' + df_gempa['Jam']

        df_gempa.insert(loc=0, column='Waktu Gempa', value=waktu_gempa)
        df_gempa.rename(columns={'Coordinates': 'Titik Gempa', 'Magnitude': 'Magnitudo'}, inplace=True)

        df_gempa = df_gempa.drop(['Tanggal', 'Jam', 'Lintang', 'Bujur', 'DateTime'], axis=1)
    else:
        print('Something wrong, cannot proceed')

    return df_gempa

def gempaTerbaru():
    urlData = 'https://data.bmkg.go.id/DataMKG/TEWS/autogempa.json'
    web = urllib.request.urlopen(urlData)
    if web.getcode() == 200:
        data = web.read()
        JSON = json.loads(data)

        wilayah = JSON['Infogempa']['gempa']['Wilayah']
        shakemap = 'https://ews.bmkg.go.id/TEWS/data/' + JSON['Infogempa']['gempa']['Shakemap']
        dirasakan = JSON['Infogempa']['gempa']['Dirasakan']
        waktu = JSON['Infogempa']['gempa']['Tanggal'] + ' - ' + JSON['Infogempa']['gempa']['Jam']
        magnitudo = JSON['Infogempa']['gempa']['Magnitude']
        kedalaman = JSON['Infogempa']['gempa']['Kedalaman']
        lokasi = JSON['Infogempa']['gempa']['Lintang'] + ' ' + JSON['Infogempa']['gempa']['Bujur']
        map = 'https://www.google.com/maps/search/' + JSON['Infogempa']['gempa']['Coordinates']

        terbaru = dbc.Card(
            dbc.CardBody(
                [
                    html.H5('Gempabumi Terkini', className='card-title'),
                    html.P(wilayah),
                    html.Label('Dirasakan (Skala MMI)'),
                    dbc.Badge(dirasakan, color='warning', className='mr-1'),
                    html.Br(),
                    dbc.Button('Detail Gempa', id='open', color='primary'),
                    dbc.Modal(
                        [
                            dbc.ModalHeader('Informasi Gempabumi'),
                            dbc.ModalBody([
                                    html.P('Peta Guncangan Gempabumi (Earthquake Shakemap)'),
                                    dbc.CardImg(src=shakemap, style={'width': '300px'}),
                                    dcc.Markdown(f'''
                                                **Parameter Gempabumi**
                                                {waktu}
                                                Magnitudo: {magnitudo}
                                                Kedalaman: {kedalaman}
                                                Lokasi: {lokasi}
                                                Google maps: [Lokasi Gempa]({map} 'Lokasi kejadian gempabumi')
                                                ''')
                            ]),
                            dbc.ModalFooter(
                                dbc.Button(
                                    'Tutup', id='close', className='ml-auto', n_clicks=0
                                )
                            ),
                        ],
                        id="modal",
                        is_open=False,
                    ),
                ]
            )
        )

    return terbaru

# layout web app
layout = html.Div([
    html.H1('Informasi Gempabumi'),
    html.Hr(),
    dbc.Row([
        dbc.Col(gempaTerbaru(), width='auto')
    ]),
    html.Br(),
    html.H3('Gempabumi Terkini (M≥5.0)'),
    dbc.Table.from_dataframe(getInfo(), striped=True, hover=True),
], style={'margin-top': '1%', 'margin-bottom': '1%', 'margin-left': '10%', 'margin-right': '10%'})

@app.callback(
    Output('modal', 'is_open'),
    [Input('open', 'n_clicks'), Input('close', 'n_clicks')],
    [State('modal', 'is_open')],
)
def toggle_modal(n1, n2, is_open):
    if n1 or n2:
        return not is_open
    return is_open


from django import forms
from django.forms import ModelForm
from django.utils.translation import gettext_lazy as _

from Order.models import Order
from User.models import EndUser
from Product.models import Products

class OrderCreateForm(ModelForm):
    payment= forms.TextInput(attrs={'hidden': True, })
    payment_type= forms.TextInput(attrs={'hidden': True, })

    class Meta:
        model = Order
        fields = ['orderer', 'orderdate',  'cost', 'goods', 'order_status', 'up1', 'up1_sc', 'up2', 'up2_sc']

        widgets = {
                #'orderer':forms.Select(choices=EndUser.objects.all()),
                #'orderdate': _('SelectDateWidget'),
                #'goods': forms.SelectMultiple(choices = Products.objects.all()),
                'orderdate':forms.DateInput(attrs={'type':'date'}),
            }
        labels = {
               # 'username':   _('用户名'),
            }
        help_texts = {
            # 'username':   _('可以用中文'),
            }
        field_classes = {
            #'orderdate':  _('cxCalender'),
            }