#!/usr/bin/env python3

import pandas as pd
import numpy as np
import UPGMAalgo as up
from UPGMAalgo import UPGMA

def zeros(shape):
	D = []
	for x in range(shape[0]):
	    D.append([])
	    for y in range(shape[1]):
	        D[-1].append(0)
	return D

def affineGap(s1, s2):  # Implementing needleman algorithm with affine gap penalty
	print(s1, s2)
	match = 1
	mismatch =1
	gap_penalty = 4
	gap_extend = 1
	m = len(s1)
	n = len(s2)  # length of two sequences
	# Generate DP table and traceback path pointer matrix
	midscore = zeros((m+1, n+1))      # the DP table
	upperscore = zeros((m+1, n+1))    # initializing upperscore
	lowerscore = zeros((m+1, n+1))    # initializing lowerscore
	midscore[0][0] = 0  # initializing midscore
	midscore[0][1] = -gap_penalty
	midscore[1][0] = -gap_penalty
	for i in range(2, len(midscore)):
	    midscore[i][0] = midscore[i-1][0] - gap_extend
	for j in range(2, len(midscore[0])):
	    midscore[0][j] = midscore[0][j-1] - gap_extend

	for i in range(m + 1):   # Calculate DP table
	    upperscore[i][0] = -999999
	    upperscore[0][1] = -gap_penalty
	for i in range(2,len(upperscore[0])):
	    upperscore[0][i] = upperscore[0][i-1] - gap_extend

	for i in range(0, n + 1):
	    lowerscore[0][i] = -999999
	    lowerscore[1][0] = -gap_penalty
	for i in range(2, len(lowerscore)):
	    lowerscore[i][0] = lowerscore[i-1][0] - gap_extend

	backtracktable = zeros((m+1, n+1))
	for i in range(n+1):
	    backtracktable[0][i] = 'e'
	for i in range(m+1):
	    backtracktable[i][0] = 's'
	backtracktable[0][0] = '*'
	purines='A,G'
	pyrimidines='C,T'
	w=0
	for i in range(1, m + 1):   # filling affine gap alignment tables
	    for j in range(1, n + 1):
	        lowerscore[i][j] = max(lowerscore[i-1][j] - gap_extend, midscore[i-1][j] - gap_penalty)
	        upperscore[i][j] = max(upperscore[i][j-1] - gap_extend, midscore[i][j-1] - gap_penalty)
	        score1 = midscore[i-1][j-1]
	        if s1[i-1] == s2[j-1]:
	            w = match
	        if s1[i-1] != s2[j-1]:
	            w -= mismatch
	        midscore[i][j] = max(score1+w, lowerscore[i][j], upperscore[i][j])

	#filling backtrack table
	        decision = max(score1+w, lowerscore[i][j], upperscore[i][j])
	        if decision == score1+w:
	            backtracktable[i][j] = 'd'
	        elif decision == lowerscore[i][j]:
	            backtracktable[i][j] = 's'
	        elif decision == upperscore[i][j]:
	            backtracktable[i][j] = 'e'

	# Traceback and compute the alignment
	align1, align2 = '', ''
	i,j = m,n # start from the bottom right cell
	while i > 0 and j > 0: # end touching the top or the left edge
	    if backtracktable[i][j] == 'd':
	        align1+=s1[i-1]
	        align2+=s2[j-1]
	        i-=1
	        j-=1
	    elif backtracktable[i][j] == 's':
	        align1+=s1[i-1]
	        align2+='-'
	        i-=1
	    elif backtracktable[i][j] == 'e':
	        align1+='-'
	        align2+=s2[j-1]
	        j-=1

	# Finish tracing up to the top left cell
	while i > 0:
	    align1 += s1[i-1]
	    align2 += '-'
	    i -= 1
	while j > 0:
	    align1 += '-'
	    align2 += s2[j-1]
	    j -= 1

	#alignment.finalize(align1,align2)
	finalArray = [align1[::-1], align2[::-1]]
	return (finalArray)

#def sumofPairscoring(s1,s2): #sum of pairs scoring

def findScore(s1, s2):  # scoring matrix for two sequences
	match,mismatch, gap,extend = 0, 0, 0, 0
	gapMemory, subScore, counts1, counts2 = 0, 0, 0, 0
	#len1 = len(s1) - alignment.newLength(s1) #removes 'X'
	#len2 = len(s2) - alignment.newLength(s2) #removes 'X'
	#print(type(s2))
	#print(s2)
	len1 = len(s1)
	len2 = len(s2)
	for i in range(len1):
	    if (s1[i] or s2[i] == '-'):
	        if(s1[i] != '-' and s2[i] != '-'):
	            if (s1[i] == s2[i]):
	                match += 1
	            else:
	                mismatch += 1
	            counts1 = counts1 +1
	            counts2 = counts2 +1
	            gapMemory=0
	        elif (s1[i] != '-' and s2[i] == '-'):
	            if (counts2 >= 0 and counts2 < len2):
	                if(gapMemory == 1):
	                    extend = extend+1
	                else:
	                    gap=gap+1
	                    gapMemory=1
	                counts1= counts1+1
	        elif (s1[i] == '-' and s2[i] != '-'):
	            if(counts1 >= 0 and counts1 < len1):
	                if(gapMemory == 2):
	                    extend = extend+1
	                else:
	                    gap = gap +1
	                    gapMemory =2
	                counts2 = counts2+1
	    subScore = (match*1) + (mismatch*(-1)) + (gap*(-4)) + (extend*(-1))
	    return subScore

def findTotalScore(newAligned):
	score = 0
	for i in range(len(newAligned)):
		for j in range(1,len(newAligned)):
			score += findScore(newAligned[i],newAligned[j])
	return score
# align progressively
def compareSeq(clusters):
	clusters = str(clusters)
	print(clusters)
	clusters=clusters.split(',')
	newAligned=[]
	for i in range(0,len(clusters)-1):
		s1 = clusters[i]
		print(s1)
		print(clusters[i+1])
		seq1,seq2=affineGap(s1,clusters[i+1])
		newAligned.append(seq1)
		if ((i+1)==(len(clusters))-1):
			newAligned.append(clusters[i+1])
	return newAligned

#@staticmethod
def similarity(arr): # used to find most similar sequence in a list
	n = len(arr)
	finalScores=[0 for x in range(n)]
	scores = [[0 for x in range(n)] for y in range(n)]
	for i in range(n):
	    sum=0
	    for j in range(n):
	        if i == j:
	            scores[i][j] = '-'
	        else:
	            scores[i][j] = findScore(arr[i], arr[j])
	            sum += scores[i][j]
	    finalScores[i]=sum
	return finalScores

def alignSeq(s):
	#print("alignSeq called!")
	s= s.split(',')
	newAligned = [0 for i in range(len(s))]
	tempList = ' '
	Scores =similarity(s)
	#print(Scores)
	sortedScores = sorted(Scores, reverse=True) #similarity is kept in sortedScores via index
	#print(sortedScores)
	index = Scores.index(sortedScores[0])
	Scores[index] = -99999
	# the problem is that the sequences may repeat and hence only one sequence is called
	# update the score and make sure that the sequences get placed inside
	index2 = Scores.index(sortedScores[1])
	tempList = affineGap(s[index], s[index2])
	newAligned[index] = tempList[0]
	for i in range(1,len(sortedScores)):
	    index2 = Scores.index(sortedScores[i])
	    Scores[index2] = -9999
	    tempList = affineGap(s[index], s[index2])
	    newAligned[index2] = tempList[1]
	return newAligned

if __name__=="__main__":

	#df = pd.read_csv('finaltestbackup.txt', sep=' ', header=None)
	#scores=[]
	#score=0
	#for row in df[0]:
#		row=str(row)
#		row=row.split(',')
#		score += findTotalScore(row)
#		scores.append(score)
#		score=0
#	print(scores)
	def makesmall(row):
		lists=[]
		for k in range(len(row[0])):
			#print("k",k)
			#print(len(row))
			smalllist=[]
			for j in range(len(row)):
				#print()
				smalllist.append(row[j][k])
				#print(smalllist)
			lists.append(smalllist)

			#print(lists)
		return lists
	#feature 1: dimensions of puzzle
	def countdimensions(row):
		#row=row.split(',')
		rows=len(row)
		cols=len(row[0])
		dimensions=rows*cols
		return dimensions

	def checkEqual(row):
		return row[1:]==row[:-1]

	#feature 2: gaps in puzzles - might not be a significant feature
	# gaps only registered if all sequences have gaps
	def gaps(row):
		count=0
		row=makesmall(row)
		#print(row)
		for i in row:
			if('-' in i):
				if(checkEqual(i)):
					count+=1

		return count

	#feature 3: number of GC content
	def GCcontent(row):
		count=0
		row=makesmall(row)
		for i in row:
			for j in range(len(i)):
				if (i[j] == G or i[j] == C):
					count+=1
		return count

	csv_path='inputAndOut_uniqueid_onesolution.csv'
	df = pd.read_csv(csv_path, header=None, skiprows=1)
	#df1= pd.read_excel(csv_path)
	# size of df is 1902
	#data = df.iloc[0:1902, 4]
	filtered_data=df.iloc[:,1]
	# this filters data by row
	#print(len(filtered_data))
	#s=['----TA', '---GGG', '------']
	scores1=[]
	count = 0
	score=0
	puzzlesize=[]
	#count=gaps(s)
	#print(count)
	counting=[]
	dimensions=0
	for row in filtered_data:
		row=row.split(',')
		#print(row)
		count=gaps(row)
		counting.append(count)
		#print("Gaps", count)
		#break
		#or i in row:

			#break
		dimensions=countdimensions(row)
		puzzlesize.append(dimensions)
			#for j in i:
	print("Dimensions ",puzzlesize)
	print("Counting ",counting)
		#puzzlesize.append(dimensions)

			# 152447 is the length of the file
			#while(row==i):
				#df2=df.iloc[row,4]
				#df2 = df2.split(',')

		#determine features

#		row=str(row)
#		row=row.split(',')
#		#print(row)
#		# print(len(row))
#		x= (UPGMA(row, findScore).tree)
#		# print(x)
#		seqs=compareSeq(x)
#			#print(seqs)
#		score+=findTotalScore(seqs)
#		count+=1
#		print("This is running:", count)
#		scores1.append(score)
#		score=0
#	print(scores1)
			# totaldifference=0
			# for j in range(len(scores1)):
				# totaldifference+=(Outputscores[i-1]-scores1[j])
			# difference.append(totaldifference)
		# print("Done with i:", i)
	# print(difference)

		#finds the score of each row
		#row = row.split(',')
		#print(row)
		#print(type(row))
		#break

	#for row in data:
	#	row = row.split(',')
	#	score = findTotalScore(row)
	#	scores1.append(score)
	#print(scores1)
	#df1['aligned sequence']=listseq
	#copy=df1.copy()
	#del copy['original_score']
	#del copy['best_score']
	#del copy['alignment']
	#copy.to_csv('output.csv', index=False, sep=',')

	#aligned_data=df.iloc[0:1615,5]
	#scores=[]
	#s=0
	#for row in aligned_data:
	#	s=findTotalScore(row)
	#	scores.append(s)
	#df['output aligned scores']=scores


import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler as SS

# READ DATA
dfile = '/home/resolt/Workspaces/MachineLearning/TensorFlow/RecurrentNeuralNetworks/monthly-milk-production.csv'
milk = pd.read_csv(dfile, index_col='Month')

# MAKE INDEX TIME SERIES
milk.index = pd.to_datetime(milk.index)

# PLOT OUT THE TIMESERIES DATA
# plt.plot(milk.index,milk['Milk Production'])
# plt.show()

# TRAIN TEST SPLIT
# X = milk['Milk Production'].head(milk.shape[1]-13)
# y = milk['Milk Production'].tail(12)
X_data = milk.head(milk.shape[0]-13)
y_data = milk.tail(12)

# SCALING
scaler = SS()
scaler.fit(X_data)
X_data = scaler.transform(X_data)
y_data = scaler.transform(y_data)

# FEEDER
def next_batch(training_data, batch_size, steps):
	"""
	INPUT: Data, Batch Size, Time Steps per batch
	OUTPUT: A tuple of y time series results. y[:,:-1] and y[:,1:]
	"""

	# STEP 1: Use np.random.randint to set a random starting point index for the batch.
	# Remember that each batch needs have the same number of steps in it.
	# This means you should limit the starting point to len(data)-steps
	rand_start = np.random.randint(low=0, high=len(training_data) - steps)

	# STEP 2: Now that you have a starting index you'll need to index the data from
	# the random start to random start + steps + 1. Then reshape this data to be (1,steps+1)
	rand_data = np.array(training_data[rand_start:rand_start+steps+1]).reshape(1,steps+1)

	# STEP 3: Return the batches. You'll have two batches to return y[:,:-1] and y[:,1:]
	# You'll need to reshape these into tensors for the RNN to .reshape(-1,steps,1)
	a = rand_data[:, :-1].reshape(-1, steps, 1)
	b = rand_data[:, 1:].reshape(-1, steps, 1)

	return a, b

# SETTINGS
num_inputs = 1
num_time_steps = 12
num_neurons = 100
num_outputs = 1
learning_rate = 0.03
train_iters = 4000
batch_size = 1

# PLACEHOLDERS
X = tf.placeholder(tf.float32, shape=[None, num_time_steps, num_inputs])
y = tf.placeholder(tf.float32, shape=[None, num_time_steps, num_outputs])

# RNN LAYER
cell = tf.contrib.rnn.BasicRNNCell(num_units=num_neurons, activation=tf.nn.relu)
cellw = tf.contrib.rnn.OutputProjectionWrapper(cell, output_size=num_outputs)

outputs, states = tf.nn.dynamic_rnn(cellw, X, dtype=tf.float32)

# MSE
loss_func = tf.reduce_mean(tf.square(tf.subtract(outputs, y)))

# OPTIMIZER
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
train = optimizer.minimize(loss_func)

# INITALIZER
init = tf.global_variables_initializer()

# SAVER
saver = tf.train.Saver()
savedir = savedir = '/home/resolt/Workspaces/MachineLearning/MODELS/rnn_timeseries_excercise/'

# SESSION
with tf.Session() as sess:
	sess.run(init)

	for iteration in range(1, train_iters+1):
		X_batch, y_batch = next_batch(training_data = X_data, batch_size = 1, steps = num_time_steps)
		sess.run(train, feed_dict={X: X_batch, y: y_batch})

		if iteration % 100 == 0:
			mse = loss_func.eval(feed_dict={X: X_batch, y: y_batch})
			# print("ITERATION: {}\tMSE: {}".format(iteration, str(round(mse,8))[:8]))
			print("ITERATION: {}\tMSE: {:8f}".format(iteration, mse))

	saver.save(sess,savedir)

# PREDICTION
print(y_data)

train_inst = np.array(X_data[-num_time_steps:]).reshape(-1, num_time_steps, 1)

with tf.Session() as sess:

	saver.restore(sess, savedir)

	for i in range(12):
		print(i)
		y_pred = sess.run(outputs, feed_dict={X: train_inst[:,-num_time_steps:,:]})
		train_inst = np.append(train_inst, y_pred[0,-1:,0]).reshape(-1,num_time_steps+i+1,num_inputs)

print(train_inst[0, -num_time_steps:, :])

# THIS SUCKS - SKIPPING AHEAD


import random
import math as m

def direct_gamma(gamma, N):
        S=0.
        s=0.
        for i in range(N):
                x=random.uniform(0, 1)
                S+=x**gamma
                s+=x**(2*gamma)
        mean=S*1./N
        y=s*1./N
        var=m.sqrt(y-mean**2)
        error=var/m.sqrt(N)
        return [mean, error]

import pylab
S=0.
RE=[]
for i in range(1, 800001):
        x=random.uniform(0, 1)
        S+=x**(-0.8)
        RE+=[S/i, ]
pylab.plot(range(1, 800001), RE, 'r-')
pylab.ylabel("$\Sigma_i$/i")
pylab.xlabel("Number of samples i")


try:
    import requests #尝试导入requests，若库不存在会报错抛出异常
except ImportError: #判断是否抛出异常
    print("requests模块不存在，正在自动安装……")
    import os
    package_name = 'requests' #定义变量package_name（用于下面安装requests模块）
    os.system(f'pip install -i https://pypi.tuna.tsinghua.edu.cn/simple {package_name}')#调用pip安装requests模块
    import requests
try:
    import tkinter #尝试导入requests，若库不存在会报错抛出异常
except ImportError: #判断是否抛出异常
    print("tkinter模块不存在，正在自动安装……")
    import os
    package_name = 'tkinter' #定义变量package_name（用于下面安装tkinter模块）
    os.system(f'pip install -i https://pypi.tuna.tsinghua.edu.cn/simple {package_name}')#调用pip安装tkinter模块
    import requests
try:
    import jieba #尝试导入jieba，若库不存在会报错抛出异常
except ImportError: #判断是否抛出异常
    print("jieba模块不存在，正在自动安装……")
    import os
    package_name = 'jieba' #定义变量package_name（用于下面安装jieba模块）
    os.system(f'pip install -i https://pypi.tuna.tsinghua.edu.cn/simple {package_name}')#调用pip安装jieba模块
    import jieba

import requests
from requests.exceptions import RequestException
import tkinter as tk
import sys
class Translate():
    def __init__(self):
        self.window = tk.Tk()  #创建window窗口
        self.window.geometry("420x270")
        self.window.title("Chinglish翻译器")  # 定义窗口名称
        self.window.resizable(0,0)  # 禁止调整窗口大小
        self.input = tk.Entry(self.window, width=50)  # 创建一个输入框,并设置尺寸
        self.info = tk.Text(self.window, height=18,width=59)   # 创建一个文本展示框，并设置尺寸
        # 添加一个按钮，用于触发翻译功能
        self.t_button = tk.Button(self.window, text='翻译', relief=tk.RAISED, width=8, height=1, command=self.fanyi)

    def gui_arrang(self):
        """完成页面元素布局，设置各部件的位置"""
        self.input.place(x=0, y=5)
        self.info.place(x=0, y=30)
        self.t_button.place(x=355, y=0)

    def fanyi(self):
        """定义一个函数，完成翻译功能"""
        original_str = self.input.get()  # 定义一个变量，用来接收输入框输入的值
        #关闭jieba日志输出
        import logging
        jieba.setLogLevel(logging.INFO)
        t=""
        t_list=jieba.cut(original_str,cut_all=False)
        #for c in original_str:
        for c in t_list:
            string = c
            data = {
            'doctype': 'json',
            'type': 'AUTO',
            'i':string
            }
            url = "http://fanyi.youdao.com/translate"
            r = requests.get(url,params=data)
            result = r.json()
            translate_result = result['translateResult'][0][0]["tgt"]
            t=t+translate_result+" "
        translate_jg = t
        self.info.delete(1.0, "end")  # 输出翻译内容前，先清空输出框的内容
        self.info.insert('end',translate_jg)  # 将翻译结果添加到输出框中
    def cle(self):
        """定义一个函数，用于清空输出框的内容"""
        self.info.delete(1.0,"end")  # 从第一行清除到最后一行

    def cle_e(self):
        """定义一个函数，用于清空输入框的内容"""
        self.input.delete(0,"end")

def main():
    t = Translate()
    t.gui_arrang()
    tk.mainloop()

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
Created on Tue Apr 07 14:25:34 2020

@author: Timmitim (Tim Laeufer)
"""

import discord
import sqlite3

class db_handler:
    STR_SERVER = 'server'
    STR_USER = 'user'
    STR_CHANNEL = 'channel'
    STR_CATEGORY = 'category'
    STR_MESSAGE = 'message'
    STR_EDITS = 'edits'
    STR_DELETES = 'deletes'

    def log(self, rel_path, message):
        conn = sqlite3.connect('logs/messages.db')
        c = conn.cursor()
        """
        Flow:

        Check if DM
            True
                Check if user exists
                    True
                        user_info = ...
                    False
                        create user
                        user_info = ...
                Check if channel exists
                    True
                        channel_info = ...
                    False
                        create channel
                        channel_info = ...
                server_info = None/0
                category_info = None/0
            False
                Check if server exists
                    True
                        server_info = ...
                    False
                        create_server
                        server_info = ...
                Check if user exists
                    True
                        user_info = ...
                    False
                        create user
                        user_info = ...
                Check if category_exists
                    True
                        category_info = ...
                    False
                        create category
                        category_info = ...
                Check if channel exists
                    True
                        channel_info = ...
                    False
                        create_channel
                        channel_info = ...

        create_message
        """
        ins = False
        allowed = [679614550286663721] #mh2, lonesome town
        #Check if DM
        try:
            cat = message.channel.category
            is_dm = False
        except AttributeError:
            is_dm = True

        if(is_dm):
            self.insert_user(c, message.author)
            self.insert_channel(c, message.channel, is_dm = True)
        else:
            #is not a dm:
            self.insert_user(c, message.author)
            self.insert_server(c, message.guild)
            try:
                self.insert_category(c, message.channel.category)
            except:
                pass
            for el in allowed:
                if(message.guild.id == el):
                    ins = True
            self.insert_channel(c, message.channel)

        if(ins):
            self.insert_message(cursor = c, message_obj = message)
        conn.commit()
        conn.close()

    def log_join(self, member_obj, amount_users):
        conn = sqlite3.connect('logs/messages.db')
        c = conn.cursor()

        query = 'INSERT INTO join (join_id,amount_now) '
        query += 'VALUES ({mem_id},{amount});'

        c.execute(query.format(
            mem_id = member_obj.id,
            amount = amount_users))

        conn.commit()
        conn.close()

    def log_edit(self, rel_path, message_before, message):
        self.log_msg_edit(rel_path, message_before, message)

    def log_msg_edit(self, rel_path, message_before, message_obj):
        conn = sqlite3.connect(rel_path)
        c = conn.cursor()

        try: #add message, if not in db
            self.insert_message(c, message_before)
        except:
            pass

        try:
            cat = message_obj.channel.category.name
            cat_id = message_obj.channel.category.id
        except AttributeError:
            cat = 'None'
            cat_id = 0
        try:
            auth_name = message_obj.author.nick
        except AttributeError:
            auth_name = message_obj.author.name
        query = 'INSERT INTO {table} '
        query += '(message_id,channel_id,user_id,user_nick,time,old_content,new_content,server_id,category_id,channel_name,category_name) '
        query += 'VALUES ({msg_id},{ch_id},{u_id},"{u_nick}","{time}","{old_content}","{content}",{s_id},{cat_id},"{channel_name}","{category_name}");'
        query = query.format(
            table = self.STR_EDITS,
            msg_id = message_obj.id,
            ch_id = message_obj.channel.id,
            u_id = message_obj.author.id,
            u_nick = auth_name,
            time = str(message_obj.created_at) + 'UTC',
            old_content = message_before.content.replace('"', '/'),
            content = message_obj.clean_content.replace('"', '/'),
            s_id = message_obj.guild.id,
            cat_id = cat_id,
            channel_name = message_obj.channel.name,
            category_name = cat)
        c.execute(query)
        conn.commit()
        conn.close()

    def log_delete(self, rel_path, message):
        self.log_msg_delete(rel_path, message)

    def log_msg_delete(self, rel_path, message_obj):
        conn = sqlite3.connect(rel_path)
        cursor = conn.cursor()

        try:
            nick = message_obj.author.nick
        except AttributeError:
            nick = message_obj.author.name

        try: #on server, with category
            query = 'INSERT INTO {table} '
            query += '(message_id,channel_id,user_id,user_nick,time,content,server_id,category_id,channel_name,category_name) '
            query += 'VALUES ({msg_id},{ch_id},{u_id},"{u_nick}","{time}","{content}", {s_id},{cat_id},"{ch_name}","{cat_name}");'
            query = query.format(
                table = self.STR_DELETES,
                msg_id = message_obj.id,
                ch_id = message_obj.channel.id,
                u_id = message_obj.author.id,
                u_nick = nick,
                time = str(message_obj.created_at) + 'UTC',
                content = message_obj.clean_content.replace('"', '/'),
                s_id = message_obj.guild.id,
                cat_id = message_obj.channel.category.id,
                ch_name = message_obj.channel.name,
                cat_name = message_obj.channel.category.name)
        except AttributeError:
            try:#on server, no category
                query = 'INSERT INTO {table} '
                query += '(message_id,channel_id,user_id,user_nick,time,content,server_id,channel_name) '
                query += 'VALUES ({msg_id},{ch_id},{u_id},"{u_nick}","{time}","{content}", {s_id},"{ch_name}");'
                query = query.format(
                    table = self.STR_DELETES,
                    msg_id = message_obj.id,
                    ch_id = message_obj.channel.id,
                    u_id = message_obj.author.id,
                    u_nick = nick,
                    time = str(message_obj.created_at) + 'UTC',
                    content = message_obj.clean_content.replace('"', '/'),
                    s_id = message_obj.guild.id,
                    ch_name = message_obj.channel.name)
            except AttributeError: #not on server
                query = 'INSERT INTO {table} '
                query += '(message_id,channel_id,user_id,user_nick,time,content) '
                query += 'VALUES ({msg_id},{ch_id},{u_id},"{u_nick}","{time}","{content}");'
                query = query.format(
                    table = self.STR_DELETES,
                    msg_id = message_obj.id,
                    ch_id = message_obj.channel.id,
                    u_id = message_obj.author.id,
                    u_nick = nick,
                    time = str(message_obj.created_at) + 'UTC',
                    content = message_obj.clean_content.replace('"', '/'))
        cursor.execute(query)
        conn.commit()
        conn.close()

    def exists_in_db(self, cursor, table, element_id):
        # SELECT EXISTS(SELECT 1 FROM myTbl WHERE u_tag="tag");
        c = cursor

        query = 'SELECT EXISTS(SELECT 1 FROM {table} WHERE {table}_id = {row_id});'
        query = query.format(
            table = table,
            row_id = element_id)
        c.execute(query)

        if(c.fetchone()[0] == 0):
            return False
        else:
            return True

    def insert_user(self, cursor, author_obj):

        if(self.exists_in_db(cursor, self.STR_USER, author_obj.id)):
            return cursor

        query = 'INSERT INTO {table} (user_id,user_name) VALUES ({u_id},"{u_name}");'
        query = query.format(
            table = self.STR_USER,
            u_id = author_obj.id,
            u_name = author_obj.name)
        cursor.execute(query)
        return cursor

    def insert_server(self, cursor, server_obj):

        if(self.exists_in_db(cursor, self.STR_SERVER, server_obj.id)):
            return cursor

        query = 'INSERT INTO {table} (server_id,server_name) VALUES ({s_id},"{s_name}");'
        query = query.format(
            table = self.STR_SERVER,
            s_id = server_obj.id,
            s_name = server_obj.name)
        cursor.execute(query)
        return cursor

    def insert_category(self, cursor, category_obj):

        if(self.exists_in_db(cursor, self.STR_CATEGORY, category_obj.id)):
            return cursor

        query = 'INSERT INTO {table} (category_id,category_name,server_id) VALUES ({c_id},"{c_name}",{s_id});'
        query = query.format(
            table = self.STR_CATEGORY,
            c_id = category_obj.id,
            c_name = category_obj.name,
            s_id = category_obj.guild.id)
        cursor.execute(query)
        return cursor

    def insert_channel(self, cursor, channel_obj, is_dm = False):

        if(self.exists_in_db(cursor, self.STR_CHANNEL, channel_obj.id)):
            return cursor

        try:
            cat = channel_obj.category
            has_category = True
        except AttributeError:
            has_category = False

        if(is_dm):
            pass
        else:
            if(has_category):
                query = 'INSERT INTO {table} (channel_id,channel_name,category_id,server_id) VALUES '
                query += '({c_id},"{c_name}",{cat_id},{s_id});'
                query = query.format(
                    table = self.STR_CHANNEL,
                    c_id = channel_obj.id,
                    c_name = channel_obj.name,
                    cat_id = channel_obj.category.id,
                    s_id = channel_obj.guild.id)
            else:
                query = 'INSERT INTO {table} (channel_id,channel_name,category_id,server_id) VALUES '
                query += '({c_id},"{c_name}",{cat_id},{s_id});'
                query = query.format(
                    table = self.STR_CHANNEL,
                    c_id = channel_obj.id,
                    c_name = channel_obj.name,
                    cat_id = 0,
                    s_id = channel_obj.guild.id)
            cursor.execute(query)
            return cursor

    def insert_message(self, cursor, message_obj):

        try:
            nick = message_obj.author.nick
        except AttributeError:
            nick = message_obj.author.name

        try: #on server, with category
            query = 'INSERT INTO {table} '
            query += '(message_id,channel_id,user_id,user_nick,time,content,server_id,category_id,channel_name,category_name) '
            query += 'VALUES ({msg_id},{ch_id},{u_id},"{u_nick}","{time}","{content}", {s_id},{cat_id},"{ch_name}","{cat_name}");'
            query = query.format(
                table = self.STR_MESSAGE,
                msg_id = message_obj.id,
                ch_id = message_obj.channel.id,
                u_id = message_obj.author.id,
                u_nick = nick,
                time = str(message_obj.created_at) + 'UTC',
                content = message_obj.clean_content.replace('"', '/'),
                s_id = message_obj.guild.id,
                cat_id = message_obj.channel.category.id,
                ch_name = message_obj.channel.name,
                cat_name = message_obj.channel.category.name)
        except AttributeError:
            try:#on server, no category
                query = 'INSERT INTO {table} '
                query += '(message_id,channel_id,user_id,user_nick,time,content,server_id,channel_name) '
                query += 'VALUES ({msg_id},{ch_id},{u_id},"{u_nick}","{time}","{content}", {s_id},"{ch_name}");'
                query = query.format(
                    table = self.STR_MESSAGE,
                    msg_id = message_obj.id,
                    ch_id = message_obj.channel.id,
                    u_id = message_obj.author.id,
                    u_nick = nick,
                    time = str(message_obj.created_at) + 'UTC',
                    content = message_obj.clean_content.replace('"', '/'),
                    s_id = message_obj.guild.id,
                    ch_name = message_obj.channel.name)
            except AttributeError: #not on server
                query = 'INSERT INTO {table} '
                query += '(message_id,channel_id,user_id,user_nick,time,content) '
                query += 'VALUES ({msg_id},{ch_id},{u_id},"{u_nick}","{time}","{content}");'
                query = query.format(
                    table = self.STR_MESSAGE,
                    msg_id = message_obj.id,
                    ch_id = message_obj.channel.id,
                    u_id = message_obj.author.id,
                    u_nick = nick,
                    time = str(message_obj.created_at) + 'UTC',
                    content = message_obj.clean_content.replace('"', '/'))
        cursor.execute(query)
        return cursor

    def fetch_sql_rows(self, db, sql):
        conn = sqlite3.connect(db)
        conn.row_factory = sqlite3.Row
        cursor = conn.cursor()
        lis = cursor.execute(sql).fetchall()
        conn.close()
        return lis


__author__ = 'Ran'
from ..api import api
from app import Flask, db, cache
from flask import render_template, request, session, redirect
from app.Database.db_global import Partition, Category
from app.Database.db_features import Collection_video
from app.elastic.elastic_global import es_query_video
from app.elastic.elastic_query import elastic_query_indexmodule_data
from app.elastic.elastic_tag import get_videoalltag
from app.Database.db_operatingrecord import get_video_likerecord, add_video_likerecord, del_video_likerecord, get_video_likerecord_userall
from app.elastic.elastic_video_operating import video_likeadd
from app.Database.db_features import add_collection_video
from app.follow.follow_user import follow_orm, follow_gettype
from app.msg.comment import write_comment, get_commentlist, likecomment_pus
from flask_login import current_user
from app.Database.db_user import Userdata_common
from app.account import Userdata
from app.middleware.get_cacheuser import get_middleware_userdata
from app.recommend_ai.business import get_recommend_videolist
from app.tag.tag_business import write_Atag
import json

#首页
@api.route('/nav')
@cache.cached(timeout=3600, key_prefix='global_nav')
def nav_api():
    data = Partition.query.filter_by().all()
    list = [{
        'ico':i.ico,
        'title': i.name,
        'url': i.maincategory_url,
        'sort': i.sort,
    }for i in data]
    return json.dumps(sorted(list, key=lambda x:x['sort'], reverse=True))

#首页-模块接口
@api.route('/index-content/module/<gettype>/<name>')
def getapi_indexmodule(gettype, name):

    '''
    category = 子分区
    partition = 主分区
    '''

    if gettype == 'partition':
        data = elastic_query_indexmodule_data("partition",name)
        return json.dumps(data)
    elif gettype == 'category':
        data = elastic_query_indexmodule_data("category",name)
        return json.dumps(data)
    elif gettype == 'all':
        data = elastic_query_indexmodule_data("all",name)
        return json.dumps(data)
    else:
        pass
        data = []
        return json.dumps(data)

#点赞记录
@api.route('/video/videolike-type', methods=["POST"])
def videolike_typeget():
    if current_user.is_authenticated :
        video = request.args.get('video')
        data = get_video_likerecord_userall(videoid=video)
        videodata = es_query_video(video)
        list = [{
            'code':i.operating_record
        }for i in data]
        likenaber = {
            "like_frequency": videodata['_source']['like_frequency'],
            "dislike_frequency": videodata['_source']['dislike_frequency']
        }
        p = {
            "type":list,
            "naber":likenaber
        }
        return json.dumps(p)
    else:
        return json.dumps({"like_frequency": 0, "dislike_frequency": 0})

#点赞
@api.route('/video/videolike', methods=["POST"])
def videolike():
    if(current_user):
        video = request.args.get('video')
        putype = request.args.get('type')

        if putype == 'True':
            putype = True
        elif putype == 'False':
            putype = False
        else:
            return 'fuck you sb'

        #查询视频点赞记录和点赞类型
        data = get_video_likerecord(videoid=video, putype=putype)

        #如果存在则删除
        if (data):
            video_likeadd(videoid=video, settype=putype, settype2='del')
            del_video_likerecord(videoid=video, putype=putype)
            return json.dumps({'code':'ok'})

        else:
            #不存在则写入
            video_likeadd(videoid=video, settype=putype, settype2='add')
            add_video_likerecord(videoid=video, putype=putype)
            return json.dumps({'code':'ok'})
    else:
        return json.dumps({'code':'error'})

#收藏
@api.route('/video/collection', methods=["POST"])
def videocollection():
    if current_user.is_authenticated:
        video = request.args.get('video')
        add_collection_video(video)
        return json.dumps({'code':'ok'})
    else:
        return json.dumps({'code':'error'})

#收藏
@api.route('/video/collection_type', methods=["POST"])
def videocollection_type():
    if current_user.is_authenticated:
        video = request.args.get('video')
        data = Collection_video.query.filter_by(video_id=video, user_id=current_user.user_id).first()
        if (data):
            return json.dumps({'code':'1'})
        else:
            return json.dumps({'code':'0'})
    else:
        return json.dumps({'code':'error'})

#获取视频up主
@api.route('/video/up', methods=["POST"])
def video_upapi():
    key = request.args.get('key')
    data = get_middleware_userdata(user_id=int(key))
    com_userdata = Userdata_common.query.filter_by(user_id=int(key)).first()
    list = {
        'name': data.username,
        'head': data.head,
        'introduction': com_userdata.introduction,
        'fan_statistics': com_userdata.fan_statistics,
    }
    return json.dumps(list)

#视频分区类型
@api.route('/video/class/<cla>/tab/<tab>', methods=["POST"])
@cache.cached(timeout=3600, key_prefix='video_type_%s')
def video_classget(cla, tab):
    data = Partition.query.filter_by(id=cla).first()
    tabdata = Category.query.filter_by(id=tab).first()
    list = {
        'partition':data.name,
        'category':tabdata.name,
    }
    return json.dumps(list)

#订阅作者
@api.route('/video/follow_auth/<authid>', methods=["POST"])
def follow_auth(authid):
    data = follow_orm(authid)
    return json.dumps({'code':data})

#订阅状态
@api.route('/video/follow_type/<authid>', methods=["POST"])
def follow_type(authid):
    if current_user.is_authenticated:
        data = follow_gettype(authid)
        return json.dumps({'code':data})
    else:
        return json.dumps({'code':False})

#评论
@api.route('/video/comment/<id>', methods=["POST"])
def video_comment(id):
    content = request.form.get('content')
    data = write_comment(content=content, video_id=id)
    list={
        'content':content,
        'head':current_user.head,
        'id': data.id,
        'user': current_user.username,
        'time': data.date,
        'like_pcs': data.like_pcs,
        'dislike_pcs': data.dislike_pcs,
        'secondary_comment': data.secondary_comment,
    }
    return json.dumps(list)

#评论列表
@api.route('/video/comment-list/<id>/<page>/<gettype>', methods=["POST"])
def video_comment_list(id, page, gettype):
    if gettype == 'false': #like_pcs #time
        gettype == False
    data = get_commentlist(id ,page, gettype)
    return json.dumps(data)

#评论列表
@api.route('/video/comment-pus/<id>/type/<clss>', methods=["POST"])
def video_comment_pus(id,clss):
    if clss == 'like':
        data=likecomment_pus(commentid = id, pustype='like')
    elif clss == 'nolike':
        data=likecomment_pus(commentid = id, pustype='nolike')
    return json.dumps(data)

#相关视频列表
@api.route('/video/recommend/<up>/<videoname>', methods=["POST"])
def video_recommend(up, videoname):
    data = get_recommend_videolist(up, videoname)
    return json.dumps(data)

#tag获取
@api.route('/video/tag/get/<vid>', methods=["POST"])
def video_gettag(vid):
    data = get_videoalltag(vid)
    return json.dumps(data)

#tag添加
@api.route('/video/tag/add-tag', methods=["POST"])
def video_addtag():
    if(current_user):
        data = request.json
        print(data)
        videoid = data['videoid']
        tag = data['tag']
        try:
            write_Atag(tag=tag,vid=videoid)
            return json.dumps({'code':'ok'})
        except:
            return json.dumps({'code':'error'})


# L = [1,3,5,6]
#
# #it = iter(range(1,10,3))
# it = iter(L)
# while True:
#     try:
#         x = next(it)
#         print(x)
#     except StopIteration:
#         break

s = {'唐僧','悟空', '八戒', '沙僧'}
it = iter(s)
while True:
    try:
        name = next(it)
        print(name)
    except StopIteration:
        print("遍历结束")
        break


import tensorflow as tf

def restore_xor(path, model):
    restore_dict = {'w_1': model['network'][0].W,
                    'b_1': model['network'][0].b,
                    'w_2': model['network'][1].W,
                    'b_2': model['network'][1].b}
    restore_op = tf.train.Saver(restore_dict)
    restore_op.restore(tf.get_default_session(), path)

def restore_params(path, layers):
    restore_dict = {}
    for layer in layers[1:-1]:
        restore_dict.setdefault(str(layer.W.name[:-2]), layer.W)
        restore_dict.setdefault(str(layer.b.name[:-2]), layer.b)
    print(restore_dict)
    restore_op = tf.train.Saver(restore_dict)
    restore_op.restore(tf.get_default_session(), path)


# coding: utf-8
import cv2
import pandas as pd
import numpy as np

class Image:
  def __init__(self, image: cv2):
    self.cv2 = image
    self.height = image.shape[0]
    self.width = image.shape[1]

  def get_halfsize(self) -> cv2:
    size = (int(self.width/2), int(self.height/2))
    image = cv2.resize(self.cv2, size)
    return image

  def get_trimming(self, size: tuple) -> cv2:
    size = (int(size[0]), int(size[1]))
    image = cv2.resize(self.cv2, size)
    return image

class SalientRegionMap:
  def __init__(self, saliency_map: list, screenshot: list, tags: pd, custom_tags: pd):
    self.screenshot = screenshot
    self.saliency_map = saliency_map
    self.canvas = np.zeros((Image(saliency_map).height, Image(saliency_map).width, 3), dtype=np.uint8)
    self.tags = tags
    self.custom_tags = custom_tags
    self.highest_saliency = self.__GetHighestSaliency()

  def GetSortedCustomTagsIndex(self) -> list:
    print(len(self.custom_tags))
    element_area_list = []
    for i in range(len(self.custom_tags) - 1):
      element_area_list.append(int(self.custom_tags.iat[i, 8]))
    return np.argsort(element_area_list)[::-1]

  def GetSalientRegionMap(self):
    for i in self.GetSortedCustomTagsIndex():
      self.__PrintElementBySaliency(i, self.custom_tags.iat[i, 0])
    return self.canvas

  def CreateImportanceMap(self, high_element_list: list):
    high_element_list_num = len(high_element_list) #配列の長さ取得
    i = 0
    print(high_element_list_num)
    for i in range(high_element_list_num):
      high_element = high_element_list[i]
      start_x = int(self.custom_tags.iat[high_element, 2])
      start_y = int(self.custom_tags.iat[high_element, 3])
      end_x = int(self.custom_tags.iat[high_element, 2]+self.custom_tags.iat[high_element, 4])
      end_y = int(self.custom_tags.iat[high_element, 3]+self.custom_tags.iat[high_element, 5])
      cv2.rectangle(self.canvas, (start_x, start_y) , (end_x, end_y), (0, 255-(i)*20, 0), 3) #標準： 20ずつ

    cv2.namedWindow("halfImg", cv2.WINDOW_NORMAL)
    cv2.imshow("halfImg", self.canvas)
    cv2.imwrite("./output/final_importance.png", self.canvas ) #Save
    k = cv2.waitKey(0)
    if k == 27:         # wait for ESC key to exit
      cv2.destroyAllWindows()

  def __GetHighestSaliency(self) -> float:
    return self.custom_tags.iat[self.GetHighSaliencyList()[0], 7]

  def GetHighSaliencyList(self):
    tag_list_num = len(self.custom_tags)
    salient_level = []
    ng_list = []
    high_element_list = []

    # 顕著度を配列に格納
    i = 1 # タイトルの次から実行
    for i in range(tag_list_num):
      if self.custom_tags.iat[i, 0] == "id_large" or self.custom_tags.iat[i, 0] == "class_large":
        salient_level.append(-1)
      else:
        salient_level.append(self.custom_tags.iat[i, 7])

    salient_level_sort_final = np.argsort(salient_level)

    salient_num = 10 # 上位何個表示するか？
    temporal_num = 1 # 一時的な変数
    salient_num_first = salient_num

    while salient_num > 0 :
      most_salient = salient_level_sort_final[tag_list_num - temporal_num]
      print(most_salient)
      if (most_salient in ng_list) == False:
        start_x = int(self.custom_tags.iat[most_salient, 2])
        start_y = int(self.custom_tags.iat[most_salient, 3])
        end_x = int(self.custom_tags.iat[most_salient, 2]+self.custom_tags.iat[most_salient, 4])
        end_y = int(self.custom_tags.iat[most_salient, 3]+self.custom_tags.iat[most_salient, 5])
        size = int(self.custom_tags.iat[most_salient, 4]*self.custom_tags.iat[most_salient, 5])
        if start_x < 0 :
          start_x = 0
        if start_y < 0 :
          start_y = 0
        if end_x > Image(self.saliency_map).width:
          end_x = Image(self.saliency_map).width
        if end_y > Image(self.saliency_map).height:
          end_y = Image(self.saliency_map).height

        if (end_x - start_x)/(end_y - start_y) < 10: # あまりにも細長いものを排除
          clipped = self.screenshot[int(start_y):int(end_y),int(start_x):int(end_x)]
          cv2.imwrite("./working/high-saliency/img"+ str(salient_num_first - salient_num + 1) +".png", clipped ) #Save
          print("画像出力 & 顕著度高いリストに追加")
          high_element_list.append(most_salient)
          salient_num -= 1

          print("%s %s %s %s 顕著度→%s" %(start_x, start_y, end_x, end_y, self.custom_tags.iat[most_salient, 7]) )

          i = 1 # タイトルの次から実行
          for i in range(tag_list_num):
            if (i in ng_list) == False:
              research_start_x = int(self.custom_tags.iat[i, 2])
              research_start_y = int(self.custom_tags.iat[i, 3])
              research_end_x = int(self.custom_tags.iat[i, 2]+self.custom_tags.iat[i, 4])
              research_end_y = int(self.custom_tags.iat[i, 3]+self.custom_tags.iat[i, 5])
              research_size = int(self.custom_tags.iat[most_salient, 4]*self.custom_tags.iat[most_salient, 5])

              if (start_x >= research_start_x) and (start_y >= research_start_y) and (end_x <= research_end_x) and (end_y <= research_end_y):
                print("%s 番怪しい" %i)
                if research_size - size < 200:
                  ng_list.append(i)
                  print("NGリストに %s を格納" %i)
              elif (start_x <= research_start_x) and (start_y <= research_start_y) and (end_x >= research_end_x) and (end_y >= research_end_y):
                print("%s 番怪しい" %i)
                if  size - research_size < 200:
                  ng_list.append(i)
                  print("NGリストに %s を格納" %i)
        else:
          print("細長すぎます")
      else:
        print("NG Fileに入っています")
      temporal_num += 1

    return high_element_list

  # 各要素の塗り潰し関数
  def __PrintElementBySaliency(self, i: int, type: str):
    start_x = int(self.custom_tags.iat[i, 2])
    start_y = int(self.custom_tags.iat[i, 3])
    end_x = int(self.custom_tags.iat[i, 2]+self.custom_tags.iat[i, 4])
    end_y = int(self.custom_tags.iat[i, 3]+self.custom_tags.iat[i, 5])
    # 最も顕著度の高い要素を最大輝度で塗りつぶす
    salient_level_num = (self.custom_tags.iat[i, 7] / self.highest_saliency) * 256
    if salient_level_num > 0:
      cv2.rectangle(self.canvas, (start_x, start_y) , (end_x, end_y), (salient_level_num, salient_level_num, salient_level_num), -1)

    # ToDo 現在画像のみそのまま表示するシステムは使用していない
    # if type == 'img' and ((end_x - start_x)*(end_y - start_y)) / (Image(self.saliency_map).width * Image(self.saliency_map).height) > 0.1 :
    #   if start_x < 0 or start_y < 0 or end_x > Image(self.saliency_map).width or end_y > Image(self.saliency_map).height:
    #     return

    #   clipped = self.screenshot[start_y:end_y,start_x:end_x]
    #   print(clipped.shape)
    #   print(start_x, start_y)
    #   self.canvas[start_y:clipped.shape[0] + start_y, start_x:clipped.shape[1] + start_x] = clipped
    # else :
    #   if salient_level_num > 0:
    #     cv2.rectangle(self.canvas, (start_x, start_y) , (end_x, end_y), (salient_level_num, salient_level_num, salient_level_num), -1)


#program3.py

#This program prompts the user for a message and a key, then encodes the message using the key and a shift cypher.

#Trevor Nierman
#R957F692

def main():

	import string

	#Obtain key
	key = eval(input("What is the key for this message? "))

	#Obtain message
	mess = input("What is the message? ")

	#Determine message length
	length = len(mess)

	new_mess = [0 for i in range(length)]

	#Begin cypher
	for i in range(length):

		#Excludes spaces
		if(mess[i] != " "):

			#Shift characters
			char = ord(mess[i])

			shift_char = char + key

			#Upper-case character
			if char > 64 and char < 91:
				if shift_char > 90:
					shift_char = ((shift_char - 65) % 26) + 65

			#Lower-case character
			if char > 96 and char < 123:
				if shift_char > 122:
					shift_char = ((shift_char - 97) % 26) + 97

			new_mess[i] = chr(shift_char)

	#Combine string array into single string
	new_mess = ''.join(new_mess)
	print(new_mess)

main()


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import argparse
import h5py
import json
import os
import scipy.misc
import sys

import cityscapesscripts.evaluation.instances2dict as cs

import detectron.utils.segms as segms_util
import detectron.utils.boxes as bboxs_util
import numpy as np
import cv2
from pycocotools import mask

def parse_args():
    parser = argparse.ArgumentParser(description='Convert dataset')
    parser.add_argument(
        '--dataset', help="cocostuff, cityscapes", default=None, type=str)
    parser.add_argument(
        '--outdir', help="output dir for json files", default=None, type=str)
    parser.add_argument(
        '--datadir', help="data dir for annotations to be converted",
        default=None, type=str)
    # if len(sys.argv) == 1:
    #     parser.print_help()
    #     sys.exit(1)
    return parser.parse_args()

def convert_coco_stuff_mat(data_dir, out_dir):
    """Convert to png and save json with path. This currently only contains
    the segmentation labels for objects+stuff in cocostuff - if we need to
    combine with other labels from original COCO that will be a TODO."""
    sets = ['train', 'val']
    categories = []
    json_name = 'coco_stuff_%s.json'
    ann_dict = {}
    for data_set in sets:
        file_list = os.path.join(data_dir, '%s.txt')
        images = []
        with open(file_list % data_set) as f:
            for img_id, img_name in enumerate(f):
                img_name = img_name.replace('coco', 'COCO').strip('\n')
                image = {}
                mat_file = os.path.join(
                    data_dir, 'annotations/%s.mat' % img_name)
                data = h5py.File(mat_file, 'r')
                labelMap = data.get('S')
                if len(categories) == 0:
                    labelNames = data.get('names')
                    for idx, n in enumerate(labelNames):
                        categories.append(
                            {"id": idx, "name": ''.join(chr(i) for i in data[
                                n[0]])})
                    ann_dict['categories'] = categories
                scipy.misc.imsave(
                    os.path.join(data_dir, img_name + '.png'), labelMap)
                image['width'] = labelMap.shape[0]
                image['height'] = labelMap.shape[1]
                image['file_name'] = img_name
                image['seg_file_name'] = img_name
                image['id'] = img_id
                images.append(image)
        ann_dict['images'] = images
        print("Num images: %s" % len(images))
        with open(os.path.join(out_dir, json_name % data_set), 'wb') as outfile:
            outfile.write(json.dumps(ann_dict))

# for Cityscapes
def getLabelID(self, instID):
    if (instID < 1000):
        return instID
    else:
        return int(instID / 1000)

def getAreaByPoint(points, h, w, category_id):
    line_type = 1  # cv2.CV_AA
    color = category_id
    seg = []
    for j in range(len(points)):
        coordx = points[j][0]
        coordy = points[j][1]
        point = []
        point.append(int(coordx))
        point.append(int(coordy))
        seg.append(point)

    labelMask = np.zeros((h, w))
    cv2.fillPoly(labelMask, np.array([seg], dtype=np.int32), color, line_type)

    mask_new, contours, hierarchy = cv2.findContours((labelMask).astype(np.uint8), cv2.RETR_TREE,
                                                     cv2.CHAIN_APPROX_SIMPLE)

    ##----------------------------------------------
    polygons = []
    # In practice, only one element.
    for contour in contours:
        contour = contour.flatten().tolist()
        polygons.append(contour)
    labelMask[:, :] = labelMask == color

    labelMask = np.expand_dims(labelMask, axis=2)
    labelMask = labelMask.astype('uint8')
    labelMask = np.asfortranarray(labelMask)

    Rs = mask.encode(labelMask)
    return float(mask.area(Rs))

def convert_cityscapes_instance_only(
        data_dir, out_dir):
    """Convert from cityscapes format to COCO instance seg format - polygons"""
    sets = [
        'gtFine_train',
        'gtFine_val',
        # 'gtFine_test',

        # 'gtCoarse_train',
        # 'gtCoarse_val',
        # 'gtCoarse_train_extra'
    ]
    ann_dirs = [
        'labels_train',
        'labels_val',
        # 'gtFine_trainvaltest/gtFine/test',

        # 'gtCoarse/train',
        # 'gtCoarse/train_extra',
        # 'gtCoarse/val'
    ]
    json_name = 'instancesonly_filtered_%s.json'
    ends_in = '.json'
    img_id = 0
    ann_id = 0
    cat_id = 1
    category_dict = {}

    category_instancesonly = ['__background__',
        'person',
        'rider',
        'car',
        'truck',
        'bus',
        'motorcycle',
        'bicycle',
        'ground',
        'road',
        'sky'
    ]

    for data_set, ann_dir in zip(sets, ann_dirs):
        print('Starting %s' % data_set)
        ann_dict = {}
        images = []
        annotations = []
        ann_dir = os.path.join(data_dir, ann_dir)
        for root, _, files in os.walk(ann_dir):
            for filename in files:
                if filename.endswith(ends_in):
                    if len(images) % 50 == 0:
                        print("Processed %s images, %s annotations" % (
                            len(images), len(annotations)))
                    json_ann = json.load(open(os.path.join(root, filename)))
                    image = {}
                    image['id'] = img_id
                    img_id += 1

                    image['width'] = json_ann['imgWidth']
                    image['height'] = json_ann['imgHeight']
                    sub_name = filename.split('_')[0:3]
                    image['file_name'] = sub_name[0] + "_" + sub_name[1] + "_" + sub_name[2] + "_" + 'leftImg8bit.png'
                    images.append(image)

                    objects = json_ann["objects"]

                    for obj in objects:
                        if obj["label"] not in category_instancesonly:
                            continue  # skip non-instance categories

                        index = category_instancesonly.index(obj["label"])# + 184

                        ann = {}
                        ann['id'] = ann_id
                        ann_id += 1
                        ann['image_id'] = image['id']
                        ann['segmentation'] = [sum(obj['polygon'], [])]

                        ann['category_id'] = index
                        ann['iscrowd'] = 0

                        seg_points = obj["polygon"]
                        ann['area'] = getAreaByPoint(seg_points, image['height'], image['width'], ann['category_id'])
                        ann['bbox'] = bboxs_util.xyxy_to_xywh(
                            segms_util.polys_to_boxes(
                                [ann['segmentation']])).tolist()[0]

                        annotations.append(ann)
                    # break
        ann_dict['images'] = images
        categories = []
        for index, value in enumerate(category_instancesonly):
            categories.append({"id": index, "name": value})
        categories = categories[1:]
        ann_dict['categories'] = categories
        ann_dict['annotations'] = annotations
        print("Num categories: %s" % len(categories))
        print("Num images: %s" % len(images))
        print("Num annotations: %s" % len(annotations))
        with open(os.path.join(out_dir, json_name % data_set), 'wb') as outfile:
            outfile.write(json.dumps(ann_dict))

if __name__ == '__main__':
    args = parse_args()
    args.datadir = "/media/administrator/deeplearning/dataset/test_cityscape"
    args.outdir = "/media/administrator/deeplearning/dataset/test_cityscape/output"
    convert_cityscapes_instance_only(args.datadir, args.outdir)


class DataConfig:

    def __init__(
        self,
        data,
        history,
        steps,
        filters,
        filt_sizes,
        dense,
        logits
    ):
        self.data = data
        self.history = history
        self.steps = steps
        self.filters = filters
        self.filt_sizes = filt_sizes
        self.dense = dense
        self.logits = logits

    #---------------------------------------------------------------------------
    # Checks
    #---------------------------------------------------------------------------

    def isEqual(self, dataConfig):
        if (
            self.data == dataConfig.data
            and self.history == dataConfig.history
            and self.steps == dataConfig.steps
            and self.filters == dataConfig.filters
            and self.filt_sizes == dataConfig.filt_sizes
            and self.dense == dataConfig.dense
            and self.logits == dataConfig.logits
        ):
            return True
        return False

    def isNotNone(self):
        if not self.data == None:
            print("d")
            return False
        if not self.history == None:
            print("h")
            return False
        if not self.steps == None:
            print("s")
            return False
        if not self.filters == None:
            print("f")
            return False
        if not self.filt_sizes == None:
            print("fs")
            return False
        if not self.dense == None:
            print("dense")
            return False
        if not self.logits == None:
            print("l")
            return False

    #---------------------------------------------------------------------------
    # toStrnig Method
    #---------------------------------------------------------------------------

    def toString(self):
        return (
            str(self.history) +
            " | " + str(self.steps) +
            " | " + str(self.filters) +
            " | " + str(self.filt_sizes) +
            " | " + str(self.dense) +
            " | " + str(self.logits)
        )


"""
Filter airdrop receivers
========================

Version: 0.1.0

Filters input in the form of
```
{
  "items" [
    { "address": "0xabc...", "something": "else" },
    { "address": "0x123...", "moar": "things" }
  ]
}

to contain only elements connected with airdrop identified by given id,
where currently this id is in the form of claim:ethereum_transaction_hash:0.

"""

from algorithms.experimental import airdrop_receivers
from algorithms.utils import param, pipeable, filter_debug

@pipeable
@filter_debug
@param("id", required=True)
def run(conn_mgr, input, id, **ignore):
    receivers = airdrop_receivers.run(conn_mgr, None, id)
    addresses = set([item["address"] for item in receivers["items"]])
    return {
        "items": [item for item in input["items"] if item["address"] in addresses]
    }


# coding: utf-8
from fractions import gcd
import sys
input = sys.stdin.readline

x, y = map(int, input().split())
a = min(x, y)
b = max(x, y)

check = str(bin(a ^ b))
# print(check)
ans = 0

for i in range(1, len(bin(a)) - 2):
    print(i)
    if check[-i] == '1':
        ans += 1
if ans == 0:
    ans += 1
print(ans)
print(len(bin(a)))


from paraview.simple import *
paraview.simple._DisableFirstRenderCameraReset()
# To run:
#   1. Run write_list.py (on Skybridge!)
#   2. Scp the fnames.txt file to your computer and update the fnames_file variable
#       in this script
#   3. Set the Skybridge path for exofiles_dir
#   4. Set dim, t, and camera variables in this script
#   5. Open Paraview and connect to Skybridge
#   6. Click Tools > Python Shell
#   7. Click Run Script and select this script
#   8. In the pipeline browser, scroll down to the bottom and click
#      on the eye next to the GroupDatasets1 filter

# User defined quantities:
# Folder containing subgrid output (on Skybridge!)
exofiles_dir = "/gscratch/dtseidl/jobs/2017/5_24/ms_thermal_only/second_round/subgrid_data"
# File that contains names of subgrid_output* files in exofiles_dir (must be on local machine!)
fnames_file = "/Users/dtseidl/Desktop/paraview_scripting/fnames2.txt"
# spatial dimension (as string)
dim = "3D"
# time step to display
t = 8
# camera variables
# 2D
#camera_position = [0.005, 0.005, 10000.0]
#camera_focal_point = [0.005, 0.005, 0.0]
#camera_parallel_scale = 0.01
# 3D
camera_position = [0.017689521496123933, 0.010350820509929916, 0.027932216343520083]
camera_focal_point = [0.004999999888241282, 0.0012499999720603225, 0.004999999888241285]
camera_view_up = [-0.14986320289908484, 0.9446188873455699, -0.2919523558508227]
camera_parallel_scale = 0.007180703147671309

with open(fnames_file) as f:
  files = f.read().splitlines()
samp = "subgrid_output" + str(t) + ".exo."
nelem = []
ncores = 1 + max([ int(files[i].split(".")[2]) for i in range(len(files))])

for core in range(ncores):
  sample = samp + str(core) + "."
  nelem.append(len([int(files[i].replace(sample,"")) for i in  range(len(files)) if sample in files[i]]))

readers = {}
readers_list =[]

renderView1 = GetActiveViewOrCreate('RenderView')
renderView1.ResetCamera()
renderView1.InteractionMode = dim
renderView1.CameraPosition = camera_position
renderView1.CameraFocalPoint = camera_focal_point
if dim == "3D":
  renderView1.CameraViewUp = camera_view_up
renderView1.CameraParallelScale = camera_parallel_scale

for core in range(ncores):
  base = exofiles_dir + "/subgrid_time_" + str(t) + ".exo." + str(core) + "."
  name = [base + str(i) for i in range(nelem[core])]
  readers[core] = ExodusIIReader(FileName=name,FaceVariables=None,GlobalVariables=None)
  readers_list.append(readers[core])

groupDatasets1 = GroupDatasets(Input=readers_list)


import numpy as np
import glob
import itertools
# import h5py
# import h5py_cache as h5c
from threading import Lock, Thread
from queue import Queue
import time

TRAIN_TEST_SPLIT=0.95

class DataLoaderBase:
    def __init__(self):
        self.train_x = []
        self.train_y = []
        self.test_x = []
        self.test_y = []

class NextItemsDataLoader(DataLoaderBase):

    def __init__(self):
        self.train_x_filenames = sorted(glob.glob('training_data/next_items/processed/*_train_x*.npz'))
        self.test_x_filenames = sorted(glob.glob('training_data/next_items/processed/*_test_x*.npz'))
        self.train_y_filenames = sorted(glob.glob('training_data/next_items/processed/*_train_y*.npz'))
        self.test_y_filenames = sorted(glob.glob('training_data/next_items/processed/*_test_y*.npz'))
        # h5f = h5py.File('training_data/next_items/hdf5/data.h5', 'r')
        # self.train_x_hdf5 = h5f['train_x']
        # self.train_y_hdf5 = h5f['train_y']
        # self.test_x_hdf5 = h5f['test_x']
        # self.test_y_hdf5 = h5f['test_y']

        super().__init__()
        self.readFromNumpyFiles()

    def _generate_train_test(self, train_test_x, train_test_y):
        x = np.concatenate((np.tile(np.arange(5),len(train_test_x))[:,np.newaxis], np.repeat(train_test_x, 5, axis=0)), axis=1)
        y = np.ravel(train_test_y)
        valid_ind = y != 0
        result_y = y[valid_ind]
        result_x = x[valid_ind]
        return result_x, result_y

    def build_HDF5(self):
        self.readFromNumpyFiles()
        print("Files loaded")
        print("Generating training data")
        self.train_x, self.train_y = self._generate_train_test(self.train_x, self.train_y)
        self.test_x, self.test_y = self._generate_train_test(self.test_x, self.test_y)
        print("Training data complete")

        # train_x_y = np.concatenate([self.train_x, self.train_y[:,np.newaxis]], axis = 1)
        # self.train_x, self.train_y = [], []
        # print("Now shuffling")
        # np.random.shuffle(train_x_y)
        # print("Shuffling complete")
        # shuffled = []

        # for i in range(0, len(train_x_y), 10 ** 6):
        #     print("Shuffling {:.2%} complete".format(i / len(train_x_y)))
        #     seg = train_x_y[i:i + 10**6]
        #     np.random.shuffle(seg)
        #     shuffled.extend(seg)

        # print("get arrays back in shape")
        # self.train_x = train_x_y[:, :-1]
        # self.train_y = train_x_y[:, -1]

        print("writing files")
        # chunk_shape_x = (self.train_x.shape[-1],1)
        # chunk_shape_y = (1,)
        h5f = h5c.File('training_data/next_items/hdf5/data.h5', 'w')
        print("0 complete")
        h5f.create_dataset('train_x', data=self.train_x, shape=self.train_x.shape)
        print("1 complete")
        h5f.create_dataset('train_y', data=self.train_y, shape=self.train_y.shape)
        print("2 complete")
        h5f.create_dataset('test_x', data=self.test_x, shape=self.test_x.shape)
        print("3 complete")
        h5f.create_dataset('test_y', data=self.test_y, shape=self.test_y.shape)
        print("4 complete")
        h5f.close()

    class GenerateTrainingData(Thread):
        def __init__(self, queue, gen_x_y):
            Thread.__init__(self)
            self.queue = queue
            self.generate_x_y = gen_x_y

        def run(self):
            while True:
                x, y = self.generate_x_y()
                self.queue.put((x, y))

    def _get_next_examples(self):
        start_index = self.randints.pop()
        end_index = min(start_index+self.batchsize, self.train_len)
        return self.train_x_hdf5[start_index: end_index], self.train_y_hdf5[start_index: end_index]

    def get_next_train_subepoch(self, num_examples):
        if not self.threads:
            # self.
            for _ in range(1):
                t = self.GenerateTrainingData(self.queue, self._get_next_examples)
                self.threads.append(t)
                t.start()
        X,Y = [], []
        for _ in range(num_examples):
            (x,y) = self.queue.get()
            X.append(x)
            Y.append(y)
        return (X,Y)

    def get_test_data(self):
        # X,Y = [],[]
        # for i in range(self.test_x_hdf5.shape[0]):
        #     X.append(self.test_x_hdf5[i])
        #     Y.append(self.test_y_hdf5[i])

        # return self.test_x_hdf5, self.test_y_hdf5
        return self._generate_train_test(self.test_x, self.test_y)
    def get_train_data(self):
        # return self.train_x_hdf5, self.train_y_hdf5
        return self._generate_train_test(self.train_x, self.train_y)

    # def get_train_data(self, pos):
    #     self.readFromNumpyFiles()
    #     train_x, train_y = [], []
    #     for x, y in zip(self.train_x, self.train_y):
    #         y_summ = y[pos]
    #         if y_summ == 0:
    #             continue
    #         else:
    #             train_x += [x]
    #             train_y += [y_summ]
    #     self.train_x = np.array(train_x)
    #     self.train_y = np.array(train_y)
    #     return self.train_x, self.train_y
    #
    # def get_test_data(self, pos):
    #     test_x, test_y = [], []
    #     for x, y in zip(self.test_x, self.test_y):
    #         y_summ = y[pos]
    #         if y_summ == 0:
    #             continue
    #         else:
    #             test_x += [x]
    #             test_y += [y_summ]
    #     self.test_x = np.array(test_x)
    #     self.test_y = np.array(test_y)
    #     return self.test_x, self.test_y

    def readFromNumpyFiles(self):

        if not self.train_x_filenames or not self.train_y_filenames or not self.test_x_filenames or not self.test_y_filenames:
            raise FileNotFoundError("No train or test numpy files in that location")

        for i in self.train_x_filenames:
            data = np.load(i)['arr_0']
            self.train_x += list(data)
        for i in self.train_y_filenames:
            data = np.load(i)['arr_0']
            self.train_y += list(data)
        for i in self.test_x_filenames:
            data = np.load(i)['arr_0']
            self.test_x += list(data)
        for i in self.test_y_filenames:
            data = np.load(i)['arr_0']
            self.test_y += list(data)

class PositionsDataLoader(DataLoaderBase):

    def __init__(self):
        self.train_x_filenames = sorted(glob.glob('training_data/positions/processed/*_train_x*.npz'))
        self.test_x_filenames = sorted(glob.glob('training_data/positions/processed/*_test_x*.npz'))
        super().__init__()

    def get_train_data(self):
        result_x, result_y = [], []
        order = [[1,0,0,0,0],[0,1,0,0,0],[0,0,1,0,0],[0,0,0,1,0],[0,0,0,0,1]]
        for x in self.train_x:

            comp_order = ((comp,pos) for comp, pos in zip(x, order))
            for champ_position in itertools.permutations(comp_order, 5):
                champ_position = np.array(champ_position)
                team_comp = np.stack(champ_position[:, 0])
                positions = np.stack(champ_position[:, 1])
                result_x.append(
                    np.concatenate((np.ravel(team_comp[:, 0]), np.ravel(team_comp[:, 1:3]), np.ravel(team_comp[:, 3:])),
                                   axis=0))
                result_y.append(np.ravel(positions))
        return np.array(result_x), np.array(result_y)

    def get_test_data(self):
        result_x, result_y = [], []
        order = [[1,0,0,0,0],[0,1,0,0,0],[0,0,1,0,0],[0,0,0,1,0],[0,0,0,0,1]]
        for x in self.test_x:

            comp_order = ((comp,pos) for comp, pos in zip(x, order))
            for champ_position in itertools.permutations(comp_order, 5):
                champ_position = np.array(champ_position)
                team_comp = np.stack(champ_position[:, 0])
                positions = np.stack(champ_position[:, 1])
                result_x.append(
                    np.concatenate((np.ravel(team_comp[:, 0]), np.ravel(team_comp[:, 1:3]), np.ravel(team_comp[:, 3:])),
                                   axis=0))
                result_y.append(np.ravel(positions))
        return np.array(result_x), np.array(result_y)

    def readFromNumpyFiles(self):

        if not self.train_x_filenames or not self.test_x_filenames:
            raise FileNotFoundError("No train or test numpy files in that location")

        for i in self.train_x_filenames:
            data = np.load(i)['arr_0']
            self.train_x += list(data)
        for i in self.test_x_filenames:
            data = np.load(i)['arr_0']
            self.test_x += list(data)

if __name__ == "__main__":
    d = NextItemsDataLoader()
    d.build_HDF5()
    # h5f = h5py.File('training_data/next_items/hdf5/data.h5', 'r')
    # train_x = h5f['train_x']
    # print(train_x.shape)
    # rando = np.random.randint(24000000, size=128*100).reshape((100,128))
    # for i in rando:
    #     t1 = time.time()
    #     tmp = train_x[sorted(list(i))]
    #     t2 = time.time()-t1
    #     print(t2)


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right

#Problem 1: Binary Search Tree Iterator
# Time complexity: O(n) to create the iterator, next costs O(1) time
# O(h) space, h is the height of the tree
#Worked on leetcode? Yes
'''
In the initialisation of the iterator, we will do an inorder traversal,
so we know that the elements array will be sorted , then we reverse it
and pop from there
'''

class BSTIterator:

    def __init__(self, root: TreeNode):
        self.inorder=[]
        st=[]
        while root or st:
            while root:
                st.append(root)
                root=root.left
            root=st.pop()
            self.inorder.append(root.val)
            root=root.right
        self.inorder=self.inorder[::-1]

    def next(self) -> int:
        """
        @return the next smallest number
        """
        if self.hasNext():
            return self.inorder.pop()

    def hasNext(self) -> bool:
        """
        @return whether we have a next smallest number
        """
        if self.inorder:
            return True
        return False

# Definition for singly-linked list.
# class ListNode:
#     def __init__(self, val=0, next=None):
#         self.val = val
#         self.next = next

#problem 2: Reorder List
#Time Complexity: O(n)
#Space Complexity : O(n)
#Worked on leetcode? Yes
'''
Traverse the linked list and keep a hashmap with indices as keys
and values as pointer to the node. This the orginal list. Then, from the
last element, we point last-1 to None, point last to start+1 and point
start to last.We increment the start pointer and decrement the last pointer.
We do this until first is >= to last index.
'''

class Solution:
    def reorderList(self, head: ListNode) -> None:
        """
        Do not return anything, modify head in-place instead.
        """
        #edge case:
        if not head or not head.next:
            return head

        h={}
        idx=0
        while head:
            h[idx]=head
            head=head.next
            idx+=1
        start=0
        last=idx-1
        count=0

        #We don't want to more then n/2 swaps
        while count<int((idx-1)/2):
            h[last-1].next=None
            h[last].next=h[start+1]
            h[start].next=h[last]
            last-=1
            start+=1
            count+=1

#problem 3: Reorder List
#Time Complexity: O(n)
#Space Complexity : O(n)
#Worked on leetcode? Yes

#Problem 3: Delete without head pointer
#Time complexity: O(1)
#Space complexity: O(1)
#Did it work on geeksforgeeks? YES

'''
Since we only check value, we can change the present value
of our node as the next one, and point the current node to
the next next one
'''
def deleteNode(curr_node):
    #code here
    if not curr_node.next:
        curr_node=None
        return
    curr_node.data=curr_node.next.data
    curr_node.next=curr_node.next.next

#Problem 4: Intersection of Two linked Lists
#Time Complexity: O(n)
#Space Complexity: O(1)
#Worked on Leetcode? Yes

'''
Brute force: Compare each node to all the elements  in the other linked list
in an O(n^2) fashion until we find it.
'''
'''
Main idea: We see that as soon as they connect,
the length of the remaining linkedlist is the same. So if their lengths differ
by k,we only have to compare index i to index i+K in the longest array
'''
class Solution:
    def getIntersectionNode(self, headA: ListNode, headB: ListNode) -> ListNode:
        #edge case:
        if not headA or not headB:
            return None

        #find lengths
        nA=0
        nB=0
        currA=headA
        currB=headB
        while currA:
            nA+=1
            currA=currA.next
        while currB:
            nB+=1
            currB=currB.next
        diff=abs(nA-nB)
        if nA>=nB:
            big=headA
            small=headB
        else:
            big=headB
            small=headA

        for i in range(diff):
            big=big.next
        while small:
            if small==big:
                return small
            small=small.next
            big=big.next
        return None


shopping_cart = []
shopping_cart_price = []

while True:
    product_input = input("place your product in the cart:")
    product_price = int(input("Enter the price of your product:"))

    shopping_cart.append(product_input)                #adds a new product to the product list
    shopping_cart_price.append(product_price)          #adds a new price to the price list

    if product_input == "end" and product_price == 0:
        shopping_cart.pop()                            #removes "end" as a member of the list
        shopping_cart_price.pop()
        Total_shopping_price = sum(shopping_cart_price)

        print("The list of product in your cart are:",shopping_cart)
        print("The price list of your items is:",shopping_cart_price)
        print("you have",len(shopping_cart),"product in your cart")
        print("Your are paying a total of ",Total_shopping_price,"dollars")
        break

#PROGRAM TO GIVE OPTION ON REMOVING FROM THE LIST
Request_to_remove = input("do you want to remove any item?")
if Request_to_remove == "yes":
    item_to_remove = input("Select an item to remove: ")
    if item_to_remove in shopping_cart:
        shopping_cart.remove(item_to_remove)
        print("The new list of product in your cart are:",shopping_cart)
        print("you have",len(shopping_cart),"product in your cart")
        print("Your total new price is ",Total_shopping_price,"dollars")
    else:
        print("This item is not in your list")
else:
    print("The list of product in your cart are:",shopping_cart)
    print("The price list of your items is:",shopping_cart_price)
    print("you have",len(shopping_cart),"product in your cart")
    print("Your are paying a total of ",Total_shopping_price,"dollars")
Collapse


from __future__ import annotations

from .type import Type
from ..exceptions import ValidationError

class Object(Type):

    accept_types = (object, dict)
    type = "object"

    def __init__(
        self,
        properties: dict,
        title: str = None,
        description: str = None,
        required: list = None,
        nullable: bool = False,
        default=None,
        deprecated: bool = False,
        read_only: bool = None,
        write_only: bool = None
    ):
        super().__init__()
        self.write_only = write_only
        self.read_only = read_only
        self.deprecated = deprecated
        self.default = default
        self.nullable = nullable
        self.properties = properties
        self.required = required if required is not None else ()
        self.title = title
        self.description = description

    def validate(self, value):
        value = super().validate(value)
        for prop in self.required:
            if prop not in value:
                raise ValidationError(f"Missing required property `{prop}` in passed dataset `{value}`")

        for key, prop in self.properties.items():
            if key not in value:
                continue
            value[key] = prop.validate(value[key])

        return value

    def to_doc(self):
        doc = self._get_base_doc()

        if self.required is not None:
            doc["required"] = self.required

        doc["properties"] = {}
        for key, prop in self.properties.items():
            doc["properties"][key] = prop.to_doc()

        return doc


import datetime, os, sys
import constants.constants as C

class ptf:
	def __init__(self):
		self.fileName = ""
		self.root_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
		self.dateFile()

	def dateFile(self):
		self.fileName = str(datetime.date.today())
		self.jira = self.root_dir + "/" +  C.jira_op + "/" + self.fileName
		self.msc = self.root_dir + "/" + C.msc_op + "/" + self.fileName
		self.noTicket = self.root_dir + "/" + C.no_ticket_op + "/" + self.fileName
		with open(self.jira, "a+") as fo:
			fo.write("")
		with open(self.msc, "a+") as fo:
			fo.write("")
		with open(self.noTicket, "a+") as fo:
			fo.write("")

	def writeOut(self, content, type):
		if type == C.JIRA:
			floc = self.jira
		elif type == C.MSC:
			floc = self.msc
		elif type == C.NOTI:
			floc = self.noTicket
		else:
			return
		with open(floc, "a+") as fo:
			fo.write(content)
			fo.write(" ")
		return


# model settings
model = dict(
    type='LXMERT',
    params=dict(
        random_initialize=False,
        num_labels=3129,
        # BertConfig
        vocab_size=30522,
        hidden_size=768,
        num_hidden_layers=12,
        num_attention_heads=12,
        intermediate_size=3072,
        hidden_act='gelu',
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        max_position_embeddings=512,
        type_vocab_size=2,
        initializer_range=0.02,
        layer_norm_eps=1e-12,
        pad_token_id=0,
        #
        mode='lxr',
        l_layers=9,  # 12
        x_layers=5,  # 5
        r_layers=5,  # 0
        visual_feat_dim=2048,
        visual_pos_dim=4,
        freeze_base=False,
        max_seq_length=20,
        model='bert',
        training_head_type='vqa2',
        bert_model_name='bert-base-uncased',
        pretrained_path='/home/datasets/mix_data/iMIX/data/models/model_LXRT.pth',
        label2ans_path='/home/datasets/mix_data/lxmert/vqa/trainval_label2ans.json',
    ))

loss = dict(type='LogitBinaryCrossEntropy')

optimizer = dict(
    type='BertAdam1',
    lr=5e-5,
    weight_decay=0.01,
    eps=1e-6,
    betas=[0.9, 0.999],
    max_grad_norm=-1,
    training_encoder_lr_multiply=1,
)
optimizer_config = dict(grad_clip=dict(max_norm=5))
'''
fp16 = dict(
    init_scale=2.**16,
    growth_factor=2.0,
    backoff_factor=0.5,
    growth_interval=2000,
)
'''

lr_config = dict(
    warmup=0.1,
    warmup_method='warmup_linear',
    # max_iters=55472,  # ceil(totoal 443753 / batch size 32) * epoch size  datasets: train
    max_iters=79012,  # floor(totoal 632117 / batch size 32) * epoch size  datasets: train, nominival
    policy='BertWarmupLinearLR')
total_epochs = 4

seed = 9595


# -*- coding: utf-8 -*-

from datetime import datetime
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict
from flask import (
    abort,
    redirect,
    render_template,
    request,
    Response,
    url_for,
    g
    )

from .. import app, lastuser, redis_store
from ..models import db, webmail_domains, JobCategory, JobPost, JobType, POSTSTATUS, newlimit, JobLocation
from ..search import do_search
from ..views.helper import getposts, getallposts, location_geodata
from ..uploads import uploaded_logos

@app.route('/', subdomain='<subdomain>')
@app.route('/')
def index(basequery=None, type=None, category=None, md5sum=None, domain=None,
        location=None, title=None, showall=True, statuses=None):
    now = datetime.utcnow()
    if basequery is None and not (g.user or g.kiosk or (g.board and not g.board.require_login)):
        showall = False
    posts = list(getposts(basequery, pinned=True, showall=showall, statuses=statuses))

    # Cache viewcounts (admin view or not)
    redis_pipe = redis_store.connection.pipeline()
    viewcounts_keys = [p.viewcounts_key for p in posts]
    for key in viewcounts_keys:
        redis_pipe.hgetall(key)
    viewcounts_values = redis_pipe.execute()
    g.viewcounts = dict(zip(viewcounts_keys, viewcounts_values))

    if posts:
        employer_name = posts[0].company_name
    else:
        employer_name = u'a single employer'

    # Make lookup slightly faster in the loop below since 'g' is a proxy
    board = g.board

    if basequery is None and posts and not g.kiosk:
        # Group posts by email_domain on index page only, when not in kiosk mode
        grouped = OrderedDict()
        for post in posts:
            pinned = post.pinned
            if board is not None:
                blink = post.link_to_board(board)
                if blink is not None:
                    pinned = blink.pinned
            if pinned:
                # Make pinned posts appear in a group of one
                grouped.setdefault(('s', post.hashid), []).append((pinned, post))
            elif post.status == POSTSTATUS.ANNOUNCEMENT:
                # Make announcements also appear in a group of one
                grouped.setdefault(('a', post.hashid), []).append((pinned, post))
            elif post.email_domain in webmail_domains:
                grouped.setdefault(('ne', post.md5sum), []).append((pinned, post))
            else:
                grouped.setdefault(('nd', post.email_domain), []).append((pinned, post))
        pinsandposts = None
    else:
        grouped = None
        if g.board:
            pinsandposts = []
            for post in posts:
                pinned = post.pinned
                if board is not None:
                    blink = post.link_to_board(board)
                    if blink is not None:
                        pinned = blink.pinned
                pinsandposts.append((pinned, post))
        else:
            pinsandposts = [(post.pinned, post) for post in posts]

    return render_template('index.html', pinsandposts=pinsandposts, grouped=grouped, now=now,
                           newlimit=newlimit, jobtype=type, jobcategory=category, title=title,
                           md5sum=md5sum, domain=domain, employer_name=employer_name,
                           location=location, showall=showall,
                           is_siteadmin=lastuser.has_permission('siteadmin'))

@app.route('/drafts', subdomain='<subdomain>')
@app.route('/drafts')
@lastuser.requires_login
def browse_drafts():
    basequery = JobPost.query.filter_by(user=g.user)
    return index(basequery=basequery, statuses=[POSTSTATUS.DRAFT, POSTSTATUS.PENDING])

@app.route('/type/<name>', subdomain='<subdomain>')
@app.route('/type/<name>')
def browse_by_type(name):
    if name == 'all':
        return redirect(url_for('index'))
    ob = JobType.query.filter_by(name=name).first_or_404()
    basequery = JobPost.query.filter_by(type_id=ob.id)
    return index(basequery=basequery, type=ob, title=ob.title)

@app.route('/at/<domain>', subdomain='<subdomain>')
@app.route('/at/<domain>')
def browse_by_domain(domain):
    if not domain:
        abort(404)
    basequery = JobPost.query.filter_by(email_domain=domain)
    return index(basequery=basequery, domain=domain, title=domain, showall=True)

@app.route('/category/<name>', subdomain='<subdomain>')
@app.route('/category/<name>')
def browse_by_category(name):
    if name == 'all':
        return redirect(url_for('index'))
    ob = JobCategory.query.filter_by(name=name).first_or_404()
    basequery = JobPost.query.filter_by(category_id=ob.id)
    return index(basequery=basequery, category=ob, title=ob.title)

@app.route('/by/<md5sum>', subdomain='<subdomain>')
@app.route('/by/<md5sum>')
def browse_by_email(md5sum):
    if not md5sum:
        abort(404)
    basequery = JobPost.query.filter_by(md5sum=md5sum)
    return index(basequery=basequery, md5sum=md5sum, showall=True)

@app.route('/in/<location>', subdomain='<subdomain>')
@app.route('/in/<location>')
def browse_by_location(location):
    geodata = location_geodata(location)
    if not geodata:
        abort(404)
    basequery = JobPost.query.filter(db.and_(
        JobLocation.jobpost_id == JobPost.id, JobLocation.geonameid == geodata['geonameid']))
    return index(basequery=basequery, location=geodata, title=geodata['short_title'])

@app.route('/in/anywhere', subdomain='<subdomain>')
@app.route('/in/anywhere')
def browse_by_anywhere():
    basequery = JobPost.query.filter(JobPost.remote_location == True)
    return index(basequery=basequery)

@app.route('/feed', subdomain='<subdomain>')
@app.route('/feed')
def feed(basequery=None, type=None, category=None, md5sum=None, domain=None, location=None):
    title = "All jobs"
    if type:
        title = type.title
    elif category:
        title = category.title
    elif md5sum or domain:
        title = u"Jobs at a single employer"
    elif location:
        title = u"Jobs in {location}".format(location=location['short_title'])
    posts = list(getposts(basequery, showall=True))
    if posts:  # Can't do this unless posts is a list
        updated = posts[0].datetime.isoformat() + 'Z'
        if md5sum:
            title = posts[0].company_name
    else:
        updated = datetime.utcnow().isoformat() + 'Z'
    return Response(render_template('feed.xml', posts=posts, updated=updated, title=title),
        content_type='application/atom+xml; charset=utf-8')

@app.route('/type/<name>/feed', subdomain='<subdomain>')
@app.route('/type/<name>/feed')
def feed_by_type(name):
    if name == 'all':
        return redirect(url_for('feed'))
    ob = JobType.query.filter_by(name=name).first_or_404()
    basequery = JobPost.query.filter_by(type_id=ob.id)
    return feed(basequery=basequery, type=ob)

@app.route('/category/<name>/feed', subdomain='<subdomain>')
@app.route('/category/<name>/feed')
def feed_by_category(name):
    if name == 'all':
        return redirect(url_for('feed'))
    ob = JobCategory.query.filter_by(name=name).first_or_404()
    basequery = JobPost.query.filter_by(category_id=ob.id)
    return feed(basequery=basequery, category=ob)

@app.route('/by/<md5sum>/feed', subdomain='<subdomain>')
@app.route('/by/<md5sum>/feed')
def feed_by_email(md5sum):
    if not md5sum:
        abort(404)
    basequery = JobPost.query.filter_by(md5sum=md5sum)
    return feed(basequery=basequery, md5sum=md5sum)

@app.route('/at/<domain>/feed', subdomain='<subdomain>')
@app.route('/at/<domain>/feed')
def feed_by_domain(domain):
    if not domain:
        abort(404)
    basequery = JobPost.query.filter_by(email_domain=domain)
    return feed(basequery=basequery, domain=domain)

@app.route('/in/<location>/feed', subdomain='<subdomain>')
@app.route('/in/<location>/feed')
def feed_by_location(location):
    geodata = location_geodata(location)
    if not geodata:
        abort(404)
    basequery = JobPost.query.filter(db.and_(
        JobLocation.jobpost_id == JobPost.id, JobLocation.geonameid == geodata['geonameid']))
    return feed(basequery=basequery, location=geodata)

@app.route('/archive')
def archive():
    def sortarchive(order_by):
        current_order_by = request.args.get('order_by')
        reverse = request.args.get('reverse')
        if order_by == current_order_by:
            if reverse is None:
                reverse = False
            try:
                reverse = bool(int(reverse))
            except ValueError:
                reverse = False
            reverse = int(not reverse)
        return url_for('archive', order_by=order_by,
            reverse=reverse,
            start=request.args.get('start'),
            limit=request.args.get('limit'))

    order_by = {
        'date': JobPost.datetime,
        'headline': JobPost.headline,
        'company': JobPost.company_name,
        'location': JobPost.location,
        }.get(request.args.get('order_by'))
    reverse = request.args.get('reverse')
    start = request.args.get('start', 0)
    limit = request.args.get('limit', 100)
    if order_by is None and reverse is None:
        order_by = JobPost.datetime
        reverse = True
    try:
        if reverse is not None:
            reverse = bool(int(reverse))
    except ValueError:
        reverse = None
    try:
        if start is not None:
            start = int(start)
    except ValueError:
        start = 0
    try:
        if limit is not None:
            limit = int(limit)
    except ValueError:
        limit = 100
    count, posts = getallposts(order_by=order_by, desc=reverse, start=start, limit=limit)

    if request.is_xhr:
        tmpl = 'archive_inner.html'
    else:
        tmpl = 'archive.html'
    return render_template(tmpl, order_by=request.args.get('order_by'),
        posts=posts, start=start, limit=limit, count=count,
        # Pass some functions
        min=min, max=max, sortarchive=sortarchive)

@app.route('/sitemap.xml', subdomain='<subdomain>')
@app.route('/sitemap.xml')
def sitemap():
    sitemapxml = '<?xml version="1.0" encoding="UTF-8"?>\n'\
                 '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
    # Add job type pages to sitemap
    for item in JobType.query.filter_by(public=True).all():
        sitemapxml += '  <url>\n'\
                      '    <loc>%s</loc>\n' % url_for('browse_by_type', name=item.name, _external=True) + \
                      '  </url>\n'
    # Add job category pages to sitemap
    for item in JobCategory.query.filter_by(public=True).all():
        sitemapxml += '  <url>\n'\
                      '    <loc>%s</loc>\n' % url_for('browse_by_category', name=item.name, _external=True) + \
                      '  </url>\n'
    # Add live posts to sitemap
    for post in getposts(showall=True):
        sitemapxml += '  <url>\n'\
                      '    <loc>%s</loc>\n' % url_for('jobdetail', hashid=post.hashid, _external=True) + \
                      '    <lastmod>%s</lastmod>\n' % (post.datetime.isoformat() + 'Z') + \
                      '    <changefreq>monthly</changefreq>\n'\
                      '  </url>\n'
    sitemapxml += '</urlset>'
    return Response(sitemapxml, content_type='text/xml; charset=utf-8')

@app.route('/logo/<hashid>', subdomain='<subdomain>')
@app.route('/logo/<hashid>')
def logoimage(hashid):
    post = JobPost.query.filter_by(hashid=hashid).first_or_404()
    if not post.company_logo:
        # If there's no logo (perhaps it was deleted), don't try to show one
        abort(404)
    if post.status in [POSTSTATUS.REJECTED, POSTSTATUS.SPAM]:
        # Don't show logo if post has been rejected. Could be spam
        abort(410)
    return redirect(uploaded_logos.url(post.company_logo))

@app.route('/search', subdomain='<subdomain>')
@app.route('/search')
def search():
    now = datetime.utcnow()
    results = sorted(do_search(request.args.get('q', u''), expand=True),
        key=lambda r: getattr(r, 'datetime', now))
    results.reverse()
    return render_template('search.html', results=results, now=now, newlimit=newlimit)


from django.http import HttpResponse
from backend.models import Files
from django import db
from django.core.files import File
import os

def upload_file(filename):
    try:
        module_dir = os.path.dirname(__file__)
        file_path = os.path.join(module_dir, filename)
        with open(file_path, 'rb') as f:
            data = File(f).read()
        try:
            Files(filename=filename, file=data).save()
            return HttpResponse('Success', status=200)
        except db.IntegrityError as db_e:
            return HttpResponse(db_e, status=409)
    except IOError as e:
        return HttpResponse(e, status=409)

def get_file(filename):
    try:
        files = Files.objects.all().values()
        for file in files:
            if file['filename'] == filename:
                try:
                    module_dir = os.path.dirname(__file__)
                    file_path = os.path.join(module_dir, filename)
                    with open(file_path, 'wb') as f:
                        File(f).write(file['file'])
                    return HttpResponse('Success', status=200)
                except IOError as e:
                    return HttpResponse(e, status=409)
    except db.IntegrityError as db_e:
        return HttpResponse(db_e, status=409)


"""
A module consisting of sinks that the processed events will end up at.
"""

from abc import abstractmethod, ABC
from collections import deque, namedtuple
from datetime import datetime
import logging
import time
import csv

from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

class Sink(ABC):
    """
    An abstract base class for pipeline sinks.
    """

    @abstractmethod
    def handle(self, events):
        """
        Handle each of the given stream of events.
        """
        pass

    @staticmethod
    def parallel(*sinks):
        return ParallelSink(sinks)

class ParallelSink(Sink):
    def __init__(self, sinks):
        self.sinks = sinks

    def handle(self, events):
        for sink in self.sinks:
            sink.handle(events)

class CalculatesAverage(Sink):
    """
    Averaging_period and expiry_time are in seconds. Will calculate a moving average.
    """

    def __init__(self, averaging_period, expiry_time):
        self.averaging_period_ms = 1000 * averaging_period
        self.expiry_time_ms = 1000 * expiry_time

        # Creates millisecond-size bins for calculating moving averages
        current_time_ms = 1000 * time.time()
        self.bins = deque(
            [AverageBin(start=(current_time_ms - self.expiry_time_ms), end=(current_time_ms - self.expiry_time_ms),
                        values=[])])

    def handle(self, events):
        for event in events:
            self.add_to_bin(event)
            expired_bin = self.maybe_expire_first_bin_and_get_average()

            if expired_bin is not None:
                average = expired_bin.average
                logger.info(f'Average value for {average.start} to {average.end} is {average.value}')

    def add_to_bin(self, event):
        # Checks if the event is old and should be ignored (i.e. happened in the past)
        if self.bins[0].start > event.timestamp:
            logger.debug(f'Not averaging old event at timestamp {event.timestamp}')
            return

        # Checks if new bins are needed to process the event
        while self.bins[-1].end < event.timestamp:
            current_last_start, current_last_end = self.bins[-1].start, self.bins[-1].end
            logger.debug(
                f'Adding new bin to deal with event at timestamp {event.timestamp} (Current last bucket is {current_last_start} to {current_last_end})')
            self.bins.append(
                AverageBin(start=current_last_end, end=(current_last_end + self.averaging_period_ms), values=[]))

        # Find the correct bin for an event and add the event's value
        bin_index = int(event.timestamp - self.bins[0].start) // self.averaging_period_ms
        self.bins[bin_index].values.append(event.value)
        self.write_bins_to_file()

    def write_bins_to_file(self):
        # Write bin values to a csv file so overall intensity trends can be identified
        with open('Gas_Averages.csv', mode='w') as csv_file:
            fieldnames = ['Bin Start', 'Bin End', 'Average Value']
            writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
            writer.writeheader()
            for i in range(len(self.bins)):
                if len(self.bins[i].values) != 0:
                    writer.writerow(
                        {'Bin Start': self.bins[i].average.start,
                         'Bin End': self.bins[i].average.end,
                         'Average Value': self.bins[i].average.value})

    def maybe_expire_first_bin_and_get_average(self):
        current_time_ms = 1000 * time.time()
        if current_time_ms - self.expiry_time_ms > self.bins[0].end:
            return self.bins.popleft()

class AverageBin(namedtuple('AverageBucket', 'start end values')):

    @property
    def average(self):
        start_datetime = datetime.fromtimestamp(self.start / 1000)
        end_datetime = datetime.fromtimestamp(self.end / 1000)
        average_value = (sum(self.values) / len(self.values)) if self.values else 0
        return Average(start=start_datetime, end=end_datetime, value=average_value)

class Average(namedtuple('Average', 'start end value')):
    """
    A record of the average value of events between two timestamps.
    """

class EventLocation(namedtuple('EventLocation', 'x y value')):
    """
    An event location, consisting of x and y coordinates the intensity of gas at that event.
    """

class LocationAverage(Sink):
    """
    Calculates the weighted average of each location with an event registered there.
    """

    def __init__(self, locations):
        self.locations = locations

    def handle(self, events):
        self.find_event_locations(events)

    def find_event_locations(self, events):
        event_locations = []
        for event in events:
            for location in self.locations:
                if location.id == event.location_id:
                    event_locations.append(EventLocation(
                        x=location.x, y=location.y, value=event.value
                    ))

        self.plot_results(event_locations)

    @staticmethod
    def plot_results(event_locations):
        x_points = []
        y_points = []
        values = []
        for i in range(len(event_locations)):
            x_points.append(event_locations[i].x)
            y_points.append(event_locations[i].y)
            values.append(event_locations[i].value)

        fig = plt.figure()
        ax = fig.gca(projection='3d')

        surf = ax.plot_trisurf(x_points, y_points, values, cmap='viridis', linewidth=0, antialiased=True)

        fig.colorbar(surf)
        ax.set_xlabel(r'x')
        ax.set_ylabel(r'y')
        ax.set_zlabel(r'Value')
        ax.set_title('GasMon')

        plt.show()