import io
import os

import pandas as pd
import pyfits
from astropy.io.votable import parse
from astropy.io.votable.tree import VOTableFile, Resource, Table, Field
from sklearn.cross_validation import StratifiedKFold, KFold
from sklearn.cross_validation import train_test_split

def load_header(uri):
    votable = parse(uri)
    table = votable.get_table_by_index(0)
    values =  table.array[0][1].tolist()
    return [str(value) for value in values] + ["class"]

def load_set(uri, format='csv', header=None, delimiter=','):
    print(str(header))
    return pd.read_csv(uri, header=None, names=header, sep=None, dtype=None,
                       na_values='?', skipinitialspace=True)

def _to_votable(data, file_name):
    votable = VOTableFile()
    resource = Resource()
    votable.resources.append(resource)
    table = Table(votable)
    resource.tables.append(table)
    columns = data.columns
    if data.columns[-1] == 'class':
        columns = columns[:-1]
    fields = [
        Field(votable, name="intensities", datatype="double", arraysize='*')]
    table.fields.extend(fields)
    table.create_arrays(1)
    table.array[0] = columns.tolist()
    votable.to_xml(file_name)

def _write_csv(data, uri, header=None, separator=',', dtypes=None):
    with io.open(uri, 'w', encoding='utf-8') as out:
        if header is not None:
            _to_votable(header,
                        "meta.xml")

        for row in data:
            rec_num = 0
            for record in row:
                val = record
                if (dtypes is not None and 'int' in str(dtypes[rec_num])):
                    val = int(val)
                elif (dtypes is not None and 'float' in str(dtypes[rec_num])):
                    val = float(val)
                out.write(str(val))
                if (rec_num != len(row) - 1):
                    out.write(separator)
                rec_num += 1
            out.write('\n')

def _parse_fits(uri):
    fits = pyfits.open(uri, memmap=False)
    dat = fits[1].data
    fits.close()
    return dat.tolist()

def split_train_set(uri, label=-1, ratio=0.67, sep=',', header=None):
    header_num = None if not header else 0
    array = pd.read_csv(uri, delimiter=sep, skipinitialspace=True,
                        na_values=['?'])
    train, test = train_test_split(array.values, train_size=ratio)
    base_name, ext = os.path.splitext(os.path.basename(uri))
    directory = os.path.dirname(uri)
    train_name = directory + base_name + '_train' + ext
    test_name = directory + base_name + '_score' + ext
    _write_csv(train, train_name, separator=sep, header=None)
    _write_csv(test, test_name, separator=sep, header=None)
    return (train_name, test_name)

def create_xvalidation_files(data_uri, data_conf, header, configuration,
                             target=None, base_folder='./result/xvalidation'):
    df = pd.read_csv(data_uri, sep=None, header=None, names=header,
                     skipinitialspace=True, na_values=['?'])
    kfold = None
    labels = df[target].values
    folds = configuration['folds']
    if (target is not None):
        kfold = StratifiedKFold(labels, folds)
    else:
        kfold = KFold(n=len(labels), n_folds=folds)

    if not os.path.exists(base_folder):
        os.makedirs(os.path.abspath(base_folder))
    i = 1
    uris = []
    for train, test in kfold:
        train_uri = base_folder + '/train_' + str(i)
        test_uri = base_folder + '/test_' + str(i)
        _write_csv(df.values[train], train_uri, header=header, dtypes=df.dtypes)
        _write_csv(df.values[test], test_uri, header=header, dtypes=df.dtypes)
        uris.append((train_uri, test_uri))
        i += 1
    return uris


"""
Постановка задачи

Разберем архитектуру LeNet и AlexNet для решения задач распознавания
изображений. Применим их для анализа исходных изображений.

Обучим модели, используя последовательную загрузку данных. Проведем оценку
качества предсказания по коэффициенту сходства.

Данные:

https://video.ittensive.com/machine-learning/clouds/train.csv.gz (54 Мб)
https://video.ittensive.com/machine-learning/clouds/train_images_small.tar.gz
(212 Мб)

Соревнование: https://www.kaggle.com/c/understanding_cloud_organization/

© ITtensive, 2020
"""

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import keras
from skimage import io
from sklearn.model_selection import train_test_split
from keras.models import Sequential
from keras.layers import Dense, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D, AveragePooling2D
from keras.layers import BatchNormalization, ZeroPadding2D
from keras import optimizers
import os

os.environ["KERAS_BACKEND"] = "plaidml.keras.backend"

"""Используемые функции"""

filesDir = "C:/PythonProject/ITtensive/machine_learning/train_images_small"
batch_size = 20
image_x = 525
image_y = 350
image_ch = 3

def mask_rate(a, x, y):
    b = a // 1400 + 0.0
    return np.round(
        x * (b * x // 2100) + y * (a % 1400) // 1400
    ).astype("uint32")

def calc_mask(px, x=image_x, y=image_y):
    p = np.array([int(n) for n in px.split(' ')]).reshape(-1, 2)
    mask = np.zeros(x * y, dtype='uint8')
    for i, l in p:
        mask[mask_rate(i, x, y) - 1: mask_rate(l + i, x, y)] = 1
    return mask.reshape(y, x).transpose()

def calc_dice(x):
    dice = 0
    px = x["EncodedPixels"]
    if px != px and x["target"] == 0:
        dice = 1
    elif px == px and x["target"] == 1:
        mask = calc_mask(px).flatten()
        target = np.ones(image_x*image_y, dtype='uint8')
        dice += 2 * np.sum(target[mask == 1]) / (np.sum(target) + np.sum(mask))
    return dice

def load_y(df):
    return np.array(
        df["EncodedPixels"].notnull().astype("int8")
    ).reshape(len(df), 1)

def load_x(df):
    x = [[]] * len(df)
    for j, file in enumerate(df["Image"]):
        x[j] = io.imread(os.path.join(filesDir, file))
    return np.array(x).reshape(len(df), image_y, image_x, image_ch)

def load_data(df, batch_size):
    while True:
        batch_start = 0
        batch_end = batch_size
        while batch_start < len(df):
            limit = min(batch_end, len(df))
            yield (load_x(df[batch_start:limit]),
                   load_y(df[batch_start:limit]))
            batch_start += batch_size
            batch_end += batch_size

"""Загрузка данных"""

data = pd.read_csv(
    'https://video.ittensive.com/machine-learning/clouds/train.csv.gz'
)

data["Image"] = data["Image_Label"].str.split("_").str[0]
data["Label"] = data["Image_Label"].str.split("_").str[1]
data.drop(labels=["Image_Label"], axis=1, inplace=True)
data_fish = data[data["Label"] == "Fish"]
print(data_fish.head())

"""Разделение данных"""

# Разделим всю выборку на 2 части случайным образом: 80% - для обучения модели,
# 20% - для проверки точности модели.

train, test = train_test_split(data_fish, test_size=0.2)
train = pd.DataFrame(train)
test = pd.DataFrame(test)
del data
print(train.head())

"""LeNet5"""

# Первая успешная архитектура сверточной нейросети, 1998

lenet = Sequential([
    Conv2D(
        6,
        (5, 5),
        input_shape=(image_y, image_x, image_ch),
        kernel_initializer="glorot_uniform",
        strides=(1, 1)
    ),
    Activation("relu"),
    AveragePooling2D(pool_size=(2, 2)),
    Conv2D(
        16,
        (5, 5),
        kernel_initializer="glorot_uniform",
        strides=(1, 1)
    ),
    Activation("relu"),
    AveragePooling2D(pool_size=(2, 2)),
    Flatten(),
    Activation("tanh"),
    Dense(120),
    Activation("tanh"),
    Dense(84),
    Activation("softmax"),
    Dense(1)
])

"""Обучение модели"""

# Обучим построенную модель и вычислим ее точность, используя самый лучший
# разделитель значений

def train_evaluate_model(model):
    model.compile(
        optimizer=optimizers.nadam_v2.Nadam(lr=0.05),
        loss="mean_absolute_error"
    )
    model.fit_generator(
        load_data(train, batch_size),
        epochs=50,
        steps_per_epoch=len(train) // batch_size
    )
    prediction = model.predict_generator(
        load_data(test, 1),
        steps=len(test),
        verbose=1
    )
    prediction = np.transpose(prediction)
    fig = plt.figure(figsize=(16, 8))
    ax = fig.add_subplot(1, 1, 1)
    ax.hist(prediction[0])
    ax.set_title("Fish")
    plt.show()
    acc = prediction[0].mean()
    acc_max = prediction[0].max()
    if acc == acc_max:
        test["target"] = np.round(prediction[0])
        return test.apply(calc_dice, axis=1, result_type="expand").mean()
    else:
        dice_best = 0
        for i in range(0, 20):
            acc += (acc_max - acc) * i / 20
            test["target"] = (prediction[0] >= acc).astype("int8")
            dice = test.apply(calc_dice, axis=1, result_type="expand")
            if dice_best < dice.mean():
                dice_best = dice.mean()
            else:
                break
        return dice_best

print("Keras, LeNet:", round(train_evaluate_model(lenet), 3))

del lenet

"""AlexNet и CaffeNet"""

# Первая сверточная нейросеть, победившая в ImageNetCaffeNet - однопроцессорная
# версия AlexNet

# Для реализации потребуется задать шаг свертки (strides) и дополнительные слои
# для заполнения границ изображения нулями, чтобы не уменьшать область после
# свертки.

alexnet = Sequential([
    Conv2D(
        96,
        (11, 11),
        input_shape=(image_y, image_x, image_ch),
        kernel_initializer='glorot_uniform',
        strides=(4, 4)
    ),
    Activation("relu"),
    BatchNormalization(),
    MaxPooling2D(pool_size=(3, 3), strides=(2, 2)),
    ZeroPadding2D(padding=(2, 2)),
    Conv2D(256, (5, 5), kernel_initializer='glorot_uniform'),
    Activation("relu"),
    BatchNormalization(),
    MaxPooling2D(pool_size=(3, 3), strides=(2, 2)),
    ZeroPadding2D(padding=(1, 1)),
    Conv2D(384, (3, 3), kernel_initializer='glorot_uniform'),
    Activation("relu"),
    BatchNormalization(),
    ZeroPadding2D(padding=(1, 1)),
    Conv2D(384, (3, 3), kernel_initializer='glorot_uniform'),
    Activation("relu"),
    BatchNormalization(),
    ZeroPadding2D(padding=(1, 1)),
    Conv2D(256, (3, 3), kernel_initializer='glorot_uniform'),
    Activation("relu"),
    BatchNormalization(),
    MaxPooling2D(pool_size=(3, 3), strides=(2, 2)),
    Flatten(),
    Activation("relu"),
    Dense(1024),
    Activation("relu"),
    Dense(1024),
    Activation("softmax"),
    Dense(1)
])

print("Keras, AlexNet:", round(train_evaluate_model(alexnet), 3))

Машинное обучение_3_сверточные нейросети/Часть_3_архитектуры нейронных сетей/LeNet_AlexNet_CaffeNet/LeNet_AlexNet_CaffeNet.py


import json
from typing import Tuple, Union
from telebot import TeleBot, types
from local_modules.path_worker.path_search import pathSearcher
from local_modules.site_parser.NTGSPIparser import Parser

from local_modules.vars.project_config import *
from local_modules.database_handler import dbHandler, router

# TODO:
# - add user create

def pp(o): print(json.dumps(o, indent=4, sort_keys=True, default=lambda o: o.__dict__))

with open(TELEGRAM_CRED, "r") as f:
    cred = json.load(f)

dbh = dbHandler.DBhandler(LOCAL_DATABASE, True)
r = router.Router(LOCAL_DATABASE)
bot = TeleBot(cred["telegram"]["token"])
np = Parser()

# np.sync()

bot.set_my_commands([
    types.BotCommand("/start", "Стоит начать с этой команды")
])

def workerPath(text:str, chat_id:int, user_id:int, message_id:Union[int, None]=None, serve:str="s", wai:object={}) -> None:
    kb = types.InlineKeyboardMarkup(row_width=5)
    answerText = wai["point"]["message"]
    onEdit = message_id is not None

    savedRowNum = 0
    savedRow = []
    serveAccords = {
        "s": "Выберите номер кабинета, который ближе всего к вам:",
        "d": "Выберите номер кабинета, который вам необходим:"
    }

    for key in wai["childs"]:
        child = wai["childs"][key]

        typeData = child["type"].split("?")

        if typeData[0] == "callback":
            kb.add(types.InlineKeyboardButton(
                    child["title"], callback_data=key))
        elif typeData[0] == "link":
            kb.add(
                types.InlineKeyboardButton(
                    child["title"], url=child["data"]))
        elif typeData[0] == "message":
            answerText = child["data"]
            onEdit = True
        elif typeData[0] == "grid":
            answerText = serveAccords[serve]

            typeGridData = typeData[1].split("&")
            grid = {}
            for tgd in typeGridData:
                tmp = tgd.split("=")
                grid[tmp[0]] = int(tmp[1])

            if savedRowNum == grid["row"]:
                savedRow.append(types.InlineKeyboardButton(
                    child["title"],
                    callback_data=("%s%d" % (serve, grid["map"]))))
            else:
                savedRowNum = grid["row"]
                kb.add(*savedRow)
                savedRow = [
                    types.InlineKeyboardButton(
                        child["title"],
                        callback_data=("%s%d" % (serve, grid["map"])))]

    return kb, answerText, onEdit

def workerService(text:str, chat_id:int, user_id:int, message_id:Union[int, None]=None, serve:str="s", wai:object={}) -> None:
    kb = types.InlineKeyboardMarkup(row_width=5)
    answerText = "none"

    if wai["action"] == "calcPath":
        us = dbh.getUserStorage(user_id)
        ps = pathSearcher()

        answerText = ps.getPathLegend(
            ps.minPath(us["source"], us["destination"])
        )

    return kb, answerText, True

def worker(text:str, chat_id:int, user_id:int, message_id:Union[int, None]=None, serve:str="s") -> None:
    kb = []

    dbh.addNewUser(user_id) # MOVE
    wai = r.pwd(user_id)

    if wai["type"] == "service":
        kb, answerText, onEdit = workerService(text, chat_id, user_id, message_id, serve, wai)
    elif wai["type"] == "path":
        kb, answerText, onEdit = workerPath(text, chat_id, user_id, message_id, serve, wai)

    if wai["type"] == "service" or wai["point"]["title"] != "root":
        kb.add(types.InlineKeyboardButton("Назад", callback_data="back"))
    if len(wai["full_path"].split("/")) > 2:
        kb.add(types.InlineKeyboardButton("Основное меню", callback_data="root"))

    if onEdit:
        bot.edit_message_text(
            text=answerText,
            chat_id=chat_id,
            message_id=message_id,
            reply_markup=kb
        )
    else:
        bot.send_message(
            text=answerText,
            chat_id=chat_id,
            reply_markup=kb)

@bot.message_handler(content_types=["text"])
def messageHandle(message):
    userId = int(message.from_user.id)
    chatId = int(message.chat.id)
    text = message.text.lower()

    worker(text, chatId, userId)

@bot.callback_query_handler(func=lambda call: True)
def callbackHandle(callback):
    userId = int(callback.from_user.id)
    chatId = int(callback.message.chat.id)
    messageId = int(callback.message.id)

    dbh.addNewUser(userId) # DEL

    onNav = True

    if "back" in callback.data:
        r.back(userId)
    elif "root" in callback.data:
        r.home(userId)
    elif "s" in callback.data:
        onNav = False
        pointId = int(callback.data.replace("s", ""))
        r.touch(userId, data={"source": pointId})
        serve = "d"
    elif "d" in callback.data:
        pointId = int(callback.data.replace("d", ""))
        r.touch(userId, data={"destination": pointId})
        r.cd(userId, 9999)
    elif callback.data.isdigit():
        r.cd(userId, int(callback.data))

    if onNav:
        worker(text="", chat_id=chatId, user_id=userId, message_id=messageId)
    else:
        worker(text="", chat_id=chatId, user_id=userId, message_id=messageId, serve=serve)

bot.polling()


from cogs import word

import datetime, random, re

import config

async def decoder(time_array):
	if time_array[1] in config.TIMES[0]:
		retTime = datetime.datetime.now() + datetime.timedelta(seconds=int(time_array[0]))
	elif time_array[1] in config.TIMES[1]:
		retTime = datetime.datetime.now() + datetime.timedelta(minutes=int(time_array[0]))
	elif time_array[1] in config.TIMES[2]:
		retTime = datetime.datetime.now() + datetime.timedelta(hours=int(time_array[0]))
	elif time_array[1] in config.TIMES[3]:
		retTime = datetime.datetime.now() + datetime.timedelta(hours=24*int(time_array[0]))
	elif time_array[1] in config.TIMES[4]:
		retTime = datetime.datetime.now() + datetime.timedelta(hours=168*int(time_array[0]))
	elif time_array[1] in config.TIMES[5]:
		retTime = datetime.datetime.now() + datetime.timedelta(hours=720*int(time_array[0]))
	elif time_array[1] in config.TIMES[6]:
		retTime = datetime.datetime.now() + datetime.timedelta(hours=8760*int(time_array[0]))
	else:
		retTime = datetime.datetime.now()

	return retTime

async def decodate(date):
	date_now = datetime.datetime.now()
	result = []

	if datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f") > date_now:
		date_obj = re.split(", |:|\.| ", str(datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f") - date_now))
	else:
		date_obj = re.split(", |:|\.| ", str(date_now - datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f")))

	if len(date_obj) == 6:
		del date_obj[5]

	if len(date_obj) == 5:
		days = int(date_obj[0])

		if days >= 30:
			result.append("{} {}".format(days // 30, await word.ending("месяц|месяца|месяцев", days // 30)))
			days %= 30 * (days // 30)
		elif days >= 7:
			result.append("{} {}".format(days // 7, await word.ending("неделя|недели|недель", days // 7)))
			days %= 7 * (days // 7)
		if days != 0:
			result.append("{} {}".format(days, await word.ending("день|дня|дней", days)))

		del date_obj[1]
		del date_obj[0]

	if int(date_obj[0]) > 0:
		result.append("{} {}".format(int(date_obj[0]), await word.ending("час|часа|часов", int(date_obj[0]))))
	if int(date_obj[1]) > 0:
		result.append("{} {}".format(int(date_obj[1]), await word.ending("минута|минуты|минут", int(date_obj[1]))))
	if int(date_obj[2]) > 0:
		result.append("{} {}".format(int(date_obj[2]), await word.ending("секунда|секунды|секунд", int(date_obj[2]))))

	if len(result) == 0:
		milliseconds = random.randint(1, 1000)

		return "{} {}".format(
			milliseconds, await word.ending("миллисекунда|миллисекунды|миллисекунд", milliseconds)
		)
	else:
		return "{}".format(result[0]) if len(result) == 1 else "{} {}".format(result[0], result[1])

async def get_time_is_str(time_str):
	try:
		time_str_output = datetime.datetime.strptime(str(time_str), "%Y-%m-%d %H:%M:%S.%f")
	except ValueError:
		time_str_output = datetime.datetime.strptime(str(time_str), "%Y-%m-%d %H:%M:%S")

	return time_str_output


import tensorflow as tf
import json
import csv
import pickle
import jieba
from model import RNet
import sys

batch_size = 64
w_emb_d = './embeddings/word_emb.pickle'
c_emb_d = './embeddings/char_emb.pickle'
rf = "./data/test.tfrecords"
model_d = "./best_model/model_8000.ckpt"

def parser():
    def parse(example):
        x = {
            "context_idxs": tf.FixedLenFeature([], tf.string),
            "ques_idxs": tf.FixedLenFeature([], tf.string),
            "context_char_idxs": tf.FixedLenFeature([], tf.string),
            "ques_char_idxs": tf.FixedLenFeature([], tf.string),
            "y1": tf.FixedLenFeature([], tf.string),
            "y2": tf.FixedLenFeature([], tf.string),
            "id": tf.FixedLenFeature([], tf.int64)
                                           }
        f = tf.parse_single_example(example,features=x)
        ids = []
        ids.append(tf.reshape(tf.decode_raw(f["context_idxs"], tf.int32), [1000]))
        ids.append(tf.reshape(tf.decode_raw(f["ques_idxs"], tf.int32), [100]))
        ids.append(tf.reshape(tf.decode_raw(f["context_char_idxs"], tf.int32), [1000, 16]))
        ids.append(tf.reshape(tf.decode_raw(f["ques_char_idxs"], tf.int32), [100, 16]))
        ids.append(f["id"])
        ys = []
        ys.append(tf.reshape(tf.decode_raw(f["y1"], tf.float32), [1000]))
        ys.append(tf.reshape(tf.decode_raw(f["y2"], tf.float32), [1000]))

        return ids[0], ids[1], ids[2], ids[3], ys[0], ys[1], ids[4]
    return parse

def make_example(filename):

    test_examples = {}
    count = 0
    print()
    with open(filename, "r") as fh:
        source = json.load(fh)
        for topic in source["data"]:
            for p in topic["paragraphs"]:
                context = p["context"].replace("''", '" ').replace("``", '" ')
                context_tokens = [token for token in jieba.cut(context,cut_all=False)]

                current = 0
                spans = []
                for token in context_tokens:
                    current = context.find(token, current)
                    spans.append((current, current + len(token)))
                    current = current + len(token)
                for qa in p["qas"]:
                    count += 1

                    starts, ends = [], []
                    answer_texts = []
                    answer_texts.append("123")
                    answer_span = []
                    for idx, span in enumerate(spans):
                        if not (3 <= span[0] or 0 >= span[1]):
                            answer_span.append(idx)
                    y1, y2 = answer_span[0], answer_span[-1]
                    starts.append(y1)
                    ends.append(y2)

                    test_examples[str(count)] = {
                        "context": context, "spans": spans, "answers": answer_texts, "uuid": qa["id"]}

    return test_examples

def main(argv):
    global rf, w_emb_d, c_emb_d, model_d

    with open(w_emb_d, 'rb') as handle:
        w_emb = pickle.load(handle)
    with open(c_emb_d, 'rb') as handle:
        c_emb = pickle.load(handle)

    test = make_example(argv[0])

    model = RNet(tf.data.TFRecordDataset(rf).map(
        parser()).repeat().batch(batch_size).make_one_shot_iterator(), w_emb[1], c_emb[1], trainable=False)

    with tf.Session() as sess:

        sess.run(tf.global_variables_initializer())
        saver = tf.train.Saver()
        saver.restore(sess, model_d)
        sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))

        final = []
        for _ in range(len(test) // batch_size + 1):
            ys = model.ys
            qa_id, _, y1, y2 = sess.run([model.qa_id, model.loss, ys[0], ys[1]])

            qq = []
            for qid, p1, p2 in zip(qa_id.tolist(), y1.tolist(), y2.tolist()):
                qq.append((test[str(qid)]["uuid"],test[str(qid)]["spans"][p1][0],test[str(qid)]["spans"][p2][1]-1))

            final.append(qq)

        f = open(argv[1], 'w')
        w = csv.writer(f)
        w.writerow(['id','answer'])
        count = 0
        for l in final:
            for k in l :
                if count < len(test):
                    w.writerow([k[0], " ".join(str(x) for x in range(k[1],k[2]+1))])
                    print(k)
                count += 1
        f.close()

if __name__ == '__main__':
    main(sys.argv[1:])


from common import *

class receiver:
    ACK = 0
    SEQ = 0
    def isCorrupted(self, packet):
        ''' Checks if a received packet has been corrupted during transmission.
        Return true if computed checksum is different than packet checksum.'''

        calc_cs = checksumCalc(packet)
        return (packet.checksum != calc_cs)

    def isDuplicate(self, packet):
        return (packet.seqNum == self.SEQ)

    def getNextExpectedSeqNum(self):
        if(self.expectedSeqNum==1):
            self.expectedSeqNum = 0
        elif self.expectedSeqNum ==0:
            self.expectedSeqNum = 1
        return self.expectedSeqNum

    def __init__(self, entityName, ns):
        self.entity = entityName
        self.networkSimulator = ns
        print("Initializing receiver: B: "+str(self.entity))

    def init(self):
        '''initialize expected sequence number'''
        self.expectedSeqNum = 0
        return

    def input(self, packet):
        '''This method will be called whenever a packet sent
        from the sender arrives at the receiver. If the received
        packet is corrupted or duplicate, it sends a packet where
        the ack number is the sequence number of the  last correctly
        received packet. Since there is only 0 and 1 sequence numbers,
        you can use the sequence number that is not expected.

        If packet is OK (not a duplicate or corrupted), deliver it to the
        application layer and send an acknowledgement to the sender
        '''
        self.SEQ = self.getNextExpectedSeqNum()

        if (self.isCorrupted(packet) or self.isDuplicate(packet)):

            packet.payload = ''

            if (self.isCorrupted(packet)):
                packet.ackNum = (self.ACK+1) % 2
            else:
                packet.ackNum = packet.seqNum % 2
            packet.checksum = packet.ackNum
            packet.seqNum = 0
            self.networkSimulator.udtSend(self.entity, packet)
            self.SEQ = self.getNextExpectedSeqNum()
        else:
            self.networkSimulator.deliverData(self.entity, packet.payload)

            npacket = Packet(0, self.ACK, 0, '')
            c = checksumCalc(npacket)
            npacket.checksum = c
            self.networkSimulator.udtSend(self.entity, npacket)
            self.ACK = (self.ACK + 1) % 2

        return


import numpy as np
import tensorflow as tf
from tensorflow.keras.layers import *
from tensorflow.keras.models import Model
from tensorflow.keras.regularizers import *
from tensorflow.keras import Input
from tensorflow.keras.optimizers import Adadelta,Adam
from tensorflow.keras import regularizers
import tensorflow.keras as keras
import tensorflow.keras.backend as kb
import pickle

#define model parameters
tf.keras.backend.set_epsilon(1e-9)
cIntInputSeqLen=384
cIntOutputSeqLen=128
cIntProcFeatures=1
cIntHiddenNode=64
cIntMaskValue=0

#define custom loss function
class CustomLoss:
    @staticmethod
    def RMSE(y_true, y_pred):
        return kb.sqrt(tf.keras.losses.mean_squared_error(y_true, y_pred))
    @staticmethod
    def MaskedRMSE(y_true,y_pred):
        isMask = kb.equal(y_true, 0)
        isMask = kb.all(isMask, axis=-1)
        isMask = kb.cast(isMask, dtype=kb.floatx())
        isMask = 1 - isMask
        isMask = kb.reshape(isMask,tf.shape(y_true))
        masked_squared_error = kb.square(isMask * (y_true - y_pred))
        masked_mse = kb.sum(masked_squared_error, axis=-1) / (kb.sum(isMask, axis=-1)+kb.epsilon())
        return kb.sqrt(masked_mse)
    @staticmethod
    def MaskedMSE(y_true,y_pred):
        isMask = kb.equal(y_true, 0)
        isMask = kb.all(isMask, axis=-1)
        isMask = kb.cast(isMask, dtype=kb.floatx())
        isMask = 1 - isMask
        isMask = kb.reshape(isMask,tf.shape(y_true))
        masked_squared_error = kb.square(isMask * (y_true - y_pred))
        masked_mse = kb.sum(masked_squared_error, axis=-1) / (kb.sum(isMask, axis=-1)+kb.epsilon())
        return masked_mse
    @staticmethod
    def MaskedMAE(y_true,y_pred):
        isMask = kb.equal(y_true, 0)
        isMask = kb.all(isMask, axis=-1)
        isMask = kb.cast(isMask, dtype=kb.floatx())
        isMask = 1 - isMask
        isMask = kb.reshape(isMask,tf.shape(y_true))
        masked_AE = kb.abs(isMask * (y_true - y_pred))
        masked_mae = kb.sum(masked_AE, axis=-1)/ (kb.sum(isMask, axis=-1)+kb.epsilon())
        return masked_mae
    #numpy function wrapper
    @staticmethod
    @tf.function
    def MaskedMAPE(y_true,y_pred):
        return tf.py_function(CustomLoss.numpyMaskedMAPE ,(y_true, y_pred), tf.double)
    @staticmethod
    def numpyMaskedMAPE(y_true,y_pred):
        MapeLst=list()
        for elm_t,elm_p in zip(y_true,y_pred):
            y_t=elm_t[0:np.count_nonzero(elm_t),:]
            y_p=elm_p[0:np.count_nonzero(elm_t),:]
            MapeLst.append(np.mean(((np.abs(y_t - y_p)+1e-10) / y_t)) * 100)
        return np.array(MapeLst,dtype=np.float)

#model structure definition
#input part
InputCap=tf.keras.layers.Input(shape=(cIntInputSeqLen,cIntProcFeatures))
MaskedInputCap=tf.keras.layers.Masking(mask_value=cIntMaskValue)(InputCap)
InputIR=keras.layers.Input(shape=(cIntInputSeqLen,cIntProcFeatures))
MaskedInputIR=tf.keras.layers.Masking(mask_value=cIntMaskValue)(InputIR)
CombInput = keras.layers.Concatenate(axis=-1)([MaskedInputCap, MaskedInputIR])
#encoder part
EncCtext=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(cIntHiddenNode,return_sequences=True))(CombInput)
EncCtext=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(cIntHiddenNode,return_sequences=True))(EncCtext)
EncCtext=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(cIntHiddenNode,return_sequences=True))(EncCtext)
EncCtextOut=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(cIntHiddenNode,return_sequences=False))(EncCtext)
CombCtext=tf.keras.layers.RepeatVector(cIntOutputSeqLen)(EncCtextOut)
#decoder part for capacity
Dec1=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(cIntHiddenNode,return_sequences=True))(CombCtext)
Dec1=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(cIntHiddenNode,return_sequences=True))(Dec1)
Dec1=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(cIntHiddenNode,return_sequences=True))(Dec1)
Dec1=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(cIntHiddenNode,return_sequences=True))(Dec1)
Dec1=tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(cIntHiddenNode*2, activation="relu",kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4)))(Dec1)
Dec1=tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(cIntHiddenNode/2, activation="relu"))(Dec1)
DecOutCap=tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(1, activation="relu"))(Dec1)
#decoder part for IR
Dec2=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(cIntHiddenNode,return_sequences=True))(CombCtext)
Dec2=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(cIntHiddenNode,return_sequences=True))(Dec2)
Dec2=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(cIntHiddenNode,return_sequences=True))(Dec2)
Dec2=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(cIntHiddenNode,return_sequences=True))(Dec2)
Dec2=tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(cIntHiddenNode*2, activation="relu",kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4)))(Dec2)
Dec2=tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(cIntHiddenNode/2, activation="relu"))(Dec2)
DecOutIR=tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(1, activation="relu"))(Dec2)

model=tf.keras.Model(inputs=[InputCap,InputIR],outputs=[DecOutCap,DecOutIR])

#load training data
trCap=pickle.load(open('trCap.p',"rb"))
trIR=pickle.load(open('trIR.p',"rb"))
vaCap=pickle.load(open('vaCap.p',"rb"))
vaIR=pickle.load(open('vaIR.p',"rb"))

#function to build training data
def BuildSeqs(Cap,IR):
    #declare list for input capacity and input IR as ls1 and ls2
    #declare list for output capacity and output IR as ls3 and ls4
    ls1,ls2,ls3,ls4=list(),list(),list(),list()
    for SelectCap,SelectIR in zip(Cap,IR):
        if(len(SelectIR)<len(SelectCap)):
            SelectCap=SelectCap[0:len(SelectIR)]
        elif(len(SelectCap)<len(SelectIR)):
            SelectIR=SelectIR[0:len(SelectCap)]
        SelectIR=SelectIR/0.04*100
        SelectCap=SelectCap/1.85*100
        x_lst=[]
        x_lst2=[]
        y_lst=[]
        y_lst2=[]
        for i in range(20,len(SelectIR)-20,1):
            splitPos = i
            inputSeq=SelectCap[0:splitPos]
            x_lst.append(inputSeq.reshape(-1,1))
            inputSeq2=SelectIR[0:splitPos]
            x_lst2.append(inputSeq2.reshape(-1,1))
            OutputSeq=SelectCap[splitPos-1::4].tolist()
            y_lst.append(OutputSeq)
            OutputSeq2=SelectIR[splitPos-1::4].tolist()
            y_lst2.append(OutputSeq2)
        #zero padding
        Proc_X=tf.keras.preprocessing.sequence.pad_sequences(x_lst,maxlen=cIntInputSeqLen,dtype='float64',padding='pre',value=0)
        Proc_X2=tf.keras.preprocessing.sequence.pad_sequences(x_lst2,maxlen=cIntInputSeqLen,dtype='float64',padding='pre',value=0)
        Proc_Y1=tf.keras.preprocessing.sequence.pad_sequences(y_lst,maxlen=cIntOutputSeqLen,dtype='float64',padding='post',value=0)
        Proc_Y2=tf.keras.preprocessing.sequence.pad_sequences(y_lst2,maxlen=cIntOutputSeqLen,dtype='float64',padding='post',value=0)
        Proc_X=Proc_X.reshape(-1,cIntInputSeqLen,cIntProcFeatures)
        Proc_X2=Proc_X2.reshape(-1,cIntInputSeqLen,cIntProcFeatures)
        Proc_Y1=Proc_Y1.reshape(-1,cIntOutputSeqLen,cIntProcFeatures)
        Proc_Y2=Proc_Y2.reshape(-1,cIntOutputSeqLen,cIntProcFeatures)
        for a,b,c,d in zip(Proc_X,Proc_X2,Proc_Y1,Proc_Y2):
            ls1.append(a)
            ls2.append(b)
            ls3.append(c)
            ls4.append(d)

    return (np.array(ls1,dtype=np.float),np.array(ls2,dtype=np.float)),(np.array(ls3,dtype=np.float),np.array(ls4,dtype=np.float))
#generate training data
x0,y0=BuildSeqs(trCap,trIR)
x1,y1=BuildSeqs(vaCap,vaIR)
#set checkpoint path
checkpoint_path0 = "capir/weight0_{epoch:04d}-{loss:.2f}-{val_loss:.2f}.hdf5"
checkpoint_path1 = "capir/weight1_{epoch:04d}-{loss:.2f}-{val_loss:.2f}.hdf5"
checkpoint_path2 = "capir/weight2_{epoch:04d}-{loss:.2f}-{val_loss:.2f}.hdf5"
#define checkpoint and early stopping callback
cp_callback0 = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path0, save_weights_only=True,verbose=1,save_freq='epoch')
cp_callback1 = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path1, save_weights_only=True,verbose=1,save_freq='epoch')
cp_callback2 = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path2, save_weights_only=True,verbose=1,save_freq='epoch')
callback2 = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=32,restore_best_weights=True)
#train model for stage 0
model.compile(optimizer=tf.keras.optimizers.Adam(lr=1e-4),loss=CustomLoss.MaskedMAE,metrics=[],loss_weights=([1.0,0.0]))
model.fit(x0,y0,batch_size=384,epochs=450,verbose=2,validation_data=(x1,y1),shuffle=True,callbacks=[cp_callback0,callback2])
model.save("2CapIR2_stg0.h5")
#freeze weights for stage 1
for idx in range(len(model.layers)):
    model.layers[idx].trainable=False
model.layers[11].trainable=True
model.layers[13].trainable=True
model.layers[15].trainable=True
model.layers[17].trainable=True
model.layers[19].trainable=True
model.layers[21].trainable=True
model.layers[23].trainable=True
#train model for stage 1
#sometimes need load checkpoint because the local best solution may not be the best solution for remaining stages.
#model.load_weights('capir/weight0_best.hdf5')
model.compile(optimizer=tf.keras.optimizers.Adam(lr=1e-4),loss=CustomLoss.MaskedMAE,metrics=[],loss_weights=([0.0,1.0]))
model.fit(x0,y0,batch_size=384,epochs=450,verbose=2,validation_data=(x1,y1),shuffle=True,callbacks=[cp_callback1,callback2])
model.save("2CapIR2_stg1.h5")
#defreeze all weights for stage 2
for idx in range(len(model.layers)):
    model.layers[idx ].trainable=True
#train model for stage 2
#model.load_weights('capir/weight1_best.hdf5')
model.compile(optimizer=tf.keras.optimizers.Adam(lr=1e-5),loss=CustomLoss.MaskedMAE,metrics=[],loss_weights=([1.0,1.0]))
#model.load_weights('capir/weight2_best.hdf5')
model.fit(x0,y0,batch_size=512,epochs=300,verbose=1,validation_data=(x1,y1),shuffle=True,callbacks=[cp_callback2,callback2])

#save model
model.save("2CapIR2.h5")
tf.keras.utils.plot_model(model,"2CapIR2.png",show_shapes=True,expand_nested=True)


# -*- coding: utf-8 -*-
"""Unit Tests for ga4gh.testbed.models.report_case module"""

from ga4gh.testbed.models.report_case import ReportCase
from ga4gh.testbed import constants as c

data = [
    {
        "name": "test case a",
        "status": c.RESULT_FAILURE,
        "debug": [
            "first debug message",
            "second debug message"
        ],
        "error": "an error occurred"
    },
    {
        "name": "htsget test case",
        "status": c.RESULT_SUCCESS,
        "info": [
            "first info message",
            "second info message"
        ]
    },
    {
        "name": "test refget service",
        "status": c.RESULT_WARNING,
        "warn": [
            "first warning message",
            "second warning message"
        ]
    }
]

def test_name():
    """pytest: test set/get name"""

    case = ReportCase()
    for case_info in data:
        case.set_name(case_info["name"])
        assert case.get_name() == case_info["name"]

def test_status():
    """pytest: test set/get status"""

    case = ReportCase()
    assert case.get_status() == None
    case.set_status_success()
    assert case.get_status() == c.RESULT_SUCCESS
    case.set_status_warning()
    assert case.get_status() == c.RESULT_WARNING
    case.set_status_failure()
    assert case.get_status() == c.RESULT_FAILURE
    case.set_status_skipped()
    assert case.get_status() == c.RESULT_SKIPPED

def test_messages():
    """pytest: test set/get different log level messages"""

    for case_info in data:
        case = ReportCase()

        keys = ["debug", "info", "warn"]
        add = {
            "debug": case.add_debug_msg,
            "info": case.add_info_msg,
            "warn": case.add_warn_msg
        }
        get = {
            "debug": case.get_debug,
            "info": case.get_info,
            "warn": case.get_warn
        }

        for key in ["debug", "info", "warn"]:
            if key in case_info.keys():
                for msg in case_info[key]:
                    add[key](msg)
                logs = get[key]()
                assert len(logs) == len(case_info[key])
                assert logs[0] == case_info[key][0]

        if "error" in case_info.keys():
            case.set_error(case_info["error"])
            assert case.get_error() == case_info["error"]

def test_as_json():
    """pytest: test as_json method"""

    for case_info in data:
        case = ReportCase()
        case.set_name(case_info["name"])
        set_status = {
            c.RESULT_SUCCESS: case.set_status_success,
            c.RESULT_WARNING: case.set_status_warning,
            c.RESULT_FAILURE: case.set_status_failure,
            c.RESULT_SKIPPED: case.set_status_skipped
        }
        add = {
            "debug": case.add_debug_msg,
            "info": case.add_info_msg,
            "warn": case.add_warn_msg
        }

        set_status[case_info["status"]]()

        for key in ["debug", "info", "warn"]:
            if key in case_info.keys():
                for msg in case_info[key]:
                    add[key](msg)

        if "error" in case_info.keys():
            case.set_error(case_info["error"])

        json = case.as_json()
        assert json["name"] == case_info["name"]
        assert json["status"] == case_info["status"]


import time
import asyncio

try:

    from telethon import TelegramClient, events, functions
except:
    import os

    os.system('pip install telethon')
    from telethon import TelegramClient, events, functions
    import asyncio

api_id = 00
api_hash = ''
phone = ''
chat_id =00   # group chat ID

client = TelegramClient('session_name22', api_id, api_hash)
client.start(phone=phone)

async def scrap_members():
    dilogs = await client.get_dialogs()
    for dilog in dilogs:
        print(f'name = {dilog.name},  id = {dilog.id}')
    groub = input('inter group id : ')
    users = await client.get_participants(int(groub))
    for user in users:
        user_id = user.username  # if you need to scrap id repace username with id
        if user_id is not None:
            print(user_id)
            # save user
            open('users.txt', 'a').write(f'{user_id}\n')

async def add_member():
    await client.start(phone)
    with open('users.txt', 'r') as f:
        user_ids = [line.strip() for line in f.readlines()]

    for user_name in user_ids:
        try:
            print(user_name)
            # group chat ID
            result = await client(
                functions.channels.InviteToChannelRequest(channel=chat_id,
                                                          users=[user_name]))

            # to not get banned by telegram

            if result:
                print("User %d successfully added to the group", user_name)
                time.sleep(50)
            else:
                print("Failed to add user %d to the group", user_name)
                time.sleep(50)

        except Exception as e:
            print(e)
            time.sleep(50)
            continue

print('for scrap members from group enter 1')
print('for add members to group enter 2')
need = int(input('what you need : '))
if need == 1:
    asyncio.get_event_loop().run_until_complete(scrap_members())
if need == 2:
    asyncio.get_event_loop().run_until_complete(add_member())


class Solution(object):
    def isValid(self, s):
        """
        :type s: str
        :rtype: bool
        """
        stack = []
        for c in s:
            if c == "(" or c == "{" or c == "[":
                stack.append(c)
            else:
                if not stack:
                    return False
                topVal = stack.pop()
                if (c == ")" and topVal != "(") or (c == "}" and topVal != "{") or (c == "]" and topVal != "["):
                    return False

        return True if len(stack) == 0 else False

solution = Solution()
print(solution.isValid("{[()]}"))


from nltk.corpus import propbank
import nltk
from nltk.corpus import wordnet as wn
import re

class Features:
    pass

def readFile(file_path="./CoNLL2009-ST-English-train.txt"):
    sentences = []
    sentence = []
    verbsWithLabel = []
    verbsNoLabel = []
    for line in open(file_path, 'r'):
        line = line.strip().lower()
        words = re.split(r'\t+', line)
        if (words[0] != None and words[0] != ''):  # with space as words[0] starts a new sentence
            sentence.append(words[1])
            if (words[12] == 'y'):
                verbsNoLabel.append(words[1])
                verbsWithLabel.append(words[13])
        else:
            obj = Features()
            obj.sentence = sentence
            obj.verbLabel = verbsWithLabel
            obj.verbNoLabel = verbsNoLabel
            sentences.append(obj)
            sentence = []
            verbsWithLabel = []
            verbsNoLabel = []
    return sentences

def senseFormat(sense):
    if (sense is not None):
        sense = sense.name()
        sense = sense[:sense.index('.v')] + sense[sense.index('v.') + 1:]
    else:
        sense = "?"
    return sense

def findDative(sentence, where, when, who, tokenized=False):
    if tokenized:
        input = sentence
    else :
        input = nltk.word_tokenize(sentence)
        where = nltk.word_tokenize(where)
        when = nltk.word_tokenize(when)
        who = nltk.word_tokenize(who)
    tags = nltk.pos_tag(input)
    out = None
    for word, tag in tags:
        if (tag[:2] == 'NN' and word not in where and word not in when and word not in who):
            out = word
            break

    return out

def readFileCsv():
    sentences = []
    sentence = []
    for line in open('./testverbs.csv', 'r'):
        line = line.strip().lower()
        words = re.split(r'\t+', line)
        if (words[0] != None and words[0] != ''):  # with space as words[0] starts a new sentence
            sentence.append(words[1])
        else:
            sentences.append(sentence)
            sentence = []
    return sentences

def lesk(context_sentence, ambiguous_word, pos=None, synsets=None):

    context = set(context_sentence)
    if synsets is None:
        synsets = wn.synsets(ambiguous_word)

    if pos:
        synsets = [ss for ss in synsets if str(ss.pos()) == pos]

    if not synsets:
        return None

    _, sense = max(
        (len(context.intersection(ss.definition().split())), ss) for ss in synsets
    )

    return sense

def start(test=True):

    print('Let  us start !!!')
    if test:
        # for ss in wn.synsets('play'):
        #    print(ss, ss.definition())

        sentences = readFile()
        gold = []
        pred = []
        tot = 0
        corr = 0
        numCorr = 0;
        nonecount = 0
        for i in range(0, len(sentences)):
            for j in range(0, len(sentences[i].verbNoLabel)):
                ## PREDICATE IDENTIFICATION
                #NO NEED WITH CONLL

                ## PREDICATE DISAMBIGUATION
                sense = lesk(sentences[i].sentence, sentences[i].verbNoLabel[j], 'v')
                if (sense is not None):
                    sense = sense.name()
                    sense = sense[:sense.index('.v')] + sense[sense.index('v.') + 1:]
                pred.append(sense)
                gold.append(sentences[i].verbLabel[j])
                if (sense is None):
                    nonecount += 1
                elif (sense == sentences[i].verbLabel[j]):
                    corr += 1
                elif (sense[-2:] == sentences[i].verbLabel[j][-2:]):
                    numCorr += 1
                tot += 1
        print("Total verbs = {} \n"
              "Correct matches = {} \n"
              "Nones = {} \n"
              "Number of sens = {} \n"
              "Score {}%".format(tot, corr, nonecount, numCorr, corr / tot * 100))
    else:
        sentences = readFileCsv()
        predicted = []
        tot = 0
        nonecount = 0
        counter = 0;
        for i in range(0, len(sentences)):
            for pred in extractPred(sentences[i], True, counter):
                sense = lesk(sentences[i], pred.verb, 'v')
                if (sense is not None):
                    sense = sense.name()
                    sense = sense[:sense.index('.v')] + sense[sense.index('v.') + 1:]
                else :
                    nonecount += 1
                    sense = "?"
                tot += 1
                obj = Features()
                obj.sense = sense
                obj.index = pred.index
                predicted.append(obj)
            counter += len(sentences[i])
        writeOutput(predicted)
        print('**************')
        print("Total verbs = {} \n"
              "Nones = {} \n"
              "Score {}%".format(tot, nonecount, (1 - (nonecount / tot)) * 100))

def writeOutput(pred):
    with open('./1766342_test.txt', 'w') as out, open('./testverbs.csv', 'r') as input:
        counter = 0;
        lines = [line.rstrip('\n') for line in input]
        for line in lines:
            words = re.split(r'\t+', line)
            if (words[0] is None or words[0] == ''):  # with space as words[0] starts a new sentence
                out.write('\n')
                continue
            found = False
            for i in range(0, len(pred)):
                if (pred[i].index == counter):
                    out.write(line + '\t' + 'y' + '\t' + pred[i].sense + '\n')
                    found = True
                    break
            if (found == False):
                out.write(line + '\t_\t_\n')
            counter += 1;

def extractPred(sentence, tokenized=False, counter=0):
    if tokenized:
        input = sentence
    else :
        input = nltk.word_tokenize(sentence)
    tags = nltk.pos_tag(input)
    pred = []
    index = 0;
    for word, tag in tags:
        if (tag[:2] == 'VB' and notAuziliaryVerb(word) ):
            obj = Features()
            obj.verb = word
            obj.index= index + counter
            pred.append(obj)
        index +=1;

    return pred

def notAuziliaryVerb(verb):
    if (verb == 'be' or verb == 'am' or verb == 'was' or verb == 'are' or verb == 'were' or verb == 'is' or verb == 'have' or verb == 'has' or verb == 'had'):
        return False
    else: return True

def listSynset():
    syns = list(wn.all_synsets())
    offsets_list = [(s.offset(), s) for s in syns]
    offsets_dict = dict(offsets_list)
    return offsets_dict


import discord
from discord.ext import commands
from pymongo import MongoClient

# Getting the NorthBot mongoDB connection URL
file = open('mongoURL.txt')
connectionURL = file.read()
file.close()

# MongoDB initialization
cluster = MongoClient(connectionURL)

# Embed Color
embedColor = discord.Color.green()

class guildStatistics(commands.Cog, name='Server Statistics'):

    def __init__(self, client):
        self.client = client

    """###################################################
    #                      Events                        #
    ###################################################"""

    # Event Showing guildStatistics is loaded
    @commands.Cog.listener()
    async def on_ready(self):
        print('\t- Loaded guildStatistics')

    # Creates the dataBase for when the bot joins a discord guild
    @commands.Cog.listener()
    async def on_guild_join(self, guild):

        guildID = guild.id
        members = guild.members
        guildName = guild.Name
        guildOwner = guild.owner

        print(f'Joined {guildName}!')

        dataBase = cluster[guildID]
        guildInfo = dataBase['guildInfo']
        channelData = dataBase['channelData']

        # Creates default information for a guild on join
        post = {'_id': guildID, 'guildName': guildName,
                'numMembers': len(members), 'streamChannel': int(), 'modChat': int(), 'baseRole': str(),
                'messageRestrictions': False, 'reputation': False, 'announceStreams': False, 'defaultRole': False}
        guildInfo.insert_one(post)

        # Goes through and checks/adds the channelData documents for every channel
        for channel in guild.channels:
            if channelData.count_documents({'_id': channel.id}) == 0:
                post = {'_id': channel.id, 'cName': channel.name, 'numMessages': 0}
                channelData.insert_one(post)

        await guildOwner.create_dm()
        await guildOwner.dm_channel.send(
            f'Hi {guildOwner.mention} thank you for adding me to your guild!'
            f'\nMake sure to do a few things first:'
            f'\n - .setModChat'
            f'\n - .profanityFilterOnOff [On/Off]'
            f'\n - .repOnOff [On/Off]'
            f'\n - .streamAnnounceOnOff [On/Off]'
            f'\n - ')

    # Deletes the data base for the guild?!?
    @commands.Cog.listener()
    async def on_guild_leave(self, guild):
        guildID = guild.id
        guildName = guild.Name

        print(f'Left {guildName}!')

        # This should delete the guild data
        cluster.drop_database(guildID)

    # If a guild updates any relevant information
    @commands.Cog.listener()
    async def on_guild_update(self, guild):
        guildID = guild.id
        guildName = guild.Name

        dataBase = cluster[str(guildID)]
        guildInfo = dataBase['guildInfo']
        guildData = guildInfo.find_one({'_id': guildID})

        if guildName != guildData['guildName']:
            guildInfo.update_one({'_id': guildID}, {'$set': {'guildName': guildName}})

    # Add one to numMembers on guild
    @commands.Cog.listener()
    async def on_member_join(self, member):
        guildID = member.guild.id
        dataBase = cluster[guildID]
        guildInfo = dataBase['guildInfo']
        modChannel = self.client.get_channel(guildInfo['modChat'])

        guildInfo.update_one({'_id': guildID}, {'$inc': {'numMembers': 1}})

        await modChannel.send(f'{member.mention} has joined the guild!')

    # Subtract one on member leave
    @commands.Cog.listener()
    async def on_member_leave(self, member):
        guildID = member.guild.id
        dataBase = cluster[guildID]
        guildInfo = dataBase['guildInfo']
        modChannel = self.client.get_channel(guildInfo['modChat'])

        guildInfo.update_one({'_id': guildID}, {'$inc': {'numMembers': -1}})

        await modChannel.send(f'{member} has left the guild... :(')

    # Keeps track of all the messages sent in guilds and adds them to a mongoDB
    # DOES NOT TRACK USER MESSAGES THAT THEY SEND
    @commands.Cog.listener()
    async def on_message(self, ctx):
        if ctx.author.bot:
            return

        dataBase = cluster[str(ctx.guild.id)]
        channelData = dataBase['channelData']

        print(f'{ctx.channel}: {ctx.author}: {ctx.author.name}: {ctx.content}')

        channelData.update_one({'_id': ctx.channel.id}, {'$inc': {'numMessages': 1}})

    # When a channel is created in the guild
    @commands.Cog.listener()
    async def on_guild_channel_create(self, channel):
        channelID = channel.id
        channelName = channel.name

        dataBase = cluster[str(channelID)]
        channelData = dataBase['channelData']

        channelData.insert_one({'_id': channelID, 'cName': channelName, 'numMessages': 0})

    # When a channel is deleted from the guild
    @commands.Cog.listener()
    async def on_guild_channel_remove(self, channel):
        channelID = channel.id
        channelName = channel.name

        dataBase = cluster[str(channelID)]
        channelData = dataBase['channelData']

        channelData.delete_one({'_id': channelID, 'cName': channelName, 'numMessages': 0})

    # If there was any relevant information changed on the channel
    @commands.Cog.listener()
    async def on_guild_channel_update(self, channel):
        channelID = channel.id
        channelName = channel.name

        dataBase = cluster[str(channelID)]
        channelData = dataBase['channelData']

        update = channelData.find_one({'_id': channelID})

        # If the channel name was changed
        if update['cName'] != channelName:
            channelData.update_one({'_id': channelID}, {'$set': {'cName': channelName}})

    """###################################################
    #                     Commands                       #
    ###################################################"""

    # TODO Gets the most talked in channel

    # TODO Gets the least talked in channel

    # TODO Gets general guild statistics (responds in embed)
    @commands.command()
    async def guildStats(self, ctx):
        guildID = ctx.guild.id
        guildName = ctx.guild.id

        dataBase = cluster[str(guildID)]
        channelData = dataBase['channelData']
        memberData = dataBase['memberData']

    @commands.command()
    @commands.has_permissions(ban_members=True)
    async def ban(self, ctx, member: discord.Member, *, reason=None):
        guildID = ctx.guild.id
        guildName = ctx.guild.name

        await member.create_dm()
        await member.dm_channel.send(f'You have been banned from {guildName} for reason: {reason}')
        await member.ban(reason=reason)

    @commands.command()
    @commands.has_permissions(kick_members=True)
    async def kick(self, ctx, member: discord.Member, *, reason=None):
        guildID = ctx.guild.id
        guildName = ctx.guild.name

        await member.create_dm()
        await member.dm_channel.send(f'You have been kicked from {guildName} for reason: {reason}')
        await member.kick(reason=reason)

    @commands.command(hidden=True)
    @commands.has_permissions(administrator=True)
    async def guildRefresh(self, ctx):
        guildID = ctx.guild.id
        guildName = ctx.guild.name
        members = ctx.guild.members  # This probably doesn't work
        channels = ctx.guild.channels
        dataBase = cluster[str(guildID)]
        guildInfo = dataBase['guildInfo']
        channelData = dataBase['channelData']
        query = {'_id': guildID}

        # If the guild document is not found in the collection then we insert a new one
        if guildInfo.count_documents(query) == 0:
            post = {'_id': guildID, 'guildName': guildName,
                    'numMembers': len(members), 'streamChannel': int(), 'modChat': int(), 'baseRole': str(),
                    'messageRestrictions': False, 'reputation': False, 'announceStreams': False, 'defaultRole': False}
            guildInfo.insert_one(post)

            await ctx.channel.send('Server info refreshed')
        else:
            return

        # Gets all the channel data put back in if some was deleted
        for channel in channels:
            if channelData.count_documents({'_id': channel.id}) == 0:
                post = {'_id': ctx.channel.id, 'cName': ctx.channel.name, 'numMessages': 0}
                channelData.insert_one(post)

    # Setting the modChat channel
    @commands.command()
    @commands.has_permissions(administrator=True)
    async def setModChat(self, ctx, channel):
        channelID = channel.id
        guildID = ctx.guild.id

        dataBase = cluster[str(guildID)]
        guildInfo = dataBase['guildInfo']

        guildInfo.update_one({'id': guildID}, {'$set': {'modChat': channelID}})

    # Ping command to see the latency of the bot
    @commands.command()
    async def ping(self, ctx):
        embed = discord.Embed(
            description='Pong!',
            color=embedColor
        )
        embed.set_footer(text=f'{round (self.client.latency * 1000)}ms')
        await ctx.channel.send(embed=embed)

    # Ping command to see if the file is loaded
    @commands.command(hidden=True)
    async def ping2(self, ctx):
        embed = discord.Embed(
            description='Pong!',
            color=embedColor
        )
        embed.set_footer(text=f'guildStatistics.py online')
        await ctx.channel.send(embed=embed)

def setup(client):
    client.add_cog(guildStatistics(client))


import webapp2
import uq
import qut
import gr
from mako.template import Template
from mako.lookup import TemplateLookup

lookup=TemplateLookup(directories=[".","./templates/"])
landing=Template(filename="templates/landing.mako",lookup=lookup)
aboutpage=Template(filename="templates/about.mako",lookup=lookup)
grifwarning=Template(filename="templates/grifwarning.mako",lookup=lookup)

class index(webapp2.RequestHandler):
	def get(self):
		self.response.write(landing.render(title="No TTCC?",bgcolor="#68a427"))

class about(webapp2.RequestHandler):
	def get(self):
		self.response.write(aboutpage.render(title="About No TTCC?",bgcolor="#68a427"))

class favicon(webapp2.RequestHandler):
	def get(self):
		f=open('favicon.ico','rb')
		self.response.headers['Content-Type'] ="image/x-icon"
		self.response.write(f.read())
		f.close()

class loading(webapp2.RequestHandler):
	def get(self):
		f=open('loading.gif','rb')
		self.response.headers['Content-Type'] ="image/gif"
		self.response.headers['Cache-Control']="max-age=31536000;"
		self.response.write(f.read())
		f.close()

class grWarn(webapp2.RequestHandler):
	def get(self):

		self.response.write(grifwarning.render(title="Griffith | No TTCC?",bgcolor="#D00"))

urls = [
	('/', index),
	('/about', about),
	('/uq',uq.ReqHandler),
	('/gr',gr.ReqHandler),
	('/gr/warn',grWarn),
	('/qut',qut.ReqHandler),
	('/qut/ttcc.pdf',qut.Pdf),
	('/uq/ttcc.pdf',uq.Pdf),
	('/gr/ttcc.pdf',gr.Pdf),
	('/favicon.ico',favicon),
	('/loading.gif',loading),
	]

app=webapp2.WSGIApplication(urls,debug=True)
if __name__=="__main__":
	from paste import httpserver
	httpserver.serve(app)


r"""
.. moduleauthor:: Tom L. Underwood

This module contains functions for performing simple minimpulations of
free energy profiles.
By *free energy profile* here I mean a function :math:`F(S)=-\ln p(S)`,
where :math:`p(S)` is the probability of the system exhibiting some
*order parameter* :math:`S`.
Note that the free energy profile :math:`F(S)` just defined is
*dimensionless* and does not include the usual :math:`k_BT` factor.

Most of the functions here deal with thermodynamic ensembles
in which the probability of the system being in some configuration
:math:`\sigma` is :math:`p_{\sigma}\propto\exp(-bS_{\sigma})`, where
:math:`b` is the thermodynamic 'force' conjugate to the order parameter
:math:`S`. For example, in the :math:`NVT` ensemble
:math:`p_{\sigma}\propto\exp(-\beta E_{\sigma})`, where :math:`\beta`
is the thermodynamic beta and :math:`E` denotes the system energy. Hence
the below functions can be used for free energy profiles over :math:`E`
obtained from :math:`NVT` simulations, with :math:`b=\beta`.
The same is true for :math:`NPT` and :math:`\mu VT` simulations, where
:math:`p_{\sigma}` is given by
:math:`p_{\sigma}\propto\exp(-\beta E_{\sigma}+\beta PV_{\sigma})` and
:math:`p_{\sigma}\propto\exp(-\beta E_{\sigma}+\beta \mu N_{\sigma})`
respectively, where :math:`P` denotes the system pressure, :math:`V`
denotes the system volume, :math:`\mu` denotes the chemical potential,
and :math:`N` denotes the number of particles in the system. Moreover
in the :math:`NPT` ensemble one could alternatively consider free energy
profiles over :math:`V`, in which case :math:`b=\beta P`; and in the
:math:`\mu VT` ensemble one could alternatively consider free energy
profiles over :math:`N`, in which case :math:`b=\beta \mu`.

Perhaps the most useful functions below perform single-histogram reweighting
over the quantity :math:`b`: the functions take the free energy
profile over :math:`S` at a given :math:`b`, and return the free energy
profile at a different :math:`b`.

Note:
   By *array* below we mean the ``numpy.array`` data type.

"""

import numpy as np
import warnings

import scipy.optimize

def from_file(filename):

    r""" Import a free energy profile from a file.

    The format of the file should be as follows. Each line corresponds to an
    order parameter, with the first token on the line being the order
    parameter, and the second token being the dimensionless free energy.
    The line may contain further tokens, but these are not used here.
    Moreover lines in the file beginning with '#' are treated as
    comments and ignored.

    Args:
       filename (str): The name of the file to import.

    Returns:
       array: An array containing the order parameter values read from the file.

       array: An array containing the free energy values read from the file.

    """

    # TO DO: Perform some safety checks on the file? Or is the below sufficient?

    with open(filename,'r') as f:

        string = f.read()

    return from_str(string)

def from_str(string):

    r"""Import a free energy profile from a string.

    The format of the string should be as follows. Each line corresponds to
    an order parameter, with the first token on the line being the order
    parameter, and the second token being the dimensionless free energy.
    The line may contain further tokens, but these are not used here.
    Moreover lines beginning with '#' are treated as comments and
    ignored.

    Args:
       string (str): The string containing the free energy profile.

    Returns:
       array: An array containing the order parameter values read from the string.

       array: An array containing the free energy values read from the string.

    """

    # TO DO: Perform some safety checks on the read input?

    op = []
    fe = []

    lines = string.splitlines()

    for line in lines:

        # Keep only the content before the first comment character '#'
        line = line.split('#')[0]

        # If what's left isn't whitespace...
        if line.strip() != "":

            op.append( float(line.split()[0])  )
            fe.append( float(line.split()[1])  )

    return (np.array(op), np.array(fe))

def fep_str(op, fe):

    r"""Return a readable string representation of a free energy profile.

    The string has two columns. The first column contains the order parameters, while
    the second column contains the corresponding values of the free energy.

    Args:
       op (array): An array containing the order parameter values.

       fe (array): An array containing the free energy values.

    Returns:
       string: The string representation of the free energy profile.

    """

    assert len(op) == len(fe), "op and fe must be of same length"

    # TO DO: There is probably a more elegant way to do this.
    # Also, control of the format of the output is important; this
    # should be added to this function eventually

    string = ""

    for i in range(0,len(op)):

        string += str(op[i])+" "+str(fe[i])+"\n"

    return string

def to_file(op, fe, filename):

    r"""Export a free energy profile to a file.

    In the file the free energy profile is represented as two columns. The first
    column contains the order parameters, while the second column contains the
    corresponding values of the free energy.

    Args:
       op (array): An array containing the order parameter values.

       fe (array): An array containing the free energy values.

       filename (string):  The name of the file to export the free energy profile to.

    """

    # TO DO: Perform some safety checks on the file?
    # Also, control of the format of the output is important; this
    # should be added to this function eventually

    with open(filename,'w') as file:

        file.write( fep_str(op,fe) )

def pdf(fe):

    r"""Calculate the probability distribution for a free energy profile.

    Args:
       fe (array): An array containing the free energy values.

    Returns:
       array: An array of floats giving the probability corresponding to each
          element in `fe`.

    Note:
       The returned probability distribution is such that the sum over all elements is
       1 (which is not the same as the integral over the interpolated profile
       being 1!)

    """

    # Numerical issues will occur with very negative values of fe (it blows up).
    # Hence we shift it so that the minimum is at 0.
    fe = shift(fe)

    normconst = np.sum( np.exp( -fe ) )

    return np.exp(-fe) / normconst

def shift(fe):

    r"""Shifts a free energy profile so that its minimum is at 0.

    Args:
       fe (array): An array containing the free energy values.

    Returns:
       array: The free energy values shifted so that the minimum value is 0.

    """

    return fe - fe.min()

def phase_probs(op, fe, op_thresh):

    r"""Returns the probabilities of two phases for a free energy profile.

    Phase 1 is defined to be configurations for which the order
    parameter is less than or equal to some threshold order parameter
    :math:`S_{thresh}`; and phase 2 corresponds to order parameters :math:`>S_{thresh}`.

    Args:
       op (array): An array containing the order parameter values.

       fe (array): An array containing the free energy values.

       op_thresh (float): The threshold order parameter :math:`S_{thresh}`
          defining the two phases.

    Returns:
       float: The probability of phase 1.

       float: The probability of phase 2.

    """

    assert len(op) == len(fe), "op and fe must be of same length"

    # Get the probability corresponding to each array element (i.e. pdf(fe)),
    # and sum over all those elements for which the order parameter is <= op_thresh.
    # Note that op > op_thresh achieves this (NOT op <= op_thresh)
    p1 = np.sum (  np.ma.masked_where(op > op_thresh, pdf(fe) ) )
    p2 = 1.0 - p1

    return p1, p2

def expected_op(op, fe, op_thresh):

    r"""Returns the expected order parameter of two phases for a free energy profile.

    Phase 1 is defined to be configurations for which the order
    parameter is less than or equal to some threshold order parameter
    :math:`S_{thresh}`; and phase 2 corresponds to order parameters :math:`>S_{thresh}`.

    Args:
       op (array): An array containing the order parameter values.

       fe (array): An array containing the free energy values.

       op_thresh (float): The threshold order parameter :math`S_{thresh}`
          defining the two phases.

    Returns:
       float: The expected value of the order parameter for phase 1.

       float: The expected value of the order parameter for phase 2.

    Note:
       An expected value for a phase will be nan, and a ``RuntimeWarning`` will
       be thrown if the probability of the phase is 0 to within the precision
       of the machine. In this case this function raises an additional warning
       to provide further context to the ``RuntimeWarning``.

    """

    assert len(op) == len(fe), "op and fe must be of same length"

    p1, p2 = phase_probs(op, fe, op_thresh)

    # op1 = (\sum_{i\in 1} op_i p_i) / (\sum_{i\in 1} p_i), where op_i is the order parameter
    # for array element i, p_i is the probability for element i, and \sum_{i\in 1}
    # is the sum over all p_i such that the op <= op_thresh. This is implemented
    # below.
    # Note that op < op_thresh is correct for op1 (NOT op <= op_thresh)
    op1 = np.sum (  np.ma.masked_where(op > op_thresh, pdf(fe) * op) ) / p1
    op2 = np.sum (  np.ma.masked_where(op <= op_thresh, pdf(fe) * op) ) / p2

    # The calculation of op1 and op2 above will encounter a divide-by-0 problem if p1 or
    # p2 is 0. The RuntimeWarning raised naturally in this case does not provide insight
    # into the source of the problem; hence I've raised another warning to provide further
    # information if it happens...
    if( p1==0 or p2==0 ):
        warnings.warn("'fep.expected_op' encountered p1=0 or p2=0. The function will return "+
                      "a 'nan' and may trigger a RuntimeWarning")

    return op1, op2

def reweight_add_lrc_lj(op, fe, epsilon, sigma, rc, volume, molsize, exclude=False):

    r"""Reweight a free energy profile over :math:`N` to add Lennard-Jones long-range corrections.

    This function reweights a free energy profile, assuming that the order parameter is
    the number of molecules in the system :math:`N` (as cwould be obtained from, e.g.
    a :math:`\mu VT` simulation), to add long-range corrections for Lennard-Jones interactions
    between particles (assuming that the profile currently corresponds to a system where
    they are absent).

    The long-range correction to the energy per Lennard-Jones particle is

    .. math::

       E_{lrc}={\frac{8\pi\rho\epsilon}{3}\Biggl[\Bigl(\frac{\sigma}{r_c}\Bigr)^3-\frac{1}{3}\Bigl(\frac{\sigma}{r_c}\Bigr)^9\Biggr]},

    where :math:`\epsilon` and :math:`\sigma` are the Lennard-Jones parameters, and :math:`\rho`
    is the density of Lennard-Jones particles. We assume the system is comprised of :math:`N` molecules
    comprised of :math:`n` atoms interacting via the Lennard-Jones potential, in which case
    :math:`\rho=Nn/V`, where :math:`V` is the volume of the system.

    Here the free energy profile is assumed to pertain to a simulation where the
    volume is constant, the system is comprised of molecules with `molsize` atoms Lennard-Jones
    interaction centres, and long-range corrections have not been included. Moreover the
    system is assumed to be homogenous (as is always the assumption when applying standard
    long-range corrections).

    Args:
       op (array): An array containing the order parameter values.

       fe (array): An array containing the free energy values.

       epsilon (float): The value of the Lennard-Jones parameter :math:`\epsilon`, in units of
          :math:`k_BT` (i.e. :math:`\epsilon/kT`).

       sigma (float): The value of the Lennard-Jones parameter :math:`\sigma`.

       rc (float): The cut-off distance for the Lennard-Jones potential.

       volume (float): The volume of the system.

       molsize (int): The number of Lennard-Jones interaction centres per molecule.

       exclude (boolean): If `true` then :math:`\rho` used in the above equation is chosen
          to be :math:`\rho=(Nn-n)/V` instead of :math:`Nn/V`. If the intramolecular
          contribution to the Lennard-Jones energy is *excluded* from the total energy, then
          :math:`\rho=(Nn-n)/V` corresponds to also excluding interactions with all images of
          the molecule with regards to the long-range corrections (though this is a weird thing
          to do).

    Returns:
       array: The free energies corresponding to `op`, but with long-range corrections added
          via reweighting

    Note:
       By passing a negative value for `epsilon`, this function can be used to remove
       long-range corrections from a free energy profile corresponding to a simulation which
       incorporates long-range corrections.

    """

    # TO DO: Is the 'exclude' option in this function necessary?

    assert len(op) == len(fe), "op and fe must be of same length"

    tailconst = (8.0*np.pi/3.0) * ( (sigma/rc)**9/3.0 - (sigma/rc)**3 ) \
                * epsilon * sigma**3 / volume

    nexcluded = 0

    if exclude:
        nexcluded = molsize

    # TO DO: Explain this equation better
    return fe + tailconst * molsize*op * ( molsize*op - np.ones(len(op))*nexcluded )

def reweight(op, fe, b_current, b_new):

    r"""Reweight a free energy profile to a new thermodynamic force.

    Args:
       op (array): An array containing the order parameter values.

       fe (array): An array containing the free energy values.

       b_current (float): The value of the force, which the current free energy
          profile corresponds to.

       b_new (float): The value of the force to reweight the free energy profile to

    Returns:
       array: The free energies corresponding to `op`, but reweighted to the chemical
          potential `mu_new`

    """

    assert len(op) == len(fe), "op and fe must be of same length"

    return fe - (b_new - b_current) * op

def reweight_to_coexistence(op, fe, b_current, b_lbound, b_ubound, op_thresh, tol=1.48e-08, maxiter=50, warn_tol=0.05):

    r"""Reweight a free energy profile to a thermodynamic force which corresponds to
    coexistence.

    Here coexistence corresponds to the system being in phase 1 and phase 2 with
    equal probability, where phase 1 is defined as the set of configurations for
    which the order parameter is less than or equal to some threshold order parameter
    :math:`S_{thresh}`, and phase 2 corresponds to order parameters :math:`>S_{thresh}`.

    Brent's method is used to locate :math:`b_{co}`, the thermodynamic force corresponding
    to coexistence. This is achieved by determining the minimum in :math:`|p_1(b)-0.5|`, where
    :math:`p_1(b)` denotes the phase 1 probability at thermodynamic force :math:`b`. The
    function ``scipy.optimize.brent`` is used to perform the optimisation.

    Args:
       op (array): An array containing the order parameter values.

       fe (array): An array containing the free energy values.

       b_current (float): The thermodynamic force :math:`b` which the free energy
          profile `fe` corresponds to.

       b_lbound (float): Lower bound on :math:`b` used in the search for coexistence.

       b_ubound (float): Upper bound on :math:`b` used in the search for coexistence.

       op_thresh (float): The threshold order parameter :math:`S_{thresh}` defining the two phases.

       tol (float, optional): Convergence threshold for Brent's method.
          The optimisation is stopped if subsequent iterations yield a change in the objective function
          of less than `tol`. Thus `tol` is effectively the precision in to which :math:`b_{co}` is
          to be determined; :math:`b_{co}` corresponds to the probability of phase 1 being 0.5 to a
          precision of approximately `tol`.

       maxiter (int, optional): Maximum number of iterations to use in Brent's method.

       warn_tol (float, optional): A warning is raised if the probability of phase 1 at the :math:`b_co`
          determined by this function is greater than `warn_tol` away from 0.5.

    Returns:
       float: The thermodynamic force :math:`b_{co}` deemed to correspond to coexistence.

       float: The probability of phase 1 at :math:`b_{co}`. Note that
          this should be very close to 0.5, otherwise the optimisation has not gone
          according to plan for some reason.

       array: The free energy profile at :math:`b_{co}`.

    Note:
       The optimisation method assumes that `b_lbound` and `b_ubound` bracket
       :math:`b_{co}`. If this is not the case then a warning is raised.

       A warning is raised if the :math:`b_co` determined by this function is greater than
       `warn_tol` away from 0.5.

       A warning is raised if the maximum number of iterations in the optimisation procedure is
       reached.
    """

    assert len(op) == len(fe), "op and fe must be of same length"
    assert b_lbound < b_ubound, "b_lbound must be < b_ubound"

    # Function which returns the absolute difference in the probability of phase 1,
    # relative to 0.5, for a given chemical potential - calculated by
    # reweighting self to that chemical potential
    # Note that the tolerance amounts to the precision to which p1 is close to 0.5??
    def objective(b):

        p1, p2 = phase_probs(op, reweight(op, fe, b_current, b), op_thresh)
        return abs(p1 - 0.5)

    # Check that b_co is bracketed by b_lbound and b_ubound
    p1_lbound, p2_tmp = phase_probs(op, reweight(op, fe, b_current, b_lbound), op_thresh)
    p1_ubound, p2_tmp = phase_probs(op, reweight(op, fe, b_current, b_ubound), op_thresh)
    if (p1_lbound-0.5) * (p1_ubound-0.5) > 0:
        warnings.warn("reweight_to_coexistence: b_lbound and b_ubound do not bracket b_co")

    # The Brent method determines where the objective function is 0 (which corresponds
    # to coexistence)
    b_opt, obj_opt, niters, fcalls = scipy.optimize.brent(objective, brack = (b_lbound,b_ubound),
                                                          tol=tol, full_output=True, maxiter=maxiter)

    if niters >= maxiter:
        warnings.warn("reweight_to_coexistence: maximum number of iterations reached!")

    # Now reweight to the optimum b to retrieve the free energy profile
    fe_opt = reweight(op, fe, b_current, b_opt)
    p1_opt, p2_opt = phase_probs(op, fe_opt, op_thresh)

    # Check that p1_opt is close to 0.5, as it should be.
    if abs(p1_opt - 0.5) > warn_tol:
        warnings.warn("reweight_to_coexistence: coexistence probability not close to 0.5!")

    return b_opt, p1_opt, fe_opt


"""
    This Python file is used to convert the old log files of acceleration to the latest log files with new acceleration
data format.

    Noticed that the old log files have data format as following:
        person_id, time_step_window, acceleration
    Each old file contains all records of all people in exact one experiment.
    The old file name is like "data2_1.csv" in which the first digit "2" denotes the id of experiment while the second
means that it is the acceleration data that are stored in the file.

    And the new log files have data format like this:
        time_step_window, acceleration_in_x, acceleration_in_y, acceleration_in_z
    Each new type file contains all records of one person in one experiment.
    The name of this kind of file is like "2.1_accele.csv" in which digit "2" denotes the experiment id and the other
digit "1" denotes the person id.

    The old log files need to be placed in the directory "input" which is in the same directory with this Python file.
And the output files will be output into the directory "output" which is also in the same directory with this Python
file.
"""

from os import mkdir, path, remove

# The minimum experiment id.
min_exp_id = 1
# The maximum experiment id.
max_exp_id = 6
# The set of IDs of missing experiments whose IDs are between "min_exp_id" and "min_exp_id".
missing_exp_id_set = set([])

def new_file_writer(exp_id, person_id, tmp_records_of_individual):
    """
    Write to new log files.
    :param exp_id:
    :param person_id:
    :param tmp_records_of_individual:
    :return:
    """
    if not path.exists("output") or path.isfile("output"):
        mkdir("output")
    file_name = str(exp_id) + "."+ person_id + "_accele.csv"
    if path.isfile(file_name):
        remove(file_name)
    with open("output\\" + file_name, "a", True) as new_file:
        for time_step_window, acceleration in tmp_records_of_individual:
            new_row_data = time_step_window + ", " + acceleration + ", 0, 0\n"
            new_file.write(new_row_data)

def old_file_processor(exp_id):
    """
    Read old log file that has records of the exp_id-th experiment and then convert it to the new log files.
    :param exp_id:
    :return:
    """
    file_name = "input\\data" + str(exp_id) + "_1.csv"
    current_person_id = None
    tmp_records_of_individual = []
    with open(file_name, "r", True) as old_file:
        while True:
            str_with_line_break = old_file.readline()
            if str_with_line_break == "":
                new_file_writer(exp_id, current_person_id, tmp_records_of_individual)
                break
            line_str = str_with_line_break.rstrip("\n")
            data_list = line_str.split(",")
            if len(data_list) != 3:
                print("Warning: Invalid row data which should be like this form, \"person_id, time_step_window, acceleration\".")
                continue
            person_id = data_list[0]
            time_step_window = data_list[1]
            acceleration = data_list[2]
            if current_person_id is None:
                current_person_id = person_id
            if person_id != current_person_id:
                new_file_writer(exp_id, current_person_id, tmp_records_of_individual)
                current_person_id = person_id
                tmp_records_of_individual = []
            tmp_records_of_individual.append([time_step_window, acceleration])

def conversion_start(min_exp_id, max_exp_id, missing_exp_id_set = None):
    """
    Run the conversion procedure according to the range of given experiment IDs.
    :param min_exp_id:
    :param max_exp_id:
    :param missing_exp_id_set:
    :return:
    """
    if missing_exp_id_set is None or len(missing_exp_id_set) == 0:
        for exp_id in range(min_exp_id, max_exp_id + 1):
            old_file_processor(exp_id)
    else:
        for exp_id in range(min_exp_id, max_exp_id + 1):
            if exp_id not in missing_exp_id_set:
                old_file_processor(exp_id)

# Here is the entrance of this program.
conversion_start(min_exp_id, max_exp_id, missing_exp_id_set)


import torch
from PIL import Image
import requests
import sys
sys.path.append("..")
from utils.utils import AI
#import the py file utils in folder utils as name utils
import utils.utils as utils

dirName = utils.makeDir("captions_clip/"+utils.getTimestamp())

img = utils.loremImage()
i = 0
filename = f"{str(i).zfill(4)}.jpg"
img.save(dirName / filename)

with open(dirName / 'clip.txt', 'w', encoding='utf-8') as f:
    f.write(f"{filename}@init\n")

while i < 1000:

    i = i + 1

    model = AI('img2txt_clip')
    txt = model.run(img)
    print(txt)

    filename = f"{str(i).zfill(4)}-{utils.fixFilename(txt)}.jpg"

    with open(dirName / 'clip.txt', 'a',encoding='utf-8') as f:
        f.write(f"{filename}@{txt}\n")

    model = AI('txt2img_stablediffusion')
    img = model.run(prompt=txt)

    img.save(dirName / filename)


import os
from tkinter import *

nordConnectUS = 'nordvpn connect'
nordConnectDE = 'nordvpn connect germany'
nordConnectP2P = 'nordvpn connect p2p'
nordConnectOnion = 'nordvpn connect onion_over_vpn'
nordStatus = 'nordvpn status'
nordDiconnect = 'nordvpn disconnect'

root = Tk()
root.geometry('400x400')
root.title('NordVPN')

def connectUS():
    os.system(nordConnectUS)

def connectDE():
    os.system(nordConnectDE)

def connectP2P():
    os.system(nordConnectP2P)

def connectOnion():
    os.system(nordConnectOnion)

def vpnStatus():
    os.system(nordStatus)

def disconnect():
    os.system(nordDiconnect)

connectUSButton = Button(root, text='Connect US', command=connectUS)
connectUSButton.pack()
connectDEButton = Button(root, text="Connect DE", command=connectDE)
connectDEButton.pack()
connectP2PButton = Button(root, text='Connect P2P', command=connectP2P)
connectP2PButton.pack()
connectOnionButton = Button(root, text="Connect Onion", command=connectOnion)
connectOnionButton.pack()
vpnStatusButton = Button(root, text="Status", command=vpnStatus)
vpnStatusButton.pack()
disconnectButton = Button(root, text="Disconnect", command=disconnect)
disconnectButton.pack()

root.mainloop()


import sys
a = input()
b = a.split('|')
c = input()

n = abs(len(b[0])-len(b[1]))

if n==0 :
    print('Impossible')
    sys.exit()
if n != len(c) :
    print('Impossible')
    sys.exit()
if len(c) >= n :
    if len(b[0]) < len(b[1]) :
        s = b[0] + c + '|' + b[1]
    else :
        s = b[0] + '|' + b[1] + c
    print(s)
else :
    print('Impossible')


from tensorflow.keras.models import Model, load_model
from tensorflow.keras import layers as L
from tensorflow.keras import backend as K
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler
from tensorflow.keras import backend as K
from tensorflow.keras import optimizers
from vggish import VGGish
from kapre.time_frequency import Melspectrogram, Spectrogram
from kapre.utils import Normalization2D

def AttRNNSpeechModel(nCategories, samplingrate=16000,
                      inputLength=16000, unet = False, rnn_func=L.LSTM):
    # simple LSTM
    sr = samplingrate
    iLen = inputLength

    inputs = L.Input((inputLength,), name='input')

    x = L.Reshape((1, -1))(inputs)

    m = Melspectrogram(n_dft=1024, n_hop=128, input_shape=(1, iLen),
                       padding='same', sr=sr, n_mels=80,
                       fmin=40.0, fmax=sr / 2, power_melgram=1.0,
                       return_decibel_melgram=True, trainable_fb=False,
                       trainable_kernel=False,
                       name='mel_stft')
    m.trainable = False

    x = m(x)

    x = Normalization2D(int_axis=0, name='mel_stft_norm')(x)

    # note that Melspectrogram puts the sequence in shape (batch_size, melDim, timeSteps, 1)
    # we would rather have it the other way around for LSTMs

    x = L.Permute((2, 1, 3))(x)
    if unet == True:
        x = L.Conv2D(16, (5, 1), activation='relu', padding='same')(x)
        up = L.BatchNormalization()(x)
        x = L.Conv2D(32, (5, 1), activation='relu', padding='same')(up)
        x = L.BatchNormalization()(x)
        x = L.Conv2D(16, (5, 1), activation='relu', padding='same')(x)
        down = L.BatchNormalization()(x)
        merge = L.Concatenate(axis=3)([up,down])
        x = L.Conv2D(1, (5, 1), activation='relu', padding='same')(merge)
        x = L.BatchNormalization()(x)
    else:
        x = L.Conv2D(10, (5, 1), activation='relu', padding='same')(x)
        x = L.BatchNormalization()(x)
        x = L.Conv2D(1, (5, 1), activation='relu', padding='same')(x)
        x = L.BatchNormalization()(x)

    # x = Reshape((125, 80)) (x)
    # keras.backend.squeeze(x, axis)
    x = L.Lambda(lambda q: K.squeeze(q, -1), name='squeeze_last_dim')(x)

    x = L.Bidirectional(rnn_func(64, return_sequences=True)
                        )(x)  # [b_s, seq_len, vec_dim]
    x = L.Bidirectional(rnn_func(64, return_sequences=True)
                        )(x)  # [b_s, seq_len, vec_dim]

    xFirst = L.Lambda(lambda q: q[:, -1])(x)  # [b_s, vec_dim]
    query = L.Dense(128)(xFirst)

    # dot product attention
    attScores = L.Dot(axes=[1, 2])([query, x])
    attScores = L.Softmax(name='attSoftmax')(attScores)  # [b_s, seq_len]

    # rescale sequence
    attVector = L.Dot(axes=[1, 1])([attScores, x])  # [b_s, vec_dim]

    x = L.Dense(64, activation='relu')(attVector)
    x = L.Dense(32)(x)

    output = L.Dense(nCategories, activation='softmax', name='output')(x)

    model = Model(inputs=[inputs], outputs=[output])

    return model

def VggishModel(nCategories = 36,iLen= 16000, sr= 16000, audioset = False):

    inputs = L.Input((iLen,), name='input')

    x = L.Reshape((1, iLen))(inputs)

    if audioset == True:
        para_m = [128, 3800, 29]

    else: # Google pre-trained
        para_m = [32, 7500, 31]

    m = Melspectrogram(n_dft=1024, n_hop=para_m[0], input_shape=(1, iLen),
                               padding='same', sr=sr, n_mels=64,

                               fmin=125.0, fmax=para_m[1], power_melgram=1.0,
                               return_decibel_melgram=True, trainable_fb=False,
                               trainable_kernel=False,
                               name='mel_stft')
    m.trainable = False
    mfs = m(x)
    mfs = L.Permute((2,1,3))(mfs)
    mfs = L.Cropping2D(cropping=((0, para_m[2]), (0, 0)))(mfs)

    if audioset == True:
        vgg_model = VGGish(include_top=True, load_weights=True)
    else:
        vgg_model = VGGish(include_top=False, load_weights=False)

    vgg_emd = vgg_model(mfs)
    if audioset == True:
        output = vgg_emd
    else:
        output = L.Dense(nCategories, activation='softmax', name='output')(vgg_emd)

    model = Model(inputs=[inputs], outputs=[output], name='VGG_pre')
    # model.summary()
    return model

## Updated from 2021 March with TF Hub
import y_params as yamnet_params
import yamnet as yamnet_model

def Yamnet():

    params = yamnet_params.Params(sample_rate=16000, patch_hop_seconds=0.1)
    # Set up the YAMNet model.
    class_names = yamnet_model.class_names('weight/yamnet_class_map.csv')
    yamnet = yamnet_model.yamnet_frames_model(params)
    yamnet.load_weights('weight/yamnet.h5')

    return yamnet


from entities.room import Room

class RoomReservation:

    def __init__(self):
        self.current_id = 0
        self.rooms = []

    def addRoom(self, room: Room) -> bool:
        room.set_id(self.current_id)
        self.current_id += 1

        self.rooms.append(room)
        for room in self.rooms:
            print(room.get_room_and_section())

    def searchRoom(self, sectionNumber, roomNumber):
        for room in self.rooms:
            if room.sectionNumber == sectionNumber and room.roomNumber == roomNumber:
                return room
        return None

    def getAllRooms(self):
        return self.rooms

    def removeRoom(self, IDRoom):
        for index, room in enumerate(self.rooms):
            if room.ID == int(IDRoom):
                return self.rooms.pop(index)

        return None

    def reserveRoom(self, IDRoom):
        for room in self.rooms:
            if room.ID == int(IDRoom):
                if room.is_occupied():
                    return False
                else:
                    room.set_occupied(True)
                    return True

        return None

    def cancelReservation(self, IDRoom):
        for room in self.rooms:
            if room.ID == int(IDRoom):
                if not room.is_occupied():
                    return False
                else:
                    room.set_occupied(False)
                    return True

        return None


# coding=utf-8

from sqlalchemy import func
from flask import request, render_template, redirect, url_for, flash, current_app, session
from flask_user import login_required, roles_required, current_user
from flask_admin import Admin, expose
from flask_admin.form import rules
from flask_admin.actions import action
from flask_admin.model.template import EndpointLinkRowAction, LinkRowAction

# db
from app.database import db, Column, reference_col, relationship

# model
from app.models.user import SexEnum
from app.models import User, Role

# flask admin base mode view
from .base_view import CustomBaseModelView

# cache
from app.cache import cache

##########################################################################
# amdin model view

class UsersModelView(CustomBaseModelView):
    """Users Model View"""
    can_create = True
    can_delete = False
    can_view_details = True

    column_display_pk = False

    column_list = (
        User.username,
        User.nickname,
        User.sex,
        User.phone,
        User.email,
        User.active,
    )

    column_labels = {
        "username": u"用户名",
        "email": u"邮箱",
        "nickname": u"昵称",
        "group": u"用户组",
        "groups": u"用户组",
        "roles": u"角色",
        "phone": u"手机号码",
        "sex": u"性别",
        "active": u"激活",
        "create_datetime": u"创建时间",
        "current_login_datetime": u"本次登录时间",
        "last_login_datetime": u"上次登录时间",
        "current_login_ip": u"本次登录IP",
        "last_login_ip": u"上次登录IP",
        "api_key": u"API密钥",
    }

    # column_choices = {
    #     'sex': [
    #         (SexEnum.male, u"男"),
    #         (SexEnum.female, u"女"),
    #         (SexEnum.other, u"其他"),
    #         (SexEnum.empty, u"未填"),
    #     ]
    # }

    column_searchable_list = ("username", )

    # 不显示password
    column_exclude_list = ("password_hash", "password", "avatar_hash")

    # 在表单中排除相关的字段
    form_excluded_columns = (
        # "username",
        # "email",
        # "phone",
        "password_hash",
        "avatar_hash",
        "api_key",
    )

    # 表单参数
    form_widget_args = {
        'username': {
            'disabled': True  # 禁止修改
        },
        "password_hash": {
            'disabled': True
        },
    }

    # @cache.cached(60, key_prefix='user_admin_view_get_query')
    def get_query(self):
        return super(UsersModelView, self).get_query()

    # @cache.cached(60)
    def get_count_query(self):
        return super(UsersModelView, self).get_count_query()

    # def get_one(self, id):
    #     prefix = "user"
    #     if self.model.has_cache(prefix, id):
    #         current_app.logger.debug("get model form CACHE: id={}".format(id))
    #         return self.model.get_cache_by_id(prefix, id)
    #     current_app.logger.debug("get model form DATABASE: id={}".format(id))
    #     model = super(UsersModelView, self).get_one(id)
    #     model.add_cache(prefix)  # add cache
    #     return model

class RolesModelView(CustomBaseModelView):
    """
    Roles Model View
    """
    can_create = True
    can_delete = True
    can_edit = True
    can_view_details = False

    column_labels = {"name": u"角色名称", "description": u"描述"}

    # action_disallowed_list = ["delete"]

def init_admin_view(admin: Admin):
    admin.add_view(UsersModelView(User, db.session, name=u"用户",
                                  endpoint="users", menu_icon_type="glyph",
                                  menu_icon_value="glyphicon-user",
                                  category="用户管理"))
    admin.add_view(RolesModelView(
        Role, db.session, name=u"角色", endpoint="roles", category="用户管理"))


# Run time complexity: O(n), Space complexity: O(1)

def findThreeLargestNumbers(array):
    result = [None, None, None]
    for num in array:
        for i in range(2, -1, -1):
            if result[i] is None or num > result[i]:
                insert(result, i, num)
                break

    return result

def insert(array, i, num):
    for j in range(0, i):
        array[j] = array[j+1]
    array[i] = num

if __name__ == "__main__":
    result = findThreeLargestNumbers(
        [141, 1, 17, -7, -17, -27, 18, 541, 8, 7, 7])
    print(result)


import unicodedata, string
import numpy as np

def toascii(unicode_text):
    """convert the unicode string to plain ascii
       removing all the null bytes
    """
    oldtext=unicodedata.normalize('NFKC',unicode_text)
    #convert from unicode to ascii dropping characters
    #if needed
    text=oldtext.encode('ascii','ignore')
    #
    # still lots of null bytes in the file so
    # removing everything that isn't an ascii letter
    # (positions 32 to 126) or whitespace
    #
    allowed=chr(32)
    for i in np.arange(33,127):
        allowed = allowed + chr(i)
    allowed=allowed + '\t\n\x0b\x0c\r '  #formfeed, vertical tab
    deletions = ''.join(ch for ch in map(chr,range(256)) if ch not in allowed)
    #
    # table doesn't remap, just passes allowed to allowed
    #
    tab = string.maketrans(allowed,allowed)
    #
    # remove bad characters (everything except allowed)
    #
    final_ascii=string.translate(text, tab, deletions=deletions)
    return final_ascii