import pandas as pd
from website.ml_pipeline.ml import *

def timeData(data):
    df = pd.DataFrame(data)
    df.sort_values(by='time')
    # Get time difference between top and bottom value

    t1 = df.iloc[-1][1]  # last time in time column
    t2 = df.iloc[0][1]  # first value in time column
    delta = t2 - t1  # time difference
    ms = delta.total_seconds() * 1000  # difference in milliseconds
    unit = "millisecond"  # default is millisecond
    if (ms < 1000):
        unit = "millisecond"
        df2 = (df.groupby([pd.Grouper(freq='L', key='time', closed="left"), 'sentiment'])[
               'value'].count().unstack(fill_value=0)).reset_index()

    else:  # must be a bigger scale
        if (ms < 60000):
            unit = "second"
            df2 = (df.groupby([pd.Grouper(freq='S', key='time', closed="left"), 'sentiment'])[
                   'value'].count().unstack(fill_value=0)).reset_index()

        else:  # must be a bigger scale
            if (ms < 3600000):
                unit = "minute"
                df2 = (df.groupby([pd.Grouper(freq='T', key='time', closed="left"), 'sentiment'])[
                       'value'].count().unstack(fill_value=0)).reset_index()

            else:
                unit = "hour"  # Max group by hour
                df2 = (df.groupby([pd.Grouper(freq='H', key='time', closed="left"), 'sentiment'])[
                       'value'].count().unstack(fill_value=0)).reset_index()

    # Return
    return {"unit": unit, "df": df2}

def parsechartpoints(df):
    #results : context
    # df : contains the groupings for the time splits

    pos_ds = []  # positive data points
    neg_ds = []
    neu_ds = []
    # convert datetime to string for json
    df['time'] = df['time'].astype(str)
    for i in df.index:

        if 'Positive' in df.columns:
            pos_ds.append({"x": df['time'][i], "y": int(df['Positive'][i])})
        if 'Negative' in df.columns:
            neg_ds.append({"x": df['time'][i], "y": int(df['Negative'][i])})
        if 'Unsure' in df.columns:
            neu_ds.append({"x": df['time'][i], "y": int(df['Unsure'][i])})

    return {"p": pos_ds, "n": neg_ds, "u": neu_ds}

def wordFrequency(tweets, hashtag):

    processed = cleantweets(tweets, hashtag)
    words = []
    for tweet in processed:
        for j in tweet.split():
            words.append(j)
    dict_ = wordListToFreqDict(words)  # split the tweets into words

    sorted = sortFreqDict(dict_)

    return sorted

def wordListToFreqDict(wordlist):
    wordfreq = [wordlist.count(p) for p in wordlist]
    return dict(list(zip(wordlist, wordfreq)))

def sortFreqDict(freqdict):
    aux = [(freqdict[key], key)
           for key in freqdict]  # tupple of word with frequency
    aux.sort()  # sort by count
    aux.reverse()
    return aux

def cleantweets(tweets, hashtag):
    cleaned = []
    for i in tweets:  # for each tweet
        i = preprocess(i)  # preprocess the tweet here
        # remove the hastag cause it will apppear the most
        i = i.replace(str(hashtag).lower(), '')
        cleaned.append(i)
    return cleaned  # cleaned tweets

def wordFrequency_Sentiment(words, alltweets):
    pos = []
    neu = []
    neg = []

    for word in words:
        num_in = 0
        sentiment = 0
        for row in alltweets:
            ans = preprocess(row['tweet'])

            if word in ans:
                num_in = num_in + 1
                sentiment = sentiment + row['value']

        avg = (sentiment/num_in) if num_in > 0 else 0
        if (avg < 0.45):
            neg.append({"word": word, "count": num_in, })
        elif (avg > 0.55):
            pos.append({"word": word, "count": num_in, })
        else:
            neu.append({"word": word, "count": num_in, })

    all = pos + neu + neg  # note order
    return {"words": [d["word"] for d in all], "pos": [d["count"] for d in pos], "neu": [d["count"] for d in neu], "neg": [d["count"] for d in neg]}

def locationData(alltweets):

    df = pd.DataFrame(alltweets)
    df2 = df.groupby('location', sort=False).agg(
        {'value': ['mean', 'count']}).reset_index()
    sortedDF = df2.sort_values([('value', 'count')], ascending=False)
    list_ = sortedDF.values.tolist()
    places = [i[0] for i in list_]
    sentiment = [i[1] for i in list_]
    counts = [i[2] for i in list_]
    pos = []
    neu = []
    neg = []
    for s in sentiment:
        if (s < 0.45):
            # The sentiment and count are at correnspinding indices
            neg.append(counts[sentiment.index(s)])
        elif (s > 0.55):
            pos.append(counts[sentiment.index(s)])
        else:
            neu.append(counts[sentiment.index(s)])

    return {"places": [d for d in places], "pos": pos, "neu": neu, "neg": neg}


import socket
import struct
import  fcntl
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sockfd = sock.fileno()
SIOCGIFADDR = 0x8915
import json
import sys
#import cgi

def get_ip(iface = 'eth0'):
    ifreq = struct.pack('16sH14s'.encode('utf-8'), iface.encode('utf-8'), socket.AF_INET, '\x00'.encode('utf-8')*14)
    try:
        res = fcntl.ioctl(sockfd, SIOCGIFADDR, ifreq)
    except:
        return None
    ip = struct.unpack('16sH2x4s8x'.encode('utf-8'), res)[2]
    return socket.inet_ntoa(ip)

mip = {}

logfile = open('testlog.txt',"w")
logfile.write("starting\n")
try:
    for i in ['wlan0','wlan1','eth0','eth1']:
        add = get_ip(i)
        if add:
            mip[i] = add
except Exception as err:
    logfile.write("{}".format(err))

logfile.write(json.dumps(mip))

print ("Content-type: text/json\n\n")
print (json.dumps(mip))
sys.stdout.flush()


from pathlib import Path

import pytest

from ipywidgets import Widget

from nbautoeval.quiz_loader import YamlLoader, run_yaml_quiz

localdir = Path(__file__).parent
yamldir = localdir.parent / "tests"

def test_rain_check():
    for file in yamldir.glob("test-ko-*.yaml"):
        with pytest.raises((ValueError, AssertionError)):
            YamlLoader(file).rain_check()

def test_exoname1():
    witness = "quiz1"
    loader = YamlLoader(yamldir / "test-ok-1.yaml")
    quiz = loader.build_quiz(witness, True)
    assert quiz.exoname == witness

def test_exoname2():
    witness = "quiz1"
    loader = YamlLoader(yamldir / "test-ok-2.yamlb")
    quiz = loader.build_quiz(witness, True)
    assert quiz.exoname == witness

# black box testing
def test_all_in_one():
    w1 = run_yaml_quiz("test-ok-1", "quiz1")
    assert isinstance(w1, Widget)
    w2 = run_yaml_quiz("test-ok-2", "quiz1")
    assert isinstance(w2, Widget)


# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
import os
import time
import requests

class MzituPipeline(object):

    def process_item(self, item, spider):
        pic_url = item['pic_url']
        # 从url取图片名称
        name = pic_url.rsplit('/', 1)[-1]
        root_path = 'd:/mzitu'
        # 取出该图片应属于哪个分类
        dir_path = item['pic_name']
        # 拼接名称确认图片路径
        file_path = '%s/%s/%s' % (root_path, dir_path, name)
        file_path = file_path.replace(' ', '-')
        # 判断是否下载过
        if not os.path.exists(file_path):
            t = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
            print(f'{t} downloading: ', file_path)
            with open(file_path, 'wb') as jpg:
                jpg.write(requests.get(pic_url, timeout=15, headers=self.getheader(pic_url)).content)
        return item

    def getheader(self, refers):
        # 要使用Referer指向，不然会被防盗链
        headers = DEFAULT_REQUEST_HEADERS = {
            'Accept-Language': 'zh-CN,zh;q=0.8',
            'Referer': '{}'.format(refers),
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36'
        }
        return headers

    def close_spider(self, spider):
        print('任务结束了！')


from .base import Processor
from googleapiclient import discovery
from google.oauth2.credentials import Credentials
from urllib.parse import urlencode
import re

class GroupsProcessor(Processor):

    def get_default_config_key():
        return 'groups'

    def process(
        self,
        output_var={
            'all_groups': 'all_groups',
            'groups_by_owner': 'groups_by_owner',
            'groups_by_manager': 'groups_by_manager'
        }):
        groups_config = self.config
        service_account = groups_config[
            'serviceAccountEmail'] if 'serviceAccountEmail' in groups_config else None

        group_credentials = Credentials(
            self.get_token_for_scopes([
                'https://www.googleapis.com/auth/cloud-identity.groups.readonly'
            ],
                                      service_account=service_account))

        group_service = discovery.build(
            'cloudidentity',
            'v1',
            http=self._get_branded_http(group_credentials))
        query = groups_config['query'] if 'query' in groups_config else ""

        query_template = self.jinja_environment.from_string(query)
        query_template.name = 'query'
        query_output = query_template.render()

        page_token = None
        all_groups = {}
        groups_by_owner = {}
        groups_by_manager = {}
        group_filter = None
        if 'filter' in groups_config:
            group_filter_template = self.jinja_environment.from_string(
                groups_config['filter'])
            group_filter_template.name = 'group_filter'
            group_filter_output = group_filter_template.render()
            group_filter = re.compile(group_filter_output)
        while True:
            search_query = urlencode({"query": query_output})
            search_group_request = group_service.groups().search(
                pageToken=page_token, pageSize=1, view="FULL")
            param = "&" + search_query
            search_group_request.uri += param
            response = search_group_request.execute()
            if 'groups' in response:
                for group in response['groups']:
                    group_key = group['groupKey']['id']
                    if group_filter:
                        if not group_filter.match(group_key):
                            continue

                    group['owners'] = []
                    group['managers'] = []
                    membership_page_token = None
                    while True:
                        membership_request = group_service.groups().memberships(
                        ).list(parent=group['name'],
                               pageToken=membership_page_token)
                        membership_response = membership_request.execute()
                        group['memberships'] = {}
                        if 'memberships' in membership_response:
                            owners = []
                            managers = []
                            for member in membership_response['memberships']:
                                member_key = member['preferredMemberKey']['id']
                                group['memberships'][member_key] = member
                                if 'roles' in member:
                                    for role in member['roles']:
                                        if role['name'] == 'OWNER':
                                            owners.append(member_key)
                                            group['owners'].append(member_key)
                                        if role['name'] == 'MANAGER':
                                            managers.append(member_key)
                                            group['managers'].append(member_key)
                            for owner in owners:
                                if owner not in groups_by_owner:
                                    groups_by_owner[owner] = [group]
                                else:
                                    groups_by_owner[owner].append(group)
                            for manager in managers:
                                if manager not in groups_by_manager:
                                    groups_by_manager[manager] = [group]
                                else:
                                    groups_by_manager[manager].append(group)

                        if 'nextPageToken' in membership_response:
                            membership_page_token = membership_response[
                                'nextPageToken']
                        else:
                            break

                    all_groups[group_key] = group
            if 'nextPageToken' in response:
                page_token = response['nextPageToken']
            else:
                break

        res = {
            'all_groups': all_groups,
            'groups_by_owner': groups_by_owner,
            'groups_by_manager': groups_by_manager
        }
        return res


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# import sys
# import os

from colemaktutor import colemaktutor

project = 'colemaktutor'
copyright = '2014, Benjamin Althues'
release = version = colemaktutor.__version__

extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.doctest',
    'sphinx.ext.todo',
    'sphinx.ext.ifconfig',
    'sphinx.ext.viewcode',
]
templates_path = ['_templates']
source_suffix = '.rst'
master_doc = 'index'
exclude_patterns = ['_build']
pygments_style = 'sphinx'

html_theme = 'nature'  # default, nature
#html_theme_options = {}
#html_theme_path = []
#html_title = None
#html_short_title = None
#html_logo = None
#html_favicon = None
html_static_path = ['_static']
#html_extra_path = []
html_show_sourcelink = False

htmlhelp_basename = 'colemaktutordoc'

latex_elements = {
    'papersize': 'a4paper',
    'pointsize': '10pt',
    'preamble': '',
}
latex_documents = [
    ('index', 'colemaktutor.tex', 'colemaktutor Documentation',
     'Benjamin Althues', 'manual'),
]
#latex_logo = None
#latex_use_parts = False
#latex_show_pagerefs = False
#latex_show_urls = False
#latex_appendices = []
#latex_domain_indices = True

man_pages = [
    ('index', 'colemaktutor', 'colemaktutor Documentation',
     ['Benjamin Althues'], 1)
]
#man_show_urls = False

texinfo_documents = [
    ('index', 'colemaktutor', 'colemaktutor Documentation',
     'Benjamin Althues', 'colemaktutor', 'One line description of project.',
     'Miscellaneous'),
]

epub_title = 'colemaktutor'
epub_author = 'Benjamin Althues'
epub_publisher = 'Benjamin Althues'
epub_copyright = '2014, Benjamin Althues'
#epub_basename = 'colemaktutor'
epub_exclude_files = ['search.html']
#epub_tocdepth = 3
#epub_tocdup = True
#epub_tocscope = 'default'
#epub_fix_images = False
#epub_max_image_width = 0
#epub_show_urls = 'inline'
#epub_use_index = True


import os
import requests

from flask import redirect, render_template, request, session
from functools import wraps

def login_required(f):
    """

    Decorate routes to require login.

    https://flask.palletsprojects.com/en/1.1.x/patterns/viewdecorators/

    """

    @wraps(f)
    def decorated_function(*args, **kwargs):

        if session.get("user_id") is None:

            return render_template("login.html", message="Please log in")

        return f(*args, **kwargs)

    return decorated_function

def formatProject(project):
    return {
        "id": project[0],
        "user_id": project[1],
        "name": project[2],
        "purpose": project[3],
        "description": project[4],
        "languages": project[5],
        "time": project[6],
        "note": project[7],
        "status": project[8]
    }

class postgresSQLConnection:
    def __init__(self, connection):
        self.connection = connection
        self.dbcursor = connection.cursor()

    def execute(self, command, query=None):
        try:
            if query is None:
                self.dbcursor.execute(command)
            else:
                self.dbcursor.execute(command, query)
            if "SELECT" in command:
                result = list(self.dbcursor.fetchall())
                return result
            else:
                self.connection.commit()
        except:
            self.dbcursor.execute("rollback")

    def __del__(self):
        self.dbcursor.close()
        self.connection.close()


import numpy as np
import pandas
from keras.models import Sequential
from keras.layers import Dense
from theano import *
from keras.wrappers.scikit_learn import KerasClassifier
from keras.utils import np_utils
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
from sklearn.preprocessing import LabelEncoder

#seed do badania parametrów
seed = 7
np.random.seed(seed)

#dane
#treningowe
dataframe = pandas.read_csv("train.csv", header=None)
dataset = dataframe.values

#testowe
dataframe1 = pandas.read_csv("test.csv", header=None)
dataset1 = dataframe1.values

#ankiety
#dataframe1 = pandas.read_csv("ank.csv", header=None)
#dataset2 = dataframe1.values

#dzielenie na output i input
X=dataset[:, 0:6].astype(float)
Y=dataset[:, 6]

# Z=dataset1[:,0:6].astype(float)
# Z1=dataset1[:,6]
# ankieta1=dataset2[:,0:6].astype(float)
# ankieta2=dataset2[:,6]

#ecnoding etykiet z nazwami emocji  na floaty
encoder = LabelEncoder()
encoder.fit(Y)
encoded_Y = encoder.transform(Y)
dummy_y = np_utils.to_categorical(encoded_Y)

# encoder.fit(Z1)
# encoded_Z = encoder.transform(Z1)
# dummy_z = np_utils.to_categorical(encoded_Z)
# encoder.fit(Z1)
# encoded_Z1 = encoder.transform(Z1)
# dummy_z1 = np_utils.to_categorical(encoded_Z1)

#budowa modelu
def baseline_model():
    model = Sequential()
    model.add(Dense(8, input_dim=6, activation='relu'))
    model.add(Dense(4, activation='softmax'))

#kompilacja modelu
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model

#fitting
estimator = KerasClassifier(build_fn=baseline_model, epochs=10000, batch_size=5)

    #evaluation
    # kfold = KFold(n_splits=10, shuffle=True, random_state=seed)
    # results = cross_val_score(estimator, X, dummy_y, cv=kfold)
    # print(" ")
    # print("Accuracy: %.2f%% (%.2f%%)" % (results.mean()*100, results.std()*100))
    # print(' ')

 #podzielenie zbioru treningowego na zbior testowy 0.23 i treningowy 0.75
X_train, X_test, Y_train, Y_test = train_test_split(X, dummy_y, test_size=0.25, random_state=seed)

estimator.fit(X_train, Y_train)

#predykcja
predictions = estimator.predict(X_test)

#ocena
kfold = KFold(n_splits=10, shuffle=True, random_state=seed)
results = cross_val_score(estimator, X_test, Y_test, cv=kfold)

print(predictions)
print(encoder.inverse_transform(predictions))
print("____________")
print("Accuracy: %.2f%% (%.2f%%)" % (results.mean()*100, results.std()*100))


from __future__ import print_function

import argparse
import os
import torch
import torchvision.transforms as transforms
import torch.optim as optim
import torch.nn as nn
from torch.autograd import Variable

from data import DataSet
from vae import VAE
import config as CONFIG

# Parameters
parser = argparse.ArgumentParser(description='VAE train')
parser.add_argument('--epochs', default=100, type=int)
parser.add_argument('--batch_size', type=int, default=128)
parser.add_argument('--lr', default=0.001, type=float)
parser.add_argument('--cuda', default=True, type=bool)
parser.add_argument('--resume', default=False, type=bool)
parser.add_argument('--seed', type=int, default=1, metavar='S',
                    help='random seed (default: 1)')
parser.add_argument('--log-interval', type=int, default=10, metavar='N',
                    help='how many batches to wait before logging training status')
args, unknown = parser.parse_known_args()
kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}

# Setup
model = VAE()
if args.cuda:
    model.cuda()

reconstruction_function = nn.BCELoss()
reconstruction_function.size_average = False
optimizer = optim.Adam(model.parameters(), lr=1e-3)

if args.resume:
    resume_path = CONFIG.model.dir + '/model.checkpoint.tar'
    if os.path.isfile(resume_path):
        print("=> loading checkpoint '{}'".format(resume_path))
        checkpoint = torch.load(resume_path)
        args.start_epoch = checkpoint['epoch']
        best_loss = checkpoint['best_loss']
        model.load_state_dict(checkpoint['state_dict'])
        optimizer.load_state_dict(checkpoint['optimizer'])
        print("=> loaded checkpoint '{}' (epoch {})"
              .format(resume_path, checkpoint['epoch']))
    else:
        print("=> no checkpoint found at '{}'".format(resume_path))

def loss_function(recon_x, x, mu, logvar):
    BCE = reconstruction_function(recon_x, x)

    # see Appendix B from VAE paper:
    # Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014
    # https://arxiv.org/abs/1312.6114
    # 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)
    KLD_element = mu.pow(2).add_(logvar.exp()).mul_(-1).add_(1).add_(logvar)
    KLD = torch.sum(KLD_element).mul_(-0.5)

    return BCE + KLD

def train(epoch):
    model.train()
    train_loss = 0
    for batch_idx, (data, _) in enumerate(train_loader):
        data = Variable(data)
        if args.cuda:
            data = data.cuda()
        optimizer.zero_grad()
        recon_batch, mu, logvar = model(data)
        loss = loss_function(recon_batch, data, mu, logvar)
        loss.backward()
        train_loss += loss.data[0]
        optimizer.step()
        if batch_idx % args.log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader),
                loss.data[0] / len(data)))

    print('====> Epoch: {} Average loss: {:.4f}'.format(
          epoch, train_loss / len(train_loader.dataset)))

def test(epoch):
    model.eval()
    test_loss = 0
    for data, _ in test_loader:
        if args.cuda:
            data = data.cuda()
        data = Variable(data, volatile=True)
        recon_batch, mu, logvar = model(data)
        test_loss += loss_function(recon_batch, data, mu, logvar).data[0]

    test_loss /= len(test_loader.dataset)
    print('====> Test set loss: {:.4f}'.format(test_loss))
    return test_loss

def save_checkpoint(state, is_best, filename):
    if is_best:
        torch.save(state, filename)

if __name__ == '__main__':
    transformers = transforms.Compose([
        transforms.ToTensor()
    ])
    model_path = CONFIG.model.dir
    train_path = CONFIG.train.img.dir
    test_path = CONFIG.val.img.dir
    train_dataset = DataSet(train_path, transform=transformers)
    test_dataset = DataSet(test_path, transform=transformers)

    train_loader = torch.utils.data.DataLoader(
        dataset=train_dataset, batch_size=args.batch_size, shuffle=True,
        **kwargs
    )

    test_loader = torch.utils.data.DataLoader(
        dataset=test_dataset, batch_size=args.batch_size, shuffle=True,
        **kwargs
    )

    # train and test model
    best_loss = 50000
    for epoch in range(1, args.epochs + 1):
        train(epoch)
        test_loss = test(epoch)

        # remember best loss and save checkpoint
        is_best = best_loss > test_loss
        best_loss = min(best_loss, test_loss)

        save_checkpoint({
            'epoch': epoch + 1,
            'state_dict': model.state_dict(),
            'best_loss': best_loss,
            'optimizer': optimizer.state_dict(),
        }, is_best=is_best, filename=CONFIG.model.dir + '/model.checkpoint.tar')


from fastapi import APIRouter, Body, Depends

from src.auth.handler import get_current_user
from .schema import CreateMeet, UpdateMeet
from .service import MeetServices

router = APIRouter()

@router.post('/')
async def create_meet(
  dto: CreateMeet = Body(...),
  username: str = Depends(get_current_user),
  service: MeetServices = Depends(MeetServices)):

  return service.create_meet(dto, username)

@router.get('/')
async def get_all(
  username: str = Depends(get_current_user),
  service: MeetServices = Depends(MeetServices)):

  return service.get_all()

@router.get('/{id}')
async def get_by_id(
  id: str,
  username: str = Depends(get_current_user),
  service: MeetServices = Depends(MeetServices)):

  return service.get_by_id(id)

@router.put('/{id}')
async def update_meet(
  id: str,
  dto: UpdateMeet = Body(embed=False),
  username: str = Depends(get_current_user),
  service: MeetServices = Depends(MeetServices)):
  return service.update_meet(id, dto)

@router.delete('/{id}')
async def delete_meet(id: str, username: str = Depends(get_current_user), service: MeetServices =  Depends(MeetServices)):
  return service.delete_meet(id)

@router.get('/{id}/object')
async def get_objects(
  id: str,
  username: str = Depends(get_current_user),
  service: MeetServices = Depends(MeetServices)):

  return service.get_objects(id)


import sys

"""
Binary search algorithm implemented in python
"""

def callback(array):
    start = 0
    end = len(array) - 1

    return (start, end)

def binary_search(array, target, callback):  # assumes array is sorted
    start, end = callback(array)

    if start > end:
        return False

    mid_index = int((start + end) / 2)

    if array[mid_index] == target:
        return True

    if array[mid_index] > target:  # recurse left array
        return binary_search(array[start : mid_index - 1], target, callback)

    else:
        return binary_search(array[mid_index + 1 : end], target, callback)

if __name__ == "__main__":
    string_nums = sys.argv[1]
    array = [int(num) for num in string_nums]
    target = int(sys.argv[2])

    result = binary_search(array, target, callback)
    print(result)


import torch
import os
import csv
from torch.utils.data import Dataset
import numpy as np

import random

debug_opt = True
no_monitor = True

DATASET_PATH = os.path.join(os.path.dirname(__file__),"../../datasets/")
TRAIN_PATH = os.path.join(DATASET_PATH, "train.csv")
TEST_PATH = os.path.join(DATASET_PATH, "test.csv")

VAL_PATH = os.path.join(DATASET_PATH,"splitted/82val.csv")
TRA_PATH = os.path.join(DATASET_PATH,"splitted/82train.csv")

def train_split(split_ratio = [0.2, 0.8]):
    train_converter = {2: lambda s: (ord(s)-ord('A'))}
    test_converter = {1: lambda s: (ord(s)-ord('A'))}

    data = np.loadtxt(
        TRAIN_PATH,
        delimiter = ',',
        dtype = np.int32,
        skiprows=1,
        converters= train_converter
        )
    data = np.transpose(data)

    id = data[0]
    digit = data[1]
    letter = data[2]
    input = np.transpose(data[3:])

    len_data = len(id)

    num_map = np.zeros((10,26))

    # get the statistic
    for i in range(len(id)):
        num_map[
            int(digit[i]),
            int(letter[i])
            ]+= 1

    val_id = []
    train_id = []

    train_num_map = np.zeros((10,26))

    for i in range(len(id)):
        #Thinks without shuffling.
        #Or I can shuffle it by shuffle the "id"
        # -> but it's little bit complex.

        if train_num_map[
            int(digit[i]),
            int(letter[i])
            ] == 0:
            # First, save it to train_id.
            train_num_map[int(digit[i]), int(letter[i])] += 1
            train_id.append(id[i])
        else:
            # Second, randomly select one.
            ran = np.random.binomial(1,split_ratio[0])
            if ran < 0.5:
                train_num_map[int(digit[i]), int(letter[i])] += 1
                train_id.append(id[i])
            else:
                val_id.append(id[i])

    with open(VAL_PATH, 'w', newline='') as valfile:
        valfile.write('id,digit,letter')
        for i in range(28*28):
            valfile.write(','+str(i))
        valfile.write('\n')

        for i in val_id:
            valfile.write(str(i))
            valfile.write(','+str(digit[i-1]))
            valfile.write(','+chr(ord('A')+letter[i-1]))
            for j in range(28*28):
                valfile.write(','+str(input[i-1][j]))
                # it call sys-call 700 times so it would be slow
            valfile.write('\n')

    with open(TRA_PATH, 'w', newline='') as trainfile:
        trainfile.write('id,digit,letter')
        for i in range(28*28):
            trainfile.write(','+str(i))
        trainfile.write('\n')

        for i in train_id:
            trainfile.write(str(i))
            trainfile.write(','+str(digit[i-1]))
            trainfile.write(','+chr(ord('A')+letter[i-1]))
            for j in range(28*28):
                trainfile.write(','+str(input[i-1][j]))
                # it call sys-call 700 times so it would be slow
            trainfile.write('\n')

    if debug_opt:
        print(id[0:10])
        print(digit[0:10])
        print(letter[0:10])
        print(input[0].shape)
        print(len(val_id))
        print(len(train_id))

if __name__ == '__main__':
    train_split()


import pandas as pd
pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns', None)
import os
from sklearn import ensemble
from sklearn import preprocessing
from sklearn import metrics
import joblib
import numpy as np

from . import dispatcher

TEST_DATA = os.environ.get("TEST_DATA")
MODEL = os.environ.get("MODEL")

def predict():
    df = pd.read_csv(TEST_DATA)
    test_idx = df["enrollee_id"].values
    df = df.drop('enrollee_id', axis = 1)
    predictions = None

    for FOLD in range(5):
        print(FOLD)
        df = pd.read_csv(TEST_DATA)

        # data is ready to train
        clf = joblib.load(os.path.join("models", f"{MODEL}_{FOLD}.pkl"))
        cols = joblib.load(os.path.join('models',f"{MODEL}_{FOLD}_columns.pkl"))
        df = df[cols]
        print(df.shape)
        preds = clf.predict_proba(df)[:, 1]

        if FOLD == 0:
            predictions = preds
        else:
            predictions += preds

    predictions /= 5

    sub = pd.DataFrame(np.column_stack((test_idx, predictions)), columns=["enrollee_id", "target"])
    return sub

if __name__ == '__main__':
    submission = predict()
    submission['enrollee_id'] = submission['enrollee_id'].astype(int)
    submission.to_csv(f'output/{MODEL}.csv', index = False)


from typing import Tuple

import ipywidgets as W
import numpy as N
import pandas as P
import traitlets as T
import traittypes as TT

from .._base import ForceBase
from ..serializers import dataframe_serialization

@W.register
class DataFrameSource(ForceBase):
    """A Graph Source that stores the ``nodes`` and ``links`` as :class:`~pandas.DataFrame` instances."""

    _model_name: str = T.Unicode("DataFrameSourceModel").tag(sync=True)

    nodes: P.DataFrame = TT.PandasType(
        klass=P.DataFrame, help="the :class:`~pandas.DataFrame` of node data"
    ).tag(sync=True, **dataframe_serialization)

    node_id_column: str = T.Unicode(
        "id",
        help="the name of the column for a node's identifier, or 0-based position in the column if `None`",
    ).tag(sync=True)

    node_preserve_columns: Tuple[str, ...] = W.TypedTuple(
        T.Unicode(), help="columns to preserve when updating ``nodes``"
    ).tag(sync=True)

    link_id_column: str = T.Unicode(
        "id",
        help="the name of the column for a links's identifier, or 0-based position in the column if `None`",
    ).tag(sync=True)

    link_preserve_columns: Tuple[str, ...] = W.TypedTuple(
        T.Unicode(), help="columns to preserve when updating ``links``"
    ).tag(sync=True)

    links: P.DataFrame = TT.PandasType(
        klass=P.DataFrame, help="the :class:`~pandas.DataFrame` of link data"
    ).tag(sync=True, **dataframe_serialization)

    link_source_column: str = T.Unicode(
        "source",
        help="the name of the column for a link's source, defaulting to ``source``",
    ).tag(sync=True)

    link_target_column: str = T.Unicode(
        "target",
        help="the name of the column for a link's target, defaulting to ``target``",
    ).tag(sync=True)

    @T.validate("links")
    def _validate_links(self, proposal: T.Bunch) -> P.DataFrame:
        value: P.DataFrame = proposal.value

        if not isinstance(value, P.DataFrame):
            message = f"'links' must be a pandas.DataFrame, not {type(value)}"
            raise T.TraitError(message)

        if self.link_id_column not in value.columns:
            value[self.link_id_column] = N.arange(len(value))

        return value

    def __repr__(self) -> str:
        """A custom representation to avoid ``pandas``/``numpy`` equality issues."""
        name = self.__class__.__name__
        nodes_shape = self.nodes.shape if self.nodes is not None else None
        links_shape = self.links.shape if self.links is not None else None
        return f"{name}(nodes={nodes_shape}, links={links_shape})"


import requests

def events_api(params):
    host = "https://www2.morgenweb.de/austausch/vk_abfrage.php?out=json"
    host_params = {"ab": params["datum"], "bis": params["datum"], "rubrik": params["rubrik"], "plz": params["plz"],
                   "ort": params["ort"], "suche": params["suche"]}
    for key, value in host_params.items():
        if value is not None:
            host += "&{}={}".format(key, value)
    r = requests.get(host)
    return r.json()["ergebnis"]


from textwrap import fill
from tkinter import *
import tkinter as tk
from tkinter import ttk
from tkinter import filedialog
from tkinter import font

root = tk.Tk()
#root.iconbitmap("redumper.ico")
root.title("  redumper")
root.geometry
import glob, os, shutil, subprocess

# Row 0: "Output Title" row
myLabel_OutputTitle = Label(root, text="Output File / Folder Name:", justify=LEFT)
myLabel_OutputTitle.grid(sticky=W, column=0, columnspan=2, row=0, padx=8, pady=(7, 1))
# Telling the myButton_OutputTitle button what to do once clicked
OutputTitle = Entry(root, width=35)
OutputTitle.grid(sticky=W, row=0, column=2, columnspan=4, padx=5.5, pady=(7, 1))
OutputTitle.insert(0, "(enter game title here)")
def myClick_OutputTitle():
    myClickLabel_OutputTitle = Label(root, text=OutputTitle.get())
    myClickLabel_OutputTitle.grid(padx=8, pady=(7, 1))
# "Select Directory" button
def getFolderPath():
    folder_selected = filedialog.askdirectory(initialdir=folderPath.get())
    folderPath.set(folder_selected)

folderPath = StringVar()
folderPath.set(os.getcwd())
btnFind = ttk.Button(root, text="Select Directory", command=getFolderPath)
btnFind.grid(row=0, column=7, padx=8, pady=(7,1))

# Row 1: "Drive Letter" row -- DISABLING FOR NOW
#drives = [ chr(x) + ":" for x in range(65,91) if os.path.exists(chr(x) + ":") ]
#myLabel_DriveLetter = Label(root, text="Drive Letter:")
#myLabel_DriveLetter.grid(sticky=W, row=1, column=0, padx=8, pady=0)
#clicked_DriveLetter = StringVar()
#clicked_DriveLetter.set("Auto")
#myDrop_DriveLetter = OptionMenu(root, clicked_DriveLetter, drives)
#myDrop_DriveLetter.grid(sticky=W, row=1, column=2, padx=0, pady=0)

# Row 2: "Drive Speed" row
myLabel_DriveSpeed = Label(root, text="Drive Speed:")
myLabel_DriveSpeed.grid(sticky=W, row=2, column=0, padx=8, pady=0)
clicked_DriveSpeed = StringVar()
clicked_DriveSpeed.set("Max")
# SupportedDriveSpeeds list to be replaced with auto-detected drive speeds.
SupportedDriveSpeeds = ["Max","72","64","48","32","16","8","4","2"]
myDrop_DriveSpeed = OptionMenu(root, clicked_DriveSpeed, *SupportedDriveSpeeds)
myDrop_DriveSpeed.grid(sticky=W, row=2, column=2, padx=3, pady=0)
# Define the selected speed value
SelectedDriveSpeed = StringVar()
def change_mydrop(*args):
    SelectedDriveSpeed.set(clicked_DriveSpeed.get())
if clicked_DriveSpeed.get() == "Max":
     SelectedDriveSpeed.set("72")
else:
    SelectedDriveSpeed.set(clicked_DriveSpeed)
clicked_DriveSpeed.trace("w", change_mydrop)

# Row 3: "Allow Non-Preservation-Grade Drives" row
myLabel_AllowDrives = Label(root, text="Allow Unsupported Drives:")
myLabel_AllowDrives.grid(sticky=W, row=3, column=0, columnspan=2, padx=8, pady=0)
checkbox_AllowDrives = StringVar()
myCheckbox_AllowDrives = Checkbutton(root, text="", variable=checkbox_AllowDrives, onvalue="--unsupported", offvalue="")
myCheckbox_AllowDrives.grid(sticky=W, row=3, column=2, padx=0, pady=0)

# Row 4: Advanced Settings frame start
frame = LabelFrame(root, text="Advanced Settings", padx=8, pady=8)
frame.grid(padx=10, column=0, columnspan=20, pady=10)

# Row 5: "Leave these settings to default if you are unsure of useage."
myLabel_AdvSetLeaveDefault = Label(frame, text="Leave these settings kept at their defaults if you are unsure of useage.")
myLabel_AdvSetLeaveDefault.grid(sticky=W, row=5, column=0, columnspan=19, padx=10, pady=(3, 10))

# Row 6: "Mode" row
myLabel_Mode = Label(frame, text="Mode:")
myLabel_Mode.grid(sticky=W, row=6, column=0, padx=10, pady=1)
clicked_Mode = StringVar()
clicked_Mode.set("cd")
myDrop_Mode = OptionMenu(frame, clicked_Mode, "cd", "dump", "protection", "refine", "split", "info")
myDrop_Mode.grid(sticky=W, row=6, column=1, padx=3, pady=1)

# Row 7: "Retry Amount" row
myLabel_Retries = Label(frame, text="Retry Amount:")
myLabel_Retries.grid(sticky=W, row=7, column=0, padx=10, pady=1)
myEntry_Retries = Entry(frame, width=20, fg="#000000", borderwidth=1)
myEntry_Retries.grid(sticky=W, row=7, column=1, columnspan=19, padx=(5,0), pady=1)
myEntry_Retries.insert(0, "50")

# Row 8: "Stop LBA (for fake TOC)" [TEXT BOX]
#myLabel_StopLBA = Label(frame, text="Stop LBA (for fake TOC):")
#myLabel_StopLBA.grid(sticky=W, row=8, column=0, padx=10, pady=1)
#myEntry_StopLBA = Entry(frame, width=20, fg="#000000", borderwidth=1)
#myEntry_StopLBA.grid(sticky=W, row=8, column=1, columnspan=19, padx=(5,0), pady=1)
#myEntry_StopLBA.insert(0, "Enter sector range.")

# Row 9: "Skip LBA Range" [TEXT BOX]
#myLabel90 = Label(frame, text="Skip LBA Range (for fake TOC):")
#myLabel90.grid(sticky=W, row=9, column=0, padx=10, pady=1)
#myEntry91 = Entry(frame, width=20, fg="#000000", borderwidth=1)
#myEntry91.grid(sticky=W, row=9, column=1, columnspan=19, padx=(5,0), pady=1)
#myEntry91.insert(0, "Enter sector range.")

# Row 10: "Zero-out erroneous sectors" [CHECKBOX]
myLabel_ZeroOut = Label(frame, text="Zero-out erroneous sectors:")
myLabel_ZeroOut.grid(sticky=W, row=10, column=0, padx=10, pady=1)
checkbox_ZeroOut = StringVar()
myCheckbox_ZeroOut = Checkbutton(frame, text="", variable=checkbox_ZeroOut, onvalue="--zero-error-sectors", offvalue="")
myCheckbox_ZeroOut.grid(sticky=W, row=10, column=1, padx=(0), pady=1)

# Row 11: "Skip Lead-in:" [CHECKBOX]
myLabel_SkipLeadIn = Label(frame, text="Skip Lead-In sectors:")
myLabel_SkipLeadIn.grid(sticky=W, row=11, column=0, padx=10, pady=1)
checkbox_SkipLeadIn = StringVar()
myCheckbox_SkipLeadIn = Checkbutton(frame, text="", variable=checkbox_SkipLeadIn, onvalue="--skip-leadin", offvalue="")
myCheckbox_SkipLeadIn.grid(sticky=W, row=11, column=1, padx=0, pady=1)

# Row 12: "Skip Size (for ringed discs)": [TEXT BOX]
#myLabel_SkipSize = Label(frame, text="Skip Size (for ringed discs):")
#myLabel_SkipSize.grid(sticky=W, row=12, column=0, padx=10, pady=1)
#myEntry_SkipSize = Entry(frame, width=20, fg="#000000", borderwidth=1)
#myEntry_SkipSize.grid(sticky=W, row=12, column=1, columnspan=19, padx=(5,0), pady=1)
#myEntry_SkipSize.insert(0, "Enter value.")

# Row 13: "Ring Size (for ringed discs)": [TEXT BOX]
#myLabel_RingSize = Label(frame, text="Ring Size (for ringed discs):")
#myLabel_RingSize.grid(sticky=W, row=13, column=0, padx=10, pady=1)
#myEntry_RingSize = Entry(frame, width=20, fg="#000000", borderwidth=1)
#myEntry_RingSize.grid(sticky=W, row=13, column=1, columnspan=19, padx=(5,0), pady=1)
#myEntry_RingSize.insert(0, "Enter value.")

# Row 14: "Parameters:" [TEXT ENTRY BOX AUTO-FILLED WITH COMMAND LINE PARAMETERS AS DECIDED BY GUI SELECTIONS] -- DISABLING FOR NOW
#myLabel_Parameters = Label(frame, text="Parameters:")
#myLabel_Parameters.grid(sticky=W, row=14, column=0, padx=10)
#myEntry_Parameters = Entry(frame, width=35, fg="#000000", borderwidth=1)
#myEntry_Parameters.grid(row=14, column=1, columnspan=19)
#myEntry_Parameters.insert(0, "myButton_StartDumping values here")

# Row 15: "Start Dumping"
def myClick_StartDumping():
    myButton_StartDumping = Label(root, command=subprocess.run([
        'redumper.exe',
        clicked_Mode.get(),
        "--image-path="+folderPath.get()+"/"+OutputTitle.get(),
        checkbox_AllowDrives.get(),
        "--speed="+SelectedDriveSpeed.get(),
        "--retries="+myEntry_Retries.get(),
        checkbox_ZeroOut.get(),
        checkbox_SkipLeadIn.get()]))
    myButton_StartDumping.grid()
myButton_StartDumping = Button(root, text="Start Dumping", command=myClick_StartDumping, fg="#0057B7", bd=3)
myButton_StartDumping.grid(row=15, column=2, padx=(14, 0), pady=(10, 18))

root.mainloop()


'''
There are a total of numCourses courses you have to take, labeled from 0 to numCourses - 1. You are given an array prerequisites where prerequisites[i] = [ai, bi] indicates that you must take course ai first if you want to take course bi.

For example, the pair [0, 1] indicates that you have to take course 0 before you can take course 1.
Prerequisites can also be indirect. If course a is a prerequisite of course b, and course b is a prerequisite of course c, then course a is a prerequisite of course c.

You are also given an array queries where queries[j] = [uj, vj]. For the jth query, you should answer whether course uj is a prerequisite of course vj or not.

Return a boolean array answer, where answer[j] is the answer to the jth query.

Example 1:

Input: numCourses = 2, prerequisites = [[1,0]], queries = [[0,1],[1,0]]
Output: [false,true]
Explanation: The pair [1, 0] indicates that you have to take course 1 before you can take course 0.
Course 0 is not a prerequisite of course 1, but the opposite is true.
Example 2:

Input: numCourses = 2, prerequisites = [], queries = [[1,0],[0,1]]
Output: [false,false]
Explanation: There are no prerequisites, and each course is independent.
Example 3:

Input: numCourses = 3, prerequisites = [[1,2],[1,0],[2,0]], queries = [[1,0],[1,2]]
Output: [true,true]

Constraints:

2 <= numCourses <= 100
0 <= prerequisites.length <= (numCourses * (numCourses - 1) / 2)
prerequisites[i].length == 2
0 <= ai, bi <= n - 1
ai != bi
All the pairs [ai, bi] are unique.
The prerequisites graph has no cycles.
1 <= queries.length <= 104
0 <= ui, vi <= n - 1
ui != vi
'''
from typing import *

import unittest

class Solution:
    def checkIfPrerequisite(self, numCourses: int, prerequisites: List[List[int]], queries: List[List[int]]) -> List[bool]:
        graph = [[] for _ in range(numCourses)]
        for pre, next in prerequisites:
            graph[pre].append(next)

        mem = {}
        res = []
        for query_u, query_v in queries:
            res.append(query_v in self.dfs(graph, query_u, mem))

        return res

    def dfs(self, graph, u, mem):
        if u in mem:
            return mem[u]

        res = {u}
        for v in graph[u]:
            res |= self.dfs(graph, v, mem)
        mem[u] = res
        return res

class TestSolution(unittest.TestCase):

    def test_case(self):
        examples = (
            ((2, [[1, 0]], [[0,1], [1,0]]), [False, True]),
            ((2, [], [[0,1], [1,0]]), [False, False]),
            ((3, [[1,2], [1,0], [2,0]], [[1,0], [1,2]]), [True, True]),
        )
        for first, second in examples:
            self.assert_function(first, second)

    def assert_function(self, first, second):
        self.assertEqual(Solution().checkIfPrerequisite(*first), second,
                         msg="first: {}; second: {}".format(first, second))

unittest.main()


from myApp import app

from flask import render_template

from flask import jsonify
import pymongo

import json

def result(f):
    def show(*args, **kwargs):
        cursor = f()
        res = []
        for c in cursor:
            if "_id" in c: c["_id"] = str(c["_id"])
            res.append(c)
        return json.dumps(res)
    return show

@app.route("/json")
@result
def json1():
    db = pymongo.Connection("127.0.0.1", 27017).app
    cursor = db.groups.find()
    return cursor

@app.route("/api/json")
def getjson():
    return render_template("jd.json")


import serial
import pygame
import serial

class MainDeviceClass(object):

	All = []

	def __init__(self, init_x, init_y):
		"""Append object to self.All so all devices can be listed globally"""
		self.All.append(self)
		"""Set up UI variables"""
		self.name      = 'Device not found'
		self.loc_x     = init_x
		self.loc_y     = init_y
		self.line1     = ["Status", "Not Found"]
		self.line2     = ["Set massflow", "0"]
		self.line3     = ["Measured flow", "0"]
		self.line4     = ["Label", "0"]
		self.lenline   = 4
		self.spacing   = 100
		self.inittext()
		"""Set up variables for serial communication"""
		self.baud = 38400
		self.isconnect = False
		self.myport    = None
		self.isheater    = False

	def inittext(self):
		self.txtclr    = (  0,   0,   0)
		self.txtfontsm = pygame.font.SysFont("Ubuntu", 22)
		self.txtfontlr = pygame.font.SysFont("Ubuntu", 44)
		self.txtfontmi = pygame.font.SysFont("Ubuntu", 33)
		self.Line0     = self.txtfontsm.render(self.name, True, self.txtclr)
		self.Line1l    = self.txtfontsm.render(self.line1[0], True, self.txtclr)
		self.Line2l    = self.txtfontsm.render(self.line2[0], True, self.txtclr)
		self.Line3l    = self.txtfontsm.render(self.line3[0], True, self.txtclr)
		self.Line4l    = self.txtfontsm.render(self.line4[0], True, self.txtclr)

	def writetoscreen(self, screen):
		screen.blit(self.Line0, [self.loc_x, self.loc_y - 15])

		if self.lenline > 0:
			screen.blit(self.Line1l, [self.loc_x, self.loc_y + 22])
			self.Line1r = self.txtfontsm.render(self.line1[1], True, self.txtclr)
			screen.blit(self.Line1r, [self.loc_x + self.spacing, self.loc_y + 22])

		if self.lenline > 1:
			screen.blit(self.Line2l, [self.loc_x, self.loc_y + 44])
			self.Line2r = self.txtfontsm.render(self.line2[1], True, self.txtclr)
			screen.blit(self.Line2r, [self.loc_x + self.spacing, self.loc_y + 44])

		if self.lenline > 2:
			screen.blit(self.Line3l, [self.loc_x, self.loc_y + 66])
			self.Line3r = self.txtfontsm.render(self.line3[1], True, self.txtclr)
			screen.blit(self.Line3r, [self.loc_x + self.spacing, self.loc_y + 66])

		if self.lenline > 3:
			screen.blit(self.Line4l, [self.loc_x, self.loc_y + 88])
			self.Line3r = self.txtfontsm.render(self.line3[1], True, self.txtclr)
			screen.blit(self.Line3r, [self.loc_x + self.spacing, self.loc_y + 88])

	def connect(self, port):
		self.isconnect = True
		self.myport    = port
		self.line1[1]  = "Connected"

	def disconnect(self):
		self.isconnect = False
		self.myport    = None
		self.line1[1]  = "Not Connected"

	def getSetpoint(self):
		try:
			with serial.Serial(self.port, self.baud) as ser:
				#test connection, get name, set isconnect = True
				ser.write(':06800401210121\r\n')    #setpoint: read, Process: 1, Parameter: 1, Type: integer
				msg = msg = ser.read_until('\r\n')    #:0680020121 7D00 \r\n measure = hex 7D00 = 32000 = 100%
				"""String manipulation to isolate relevant hex and convert to setpoint"""
				msg = msg.replace(':0680020121', '')
				msg = msg.replace('\r\n', '')
				setp = int(msg, 16) / 32000 * self.maxsetpoint
				self.setpoint = setp
				return self.setpoint

		except serial.SerialTimeoutException:
			print("Port 1 is not connected, check if device is plugged in")
			self.deviceDisconnect(device)
			return self.setpoint

		except serial.SerialException:
			print("Could not find port 1, check if RS232 board is connected")
			self.deviceDisconnect(device)
			return self.setpoint
		pass

	def getMeasure(self):
		try:
			with serial.Serial(self.port, self.baud) as ser:
				#test connection, get name, set isconnect = True
				ser.write(':06800401210120\r\n')    #measure: read, Process: 1, Parameter: 0, Type: integer
				msg = ser.read_until('\r\n')    #::0680020121 7D00 \r\n measure = hex 7D00 = 32000 = 100%
				"""String manipulation to isolate relevant hex and convert to setpoint"""
				msg = msg.replace(':0680020121', '')
				msg = msg.replace('\r\n', '')
				meas = int(msg, 16) / 32000 * self.maxmeasure
				self.measure = meas

		except serial.SerialTimeoutException:
			print("Port 1 is not connected, check if device is plugged in")
			self.disconnect(device)
			pass
		except serial.SerialException:
			print("Could not find port 1, check if RS232 board is connected")
			self.disconnect(device)

	def setSetpoint(self, setpoint):
		try:
			with serial.Serial(self.port, self.baud) as ser:
				msg = str(hex(int(float(setpoint)/self.maxsetpoint*32000)))
				msg = msg.replace('0x', '')
				while not len(msg) == 4:
					msg = '0' + msg

				ser.write(':0680010121{}\r\n'.format(msg))    #read usertag: Process: 113, Parameter: 6, Type: string
				ans  = ser.read_until('\r\n')    #::0680020121 7D00 \r\n measure = hex 7D00 = 32000 = 100%

		except serial.SerialTimeoutException:
			print("Port 1 is not connected, check if device is plugged in")
			self.disconnect()
			pass
		except serial.SerialException:
			print("Could not find port 1, check if RS232 board is connected")
			self.disconnect()

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

class CoriFlow(MainDeviceClass):
	def __init__(self, init_x, init_y):
		super(CoriFlow, self).__init__(init_x, init_y)
		self.maxsetpoint  = 5
		self.usertag      = ':0D80027166005553455254414700\r\n'

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

class ElFlow(MainDeviceClass):
	def __init__(self, init_x, init_y):
		super(ElFlow, self).__init__(init_x, init_y)
		self.maxsetpoint  = 5
		self.usertag      = ':0D80027166005553455212314700\r\n'

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

class Heater(MainDeviceClass):
	def __init__(self, init_x, init_y):
		super(ElFlow, self).__init__(init_x, init_y)
		self.isheater       = True

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

if __name__ == "__main__":
	print("Started running in BronkhorstDevices.py")
	#execfile('Main.py')


import re
# import nltk.corpus
import numpy as np
# from itertools import chain

'''Create dictionary'''
d_list = []
with open("google-10000-english.txt") as dictionary:
    for line in dictionary:
        d_list.append(line.strip())

'''Create first paragraph of Jane Austen txt as a list of strings'''
text_list = []
with open("austen-sense-corrupted.txt") as text:
    for line in text:
        text_list.append(line)

austen_list = []
for i in text_list:
    austen_list.append(i.strip())

nested_list = []
for x in austen_list:
    nested_list.append(x.split(" "))

final_list = [item for sublist in nested_list for item in sublist]
# first paragraph
final_list = final_list[12:223]
# 12:223
# print(final_list)

'''Calculate minimum levenshtein distance between two word'''
def levenshtein_matrix(corruptedWord, correctedWord):
    # create empty matrix
    distances = np.zeros((len(corruptedWord) + 1, len(correctedWord) + 1), dtype = int)

    # initialize first row and column
    for w1 in range(1, len(corruptedWord) + 1):
        distances[w1][0] = w1
    for w2 in range(1, len(correctedWord) + 1):
        distances[0][w2] = w2

    for row in range(1, len(corruptedWord) + 1):
        for col in range(1, len(correctedWord) + 1):
            # calculate substitution cost
            if corruptedWord[row - 1] == correctedWord[col - 1]:
                sub_cost = 0
            else:
                sub_cost = 1
            # calculate minimum cost
            distances[row][col] = min(distances[row-1][col] + 1, distances[row-1][col-1] + sub_cost, distances[row][col-1] + 1)

    return distances[len(corruptedWord)][len(correctedWord)]

# print(levenshtein_matrix("estete", "estate"))

''' Takes a list of words and compare against dictionary. Returns word with shortest levenshtein distance'''
def correct_word(word, dictionary):
    min_dist = None
    for j in dictionary:
        current_dist = levenshtein_matrix(word, j)
        if (min_dist is None) or (current_dist <= min_dist):
            min_dist = current_dist
            current_word = j
    return current_word

'''Return list of corrected words from Jane Austen text
- If previous word ends with period, check capitalized word
- If not, pass
- If word[-1] is punctuation, remove punctuation, correct word, add punctuation
'''

def string_match(word_list, dictionary):
    corrected_story = []
    for i in word_list:
        if i in dictionary or i == "":
            corrected_story.append(i)
        elif re.findall(r"\w+[\.\,\;]$", i):
            punc = i[-1]
            only_word = i.replace(punc, "")
            c_word = correct_word(only_word, dictionary)
            corrected_story.append(c_word + punc)
        elif i[0].isupper() == True:
            corrected_story.append(i)
        else:
            corrected_story.append(correct_word(i, dictionary))
    return corrected_story

print(string_match(final_list, d_list))

'''Return final single string of story'''
def corrected_string(correct_story):
    final_story = ""
    for i in correct_story:
        final_story += str(i) + " "
    final_story = final_story[:-1]
    return final_story

# print(corrected_string(string_match(final_list, d_list)))


from selenium import webdriver
import unittest
import time
from selenium.webdriver.common.action_chains import ActionChains

# driver = webdriver.Chrome()
class post_content(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.driver = webdriver.Chrome()
        pngname = time.strftime("%b %d %Y %H:%M:%S", time.gmtime())
        cls.driver.save_screenshot(pngname+'.png')
        # print('pinduoduo')

    # def setUp(self):
    #     # self.driver = webdriver.Chrome()
    #     print('121212')

    def test_01_login(self):
        driver = self.driver
        driver.get('http://39.107.96.138:3000/signin')
        driver.find_element_by_id('name').send_keys('user1')
        driver.find_element_by_id('pass').send_keys('123456')
        driver.find_element_by_class_name('span-primary').click()

        url = driver.current_url
        self.assertEqual(url, "http://39.107.96.138:3000/")
        name = driver.find_element_by_css_selector(' div.user_card > div> span.user_name > a').text
        self.assertEqual(name, "user1")

    @unittest.skip('neednot do it')
    def test_02_post(self):
        driver = self.driver
        driver.get('http://39.107.96.138:3000/topic/create')
        driver.find_element_by_id('tab-value').click()
        driver.find_element_by_css_selector('#tab-value > option:nth-child(2)').click()
        driver.find_element_by_name('title').send_keys('0201')
        content = driver.find_element_by_class_name('CodeMirror-scroll')
        content.click()
        ActionChains(driver).move_to_element(content).send_keys("it's time !!").perform()
        driver.find_element_by_class_name('submit_btn').click()

    # def tearDown(self):
    #     driver.quit()
    #     print('343434')

    @classmethod
    def tearDownClass(cls):
        driver = cls.driver
        print('wahahah')
        driver.quit()

if __name__ == "__main__":
    unittest.main(verbosity =2)


from rest_framework import serializers
from contest import models

from user.models import User,Team

class ContestSerializer(serializers.ModelSerializer):
    def validate(self, data):
        """
        Check that start is before finish.
        """
        if data['start_time'] > data['end_time']:
            raise serializers.ValidationError("end_time must occur after start_time")
        return data
    class Meta:
        model = models.Contest
        fields = "__all__"

class ContestConfigSerializer(serializers.ModelSerializer):
    class Meta:
        model = models.ContestConfig
        fields = ("key","value")

class ScoreboardSerializer(serializers.ModelSerializer):
    class Meta:
        model = User
        fields = ("id","nickname","solved_amount","points","last_point_at","solved_challenges")
        read_only_field = [
            "id",
            "nickname",
            "solved_amount",
            "points",
            "last_point_at"
        ]
        ordering = ['points']

class TeamScoreboardSerializer(serializers.ModelSerializer):
    class Meta:
        model = Team
        fields = ("id","name","solved_amount","points","solved_challenges")
        read_only_field = [
            "id",
            "name",
            "solved_amount",
            "points",
        ]
        ordering = ['points']


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2019/8/4 22:02
# @Author  : tc
# @File    : RemoveDuplicatesFromSortedList.py
"""
给定一个排序链表，删除所有重复的元素，使得每个元素只出现一次。
1.Input:1->1->2
  Output: 1->2

2.Input:1->1->2->3->3
  Output:1->2->3

https://leetcode-cn.com/problems/remove-duplicates-from-sorted-list/submissions/

注意： 1.问题是已经排序了,所以所有重复的结点会连在一起

      2.当前结点遇到与下一个结点值一样的结点时，需要让当前结点的前一个结点指向重复结点的下一个结点，而这个前
       一个结点不能向后滑动，需要与新指向的结点再次比较。

      3.if语句中的cur.next可能会空

"""
class ListNode:
    def __init__(self, x):
        self.val = x
        self.next = None

def deleteDuplicates(head):
    if not head or not head.next:
        return head
    headNode = ListNode(1)
    headNode.next = head  # 创建一个哑结点
    cur = headNode.next  # 创建一个当前结点
    while cur and cur.next:
        if cur.val == cur.next.val:
            cur.next = cur.next.next
        else:
            cur = cur.next
    return headNode.next

if __name__ == '__main__':
    a = ListNode(1)
    b = ListNode(1)
    c = ListNode(2)
    d = ListNode(3)
    e = ListNode(3)

    a.next = b
    b.next = c
    c.next = d
    d.next = e

    result = deleteDuplicates(a)

    while (result):
        print(result.val)
        result = result.next


# coding:utf-8

file_1 = '/export/home/sunhongchao1/1-NLU/Workspace-of-NLU/corpus/sa/comment/shaver/label/test.txt'
file_2 = '/export/home/sunhongchao1/1-NLU/Workspace-of-NLU/lexical_analysis/outputs/shaver_entity_epoch_10_hidden_layer_4_max_seq_len_128_gpu_3/entity_test_results.txt'

lines_1 = []
lines_2 = []

with open(file_1, encoding='utf-8', mode='r') as f:
    lines_1 = f.readlines()

with open(file_2, encoding='utf-8', mode='r') as f:
    lines_2 = f.readlines()

print(len(lines_1))
print(len(lines_2))

idx = 0
for item1, item2 in zip(lines_1, lines_2):
    idx += 1
    if item1.strip() == ''and item2.strip() != '' :
        print("###", item1.strip())
        print('###', item2.strip())
        print(idx)
    elif item2.strip() == ''and item1.strip() != '' :
        print("$$$", item1.strip())
        print("$$$$", item2.strip())
        print(idx)
    else:
        pass


import pandas as pd
import re
import os
import sys

test_file = sys.argv[1]
filename, _ = os.path.splitext(test_file)
tmpname = "tmp-" + filename
outname = filename + ".csv"

ifn = open(test_file, "r")
ofn = open(tmpname, "w")

while True:
    line = ifn.readline()
    if not line:
        break
    line = line.replace(":1", "")
    line = line.replace(" $", "")
    line = re.sub(" $", "", line)
    line = line.replace(" ", ",")
    ofn.write(line)

ifn.close()
ofn.close()

df = pd.read_csv(tmpname, header=None)
n_rows_origin = len(df)

df = df.dropna()
df = df.astype('int64')
n_rows_dropped = len(df)
print("{}/{}".format(n_rows_dropped, n_rows_origin))

df.to_csv(outname, header=False, index=False)
if os.path.exists(tmpname):
    os.remove(tmpname)


from nltk.tokenize import word_tokenize
from util import Counter

def ngrams(corpus, n):
    model = Counter()
    for sentence in corpus:
        sent = word_tokenize(sentence)
        for i in range(len(sent)-1):
            key = tuple(sent[i:i+n])
            key = tuple(w.lower() for w in key)
            model[key] += 1
    return model


import re

import sqlalchemy as sa
from sqlalchemy import orm, event
from sqlalchemy.ext import declarative

from ldotcommons import utils

Base = declarative.declarative_base()

def _re_fn(regexp, other):
    return re.search(regexp, other, re.IGNORECASE) is not None

def create_engine(uri='sqlite:///:memory:', echo=False):
    engine = sa.create_engine(uri, echo=echo)

    # @property
    # def __monkeypatch_Base_query(self):
    #     return self.session.query(self)

    # Base.metadata.create_all(bind=engine)

    # Monkeypatch magic
    # setattr(Base, 'engine', engine)
    # setattr(Base, 'session', create_session(engine=engine))
    # setattr(Base.__class__, 'query', __monkeypatch_Base_query)

    # Enable regexp and foreign_keys functionality for sqlite connections
    if uri.startswith('sqlite:///'):
        @event.listens_for(engine, "begin")
        def do_begin(conn):
            conn.connection.create_function('regexp', 2, _re_fn)

        @event.listens_for(engine, "connect")
        def set_sqlite_pragma(conn, record):
            cursor = conn.cursor()
            cursor.execute("PRAGMA foreign_keys=ON")
            cursor.close()

    return engine

def create_session(uri=None, engine=None, echo=False):
    if not engine:
        engine = create_engine(uri=uri, echo=echo)

    sess = orm.sessionmaker()
    sess.configure(bind=engine)
    Base.metadata.create_all(engine)
    session = sess()

    # session = orm.scoped_session(orm.sessionmaker(bind=engine))

    return session

def query_from_params(conn, mapping, **params):

    # # This code can be used for add type-safety to this function
    # from sqlalchemy.sql import sqltypes

    # columns = self._mapping.__table__.columns
    # for (colname, column) in columns.items():
    #     columntype = column.type
    #     if isinstance(columntype, sqltypes.String):
    #         self._ops[(colname, None)] = self.by_string
    #         self._ops[(colname, 'like')] = self.by_string_like
    #         self._ops[(colname, 'regexp')] = self.by_string_regexp

    #     if isinstance(columntype, sqltypes.Integer):
    #         self._ops[(colname, 'min')] = self.by_amount_min
    #         self._ops[(colname, 'max')] = self.by_amount_max

    q = conn.query(mapping)

    for (prop, value) in params.items():
        if '_' in prop:
            key = '_'.join(prop.split('_')[:-1])
            mod = prop.split('_')[-1]
        else:
            key = prop
            mod = None

        attr = getattr(mapping, key, None)

        if mod == 'like':
            q = q.filter(attr.like(value))

        elif mod == 'regexp':
            q = q.filter(attr.op('regexp')(value))

        elif mod == 'min':
            value = utils.parse_size(value)
            q = q.filter(attr >= value)

        elif mod == 'max':
            value = utils.parse_size(value)
            q = q.filter(attr <= value)

        else:
            q = q.filter(attr == value)

    return q

def glob_to_like(x, wide=False):
    for (g, l) in (('*', '%'), ('.', '_')):
        x = x.replace(g, l)

    if wide:
        if not x.startswith('%'):
            x = '%' + x

        if not x.endswith('%'):
            x = x + '%'

    return x


from random import uniform
from vector import Vector
from cult import Cult
from demon import Demon
from berserk import Berserk

class World:

    def __init__(self, model):
        self.bodies = []
        self.model = model
        self.time = 0
        self.data_demons = []
        self.data_gr_demons = []
        self.data_cults = []
        self.step_time = 1

        self.count_berserk = []
        self.data_berserk = -1

    # добавление тела и спрайта
    def add_body(self, b):
        self.bodies.append(b)

    # удаление
    def remove(self, b):
        self.bodies.remove(b)

    # все действия в мире
    def act(self, dt):
        self.next_step(dt)
        self.time_to_turn(dt)
        self.collisions()
        for b in self.bodies:
            if type(b.parent) == Cult:
                self.model.life(b.parent, dt)
            if b.deleted:
                self.remove(b)
            if not self.time and not b.see:
                b.velocity = Vector(b.velocity.lenght, uniform(0, 360))
            b.move(dt)
            b.parent.view_body.pos = b.pos

    # обработка пересечений тел (взаимодействий)
    def collisions(self):
        for i in range(len(self.bodies)):
            b1 = self.bodies[i]
            if b1.deleted:
                continue
            v1 = b1.parent.view_body
            for j in range(i+1, len(self.bodies)):
                b2 = self.bodies[j]
                if b2.deleted:
                    continue
                v2 = b2.parent.view_body
                if b1.intersect(b2):
                    self.model.collision(b1.parent, b2.parent)
                elif b2.intersect(b1):
                    self.model.collision(b2.parent, b1.parent)
                elif v1.intersect(b2):
                    self.model.detection(b1.parent, b2.parent)
                elif v2.intersect(b1):
                    self.model.detection(b2.parent, b1.parent)
                if b1.deleted:
                    b2.see = False
                elif b2.deleted:
                    b1.see = False

    # время для смены направления
    def time_to_turn(self, dt):
        if self.time >= 5:
            self.time = 0
            return True
        self.time += dt
        return False

    def next_step(self, dt):
        if self.step_time >= 1:
            cults = [bd.parent for bd in self.bodies if type(bd.parent) == Cult]
            count_cult = sum([cult.settlers for cult in cults])
            demons = [bd.parent for bd in self.bodies if type(bd.parent) == Demon]
            count_demon = sum([demons.count for demons in demons])
            self.data_cults.append(len(cults))
            self.data_gr_demons.append(len(demons))
            self.data_demons.append(count_cult + count_demon)

            berserk = [bd.parent for bd in self.bodies if type(bd.parent) == Berserk]
            self.count_berserk.append(len(berserk))
            for i in range(len(self.count_berserk)):
                if self.count_berserk[i] != 0:
                    self.data_berserk = i+2
                    continue

            self.step_time = 0
            self.model.time_of_life += 1
        else:
            self.step_time += dt


#!/usr/bin/env python3

from os import path
from collections import *
import re

CLASSES, MY_TICKET, NEARBY_TICKETS = range(3)

with open(path.join(path.dirname(__file__), "input.txt")) as f:
    state = CLASSES
    classes = []
    my_ticket = None
    nearby_tickets = []

    for line in f.read().splitlines():
        if not line:
            state += 1
            continue

        if line in ("your ticket:", "nearby tickets:"):
            continue

        if state == CLASSES:
            name, lo1, hi1, lo2, hi2 = re.fullmatch(
                r"(.+): (\d+)-(\d+) or (\d+)-(\d+)", line
            ).groups()
            classes.append((name, int(lo1), int(hi1), int(lo2), int(hi2)))
        elif state == MY_TICKET:
            my_ticket = tuple(map(int, line.split(",")))
        else:
            nearby_tickets.append(tuple(map(int, line.split(","))))

    part1 = 0
    valid_tickets = []

    for ticket in nearby_tickets:
        valid = True
        for field in ticket:
            for _, lo1, hi1, lo2, hi2 in classes:
                if lo1 <= field <= hi1 or lo2 <= field <= hi2:
                    break
            else:
                valid = False
                part1 += field
        if valid:
            valid_tickets.append(ticket)

    print("Part 1:", part1)

    options = {i: set(range(len(classes))) for i in range(len(classes))}

    for ticket in valid_tickets:
        for i, field in enumerate(ticket):
            for j, (_, lo1, hi1, lo2, hi2) in enumerate(classes):
                if not (lo1 <= field <= hi1 or lo2 <= field <= hi2):
                    options[i].remove(j)

    part2 = 1

    while options:
        remove_keys = []
        remove_vals = set()
        for i, op in options.items():
            if len(op) == 1:
                cls = next(iter(op))
                if classes[cls][0].startswith("departure"):
                    part2 *= my_ticket[i]
                remove_keys.append(i)
                remove_vals.add(cls)
        for key in remove_keys:
            del options[key]
        for vals in options.values():
            vals.difference_update(remove_vals)

    print("Part 2:", part2)


import time
print('Timer')
minutes = int(input('Minutes: '))
seconds = int(input('Seconds: '))
timer = (minutes * 60) + seconds
x = seconds
y = minutes
while timer > 0:
    print(y, ':', x)
    time.sleep(1)
    x -= 1
    if x == 0 or x == -1:
        x = 59
        y -= 1
    timer -= 1


from NEAT_Individual import *
import time
import copy
import numpy as np

try:
    import cPickle as pickle
except ModuleNotFoundError:
    import pickle

# individuals
INPUT_SIZE = 8
OUTPUT_SIZE = 4

# population
POPULATION_SIZE = 200
NUMBER_OF_TRIALS = 2
MAX_STEPS = 5000
MAX_GENERATIONS = 1000

# evolutionary
SURVIVOR_PROPORTION = 0.25
RE_EXPRESSION_CHANCE = 0.25

# mutation
MUTATE_ALL_WEIGHTS_CHANCE = 0.80
MUTATE_WEIGHT_CHANCE = 0.90
MUTATE_WEIGHT_STRENGTH = 0.10

MUTATE_NODE_CHANCE = 0.01
MUTATE_CONNECTION_CHANCE = 0.10

# speciation
compatibility_threshold = 0.17

# global variables (sue me)
node_number = INPUT_SIZE+OUTPUT_SIZE+1
#connection_number = INPUT_SIZE*OUTPUT_SIZE+1
connection_number = 0

# filenames
timestamp = time.strftime("%Y-%m-%d-%H%M%S", time.localtime())
filename = "data/average_fitness_log_" + timestamp + ".txt"
filename2 = "data/best_individual_log_" + timestamp + ".txt"

pickle_file = "data/best_individuals_" + timestamp + ".dat"

# loading bars
spacer = "_"
bit = "/"
inds = 4

def main():
    # global values
    global node_number
    global connection_number
    global best_individuals

    # pickle stuff
    best_individuals = []

    print("-"*60)
    print("SAVING TO FILES:")
    print(filename)
    print(pickle_file)
    print()

    # generate initial population
    population = gen_rand_pop(POPULATION_SIZE)

    for gen_number in range(MAX_GENERATIONS):
        print("GENERATION:", gen_number, "\n")
        raw_fits = assess_pop_fits(population)
        adj_fits = calc_pop_adj_fits(population)
        sorted_pop = sort_pop(population)

        fit_stats(adj_fits, sorted_pop)

        # prep for next generation
        print("-"*40)
        culled_pop = cull_population(sorted_pop)
        species_list = speciate_pop(culled_pop)
        allocations = allocate_offspring(species_list)

        # save best individual to file
        best_individuals = []
        for species in species_list:
            best_individuals.append(copy.deepcopy(species[-1]))

        file_pi = open(pickle_file, 'wb')
        pickle.dump(best_individuals, file_pi)
        #print("PICKLED BEST INDIVIDUALS")

        # creates the next generation
        population = create_next_gen(species_list, allocations)

        print("COMPATIBILITY THRESHOLD:", compatibility_threshold)
        print("  OFFSPRING ALLOCATIONS:", allocations)
        print("   NEXT GENERATION SIZE:", len(population))
        print("="*80+"")

def gen_rand_pop(size: int) -> list:
    print(end="SEEDING POPULATION...")
    population = [Individual(input_size=INPUT_SIZE, output_size=OUTPUT_SIZE) for _ in range(size)]
    print("DONE")
    print("="*80)
    return population

def assess_fit(target: Individual) -> None:
    target.assess_fitness(max_trials=NUMBER_OF_TRIALS,  max_steps=MAX_STEPS, display=False)

def assess_pop_fits(population: list) -> list:
    start_time = time.time()
    loading("ASSESSING POPULATION FITNESS", len(population))

    for i in range(len(population)):
        assess_fit(population[i])
        blip(i)

    end_load(start_time)

    fits = [individual.fitness for individual in population]
    return fits

def get_compatibility_distance(ind_one: Individual, ind_two: Individual) -> float:

    excess_gene_weight = 1
    disjoint_gene_weight = 1
    weight_diff_weight = 0.4

    ind_one_connections = ind_one.genome.get_connection_ids()
    ind_two_connections = ind_two.genome.get_connection_ids()

    # --- deal with edge case of null individual
    if len(ind_one_connections) == 0 and len(ind_two_connections) == 0:
        return 0
    if len(ind_one_connections) == 0 and len(ind_two_connections) != 0:
        return len(ind_two_connections) * excess_gene_weight
    if len(ind_one_connections) != 0 and len(ind_two_connections) == 0:
        return len(ind_one_connections) * excess_gene_weight

    disjoints = 0

    # -- calc num of disjoints and excesses
    if len(ind_one_connections) == 0: ind_one_connections.append(0)
    if len(ind_two_connections) == 0: ind_two_connections.append(0)

    max_val = max(max(ind_one_connections), max(ind_two_connections))
    min_val = min(max(ind_one_connections), max(ind_two_connections))

    for count in range(min_val+1):
        if (count in ind_one_connections) ^ (count in ind_two_connections):
                disjoints += 1

    ran = range(min_val+1, max_val+1)
    one = len(set(ran).intersection(ind_one_connections))
    two = len(set(ran).intersection(ind_two_connections))

    excesses = max(one, two)
    # -- end

    # -- calc avg. weight differences of matching connections
    matching = set(ind_one_connections).intersection(ind_two_connections)
    number = len(matching)
    sum = 0

    for num in matching:
        weight1 = ind_one.genome.connections[num].weight
        weight2 = ind_two.genome.connections[num].weight
        difference = abs(weight1 - weight2)
        sum += difference

    average = sum/max(1, number)

    large_genome_size = max(len(ind_one_connections), len(ind_two_connections)) # N is the size of the larger genome

    compatibility_distance = (excess_gene_weight*excesses)/large_genome_size + \
                             (disjoint_gene_weight*disjoints)/large_genome_size + weight_diff_weight * average

    #print("Comp Dist: ", compatibility_distance)
    return compatibility_distance

def calc_adj_fit(target: Individual, population: list) -> float:
    global compatibility_threshold
    species_size = 0
    comp_dists = []

    # dynamically calculate comp threshold

    for individual in population:
        comp_dist = get_compatibility_distance(target, individual)
        comp_dists.append(comp_dist)
        if comp_dist <= compatibility_threshold:
            species_size += 1

    target.adj_fitness = target.fitness/species_size

    return sum(comp_dists)/len(comp_dists)

def calc_pop_adj_fits(population: list) -> list:
    global compatibility_threshold
    start_time = time.time()
    loading("CALCULATING ADJUSTED FITNESSES", len(population))
    comp_dists = []

    for i in range(len(population)):
        comp_dists.append(calc_adj_fit(population[i], population))
        blip(i)

    adj_fits = [individual.adj_fitness for individual in population]

    compatibility_threshold = sum(comp_dists)/len(comp_dists)

    end_load(start_time)
    print("-"*40+"\n")
    return adj_fits

def sort_pop(population: list) -> list:
    #print(end="          SORTING POPULATION...", flush=True)
    fits = [individual.adj_fitness for individual in population]
    sorted_zip = sorted(zip(fits, range(POPULATION_SIZE), population))
    sorted_pop = [ind[2] for ind in sorted_zip]
    #print("DONE")
    return sorted_pop

def cull_population(population: list) -> list:
    #print(end="    CULLING POPULATION [" + str(SURVIVOR_PROPORTION) + "]...", flush=True)
    culled_pop = population[round(-SURVIVOR_PROPORTION*POPULATION_SIZE):]
    #print("DONE")
    return culled_pop

def speciate_pop(population: list) -> list:
    #print(end="       SPECIATING POPULATION...", flush=True)
    global species_number
    species_list = []
    species_number = 0

    for i in range(len(population)):
        target = population[i]
        target.species = -1
        for species in species_list:
            if get_compatibility_distance(target, species[0]) <= compatibility_threshold:
                target.species = species[0].species
                species.append(target)
                break
        if target.species == -1:
            target.species = species_number
            species_list.append([target])
            species_number += 1

    #print("DONE")
    return species_list

def allocate_offspring(species_list: list) -> list:
    #print(end="ALLOCATING SPECIES OFFSPRING...", flush=True)

    allocations = []
    for species in species_list:
        allocations.append(0)
        for individual in species:
            allocations[-1] += individual.adj_fitness
    total = sum(allocations)

    for i in range(len(allocations)):
        allocations[i] = round(allocations[i]*POPULATION_SIZE/max(0.0000001, total))

    #print("DONE")
    return allocations

def create_next_gen(species_list: list, allocations: list) -> list:
    #print(end="    CREATING NEXT GENERATION...", flush=True)
    next_gen = []

    # i is the species number
    for i in range(len(species_list)):
        # create the selecition distribution for this species
        dist = create_selection_dist(species_list[i])

        # create children a number of times given in the allocations list
        for child_count in range(allocations[i]):

            # select first parent
            value = random.uniform(0, 1)
            k = 0
            while value > dist[k]:
                k += 1
            parent1 = species_list[i][k]

            # select second parent
            value = random.uniform(0, 1)
            k = 0
            while value > dist[k]:
                k += 1
            parent2 = species_list[i][k]

            # ensure parent1 is higher fitness
            if parent2.adj_fitness > parent1.adj_fitness:
                parent1, parent2 = parent2, parent1

            # create child and add to next generation list
            child = crossover(parent1, parent2)

            # mutate child
            if random.uniform(0, 1) < MUTATE_ALL_WEIGHTS_CHANCE:
                child.mutate_weights(MUTATE_WEIGHT_CHANCE, MUTATE_WEIGHT_STRENGTH)

            if random.uniform(0, 1) < MUTATE_NODE_CHANCE:
                mutate_node(child)

            if random.uniform(0, 1) < MUTATE_CONNECTION_CHANCE:
                mutate_connection(child, species_list)

            # add child to list
            next_gen.append(child)

    #print("DONE\n"+"-"*40+"\n")
    # return new population
    return next_gen

def create_selection_dist(group: list) -> list:
    dist = []
    for individual in group:
        dist.append(individual.adj_fitness)

    total = sum(dist)
    running_sum = 0
    for i in range(len(dist)):
        running_sum += dist[i]
        dist[i] = running_sum/max(0.0000001, total)

    return dist

def crossover(high: Individual, low: Individual) -> Individual:
    child = copy.deepcopy(high)

    for con_id in child.genome.get_connection_ids():

        if con_id in low.genome.get_connection_ids():

            isExpressed = True

            if not child.genome.connections[con_id].expressed:
                rand = random.uniform(0, 1)
                if rand >= RE_EXPRESSION_CHANCE:
                    isExpressed = False

            rand = random.randint(0, 1)
            if rand == 0:
                child.genome.connections[con_id].weight = low.genome.connections[con_id].weight

            child.genome.connections[con_id].expressed = isExpressed

    return child

def mutate_node(individual: Individual) -> None:
    global node_number
    global connection_number
    if individual.mutate_node(node_number, connection_number):
        node_number += 1
        connection_number += 2
    else:
        pass

def mutate_connection(individual: Individual, species_list: list) -> bool:
    global connection_number
    if individual.mutate_connection(connection_number):
        target_con = individual.genome.connections[connection_number]

        for species in species_list:
            for ind in species:
                for connection in ind.genome.get_connections():
                    if connection.in_node == target_con.in_node and connection.out_node == target_con.out_node and \
                            not connection.id == target_con.id:
                        del individual.genome.connections[target_con.id]

                        target_con.id = connection.id
                        individual.genome.connections[target_con.id] = target_con

                        return True

        connection_number += 1
    else:
        pass
        #print("CONNECTION FAILED")

# formatting shit -----------------------------------------
def blip(value: int) -> None:
    if value % inds == inds - 1:
        print(end=bit, flush=True)

def loading(title: str, length: int) -> None:
    top = " "+title
    top += spacer * (length // inds - len(title))
    top += "\n|"
    print(end=top)

def end_load(start_time: float) -> None:
    print("| %ds\n" % (time.time() - start_time))

def fit_stats(fit_list: list, sorted_pop: list) ->None:
    avg_adj_fit = sum(fit_list) / len(fit_list)
    print("AVG ADJ FITNESS:", avg_adj_fit, "\n")
    f = open(filename, "a")
    f.write("%f\n" % avg_adj_fit)
    f.close()

    print("BEST FITNESS:", sorted_pop[-1].adj_fitness)
    print()

# run main
if __name__ == '__main__':
    start_time = time.time()
    main()
    print("\n\nTOTAL EVALUATION TIME:", time.time()-start_time)


import logging
from logging import config as logger_config

class Metrics:
    """
    Класс для работы с метриками
    В конструкторе необходимо передать объект кеша
    Ибо часть метрик храниться в редисе
    И нужно еще и БД
    """

    def __init__(self, app, config):
        self.cache = app.get('cache')
        self.db = app.get('db')
        logger_config.dictConfig(config.get('logging'))
        logger = logging.getLogger()
        self.logger = logger
        #self.pref = config.get('metrics').get('default_prefix') or ''
        # список доступных метрик
        self.available = ['jwt_issued_count', 'crash_500', 'data_api_failed_request_count', 'data_api_added_tag_count',
                          'listened_call_count'
                          ]

    async def renew(self, app = None):
        """
        Очищаем все метрики и начинаем снова
        Это выполняется один раз при запуске приложения
        :return:
        """

        for x in self.available:
            await self.cache.set_cache_wo_expire(str(x).strip(), str(0))

    async def inc(self, name):
        """
        Увеличить счетчик метрики
        :param name:
        :return:
        """
        if name not in self.available:
            return 0

        param = await self.cache.get_cache(str(name).strip())
        if not param:
            param = 0
        param = int(param)
        param += 1
        self.logger.debug('Metric %s incremented' % name)
        await self.cache.set_cache_wo_expire(str(name).strip(), str(param))
        return param

    async def get(self, name):
        """
        Получить метрику
        :param name:
        :return:
        """
        if name not in self.available:
            return 0
        self.logger.debug('Metric %s fetched' % name)

        return await self.cache.get_cache(str(name).strip() or 0)

    async def all(self):
        """
        Вернуть все метрики в виде словарика :)
        :return:
        """
        self.logger.debug('All Metric fetched')
        ret = {}
        for x in self.available:
            name = str(x).strip()
            ret[name] = int(await self.cache.get_cache(name) or 0)
        return ret