from datetime import datetime
import pandas
import json

import luigi
import urllib

import os
from itertools import groupby
from SlackNotifier import SlackNotifier

from ParquetFileInsertion import IngestParquetFile
from LoggerFactory import LoggerFactory

import argparse
import logging
import traceback

class RunAllEolHostingIngestions(luigi.WrapperTask):
    data_folder = luigi.Parameter()
    eol_tables_config_path = luigi.Parameter()
    environments_to_ingest = luigi.ListParameter()
    ingest_to = luigi.Parameter()
    ingestion_date = luigi.DateParameter()
    logs_file_path = luigi.Parameter()

    # The logic of starting multiple tasks basing on our files/partitioning structure
    # Warning! Dependent workers (for IngestParquetFile) will run in a separate process. Unfortunatelly, that complicates troubleshooting and logging.
    # To debug code logic, change luigi.cfg [core] workers = 1. Don't forget to undo this change before commiting.
    # Note, that for multiprocess behaviour number of workers should be > 1.
    # To troubleshoot multiprocess issues use logging since debugging will be unable.
    def requires(self):
        # Logger for this task somehow breaking the multi-thread execution.
        # Therefore, no logger is used for this task.
        eol_tables_config = self.read_tables_configuration(self.eol_tables_config_path)

        configured_files_to_ingest = self.aggregate_files_configured_to_ingest(self.data_folder, eol_tables_config)
        ### For debugging a specific parquet file
        #configured_files_to_ingest = [f for f in configured_files_to_ingest if f.FileName=='33413888-000000-0000.parquet']

        for environment, files_to_ingest_for_environement in groupby(configured_files_to_ingest, lambda x: x.Environment):
            if (not environment in self.environments_to_ingest):
                continue
            for file_to_ingest in files_to_ingest_for_environement:
                # Skip not enabled tables and files for not today
                if(file_to_ingest.Date < self.ingestion_date or
                        not file_to_ingest.IsEnabled):
                    continue
                yield IngestParquetFile(
                    ingest_to = self.make_db_connection_string(),

                    ingestion_date = self.ingestion_date,
                    environment = file_to_ingest.Environment,

                    parquet_source = file_to_ingest.FileName,
                    parquet_source_location = file_to_ingest.FullPath,
                    backup_date = file_to_ingest.Date,
                    table = file_to_ingest.TargetTable,
                    column_names = file_to_ingest.TargetColumns,

                    logs_file_path = self.logs_file_path
                )

    def make_db_connection_string(self):
        connection_string = "Driver={SQL Server};Server=" + self.ingest_to + ";Database=CustomerIntelligence;UID=;PWD="
        connection_string = urllib.parse.quote_plus(connection_string)
        connection_string = "mssql+pyodbc:///?odbc_connect=%s" % connection_string
        return connection_string

    def read_tables_configuration(self, config_file_path):
        if config_file_path is None or not os.path.isfile(config_file_path):
            raise FileNotFoundError(config_file_path)
        tables = pandas.read_json(config_file_path)
        if len(tables) < 1:
            raise Exception("List of tables to ingest is empty at: {}".format(config_file_path))
        return json.loads(tables.to_json(orient='records'))

    def aggregate_files_configured_to_ingest(self, root_folder, eol_tables_config):
        local_files = self.list_all_files_of_eol_hosting_structure(root_folder)
        files_to_ingest = [f for f in local_files
                                if any(table['source'] == f.EntityName for table in eol_tables_config)]
        for file in files_to_ingest:
            config = ([c for c in eol_tables_config if c['source'] == file.EntityName])[0]
            file.ExtendWithIngestionConfig(config)
        return files_to_ingest

    def check_path(self, file_path):
        # path structure: data folder path\\environment=NL\\Tables\\2019\\12\\31
        return len(file_path.split('\\')[-5:][0].split('=')) > 1

    def list_all_files_of_eol_hosting_structure(self, folder_path):
        paths = []
        for path, subdirs, files in os.walk(folder_path):
            if not self.check_path(path):
                continue
            for name in files:
                paths.append(os.path.join(path, name))
        cig_files = [CigIngestionFile(p) for p in paths]
        return cig_files

class CigIngestionFile():
    def __init__(self, file_path):
        if file_path is None:
            raise FileNotFoundError(file_path)
        # path structure: data folder path\\environment=NL\\2019\\12\\31\\file.parquet
        splitted_path = file_path.split('\\')
        splitted_path = splitted_path[-6:]

        if len(splitted_path) < 1:
            raise FileNotFoundError(file_path)

        environment = splitted_path[0].split('=')[1]
        date_string = "{}-{}-{}".format(splitted_path[2], splitted_path[3], splitted_path[4])

        self.FullPath = file_path
        self.FileName = splitted_path[5]
        self.Environment = environment
        self.Date = datetime.strptime(date_string, "%Y-%m-%d").date()
        self.EntityName = splitted_path[1]

    def ExtendWithIngestionConfig(self, ingestion_config):
        self.TargetTable = ingestion_config['target_name']
        self.TargetColumns = ingestion_config['columns']
        self.IsEnabled = ingestion_config['is_enabled']

def read_ingestion_config(ingestion_config_filename):
    config = {}
    currentFolder = os.path.dirname(__file__)
    with open(os.path.join(currentFolder, ingestion_config_filename)) as file:
        config = json.load(file)
    return config

def send_execution_summary(execution_summary, environments, db_server_name, ingestion_date):
    icon = ":heavy_check_mark:" if execution_summary.worker.run_succeeded else ":fish_cake:"
    luigi_page = 'http://nlc1prodci02:8082/static/visualiser/index.html'

    header = "{}\n EOL Hosting ETL : Ingestion summary\n\nFor the following Environments: {}\nIngestion date (CIGCopyTime): {}"\
        .format(icon, ", ".join(environments), ingestion_date)
    body = "\nRun visualization page: {}\n\n{}\n".format(luigi_page, execution_summary.summary_text)
    footer = "The SQL Server DB: {}\n{}".format(db_server_name, "----------------------------------------------------------------")
    message = "{}\n{}\n{}".format(header, body, footer)
    SlackNotifier().notify_with_message(message)

# The entry point
# Before starting it luigi server must be started
# Via cmd : luigid
#      Or : python start_luigi_server.py
if __name__ == "__main__":
    try:
        ingestion_config_filename = 'ingestion_config.json'
        parser = argparse.ArgumentParser()
        parser.add_argument('--ingestion_config_filename', help='Specify filename of ingestion configuration (Default="ingestion_config.json")')
        args = parser.parse_args()
        if not args.ingestion_config_filename is None and len(args.ingestion_config_filename) > 0:
            ingestion_config_filename = args.ingestion_config_filename

        now = datetime.now()
        config = read_ingestion_config(ingestion_config_filename)
        data_folder = config['data_folder']+'\\'
        environments_to_ingest = config['environments']
        ingest_to = config['ingest_to']
        ingestion_date = datetime.strptime(config['ingestion_date'], '%Y-%m-%d') if len(config['ingestion_date']) > 0 else now
        currentFolder = os.path.dirname(__file__)
        tables_to_upload_config_file = os.path.join(currentFolder, config['tables_to_upload_config_file'])
        logs_file_path = "{}\luigi.{}.log".format(config['logs_folder'], now.strftime('%Y-%m'))

        logger = LoggerFactory.create_logger_for_multiprocess_use(logs_file_path, logging.INFO, 'main')
        logger.info("The ingestion is configured and started.")

        execution_summary = luigi.build(
            [RunAllEolHostingIngestions(
                eol_tables_config_path = tables_to_upload_config_file,
                data_folder = data_folder,
                environments_to_ingest = environments_to_ingest,
                ingest_to = ingest_to,
                ingestion_date = ingestion_date,
                logs_file_path = logs_file_path
                )],
            detailed_summary=True)

        logger.info(execution_summary.summary_text)

        if config['is_execution_summary_via_slack_enabled'] and (
            len(execution_summary.worker._scheduled_tasks) > 1 or
            not execution_summary.worker.run_succeeded):
                send_execution_summary(execution_summary, environments_to_ingest, ingest_to, ingestion_date)

    except Exception as e:
        message = ":rage4: \n EOL Hosting ETL : Unexpected failure \nError:\n{}\n{}"\
            .format(e, traceback.format_exc())
        SlackNotifier().notify_with_message(message)
        logger = LoggerFactory.create_logger_for_multiprocess_use(logs_file_path, logging.ERROR, 'main-crashed')
        logger.fatal(e, exc_info=True)


import aiohttp
from bs4 import BeautifulSoup
#import pandas as pd
import requests
import asyncio

#MODIFY YOUR INFO/REQUIREMENTS#####

ourServer = "twisting-nether"
ourRaidsPerWeek = "2"
updateThresholdDays = 10
pagesToCheck = 10

###################################
#KEYS##############################
urlBase = "https://www.wowprogress.com"
key = "/character/eu/"
nextkey = "/gearscore/char_rating/next/"
transferKey = "Yes, ready to transfer"
timeKey = "day ago"
timeKey2 = "days ago"
###################################

print("\n\n Class & spec combinations available: \n ~~~~~~~~~~~~~~~~~~\n deathknight - dd, tank \n demon hunter - dd, tank \n druid - balance, restoration, feral dd, feral tank \n hunter - beastmastery, marksmanship, survival \n mage - arcane, fire, frost \n monk - healer, dd, tank \n paladin - holy, protection, retribution \n priest - healer, dd \n rogue - assassination, outlaw, subtlety \n shaman - elemental, enhancement, restoration \n warlock - affliction, demonology, destruction \n warrior - dd, protection \n ~~~~~~~~~~~~~~~~~~\n")

ourClass = input("Class: ")
ourSpecs = input("Specs: ")
minGearscore = input("Min Gearscore: ")
minKills = input("Min mythic kills this tier: ")

async def fetch(session, url):
        try:
            async with session.get(url) as response:
                text = await response.text()
                return text, url
        except Exception as e:
            print(str(e))

async def main():

    url = urlBase + "/gearscore/class." + ourClass + "?lfg=1&raids_week=" + ourRaidsPerWeek + "&lang=en"
    theList = []

    print("Searching for recruits now...")

    ##Check the class summary pages to gather urls for all characters under the search parameters
    for x in range(pagesToCheck):
        msg = "Checking page " + str(x) + "/" + str(pagesToCheck)
        print(msg, end='\r')

        r = requests.get(url)
        data = r.text
        soup = BeautifulSoup(data,'html.parser')

        for link in soup.find_all('a'):
            target = link.get('href')
            if nextkey in target:
                url = urlBase + target

        for tr in soup.find('table').find_all('tr'):
            for link in tr.find_all('a'):
                target = link.get('href')
                if nextkey in target:
                    url = urlBase + target
                if key in target:
                    for spanText in tr.find_all('span'):
                        if timeKey in spanText.get_text() or timeKey2 in spanText.get_text():
                            timeText = spanText.get_text()
                            theTime = timeText[:timeText.find(' ')]
                            if int(theTime) < updateThresholdDays:
                                theList.append(target)
                        else:
                            theList.append(target)

    msg = "Found " + str(len(theList)) + " potential recruits."
    print(msg)

    tasks = []
    finalList = []

    headers = {
        "user-agent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
    ##Asynchronously get all the raw html data for each url
    async with aiohttp.ClientSession(headers=headers) as session:
        for item in theList:
            url = urlBase + item
            tasks.append(fetch(session, url))

        htmls = await asyncio.gather(*tasks)
        msg = "Successfully retrieved " + str(len(htmls)) + " URLs."
        print(msg)
        print("Processing...")
        index = 0
        for html in htmls:
            if html is not None:
                url = html[1]
                data = html[0]
                soup = BeautifulSoup(data,'html.parser')
                ##traverse html data for relevant match to search criteria (TODO Add more filters)
                needsTransfer = True
                if ourServer in item:
                    needsTransfer = False

                for strongText in soup.find_all('strong'):
                    if ourSpecs in strongText.get_text():
                        for gs in soup.find_all(class_='gearscore'):
                            if "Item Level" in gs.get_text():
                                gsText = gs.get_text()
                                gsF = float(gsText[12:])
                                if gsF >= float(minGearscore):
                                    bossKillTable = soup.find(id="tiers_details")
                                    if bossKillTable is not None:
                                        kills = 0
                                        for kill in bossKillTable.find_all(class_='progress_heroic'):
                                            if "Mythic" in kill.get_text():
                                                kills+=1
                                        if kills >= int(minKills):
                                            if needsTransfer:
                                                for spanText in soup.find_all('span'):
                                                    if transferKey in spanText.get_text():
                                                       finalList.append(url)
                                            else:
                                               finalList.append(url)
            else:
                continue
            index += 1
            msg = "Progress: " + str(round(len(htmls) / 100 * index, 2)) + "%"
            print(msg, end='\r')

    ##Write the relevent urls for hits to file; TODO Directly to google sheet?
    file_name = ourClass + "_" + ourSpecs + ".txt"
    with open(file_name, "w") as f:
      [f.write(f'{final_item}\n') for final_item in finalList]
    print(f"{str(len(finalList))} recruit(s) found and written to file.")

if __name__ == "__main__":
    asyncio.run(main())


#!/usr/bin/env python3

import contextlib
import time

class timeit(contextlib.AbstractContextManager):
	def __enter__(self):
		self.start = time.perf_counter()
		return self

	def __exit__(self, *excinfo):
		self.end = time.perf_counter()
		self.time = self.end - self.start

def test():
	import math

	with contextlib.suppress(ValueError):
		with timeit() as timer:
			time.sleep(2)
			raise ValueError

	assert math.floor(timer.time) == 2

if __name__ == '__main__':
	test()


#!/usr/bin/python3
# coding=utf-8
import dandan
import time
import random
import confluent_kafka as confluent
import config

logger = config.getLogger()

topic = 'test-partitions'

conf = {
    'bootstrap.servers': config.hosts,
}
producer = confluent.Producer(conf)

count = 10000
partitions = 2
future = time.time() + 100000

while future > time.time():
    message = bytes("test message", encoding='utf8')
    logger.debug('send message %s', message)
    producer.produce(
        topic=config.topic,
        value=message,
        # key=bytes(str(var % partitions), encoding='utf-8')
    )
    producer.flush()
    time.sleep(random.random() * 0.01)


import sys, os, random

# Randomly selects a given number of items in a given folder, deletes others.
if __name__ == "__main__":
    if len(sys.argv) < 3:
        print("Usage: " + sys.argv[0] + " <input folder> <final number of files>")
        sys.exit(1)
    folder = sys.argv[1]
    chosen = int(sys.argv[2])
    allFiles = os.listdir(folder)
    remove = random.sample(allFiles, k = len(allFiles) - chosen)
    for i in remove:
        os.remove(os.path.join(folder, i))


from __future__ import absolute_import
from __future__ import print_function
import os
import pandas as pd
import sys
import time
top_path=os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
# add the top-level directory of this project to sys.path so that we can import modules without error
sys.path.append(top_path)
import numpy as np
import argparse
import imp
import re
import gc
from mimic3models.in_hospital_mortality import utils
from mimic3benchmark.readers import InHospitalMortalityReader
import tensorflow as tf
from mimic3models.preprocessing import Discretizer, Normalizer
from mimic3models import metrics
from mimic3models import keras_utils
from mimic3models import common_utils
from keras.callbacks import ModelCheckpoint, CSVLogger
import random
def draw_samples(data,seed,percentage):
    positives = list(data[data['y_true'] == 1].index.values.tolist())
    negatives = list(data[data['y_true'] == 0].index.values.tolist())
    # randomly sample labels at this percentage
    n = len(data)
    np.random.seed(seed)
    if percentage > 0.05:
        samples = np.random.randint(n, size=int(n * percentage))
    else:
        np.random.seed(seed)
        positives = random.sample(positives, int(n * percentage / 2))
        np.random.seed(seed)
        negatives = random.sample(negatives, int(n * percentage / 2))
        samples = [*positives, *negatives]
    data = data[['stay','y_true']].iloc[samples, :]
    return data

#python -m mimic3models.split_train_val data/in-hospital-mortality
#   python -um mimic3models.in_hospital_mortality.main --network mimic3models/keras_models/lstm.py --dim 16 --timestep 1.0 --depth 2 --dropout 0.3 --mode train --batch_size 8 --save_every 10000 --output_dir mimic3models/in_hospital_mortality

#                   -------- argparse --------
parser = argparse.ArgumentParser()
# WTF !
common_utils.add_common_arguments(parser)
parser.add_argument('--target_repl_coef', type=float, default=0.0)

parser.add_argument('--seed', type=int, default=0)
parser.add_argument('--data', type=str, help='Path to the data of in-hospital mortality task',
                    default=os.path.abspath(os.path.join(os.path.dirname(__file__), '../../data/in-hospital-mortality/')))
parser.add_argument('--output_dir', type=str, help='Directory relative which all output files are stored',
                    default='.')
args = parser.parse_args()
physical_devices = tf.config.list_physical_devices('GPU')
# Don't pre-allocate memory; allocate as-needed
tf.config.experimental.set_memory_growth(physical_devices[0], True)

#                   --------------------------

#                   ------ not yet realize ----
if args.small_part:   #TODO:
    args.save_every = 2**30

target_repl = (args.target_repl_coef > 0.0 and args.mode == 'train') #TODO:
#                   ---------------------------
# percentages=[0.001,0.002,0.003,0.004,0.005,0.008,0.010,0.050,0.100,0.200,0.300,0.400,0.500,0.650,0.85,1.000]
percentages=[0.005]
accs=[]
roc_tests = []
prc_tests = []
aps=[]
pses = []
train_list =pd.read_csv(os.path.join(top_path,'ref','train_listfile.csv'))
seed=args.seed
for percentage in percentages:
    temp_list_file=draw_samples(train_list,seed,percentage)
    temp_list_file.to_csv(os.path.join(args.data, 'temp.csv'),index=False)
# Build readers, discretizers, normalizers
    train_reader = InHospitalMortalityReader(dataset_dir=args.data,
                                             listfile=os.path.join(args.data, 'temp.csv'),
                                             period_length=48.0)

    # * mined that dataset_dir of it is 'train'
    val_reader = InHospitalMortalityReader(dataset_dir=args.data,
                                           listfile=os.path.join(args.data, 'val_listfile.csv'),
                                           period_length=48.0)

    # default impute strategy is 'previous' here
    # default timestep is 1.0
    discretizer = Discretizer(timestep=float(args.timestep),
                              store_masks=True,
                              impute_strategy='previous',
                              start_time='zero')

    discretizer_header = discretizer.transform(train_reader.read_example(0)["X"])[1].split(',')
    cont_channels = [i for (i, x) in enumerate(discretizer_header) if x.find("->") == -1]  # only categorical data has '->'

    normalizer = Normalizer(fields=cont_channels)  # choose here which columns to standardize
    normalizer_state = args.normalizer_state
    if normalizer_state is None:
        normalizer_state = 'ihm_ts{}.input_str:{}.start_time:zero.normalizer'.format(args.timestep, args.imputation)
        normalizer_state = os.path.join(os.path.dirname(__file__), normalizer_state)
    normalizer.load_params(normalizer_state)  # ?Need to run "create_normalizer_state.py" first

    args_dict = dict(args._get_kwargs())
    args_dict['header'] = discretizer_header
    args_dict['task'] = 'ihm'
    args_dict['target_repl'] = target_repl

    # Build the model
    print("==> using model {}".format(args.network))
    model_module = imp.load_source(os.path.basename(args.network), args.network)
    model = model_module.Network(**args_dict)
    suffix = 'percentage_%f_seed_%d'%(percentage,seed)
    model.final_name = 'lstm_'+ suffix
    print("==> model.final_name:", model.final_name)

    # Compile the model
    # print("==> compiling the model")

    optimizer_config = {'class_name': args.optimizer,
                        'config': {'learning_rate': args.lr,
                                   'beta_1': args.beta_1}}
    # NOTE: one can use binary_crossentropy even for (B, T, C) shape.
    #       It will calculate binary_crossentropies for each class
    #       and then take the mean over axis=-1. Tre results is (B, T).
    if target_repl:
        loss = ['binary_crossentropy'] * 2
        loss_weights = [1 - args.target_repl_coef, args.target_repl_coef]
    else:
        loss = 'binary_crossentropy'
        loss_weights = None

    model.compile(optimizer=optimizer_config,
                  loss=loss,
                  loss_weights=loss_weights)
    model.summary()

    # Load model weights
    n_trained_chunks = 0

    # Read data
    train_raw = utils.load_data(train_reader, discretizer, normalizer, args.small_part,percentage=1)
    val_raw = utils.load_data(val_reader, discretizer, normalizer, args.small_part)

    if target_repl:
        T = train_raw[0][0].shape[0]

        def extend_labels(data):
            data = list(data)
            labels = np.array(data[1])  # (B,)
            data[1] = [labels, None]
            data[1][1] = np.expand_dims(labels, axis=-1).repeat(T, axis=1)  # (B, T)
            data[1][1] = np.expand_dims(data[1][1], axis=-1)  # (B, T, 1)
            return data

        train_raw = extend_labels(train_raw)
        val_raw = extend_labels(val_raw)

    # Prepare training
    path = os.path.join(args.output_dir, 'keras_states/' + model.final_name + '.state')

    metrics_callback = keras_utils.InHospitalMortalityMetrics(train_data=train_raw,
                                                              val_data=val_raw,
                                                              target_repl=(args.target_repl_coef > 0),
                                                              batch_size=args.batch_size,
                                                              verbose=args.verbose)
    # make sure save directory exists
    dirname = os.path.dirname(path)
    if not os.path.exists(dirname):
        os.makedirs(dirname)
    saver = ModelCheckpoint(path, verbose=1, period=args.save_every)

    keras_logs = os.path.join(args.output_dir, 'keras_logs')
    if not os.path.exists(keras_logs):
        os.makedirs(keras_logs)
    csv_logger = CSVLogger(os.path.join(keras_logs, model.final_name +'.csv'),
                           append=True, separator=';')

    print("percentage %f==> training"%percentage)
    model.fit(x=train_raw[0],
              y=train_raw[1],
              validation_data=val_raw,
              epochs=n_trained_chunks + args.epochs,
              initial_epoch=n_trained_chunks,
              callbacks=[metrics_callback, saver, csv_logger],
              shuffle=True,
              verbose=args.verbose,
              batch_size=args.batch_size)
    # gc.collect()

#

    # ensure that the code uses test_reader
    del train_reader
    del val_reader
    del train_raw
    del val_raw

    test_reader = InHospitalMortalityReader(dataset_dir=os.path.join(args.data),
                                            listfile=os.path.join(args.data, 'test_listfile.csv'),
                                            period_length=48.0)
    ret = utils.load_data(test_reader, discretizer, normalizer, args.small_part,
                          return_names=True)

    data = ret["data"][0]
    labels = ret["data"][1]
    names = ret["names"]

    predictions = model.predict(data, batch_size=args.batch_size, verbose=1)
    predictions = np.array(predictions)[:, 0]
    results=metrics.print_metrics_binary(labels, predictions)
    auroc=results['auroc']
    auprc=results['auprc']
    minpse=results['minpse']
    ap=results['ap']
    acc=results['acc']
    roc_tests.append(auroc)
    prc_tests.append(auprc)
    pses.append(minpse)
    accs.append(acc)
    aps.append(ap)
    # path = os.path.join(args.output_dir, "test_percentage%f_seed%d"%(percentage,seed) + ".csv")
    # utils.save_results(names, predictions, labels, path)
    del temp_list_file,model
    gc.collect()

print('---------  seed = %d ---------------'%seed)
print('------ percentages = %s'%percentages)
print('acc===================')
print(acc)
print('auroc=================')
print(auroc)
print("ap=======")
print(aps)
print('auprc=================')
print(auprc)
print('pse===================')
print(pses)


# uncompyle6 version 3.7.4
# Python bytecode 3.6 (3379)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.macosx-10.7-x86_64/egg/airflow/models/crypto.py
# Compiled at: 2019-09-11 03:47:34
# Size of source mod 2**32: 3042 bytes
from builtins import ImportError as BuiltinImportError
from airflow import configuration
from airflow.exceptions import AirflowException
from airflow.utils.log.logging_mixin import LoggingMixin

class InvalidFernetToken(Exception):
    pass

class NullFernet(object):
    __doc__ = '\n    A "Null" encryptor class that doesn\'t encrypt or decrypt but that presents\n    a similar interface to Fernet.\n\n    The purpose of this is to make the rest of the code not have to know the\n    difference, and to only display the message once, not 20 times when\n    `airflow initdb` is ran.\n    '
    is_encrypted = False

    def decrypt(self, b):
        return b

    def encrypt(self, b):
        return b

_fernet = None

def get_fernet():
    """
    Deferred load of Fernet key.

    This function could fail either because Cryptography is not installed
    or because the Fernet key is invalid.

    :return: Fernet object
    :raises: airflow.exceptions.AirflowException if there's a problem trying to load Fernet
    """
    global InvalidFernetToken
    global _fernet
    log = LoggingMixin().log
    if _fernet:
        return _fernet
    try:
        from cryptography.fernet import Fernet, MultiFernet, InvalidToken
        InvalidFernetToken = InvalidToken
    except BuiltinImportError:
        log.warning('cryptography not found - values will not be stored encrypted.')
        _fernet = NullFernet()
        return _fernet
    else:
        try:
            fernet_key = configuration.conf.get('core', 'FERNET_KEY')
            if not fernet_key:
                log.warning('empty cryptography key - values will not be stored encrypted.')
                _fernet = NullFernet()
            else:
                _fernet = MultiFernet([Fernet(fernet_part.encode('utf-8')) for fernet_part in fernet_key.split(',')])
                _fernet.is_encrypted = True
        except (ValueError, TypeError) as ve:
            raise AirflowException('Could not create Fernet object: {}'.format(ve))

        return _fernet


#!/usr/bin/python
# -*- coding: utf-8 -*-
import StringIO

import xlsxwriter
from django.utils.translation import ugettext

def WriteToExcel(licenses, user):
    output = StringIO.StringIO()
    workbook = xlsxwriter.Workbook(output)
    worksheet_s = workbook.add_worksheet(ugettext(licenses[0].doc_number.replace("/", ".")))

    # excel styles
    title = workbook.add_format({
        'bold': True,
        'font_size': 14,
        'align': 'center',
        'valign': 'vcenter'
    })
    header = workbook.add_format({
        'bg_color': '#F7F7F7',
        'color': 'black',
        'align': 'center',
        'valign': 'top',
        'border': 1
    })
    cell = workbook.add_format({
        'align': 'left',
        'valign': 'top',
        'text_wrap': True,
        'border': 1
    })
    cell_center = workbook.add_format({
        'align': 'center',
        'valign': 'top',
        'text_wrap': True,
        'border': 1
    })
    cell_no_border = workbook.add_format({
        'valign': 'top',
        'text_wrap': True,
        'border': 0
    })

    # write title
    title_text = u"{0}".format(ugettext(licenses[0].doc_number.replace("/", ".")))
    # merge cells
    worksheet_s.merge_range('B2:F2', title_text, title)

    # write header
    worksheet_s.write(4, 0, ugettext(u"№ п.п"), header)
    worksheet_s.write(4, 1, ugettext(u"ФИО получателя"), header)
    worksheet_s.write(4, 2, ugettext(u"Должность получателя"), header)
    worksheet_s.write(4, 3, ugettext(u"Номер электронного ключа eToken"), header)
    worksheet_s.write(4, 4, ugettext(u"Номер лицензии на право использования программного продукта КриптоАРМ"), header)
    worksheet_s.write(4, 5, ugettext(u"Номер лицензии на право использования программного продукта КриптоПро"), header)
    worksheet_s.write(4, 6, ugettext(u"Подпись"), header)
    # write data

    for idx, data in enumerate(licenses):
        row = 5 + idx
        worksheet_s.write_number(row, 0, idx + 1, cell_center)
        worksheet_s.write_string(row, 1, licenses[idx].name_of_recepient, cell)
        worksheet_s.write_string(row, 2, licenses[idx].position_of_recepient, cell)
        worksheet_s.write_string(row, 3, licenses[idx].eToken_number, cell)
        worksheet_s.write_string(row, 4, licenses[idx].ca_license[:5] + "-...-" + licenses[idx].ca_license[-5:], cell)
        worksheet_s.write_string(row, 5, licenses[idx].cp_license[:5] + "-...-" + licenses[idx].cp_license[-5:], cell)
        worksheet_s.write_string(row, 6, "", cell)

    worksheet_s.write(row + 2, 3, ugettext(u"Выдал"), cell_no_border)
    worksheet_s.write(row + 2, 4, ugettext(u""), cell_no_border)
    worksheet_s.write(row + 2, 5, ugettext(user.get_full_name()), cell_no_border)

    # close workbook
    workbook.close()
    xlsx_data = output.getvalue()
    return xlsx_data


from __future__ import absolute_import, division, print_function, unicode_literals
# basic imports
import numpy as np
import csv, sys, os
import pandas as pd
from datetime import date
#gui imports
from tkinter import filedialog
from tkinter import *
#NEURAL NET functions from models.py
from models import *

class CreateToolTip(object):
    '''
    create a tooltip for a given widget
    '''
    def __init__(self, widget, text='widget info'):
        self.widget = widget
        self.text = text
        self.widget.bind("<Enter>", self.enter)
        self.widget.bind("<Leave>", self.close)
    def enter(self, event=None):
        x = y = 0
        x, y, cx, cy = self.widget.bbox("insert")
        x += self.widget.winfo_rootx() + 25
        y += self.widget.winfo_rooty() + 20
        # creates a toplevel window
        self.tw = tkinter.Toplevel(self.widget)
        # Leaves only the label and removes the app window
        self.tw.wm_overrideredirect(True)
        self.tw.wm_geometry("+%d+%d" % (x, y))
        label = tkinter.Label(self.tw, text=self.text, justify='left',
                       background='yellow', relief='solid', borderwidth=1,
                       font=("times", "8", "normal"))
        label.pack(ipadx=1)
    #def close(self, event=None):
    #    if self.tw:
    #        self.tw.destroy()

# GUI

class popupWindow(object):
    def __init__(self,master):
        top=self.top=Toplevel(master)
        self.choose_button = Button(top, text='Choose CSV file', command=self.choose_file)
        self.choose_button.grid(row=0, column=3)
        self.fillall = Button(top, text='Add all columns', command=self.add_all_columns, state=DISABLED)
        self.fillall.grid(row=1, column=3)
        self.l_options=Label(top,text='', wraplengt=400)
        self.l_options.grid(row=0, columnspan=3)
        self.l=Label(top,text='Column to predict:', state=DISABLED)
        self.l.grid(row=1)
        self.e=Entry(top, state=DISABLED)
        self.e.grid(row=1, column=2)
        self.l2=Label(top,text='Numerical columns used to predict\n in a list: (e.g. column1,column2,etc.)', state=DISABLED)
        self.l2.grid(row=2)
        self.e2=Entry(top, state=DISABLED)
        self.e2.grid(row=2, column=2)
        self.l6=Label(top,text='Text columns used to predict\n in a list: (e.g. column1,column2,etc.)', state=DISABLED)
        self.l6.grid(row=3)
        self.e4=Entry(top, state=DISABLED)
        self.e4.grid(row=3, column=2)
        self.l3=Label(top,text='Number of epochs:', state=DISABLED)
        self.l3.grid(row=4)
        self.e3=Entry(top, state=DISABLED)
        self.e3.grid(row=4, column=2)
        self.dp_var = IntVar()
        self.ch=Checkbutton(top, text='Dropout', state=DISABLED, variable=self.dp_var)
        self.ch.grid(row=4, column=3)
        #self.ttp_ch = CreateToolTip(self.ch, 'Training stops when theres no change over 0.005 in acc over 10 steps.')

        self.l4=Label(top,text='Optimizer:', state=DISABLED)
        self.l4.grid(row=5)
        options = ['sgd','rmsprop','adagrad','adadelta','adam','adamax','nadam']
        self.optimizer = StringVar(top)
        self.optimizer.set(options[4]) # default value
        self.w = OptionMenu(top, self.optimizer, *options)
        self.w.configure(state="disabled")
        self.w.grid(row=5, column=2)

        self.l5=Label(top,text='Loss:', state=DISABLED)
        self.l5.grid(row=6)
        options_loss = ['mean_squared_error','mean_squared_logarithmic_error','mean_absolute_error','binary_crossentropy','hinge','squared_hinge','categorical_crossentropy','sparse_categorical_crossentropy','kullback_leibler_divergence']
        self.loss = StringVar(top)
        self.loss.set(options_loss[3]) # default value
        self.w2 = OptionMenu(top, self.loss, *options_loss)
        self.w2.configure(state="disabled")
        self.w2.grid(row=6, column=2)

        self.b2=Button(top,text='Train',command=self.run_network, state=DISABLED)
        self.b2.grid(row=8, column=3)
        self.b3=Button(top,text='Clear Output',command=self.clear_output)
        self.b3.grid(row=12, column=3)
        self.save_var = IntVar()
        self.ch2=Checkbutton(top, text='Save Model', state=DISABLED, variable=self.save_var)
        self.ch2.grid(row=9, column=3)
        self.analysis_var = IntVar()

        self.label_output = Label(top, text='')
        self.label_output.grid(row=8, pady=5, padx=5, column=0, columnspan=3, rowspan=4)

    def run_network(self):
        self.value=self.e.get()
        self.value2=self.e2.get()
        self.value3=self.e3.get()
        self.value4=self.e4.get()
        accuracy, prediction_result, model_info = neural_net_csv_features(root.filename,str(self.entryValue()),[x.strip() for x in self.entryValue2().split(',')],[x.strip() for x in self.entryValue4().split(',')], int(self.entryValue3()), self.optimizer.get(), self.loss.get(), self.dp_var.get(), self.save_var.get())
        self.label_output['text'] = ''
        self.label_output['text'] += 'Training done. \n{}'.format(accuracy)
        self.label_output['text'] += '\nTest Predictions:\n {}'.format(prediction_result)
        self.label_output['text'] += model_info

    def entryValue(self):
        return self.value

    def entryValue2(self):
        return self.value2

    def entryValue3(self):
        return self.value3

    def entryValue4(self):
        return self.value4

    def clear_output(self):
        self.label_output['text'] = ''

    def add_all_columns(self):
        dataframe = pd.read_csv(root.filename)
        for i in list(dataframe.columns):
            first_column = dataframe[i].iloc[1]
            if self.e.get() == list(dataframe.columns)[list(dataframe.columns).index(i)]:
                continue
            if type(first_column)==str:
                self.e4.insert(END, str(i)+',')
            else:
                self.e2.insert(END, str(i)+',')
        self.e2.delete(len(self.e2.get())-1, END)
        self.e4.delete(len(self.e4.get())-1, END)

    def choose_file(self):
        root.filename = filedialog.askopenfilename(initialdir = os.getcwd(),title = 'Select file',filetypes = (('csv files','*.csv'),('all files','*.*')))
        dataframe = pd.read_csv(root.filename)
        options_str = ''
        if len(list(dataframe.columns))>30:
            for i in list(dataframe.columns[:30]):
                if i==list(dataframe.columns)[30]:
                    options_str += i
                else:
                    options_str += i + ', '
            options_str += ' ...'
        else:
            for i in list(dataframe.columns):
                if i==list(dataframe.columns):
                    options_str += i
                else:
                    options_str += i + ', '
            options_str = options_str[1:-2]
        self.l_options['text'] = 'Columns: ' + options_str
        if len(str(root.filename)) > 2:
            self.label_output['text'] = 'File ready.\n'
            self.e['state'] = 'normal'
            self.e2['state'] = 'normal'
            self.e3['state'] = 'normal'
            self.l['state'] = 'normal'
            self.l2['state'] = 'normal'
            self.l3['state'] = 'normal'
            self.w['state'] = 'normal'
            self.l5['state'] = 'normal'
            self.w2['state'] = 'normal'
            self.l4['state'] = 'normal'
            self.b2['state'] = 'normal'
            self.e4['state'] = 'normal'
            self.l6['state'] = 'normal'
            self.ch['state'] = 'normal'
            self.ch2['state'] = 'normal'
            self.fillall['state'] = 'normal'

class popupWindow_predict(object):
    def __init__(self,master,model_fn):
        global model_filename
        model_filename=model_fn
        analysis_filename=None
        top=self.top=Toplevel(master)
        self.choose_button_predict = Button(top, text='Choose CSV file to predict', command=self.choose_predict_file)
        self.choose_button_predict.grid(row=1)
        self.choose_model_button_predict = Button(top, text='Choose model folder for prediction', command=self.choose_model_file)
        self.choose_model_button_predict.grid(row=2)
        self.l_predict_filename=Label(top,text='', wraplengt=200)
        self.l_predict_filename.grid(row=1, column=1)
        self.l_model_filename=Label(top,text=model_filename, wraplengt=200)
        self.l_model_filename.grid(row=2, column=1)
        self.l_predict=Label(top,text='Feature to predict:', state=DISABLED)
        self.l_predict.grid(row=4)
        self.e_predict=Entry(top, state=DISABLED)
        self.e_predict.grid(row=4, column=1)
        self.l2_predict=Label(top,text='Row to predict (which test sample) for analysis:', state=DISABLED)
        self.l2_predict.grid(row=5)
        self.e2_predict=Entry(top, state=DISABLED)
        self.e2_predict.grid(row=5, column=1)
        self.l3_predict=Label(top,text='nth:', state=DISABLED)
        self.l3_predict.grid(row=6)
        self.e3_predict=Entry(top, state=DISABLED)
        self.e3_predict.grid(row=6, column=1)
        self.b_predict=Button(top,text='Predict',command=self.run_predict, state=DISABLED)
        self.b_predict.grid(row=7, column=1)
        self.label_output_predict=Label(top,text='', state=DISABLED)
        self.label_output_predict.grid(row=8, columnspan=2)

    def choose_predict_file(self):
        global predict_filename
        predict_filename = filedialog.askopenfilename(initialdir = os.getcwd(),title = 'Select file',filetypes = (('csv files','*.csv'),('all files','*.*')))
        if len(str(predict_filename)) > 2:
            self.label_output_predict['text'] = 'Predict File ready.\n'
            self.e_predict['state'] = 'normal'
            self.l_predict['state'] = 'normal'
            self.e2_predict['state'] = 'normal'
            self.l2_predict['state'] = 'normal'
            self.e3_predict['state'] = 'normal'
            self.l3_predict['state'] = 'normal'
            self.b_predict['state'] = 'normal'
            self.label_output_predict['state'] = 'normal'
            self.l_predict_filename['text'] = predict_filename

    def entryValue_predict(self):
        return self.e_predict.get()

    def entryValue_predict2(self):
        return self.e2_predict.get()

    def entryValue_predict3(self):
        return self.e3_predict.get()

    def choose_model_file(self):
        global model_filename
        model_filename = filedialog.askdirectory(initialdir = os.getcwd())
        self.l_model_filename['text']=model_filename
        self.b_weights['state'] = 'normal'

    def choose_analysis_file(self):
        global analysis_filename
        analysis_filename = filedialog.askopenfilename(initialdir = os.getcwd(),title = 'Select file',filetypes = (('csv files','*.csv'),('all files','*.*')))
        self.l_analysis_filename['text']=analysis_filename

    def run_predict(self):
        global model_filename
        global analysis_filename
        results = predict_csv_features(predict_filename, self.entryValue_predict(), model_filename, self.entryValue_predict2(), int(self.entryValue_predict3()))
        self.label_output_predict['text'] = 'Prediction:\n{}'.format(results)

class mainWindow(object):
    def __init__(self,master):
        self.master=master
        self.b=Button(master,text='CSV Model Train',command=self.popup, anchor=CENTER)
        self.b.grid(row=2)
        self.label2=Label(master,text='Make Models:')
        self.label2.grid(row=1)
        self.b_p=Button(master,text='CSV Model Predict',command=self.popup_predict, anchor=CENTER)
        self.b_p.grid(row=2, column=1)
        self.label3=Label(master,text='Predict with saved models:')
        self.label3.grid(row=1, column=1)
        self.label3=Label(master,text='Saved models in cwd:')
        self.label3.grid(row=1, column=3)
        self.lb1 = Listbox(master, width=25)
        self.lb1.grid(row=5, column=3, sticky='se')
        self.lb1.bind('<Double-Button>', self.onselect)
        self.b3=Button(master,text='Refresh',command=self.refresh, anchor=CENTER).grid(row=2, column=3, sticky='e')

        # adding found files to list
        global dirs_found
        dirs_found = []
        for count,dr in enumerate(os.listdir(os.getcwd()),1):
            if 'ml_analysis' in dr:
                dirs_found.append(dr)
                self.lb1.insert(count, 'CSV Analysis: '+ dr)
            if 'ml_model' in dr:
                dirs_found.append(dr)
                self.lb1.insert(count, 'CSV Model: '+ dr)

    # refreshing the list
    def refresh(self):
        global dirs_found
        dirs_found = []
        self.lb1.delete(0,'end')
        for count,dr in enumerate(os.listdir(os.getcwd()),1):
            if 'ml_analysis' in dr:
                dirs_found.append(dr)
                self.lb1.insert(count, 'CSV Analysis: '+ dr)
            if 'ml_model' in dr:
                dirs_found.append(dr)
                self.lb1.insert(count, 'CSV Model: '+ dr)

    def popup(self):
        self.w=popupWindow(self.master)
        self.b['state'] = 'disabled'
        self.b_p['state'] = 'disabled'
        self.master.wait_window(self.w.top)
        self.b['state'] = 'normal'
        self.b_p['state'] = 'normal'

    def popup_predict(self):
        self.w=popupWindow_predict(self.master,'')
        self.b['state'] = 'disabled'
        self.b_p['state'] = 'disabled'
        self.master.wait_window(self.w.top)
        self.b['state'] = 'normal'
        self.b_p['state'] = 'normal'

    def popup_text(self):
        self.w=popupWindow_csv_text(self.master)
        self.b['state'] = 'disabled'
        self.b_p['state'] = 'disabled'
        self.master.wait_window(self.w.top)
        self.b['state'] = 'normal'
        self.b_p['state'] = 'normal'

    def onselect(self, evt):
        global dirs_found
        w = evt.widget
        index = int(w.curselection()[0])
        value = w.get(index)
        dir_name=dirs_found[index]

        self.w=popupWindow_predict(self.master, dir_name)
        self.b['state'] = 'disabled'
        self.b_p['state'] = 'disabled'
        self.master.wait_window(self.w.top)
        self.b['state'] = 'normal'
        self.b_p['state'] = 'normal'

if __name__ == '__main__':
    root = Tk(className='ml_models')
    root.resizable(False, False)
    gui = mainWindow(root)
    root.mainloop()


str = 'ABCDEFGHIJKLMN'
dict = {}

for index, value in enumerate(str):
    dict[value] = index

str = 'OPQRSTUVWXYZ'[::-1]

for index, value in enumerate(str):
    dict[value] = index + 1

def solution(name):
    cur = []
    curIndex = 0
    answer = 0
    size = len(name)

    for i in range(size):
        cur.append('A')

    while True:
        if cur[curIndex] != name[curIndex]:
            answer += dict[name[curIndex]]
            cur[curIndex] = name[curIndex]

        if ''.join(cur) == name:
            return answer
        else:
            for i in range(1, int(size / 2) + 1):
                # 오른
                r = curIndex + i
                l = curIndex - i
                if l < 0:
                    l = size + l

                if r >= size:
                    r %= size

                if cur[r] != name[r]:
                    curIndex = r
                    answer += i
                    break
                elif cur[l] != name[l]:
                    curIndex = l
                    answer += i
                    break

assert solution("JEROEN") == 56
assert solution("JAN") == 23
assert solution("AAA") == 0


# coding: utf-8
import re
import time
import asyncio
import aiohttp
import pyautogui

class DanmuServer():
	def __init__(self, roomid, csrf_token):
		self.url = "https://api.live.bilibili.com/ajax/msg"
		self.form = {"roomid":roomid,
			"csrf_token":csrf_token}
		self.allow_keyboard_dicts = {'q':'q', 'w':'w', 'e':'e', 'r':'r',
									'w':'w', 'a':'a', 's':'s', 'd':'d', 'f':'f',
									"space":" "}
		self.allow_mouse_dicts = {"mlc":"left", "mrc":"right", "mmc":"middle"}

	def danmu_execution(self, dm):
		'''
		处理鼠标/键盘
		'''
		print(dm)
		dm_cmouse = re.findall(r".*?(mlc|mrc|mmc).*?", dm)
		dm_mmouse = re.findall(r".*?mmove\s*(\d+)\s+(\d+).*?", dm)
		dm_instructions = re.findall(r".*?(\w)(\d?).*?", dm)
		if dm_cmouse:
			pyautogui.click(button=self.allow_mouse_dicts[dm_cmouse[0]])
			time.sleep(0.3)
			return
		elif dm_mmouse:
			now_x, now_y = pyautogui.position()
			x = int(dm_mmouse[0][0]) if dm_mmouse[0][0] else now_x
			y = int(dm_mmouse[0][1]) if dm_mmouse[0][1] else now_y
			pyautogui.moveRel(x, y, duration=0.25)
			return
		if "space" in dm:
			dm_instructions = [("space", "")]
		if dm_instructions:
			cmd = dm_instructions[0]
			if cmd[0] in self.allow_keyboard_dicts.keys():
				if cmd[1]:
					time_wait = int(cmd[1])
				else:
					time_wait = int(1)
				pyautogui.keyDown(self.allow_keyboard_dicts[cmd[0]])
				time.sleep(time_wait)
				pyautogui.keyUp(self.allow_keyboard_dicts[cmd[0]])

	async def getDanmu(self):
		'''
		使用异步 请求api
		'''
		async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False)) as session:
			while True:
				print("连接弹幕网...(゜-゜)つロ 干杯~")
				async with session.post(self.url, data=self.form) as resp:
					print("获取弹幕数据...(゜-゜)つロ 干杯~")
					danmuData = await resp.json()
					# dm = [ (_tmp['nickname'], _tmp['text'], _tmp['timeline']) for _tmp in danmuData['data']['room'][-1:-6:-1] ]
					dm_text = danmuData['data']['room'][-1]['text']
					# self.push_danmu(data_list)
					print("获取弹幕数据成功...(゜-゜)つロ 干杯~")
					print("解析弹幕数据;并运行...(゜-゜)つロ 干杯~")
					self.danmu_execution(dm_text)
					# 太快了。我要减减速度
					time.sleep(0.3)

roomid = 3758547
csrf_token = "088056*********584eda17"

loop = asyncio.get_event_loop()
danmuserver = DanmuServer(roomid, csrf_token)
loop.run_until_complete(danmuserver.getDanmu())
loop.close()


# -*- coding: utf-8 -*-
"""
Created py for the Python Guide to Creating your Own Functions

"""

import glob
import os
import pandas as pd
import numpy as np
import datetime

################### Initialize the py file
DIRECTORY_LOCATION = r'C:/Users/Kevin/Desktop/Data'
print(f"Database directory is {DIRECTORY_LOCATION}")

############################# Read in a file
def load_file(filename):
    """

    Attempts to load a csv file. The passed parameter, ``filename``, should be obtained by calling ``find_directory_file()``.

    If you already know the filename, then you could provide your own filename.

    In addition, this function will remove any rows in the data that have no data (NA)

    Parameters
    ----------
    **filename**: string

    Example of how to use this function.

        ``df = load_file('my_file_name.csv')``

    OR you could use the ``find_directory_file()`` function

        ``file_name = find_directory_file(your_file_name.csv)``

        ``df = load_file(file_name)``

    Returns
    -------
    DataFrame
        A DataFrame representing the contents of the passed parameter 'filename'

    """

    df = pd.read_csv(filename)
    df.dropna(how='all', inplace=True) # remove rows with no data
    return df

def find_directory_file(string_search=''):
    """

    Attempts to find a file in your current directory folder. It will use the passed parameter, ``search_string`` on all files in your directory folder.

    You do not need to ensure the whole file name is inputted into the ``search_string`` parameter, only a portion is needed if the name is unique enough.

    If there is more than one file found, it will use the most recent file based on the "Modified time" according to your operating system's timestamp on the file

    Parameters
    ----------
    **string_search** : string

    Example of how to use this function.

        ``games_file = find_directory_file(my_data_file)``

    If you work with files that is appended the date at the end based on when you've extracted it, then you could omit the date portion of the file.

    As an example, if you have a file named Games_Jan252020.csv, you could do the following.

        ``games_file = find_directory_file(Games_)``

    Returns
    -------
    string
        The path and filename of the file if found, otherwise, an exception will be raised.

    """

    csv_file_list = glob.glob(DIRECTORY_LOCATION + "\*.csv") # use glob to match filename
    files_list = [i for i in csv_file_list if string_search in i]
    num_found_files = len(files_list)

    # Situation 1: no files found
    if num_found_files == 0:
        raise Exception(f'Could not find the {string_search} file in your directory folder.')

    # Situation 2: more than one matching found
    elif num_found_files > 1:
        files_list.sort(key=lambda x: os.path.getmtime(x), reverse=True) # use the most recent file
        print(f'Found more than one {string_search} file. Will use the most recent file based on "Date Modified" on your computer.')
        directory, file = os.path.split(files_list[0])
        print(f'Using directory file: {file}')
        return files_list[0]

    # Situation 3: one file found
    else:
        directory, file = os.path.split(files_list[0])
        print(f'Using directory file: {file}')
        return files_list[0]

    print()

############### Question 1: Best selling game by region this year
def generate_top_region_games_by_year(currentYear = datetime.datetime.now().year, filename='vgsales'):
    """

    Generates the top selling game by each region by year. By default, if no parameter is passed for ``currentYear``, then the function will use the systems' current year.

    The final Data Frame will be exported to the current directory location.

    Parameters
    ----------
    currentYear : int, optional
        Will default to systems' current year if no input is given

    filename : string, optional
        Will default to ``vgsales`` if no input is given.

    Returns
    -------
    This function does not return a value but exports a dataframe to the current directory.

    """

    # Attempt to find the correct filename in the current directory
    games_file = find_directory_file(filename)

    # Load in the csv file into the dataframe, df
    df = load_file(games_file)

    # Filter for the current year
    df = df[df['Year'] == currentYear]

    # Find the top selling game by finding the highest value in each respective column using .argmax()
    df_NA = pd.DataFrame(df.iloc[df['NA_Sales'].values.argmax()])
    df_NA.columns = ['NA_Sales']

    df_EU = pd.DataFrame(df.iloc[df['EU_Sales'].values.argmax()])
    df_EU.columns = ['EU_Sales']

    df_JP = pd.DataFrame(df.iloc[df['JP_Sales'].values.argmax()])
    df_JP.columns = ['JP_Sales']

    df_Other = pd.DataFrame(df.iloc[df['Other_Sales'].values.argmax()])
    df_Other.columns = ['Other_Sales']

    # Concatenate the four pandas series into a finalized dataframe
    df_final = pd.concat([df_NA, df_EU, df_JP, df_Other], axis=1).reset_index()

    # Remove unecessary row
    df_final = df_final[df_final['index'] != 'Rank']

    # Export finalized dataframe
    filename = os.path.join(DIRECTORY_LOCATION, f'Top_Selling_Game_By_Region_{currentYear}.csv')
    df_final.to_csv(filename, index=False)
    print(f'Exporting completed file to this folder path: {filename}')

############### Question 2: Top 3 Publishers by Total Sales Each Year
def top_three_publishers_by_year(filename='vgsales'):
    """

    Generates the top three publishers for all years found in the input file. If no parameter is passed, the default filename will be ``vgsales``.

    The final Data Frame will be exported to the current directory location.

    Parameters
    ----------
    filename : string, optional
        Will default to ``vgsales`` if no input is given.

    Returns
    -------
    This function does not return a value but exports a dataframe to the current directory.

    """

    # Attempt to find the correct filename in the current directory
    games_file = find_directory_file(filename)

    # Load in the csv file into the dataframe, df
    df = load_file(games_file)

    # Calcuate the total global sales each for publisher for each year
    df2 = df.groupby(['Year', 'Publisher'])['Global_Sales'].sum().reset_index().sort_values(['Year', 'Global_Sales'], ascending=[True, False])

    # Once you've sorted by top sales per year, grab the top 3 values from each year
    df2 = df2.groupby('Year').head(3)

    # To calculate rank, create a new column with the values: [0, 1, 2]
    df2.insert(0, 'Rank', df2.groupby('Year').cumcount())

    # Drop the global sales column as we no longer need it
    del df2['Global_Sales']

    # Pivot your dataframe so the rankings are now columns for easier user readability
    df2 = df2.pivot(index='Year', columns='Rank', values='Publisher')

    # Clean the dataframe before exporting
    df2.reset_index(inplace=True)
    df2.rename(columns={0.0: 'Top_Publisher', 1.0: '2nd_Place', 2.0: '3rd_Place'}, inplace=True)
    df2.fillna("No Publisher", inplace=True)

    # Export finalized dataframe
    filename = os.path.join(DIRECTORY_LOCATION, f'Top_Three_Publishers_by_Sales_Each_Year.csv')
    df2.to_csv(filename, index=False)
    print(f'Exporting completed file to this folder path: {filename}')

############### Question 3: Total Console Sales by Year - using user inputted years
def console_sales_per_year(filename='vgsales'):
    """

    Generates the top selling game by each region by year. This function requires the user input data once it has been invoked.

    The user must input a minimum of one year as a response. If the user wishes to input more than one year, each year must be seperated by a comma: ``,``

    The final Data Frame will be exported to the current directory location.

    Parameters
    ----------
    filename : string, optional
        Will default to ``vgsales`` if no input is given.

    Returns
    -------
    This function does not return a value but exports a dataframe to the current directory.

    """

    # Attempt to find the correct filename in the current directory
    games_file = find_directory_file(filename)

    # Load in the csv file into the dataframe, df
    df = load_file(games_file)

    # Ask user to input years seperated by commas
    s = input("Type in the years you want to search up, seperated by commas:")
    years_inputted = list(map(int, s.split(',')))

    # Filter main dataframe for the years inputted by user
    df = df[df['Year'].isin(years_inputted)]

    # Groupby console to calculate total sales by year
    df = df.groupby(['Year', 'Platform'])['Global_Sales'].sum().reset_index()

    # Pivot results for easier user readability in exported file
    df = df.pivot(index='Platform', columns='Year', values='Global_Sales').reset_index().fillna(0)

    # Export finalized dataframe
    filename = os.path.join(DIRECTORY_LOCATION, f'Console_Sales_per_Year.csv')
    df.to_csv(filename, index=False)
    print(f'Exporting completed file to this folder path: {filename}')


# # num_list = [int(input("Enter int {}: ".format(i+1))) for i in range(3)]
# # print(num_list)

# # for i in range(0,10,2):
# #     if(i%2)
# #     print(i)
# # for i in range(1,10,2):
# #     if(i%3)
# t=[]
# i=1
# b1=1
# b2=0
# n=int(input("enter n"))
# while i<=n:

#     b1=b1*i
#     i=i+1
#     t.append(b1)
# print(t)
# print (b1)

# login="test"
# while login != "First":
#     login= input("enter login=")
# print("hello")

while login != "First":
    login= input("enter login=")
print("hello")


import tensorflow as tf
from . import crop
from . import rnet
from . import coeffs_to_mesh

import os
RENDER_OP_LIBRARY = 'yadira_render_cpp.so'
lib_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../')
assert os.path.isdir(lib_dir), lib_dir
lib_file = os.path.join(lib_dir, RENDER_OP_LIBRARY)
assert os.path.isfile(lib_file),lib_file
render_ops = tf.load_op_library(lib_file)
assert hasattr(render_ops, 'render_colors')

class MS(tf.Module):
    def __init__(self):
        super(MS, self).__init__()
        self.crop = crop.Crop()
        self.rnet = rnet.RNet()
        self.coeffs_to_mesh = coeffs_to_mesh.CoeffsToMesh()

        self.render_colors = render_ops.render_colors

        self.get_coeffs = tf.function(
            self.get_coeffs,
            input_signature=[
                tf.TensorSpec([1, None, None, 3], tf.float32, name='images'),
                tf.TensorSpec([1, 5, 2], tf.float32, name='facial_landmarks'),
            ])

        self.get_mesh = tf.function(
            self.get_mesh,
            input_signature=[
                tf.TensorSpec([1, None, None, 3], tf.float32, name='images'),
                tf.TensorSpec([1, 5, 2], tf.float32, name='facial_landmarks'),
                tf.TensorSpec([], tf.bool),
            ])

        self.get_depthmap = tf.function(
            self.get_depthmap,
            input_signature=[
                tf.TensorSpec([1, None, None, 3], tf.float32, name='images'),
                tf.TensorSpec([1, 5, 2], tf.float32, name='facial_landmarks'),
                tf.TensorSpec([], tf.bool),
            ])

    def get_coeffs(self, b_img, b_lms2d_n5):
        """
        Returns:
            crop [b, 224, 224, 3]
            coeffs [b, 257]
        """
        img = b_img[0]
        lms2d_n5 = b_lms2d_n5[0]

        img, lms2d_n5, t, s = self.crop(img, lms2d_n5)
        coeffs = self.rnet(tf.expand_dims(img, axis=0))
        angles = coeffs[0, 224:227]  # pitch, yaw, roll
        return {
            'crop': img,
            't': t,
            's': s,
            'coeffs': coeffs,
            'angles': angles,
        }

    def get_mesh(self, b_img, b_lms2d_n5, frontalize):
        """
        Returns:
            crop [b, 224, 224, 3]
            coeffs [b, 257]
            mesh [b, N_vertices, 3]
        """
        out = self.get_coeffs(b_img, b_lms2d_n5)
        mesh = self.coeffs_to_mesh(out['coeffs'], frontalize)
        out.update({'mesh': mesh})
        return out

    def get_depthmap(self, img_bytes, b_lms2d_n5, frontalize):
        """
        Returns face info in source crop coordinates (either frontalize=True or not):
        Возвращает информацию о лице в координатах кропа (вне зависимости от frontalize=True или False)
            crop [b, 224, 224, 3]
            coeffs [b, 257]
            coeffs_251 [b, 251]
            mesh [b, N_vertices, 3]
            depthmap [b, 224, 224, 1]
        """
        out = self.get_mesh(img_bytes, b_lms2d_n5, frontalize)
        mouth = False

        def _get_depth_from_vertices_tf(vertices):
            z = vertices[:, 2:]
            z = z - tf.reduce_min(z)
            z = z / tf.reduce_max(z)
            return z

        z = _get_depth_from_vertices_tf(out['mesh'][0])
        depthmap = self.render_colors(out['mesh'][0], self.coeffs_to_mesh.get_triangles(mouth), z, 224, 224, 1)
        out.update({'depthmap': tf.expand_dims(depthmap, axis=0),})
        return out

def test():
    import os
    import cv2
    import numpy as np
    from PIL import Image
    import matplotlib.pyplot as plt
    from copy import deepcopy
    def _get_lms2d_n5_from_txt(img_file):
        txt_file = os.path.splitext(img_file)[0]+'.txt'
        lms2d_n5 = np.loadtxt(txt_file)
        assert lms2d_n5.shape == (5, 2)
        return lms2d_n5
    def _read_img_pil(img_file):
        assert os.path.isfile(img_file)
        pil_img = Image.open(img_file)
        return pil_img

    def draw_vertices(mesh_vertices, image, vertice_color=[255, 255, 255], vertice_r=1, extra_scale=3):
        mesh_vertices *= extra_scale

        h, w, _ = image.shape
        w *= extra_scale
        h *= extra_scale
        image = cv2.resize(image, (w, h))

        def draw_r_1():
            image[y, x] = vertice_color
        def draw_r():
            cv2.circle(image, (x, y), vertice_r, vertice_color, thickness=-1)

        if vertice_r == 1:
            draw_func = draw_r_1
        else:
            draw_func = draw_r

        for vertex in mesh_vertices:
            x, y = [np.round(v).astype(int) for v in vertex[:2]]
            if 0 <= x < w and 0 <= y < h:
                draw_func()
        return image
    img_file = os.path.join(os.path.dirname(__file__), 'resources/examples/000002.jpg')
    img_file = os.path.normpath(os.path.join(os.path.dirname(__file__), '../working_dir/exmp1.jpg'))
    assert os.path.isfile(img_file)
    lms2d_n5 = _get_lms2d_n5_from_txt(img_file)

    img = _read_img_pil(img_file)
    img = np.array(img)
    plt.imshow(img); plt.show()
    img_vis = draw_vertices(deepcopy(lms2d_n5), deepcopy(img), vertice_color=[0,255,0],vertice_r=6)
    plt.imshow(img_vis); plt.show()
    img = img.astype(np.float32)

    ms = MS()
    img = np.expand_dims(img, axis=0)
    lms2d_n5 = np.expand_dims(lms2d_n5, axis=0)
    out = ms.get_depthmap(img, lms2d_n5, False)

    def _show_img(image, title='', title_color='lime', title_size=20, cmap=None):
        plt.figure(figsize=(10, 10))
        plt.suptitle(title, c=title_color, fontsize=title_size)
        plt.imshow(image, cmap=cmap)
        plt.show()

    plt.imshow(out['crop'].numpy().astype(int)); plt.show()
    mesh = out['mesh'].numpy()[0]
    vis = draw_vertices(mesh, out['crop'].numpy())
    plt.imshow(vis.astype(int)); plt.show()
    print()

from ..transform import base as tb
from ..transform import hwc
from ..fan2d import Pipe_GetLms2d5n
import numpy as np
class Pipe_MsFaceReconstruction(tb.Pipe):
    KEY_CALL_IMMUT_hwc = hwc.KEY_hwc
    KEY_CALL_IMMUT_lms2d_5n = Pipe_GetLms2d5n.KEY_OUT_lms2d_5n
    KEY_OUT_mesh = 'mesh'

    """(x,y)"""
    KEY_OUT_t = 't'

    KEY_OUT_s = 's'

    """
    pitch, yaw, roll
    """
    KEY_OUT_angles = 'angles'
    def before_keys_assertions(self, **data):pass
    def after_keys_assertions(self, **data):
        mesh = data[self.KEY_OUT_mesh]
        if mesh is not None:
            assert len(mesh.shape)==2 and mesh.shape[-1]==3
            t = data[self.KEY_OUT_t]
            assert t.shape==(2,)
            s = data[self.KEY_OUT_s]
            assert s.shape==()
class MsFaceReconstruction(tb.TransformNoDataIndependentRandomness):
    PIPE_DEFAULT = Pipe_MsFaceReconstruction
    def _init_(self, **config):
        self.ms = MS()
    def transform(self, hwc, lms2d_5n):
        if lms2d_5n is not None:
            b_img = np.expand_dims(hwc, axis=0)
            b_lms2d_n5 = np.expand_dims(lms2d_5n, axis=0)
            out = self.ms.get_mesh(b_img, b_lms2d_n5, False)
            return {
                self.PIPE_DEFAULT.KEY_OUT_mesh: out['mesh'].numpy()[0],
                self.PIPE_DEFAULT.KEY_OUT_t: out['t'].numpy(),
                self.PIPE_DEFAULT.KEY_OUT_s: out['s'].numpy(),
                self.PIPE_DEFAULT.KEY_OUT_angles: out['angles'].numpy(),
            }
        else:
            return {
                self.PIPE_DEFAULT.KEY_OUT_mesh: None,
                self.PIPE_DEFAULT.KEY_OUT_t: None,
                self.PIPE_DEFAULT.KEY_OUT_s: None,
                self.PIPE_DEFAULT.KEY_OUT_angles: None,
            }


#Encoding=UTF8

import tornado.web
import mongo_helper
import json
import logger

class InventoryHandler(tornado.web.RequestHandler):
    def get(self):
        name = self.get_argument('name', '')
        if not name:
            return

        inv = mongo_helper.get_inventory(name)
        inv['test'] = 'wangtt'
        self.write(json.dumps(inv))

    def put(self):
        name = self.get_argument('name', '')
        if not name:
            return

        count = self.get_argument('count', '100')
        count = int(count)

        mongo_helper.add_inventory(name, count)

    def post(self):
        name = self.get_argument('name', '')
        if not name:
            return

        count = self.get_argument('count', '100')
        count = int(count)

        mongo_helper.update_inventory(name, count)


import torch
import torchvision
import torchvision.transforms as transforms
from PIL import Image
import numpy as np
import cv2
import numpy
import torch.utils.data as data
import os

def default_loader(path):
    return Image.open(path).convert('RGB')

mytransform = transforms.Compose([
    transforms.Grayscale(1),
    transforms.ToTensor(),
])

class myImageFolder(data.Dataset):
    def __init__(self, root, fig_cls, fig_t, transform = None, loader = default_loader):
        fh = open(fig_cls)
        imgs = []
        for fh_line in fh.readlines():
            fn = fh_line.split()[0].split('.')[0]
            if os.path.isfile(os.path.join(root, fn + '.jpg')):
                imgs.append((str(fn), int(fh_line.split()[1])))
            else:
                print('can not find the path : ' + os.path.abspath(os.path.join(root, fn)))
        self.root = root
        self.t_root = fig_t
        self.imgs = imgs
        self.transform = transform
        self.loader = loader

    def __getitem__(self, index):
        fn, label_cls = self.imgs[index]
        img = self.loader(os.path.join(self.root, fn + '.jpg'))
        label_t = torch.from_numpy(numpy.load(os.path.join(self.t_root, fn + '.npy')))
        if self.transform is not None:
            img = self.transform(img)
        return img, torch.tensor(label_cls).view(-1), label_t

    def __len__(self):
        return len(self.imgs)

train_dataset = myImageFolder(root = './data/data2/train_fig', fig_cls = './data/data2/train_out.txt', fig_t = './data/data2/train_t', transform= mytransform)
test_dataset = myImageFolder(root = './data/data2/test_set', fig_cls = './data/data2/output_test.txt', fig_t = './data/data2/test_t', transform= mytransform)

BATCH_SIZE = 2048
NUM_WORKERS = 8

train_loader = data.DataLoader(train_dataset, batch_size= BATCH_SIZE, shuffle = True, num_workers= NUM_WORKERS)
test_loader = data.DataLoader(test_dataset, batch_size= BATCH_SIZE, shuffle = True, num_workers= NUM_WORKERS)


import torch
import time
import numpy as np
import os
import scipy

class MultiAgentCluster(object):
    """docstring for MultiAgentCluster."""

    def __init__(self, agents, store_interval,
                 reload_agents_interval, reload_playing_agents_principle,
                 log_dir, tf_summary):

        super(MultiAgentCluster, self).__init__()
        '''
            Parameters:

                agents: a list of assets.agents.Agent
                store_interval: interval (in seconds) to store MultiAgentCluster by calling MultiAgentCluster.store()

                reload_agents_interval: interval (in seconds) of reloading agents
                reload_playing_agents_principle: principle of reloading playing agents

                log_dir: directory to put log files
                tf_summary: tensorboard summary object
        '''

        # print('# INFO: [{}][Instantiate Start >>>>]'.format(
        #     self.get_log_tag(),
        # ))

        self.all_agents = agents

        self.learning_agents = []
        self.playing_agents = []

        for agent in self.all_agents:
            if agent.mode in ['learning']:
                self.learning_agents += [agent]
            elif agent.mode in ['playing']:
                self.playing_agents += [agent]
            else:
                raise Exception('agent.mode is {}, whitch is invalid'.format(
                    agent.mode
                ))

        self.num_agents = len(self.all_agents)

        self.store_interval = store_interval
        self.last_time_store = time.time()

        self.reload_agents_interval = reload_agents_interval
        self.last_time_reload_agents = time.time()
        self.reload_playing_agents_principle = reload_playing_agents_principle

        self.log_dir = log_dir
        self.tf_summary = tf_summary

        self.checkpoints_reward_record = None
        self.checkpoints_reward_record_FRAME = 0
        self.checkpoints_reward_record_REWARD = 1

        # print('# INFO: [{}][>>>> Instantiate End]'.format(
        #     self.get_log_tag(),
        # ))

    def reset(self, obs):
        '''
            reset
        '''

        # print('# INFO: [{}][reset]'.format(
        #     self.get_log_tag(),
        # ))

        for agent in self.all_agents:
            agent.reset(obs[:, agent.id])

    def before_rollout(self):
        '''
            call this before rollout will:
                1, schedule trainer for all learning agents
        '''

        # print('# INFO: [{}][before_rollout]'.format(
        #     self.get_log_tag(),
        # ))

        for agent in self.learning_agents:
            agent.schedule_trainer()

    def experience_not_enough(self):
        '''
            check if experiences of learning agent is enough or not
        '''

        # TODO: This could be improved
        # if len(self.learning_agents) != 1:
        #     input('# ERROR: currently only support one learning_agent')

        return self.learning_agents[0].experience_not_enough()

    def act(self, obs, learning_agents_deterministic, playing_agents_deterministic):
        '''
            act
        '''

        # print('# INFO: [{}][act]'.format(
        #     self.get_log_tag(),
        # ))

        action = []
        for agent in self.all_agents:
            action += [agent.act(
                obs=obs[:, agent.id],
                deterministic={
                    'learning': learning_agents_deterministic,
                    'playing': playing_agents_deterministic,
                }[agent.mode],
                update_normalize_obs={
                    'learning': True,
                    'playing': False,
                }[agent.mode],
            ).unsqueeze(1)]

        return torch.cat(action, 1)

    def observe(self, obs, reward, done, infos):
        '''
            observe
        '''

        # print('# INFO: [{}][observe]'.format(
        #     self.get_log_tag(),
        # ))

        for agent in self.all_agents:
            agent.observe(
                obs[:, agent.id], reward[:, agent.id], done[:, agent.id], infos,
            )

    def store(self):
        '''
            store will:
                1, store all learning agents
        '''

        print('# INFO: [{}][store start >>>>]'.format(
            self.get_log_tag(),
        ))

        '''store learning_agents'''
        for agent in self.learning_agents:
            agent.store()

        print('# INFO: [{}][>>>> store end]'.format(
            self.get_log_tag(),
        ))

    def restore(self):
        '''
            restore will:
                1, (if agent.population_number > 1) re-generated population id for each of all agents_cluster
                2, restore a recent checkpoint according to the re-generated population id for each of all agents
        '''

        print('# INFO: [{}][restore start >>>>]'.format(
            self.get_log_tag(),
        ))

        '''restore playing agents'''
        for agent in self.playing_agents:
            if agent.population_number > 1:
                agent.randomlize_population_id()
            agent.restore(principle='recent')

        '''restore learning agents'''
        except_population_ids = []
        for agent in self.learning_agents:
            if agent.population_number > 1:
                agent.randomlize_population_id(
                    except_population_ids=except_population_ids,
                )
                except_population_ids += [agent.population_id]
            agent.restore(principle='recent')

        print('# INFO: [{}][>>>> restore end]'.format(
            self.get_log_tag(),
        ))

    def reload_playing_agents(self):
        '''
            reload playing agent will:
                1, (if agent.population_number is more than 1) re-generated population id for each playing agent
                2, restore a new checkpoint for each playing agent, according to
                    1, (if agent.population_number is more than 1) the re-generated population id
                    2, self.reload_playing_agents_principle
        '''

        # print('# INFO: [{}][reload_playing_agents]'.format(
        #     self.get_log_tag(),
        # ))

        '''(if agent.population_number is more than 1) re-generated population id for each playing agent'''
        for agent in self.playing_agents:
            if agent.population_number > 1:
                agent.randomlize_population_id()
            agent.restore(principle=self.reload_playing_agents_principle)

    def reload_learning_agents(self):
        '''
            reload learning agent will:
                (if agent.population_number is more than 1)
                    1, store each learning agent
                    2, re-generated population id for each learning agent, there will not be the case
                        when the same population_id being selected for multiple times
                    3, restore a recent checkpoint according to the re-generated population id for each learning agent
                (else)
                    1, nothing
        '''

        # print('# INFO: [{}][reload_learning_agents]'.format(
        #     self.get_log_tag(),
        # ))

        except_population_ids = []
        for agent in self.learning_agents:
            if agent.population_number > 1:
                agent.store()
                agent.randomlize_population_id(
                    except_population_ids=except_population_ids,
                )
                except_population_ids += [agent.population_id]
                agent.restore(principle='recent')

    def after_rollout(self):
        '''
            called after each rollout:
                1, call agent.after_rollout() for each agent
                2, (every reload_agents_interval) reload agents
                3, (every store_interval) store
        '''

        # print('# INFO: [{}][after_rollout]'.format(
        #     self.get_log_tag(),
        # ))

        '''call agent.after_rollout() for each agent'''
        for agent in self.all_agents:
            agent.after_rollout()

        '''reload agents'''
        if ((time.time() - self.last_time_reload_agents) > self.reload_agents_interval):
            self.last_time_reload_agents = time.time()
            self.reload_playing_agents()
            self.reload_learning_agents()

        '''store'''
        if ((time.time() - self.last_time_store) > self.store_interval):
            self.last_time_store = time.time()
            self.store()

    def get_log_tag(self):
        '''
            get the log tag of this class
        '''

        return 'MultiAgentCluster'


from decimal import getcontext, Decimal

precision = int(input("Enter Precision: "))
getcontext().prec = precision+1

MAX = 5000

def fibonacci(n):
    a = 0
    b = 1
    for i in range(0, n):
        temp = a
        a = b
        b = temp + a
    return a

f = [Decimal(0), Decimal(1), Decimal(1), Decimal(2)]
for x in range(2, MAX):
    f.append(Decimal(fibonacci(x)))
    if f[-1]/f[-2] == f[-2]/f[-3]:
        print(f[-1] / f[-2])
        print(f"""Required {len(f)} numbers from the fibonacci series to achieve approximation
accurate to {precision} decimal places""")
        break


num_to_bits = [0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4];
def countSetBitsRec(num):
    nibble = 0;
    if (0 == num):
        return num_to_bits[0];
    nibble = num & 0xf;
    return num_to_bits[nibble] + countSetBitsRec(num >> 4);

for _ in range(int(input())):
    n, q = [int(x) for x in input().split()]
    a = list(map(int, input().split()))
    p = [];c=[]
    for i in range(q):
        p1=int(input())

        b = []
        e = 0
        o = 0
        for i in a:
            if(bin(i^p1).count('1')%2==0):
                e += 1

        c.append(e)
    for i in c:
        print(i,len(a)-i)


import torch
import torch.nn as nn
from torchvision import utils

from utils.save import save_loss, save_fake_images

class EBGAN:
    def __init__(self, Generator, Discriminator, weights_init, epochs,
                 batch_size, img_size, save_epoch, dim_z, lr, beta1,
                 beta2=0.999, lambda_pt=0.1, margin=1, device='cpu'):
        self.device = device

        self.G = Generator().to(device)
        self.D = Discriminator().to(device)
        self.G.apply(weights_init)
        self.D.apply(weights_init)

        self.criterion = nn.MSELoss()

        self.d_optimizer = torch.optim.Adam(self.D.parameters(), lr=lr,
                                            betas=(beta1, beta2))
        self.g_optimizer = torch.optim.Adam(self.G.parameters(), lr=lr,
                                            betas=(beta1, beta2))

        self.epochs = epochs
        self.batch_size = batch_size
        self.img_size = img_size
        self.save_epoch = save_epoch
        self.dim_z = dim_z
        self.lambda_pt = lambda_pt
        self.margin = margin

    def train(self, train_loader, result_root='result/mnist'):
        data_size = len(train_loader)
        iters = 0
        g_losses = []
        d_losses = []
        fixed_z = torch.randn(64, self.dim_z, device=self.device)

        for epoch in range(self.epochs):
            for i, (images, _) in enumerate(train_loader):
                b_size = images.shape[0]
                ############################
                # (1) Update D network
                ###########################
                # Train with all-real batch
                # Format batch
                real_images = images.to(self.device)
                # Forward pass real batch through D
                d_real, _ = self.D(real_images)
                # Calculate loss on all-real batch
                d_loss_real = self.criterion(d_real, real_images)

                # Train with all-fake batch
                # Generate batch of latent vectors
                z = torch.randn(b_size, self.dim_z, device=self.device)
                # Generate fake image batch with G
                fake_images = self.G(z)
                # Classify all fake batch with D
                d_fake, embeddings = self.D(fake_images.detach())
                # Calculate D's loss on the all-fake batch
                d_loss_fake = self.criterion(d_fake, fake_images.detach())
                # Calculate the gradients for this batch
                self.D.zero_grad()
                d_loss = d_loss_real
                if self.margin - d_loss_fake.item() > 0:
                    d_loss += self.margin - d_loss_fake
                d_loss.backward()
                # Update D
                self.d_optimizer.step()

                ############################
                # (2) Update G network
                ###########################
                # Since we just updated D, perform another forward pass of
                # all-fake batch through D.
                d_fake, embeddings = self.D(fake_images)
                # Calculate G's loss based on this output
                g_loss = self.criterion(d_fake, fake_images) \
                         + self.lambda_pt * self.pullaway_loss(embeddings)
                # Calculate gradients for G
                self.G.zero_grad()
                g_loss.backward()
                # Update G
                self.g_optimizer.step()

                d_losses.append(d_loss.item())
                g_losses.append(g_loss.item())

                # Output training states
                if i % 50 == 0:
                    self._show_training_state(epoch, i, data_size,
                                              d_loss, g_loss)

                # Save the trained parameters
                if (epoch + 1) % self.save_epoch == 0:
                    self.save_model(epoch + 1, result_root)
                # Check the model by saving G's output on fixed_noise
                if (iters % 500 == 0) or (
                        (epoch == 100 - 1) and (i == len(train_loader) - 1)):
                    with torch.no_grad():
                        fake = self.G(fixed_z).detach().cpu()
                        fake_images = utils.make_grid(fake, padding=2,
                                                      normalize=True)
                        save_fake_images(fake_images, iters, result_root)

                iters += 1
        # Save the losses
        save_loss(d_losses, g_losses, result_root)

    @staticmethod
    def pullaway_loss(embeddings):
        norm = torch.sqrt(torch.sum(embeddings ** 2, -1, keepdim=True))
        normalized_emb = embeddings / norm
        similarity = torch.matmul(normalized_emb,
                                  normalized_emb.transpose(1, 0))
        batch_size = embeddings.size(0)
        loss_pt = (torch.sum(similarity) - batch_size) / (
                batch_size * (batch_size - 1))
        return loss_pt

    def _show_training_state(self, epoch, iteration, data_size, d_loss,
                             g_loss):
        print(f'[{epoch}/{self.epochs}][{iteration}/{data_size}]\t\
              Loss_D: {d_loss:.4f}\tLoss_G: {g_loss:.4f}')

    def save_model(self, epoch, path):
        torch.save(self.G.state_dict(),
                   f'{path}/checkpoint/generator_{epoch}.pkl')
        torch.save(self.D.state_dict(),
                   f'{path}/checkpoint/discriminator_{epoch}.pkl')
        print(f'Models save to {path}/')

    def load_model(self, discriminator_path, generator_path):
        self.D.load_state_dict(torch.load(discriminator_path))
        self.G.load_state_dict(torch.load(generator_path))
        print('Discriminator model loaded from {}-'.format(discriminator_path))
        print('Generator model loaded from {}.'.format(generator_path))


#!/usr/bin/python

# scanall.py
# this script crawls the DB to make sure the hosts are up.
# ========================================================== #
# COLDBLUE
# PETER CHRISTIAN FRAEDRICH
# ========================================================== #

import Queue
import threading
import subprocess as subp
from socket import gethostbyaddr as rdns
import sys
import os
from codex import log, db_connect

db = db_connect()
rows  = db.find()

# GET RUNTIME ARGS
if len(sys.argv) > 1 :
	if sys.argv[1] == "-node": # -node flag sets new path for log files since script is launched from app root
		logpath = "./log/py-scanall.log"
else :
	logpath = "../log/py-scanall.log"

def pinger(ip, health):
	try:
		res = os.system('ping -W 1 -c 1 ' + ip + ' > /dev/null 2>&1')
	except:
		log("ERROR","ICMP transmit error.", logpath)
		res = 1
	#print ip, response
	if res == 0:
		try:
			dns = rdns(ip)[0]
		except:
			dns = ''
		reserved = db.find_one({'ipaddr':ip})
		if reserved['reserved'] == 'reserved.png':
			resupdate = 'clear.png'
		else:
			resupdate = 'reserved.png'
		db.update( {"ipaddr": ip}, {"$set" : {"health":"green.png","dnsname":dns,"reserved":resupdate} } )
	else:
		db.update( {"ipaddr": ip}, {"$set" : {"health":"red.png"} } )

# called by each thread
def ping_ip(q, row):
	ip = row['ipaddr']
	health = row['health']
	q.put(pinger(ip,health))

log("INFO",'Creating queue.', logpath)
q = Queue.Queue()

for i in rows:
	t = threading.Thread(target=ping_ip, args = (q,i))
	t.start()

s = q.get()
log("INFO", "db_crawler finished crawling.", logpath)

exit()


from typing import List

class Solution:
    def sortColors(self, nums: List[int]) -> None:
        """
        通过双指针把数组分为3部分[0,1,2]
        遍历数组，把所有0交换到数组左边，1保持不动放在中间，2交换到右边
        遍历数组，i从左到右，p2从右到左，i超过p2时结束
        i <= p2:
            - nums[i]==0,和nums[p0]交换，p0和i前进
            - nums[i]==1,保持不动，i前进
            - nums[i]==2,和nums[p2]交换, p2前进
                - i 保持不动，因为和nums[p2]交换后，nums[i]可能还是2，所以i要进入下一轮循环
        """
        n = len(nums)
        p0, p2 = 0, n - 1
        i = 0
        while i <= p2:
            if nums[i] == 0:
                nums[i], nums[p0] = nums[p0], nums[i]
                p0 += 1
                i += 1
            elif nums[i] == 1:
                i += 1
            elif nums[i] == 2:
                nums[i], nums[p2] = nums[p2], nums[i]
                p2 -= 1

if __name__ == '__main__':
    res = Solution()
    print(res.sortColors(nums=[2, 0, 2, 1, 1, 0]))


#!/usr/bin/env python
# -*- coding:utf-8 -*-
# author: nanata
# datetime: 2020/09/22

from ._executor import Executor

class Transformer(Executor):

    def execute(self, input_list):
        model_obj = input_list[1]
        for attr, value in self.control_dict.items():
            if hasattr(model_obj, attr):
                setattr(model_obj, attr, value)
        self.outputs.append(model_obj.transform(input_list[0]))
        return self.outputs


import numpy as np
import matplotlib.pyplot as pl
import matplotlib as mpl
from scipy.interpolate import interp2d,Rbf,griddata
from mpl_toolkits.axes_grid1 import make_axes_locatable
import sys

font = {'family' : 'sans-serif',
        'size'   : 15}

mpl.rcParams['xtick.major.size'] = 5
mpl.rcParams['xtick.major.width'] = 1
mpl.rcParams['xtick.minor.size'] = 3
mpl.rcParams['xtick.minor.width'] = 1
mpl.rcParams['ytick.major.size'] = 5
mpl.rcParams['ytick.major.width'] = 1
mpl.rcParams['ytick.minor.size'] = 3
mpl.rcParams['ytick.minor.width'] = 1
mpl.rc('font', **font)

###-----------------------------------------

#Use: python PlotROSATmaps.py bandID
#where bandID = 1,2,4,5,6,7 is the energy band to plot

bandID = int(sys.argv[1])

if (bandID == 1):
    colmap = "Blues"
if (bandID == 2):
    colmap = "GnBu"
if (bandID == 4):
    colmap = "Greens"
if (bandID == 5):
    colmap = "Oranges"
if (bandID == 6):
    colmap = "Reds"
if (bandID == 7):
    colmap = "Purples"

#Load Xray map
XrayMap = np.loadtxt("../maps/ROSAT_R"+str(bandID)+"_rate.txt")

#Set data ranges for the plot
minval = 1
maxval = 1e5
cticks = [0.0,1,2,3,4,5]
clabs = [r'$1$',r'$10$',r'$10^{2}$',r'$10^{3}$',r'$10^{4}$',r'$10^{5}$']

#Do some plotting
fig = pl.figure(figsize=(12,7))
ax1 = fig.add_subplot(111)

im1 = ax1.imshow(np.log10(np.abs(np.clip(XrayMap, minval,maxval)))
        , extent=[-180, 180, -90, 90],cmap=colmap, origin="lower")

divider = make_axes_locatable(ax1)
cax = divider.append_axes("right", size="5%", pad=0.1)

cb1 = pl.colorbar(im1, label=r"$10^{-6}$ counts per second per arcmin$^{2}$",\
            ticks=cticks, extend="both", cax=cax)
cb1.ax.set_yticklabels(clabs)

ax1.set_aspect(1.0)

ax1.set_xlim(-180,180)
ax1.set_ylim(-90, 90)

ax1.set_xticks([-180, -120, -60, 0, 60, 120, 180])
ax1.set_yticks([-90, -45, 0, 45, 90])

bandtext = "..."

if (bandID == 1):
    bandtext = "(Band R1: 0.11 - 0.284 keV)"
if (bandID == 2):
    bandtext = "(Band R2: 0.14 - 0.284 keV)"
if (bandID == 4):
    bandtext = "(Band R4: 0.44 - 1.01 keV)"
if (bandID == 5):
    bandtext = "(Band R5: 0.56 - 1.21 keV)"
if (bandID == 6):
    bandtext = "(Band R6: 0.73 - 1.56 keV)"
if (bandID == 7):
    bandtext = "(Band R7: 1.05 - 2.04 keV)"

ax1.set_title('ROSAT Soft X-ray Background '+bandtext, fontsize=12)

#Put a box around region 12...
#ax1.plot([-80, -80, 80, 80, -80], [+5, +15, +15, +5, +5], 'k:')

ax1.set_xlabel(r'$l$ [degrees]', labelpad=5)
ax1.set_ylabel(r'$b$ [degrees]', labelpad=5)

pl.savefig('../plots/ROSAT_R' + str(bandID) +'_fullsky.pdf', bbox_inches="tight")


import os
import sys
import json
from PyQt5 import QtCore, QtWidgets
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *

class WindowClassificationMain(QtWidgets.QWidget):

    backward_main = QtCore.pyqtSignal();

    def __init__(self):
        super().__init__()
        self.title = 'Monk Classification'
        self.left = 100
        self.top = 100
        self.width = 500
        self.height = 220
        self.initUI()

    def initUI(self):
        self.setWindowTitle(self.title)
        self.setGeometry(self.left, self.top, self.width, self.height);

        # Forward
        self.b1 = QPushButton('Next', self)
        self.b1.move(300,180)
        self.b1.clicked.connect(self.forward)
        self.b1.setEnabled(False)

        # Backward
        self.b2 = QPushButton('Back', self)
        self.b2.move(200,180)
        self.b2.clicked.connect(self.backward)

        # Quit
        self.b3 = QPushButton('Quit', self)
        self.b3.move(400,180)
        self.b3.clicked.connect(self.close)

    def forward(self):
    	pass;

    def backward(self):
    	self.backward_main.emit();