import asyncio
import curses
from curses import textpad, ascii
import datetime

class Prompt():
    def __init__(self, prompt_string='> '):
        self.screen = curses.initscr()
        curses.noecho()
        curses.cbreak()
        self.prompt_string = prompt_string
        self.output_window = None
        self.prompt_window = None
        self._initialize()
        self.rebuild_prompt()
        self.loop = asyncio.get_event_loop()
        # self.data = asyncio.Queue()
        self.task = asyncio.gather(self.get_time())
        self.loop.run_until_complete(self.task)

    def _initialize(self):
        max_y, max_x = self.screen.getmaxyx()
        self.output_window = self.screen.subwin(max_y - 1, max_x, 0, 0)
        self.prompt_window = curses.newwin(1, max_x, max_y - 1, 0)
        self.edit = textpad.Textbox(self.prompt_window, insert_mode=True)
        self.output_window.scrollok(True)
        self.prompt_window.scrollok(True)

    def rebuild_prompt(self, default_text=None):
        self.prompt_window.clear()
        self.prompt_window.addstr(self.prompt_string)
        if default_text:
            self.prompt_window.addstr(default_text)
        self.prompt_window.refresh()

    def _validate_input(self, key):
        if key == ord('\n'):
            return curses.ascii.BEL
        if key == 127:
            key = curses.KEY_BACKSPACE
        if key in (curses.ascii.STX, curses.KEY_LEFT, curses.ascii.BS, curses.KEY_BACKSPACE):
            minx = len(self.prompt_string)
            (y, x) = self.prompt_window.getyx()
            if x == minx:
                return None
        if key == curses.KEY_RESIZE:
            self.resize()
            return None
        return key

    def resize(self):
        max_y, max_x = self.screen.getmaxyx()
        self.output_window.resize(max_y - 1, max_x)
        self.prompt_window.resize(1, max_x)
        self.prompt_window.mvwin(max_y - 1, 0)
        self.output_window.refresh()
        self.prompt_window.refresh()

    def readline(self, handle_interrupt=True):
        self.prompt_window.keypad(1)
        try:
            self.input_string = self.edit.edit(self._validate_input)
            self.input_string = self.input_string[len(self.prompt_string):len(self.input_string) - 1]
        except KeyboardInterrupt:
            if handle_interrupt:
                return False
            else:
                raise KeyboardInterrupt
        self.rebuild_prompt()
        return True

    def addline(self, line):
        self.output_window.addstr(line + '\n')
        self.output_window.refresh()
        self.prompt_window.refresh()

    async def get_time(self):
        while True:
            now = datetime.datetime.now()
            today = now.today()
            today = str(today.strftime("%m-%d-%y"))
            current_time = str(now.strftime("%H-%M-%S"))
            test = "[" + today + " " + current_time + "]:"
            self.output_window.addline(test)
            await asyncio.sleep(1)

# def main(stdscr):
#     console = Prompt(stdscr)

if __name__ == '__main__':
    console = Prompt()


import subprocess
import os
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

def get_gsea_enrichment(input_file, library, out_folder):
    """Runs GSEA based on input set of genes with associated logFC values
    using local jar file and gene set library (.gmt file)

    Parameters
    ----------
    input_file : str
        path to 2-column tab-seperated file  of gene (or phospshosite)
        and associated log2 fold-change values. Extension should be .rnk
    library : str
        path to gene set library
    out_folder : str
        path to output folder where all output files will be saved

    Returns
    -------
    df : pandas dataframe
       Summary of results from enrichment analysis that includes
       the term enriched, enrichment value, p-value and FDR.
    """
    path = out_folder
    os.makedirs(path)
    if '.gmt' not in library:
        lib = '%s.gmt' % library
    else:
        lib = library
    msda_path = os.path.dirname(os.path.abspath(__file__))
    arg_list = ['java', '-cp', os.path.join(msda_path, 'jars/gsea2-2.2.3.jar'),
                '-Xmx1024m',  'xtools.gsea.GseaPreranked',
                '-param_file', os.path.join(msda_path, 'gsea_parameters.txt'),
                '-rnk', input_file,
                '-gmx', lib,
                '-out', path]
    # if out_folder:
    #     arg_list += ['-out', path]
    # if out_file:
    #     arg_list += ['-rpt_label', out_file]
    subprocess.call(arg_list)
    terminal_folder = os.listdir(path)[0]
    timestamp = terminal_folder.split('.')[2]
    terminal_path = "%s/%s" % (path, terminal_folder)

    file1 = '%s/gsea_report_for_na_neg_%s.xls' % (terminal_path, timestamp)
    file2 = '%s/gsea_report_for_na_pos_%s.xls' % (terminal_path, timestamp)
    dfl = []
    if os.path.isfile(file1):
        dfl.append(pd.read_table(file1))
    if os.path.isfile(file2):
        dfl.append(pd.read_table(file2))
    # try:
    #     df_pos = pd.read_table(file1)
    #     df_neg = pd.read_table(file2)
    # except IOError:
    #     pass
    try:
        df = pd.concat(dfl)
    except ValueError:
        df = pd.DataFrame()
    return df

def make_rnkfile(df, sample1, sample2, identifier):
    """ Generated 2-column dataframe of identifier (gene or phosphosite)
    and associated log2 fold-change values

    Parameters
    ----------
    df : pandas dataframe
        proteomics or phosphoproteomics dataframe of dataset.
    sample1 : str
        column name of test sample.
    sample2 : str
        column name of control sample.
    identifier : str
       name of column that contains identifiers.

    Results
    -------
    df2 : pandas dataframe
       2-column dataframe of identifier (gene or phosphosite)
       and associated log2 fold-change values
    """
    df2 = df.copy()
    df2[sample1] = df2[sample1].div(df2[sample2])
    df2[sample1] = df2[sample1].apply(np.log2)
    df2 = df2.sort_values([sample1])
    df2 = df2[[identifier, sample1]]
    df2[identifier] = [g.upper() for g in df2[identifier].tolist()]
    df2 = df2.replace([-np.inf, np.inf], np.nan)
    df2 = df2.dropna()
    return df2

def plot_nes(df, filter=True, top=10, outfile=None,
             pval_label='NOM p-val', fdr_label='FDR q-val',
             regulon='NAME',
             fontsize=12, show_pval=False):
    """Plots gene set terms and associated enrichment scores
       from enrichment analyses.

    Parameters
    ----------
    df : pandas dataframe
       output dataframe from running enrichment analysis.
       Should contain term name, enrichment score, associated p-value and FDR.
    filter : Optional[bool]
       If True, terms for which the p-value is less than 0.05
       and FDR is less than 0.2 are filtered out. Default is True.
    top : Optional[int]
       Number of terms from downregulated and upregulated terms
       to be included in the plot. Default is 10.
    outfile : Optional[str]
       name of file to save the plot. Default is None
    pval_label : Optional[str]
       Name of column that contains p-values. Default is 'NOM p-val'
    fdr_label : Optional[str]
       Name of column that contains FDR values. Default is 'FDR q-val'
    regulon : Optional[str]
       Name of column that contains name of terms. Default is 'NAME'.
    fontsize : Optional[int]
       Size of fonts in plot. Default is 10.
    show_pval : bool
       p-values shown on plot if True. Default is False.
    """
    sns.set_style('dark')
    font = {'family': 'sans-serif',
            'color':  'black',
            'weight': 'normal',
            'size': fontsize,
            }
    sns.despine()
    if filter is True:
        df = df[(df[pval_label] <= 0.05) & (df[fdr_label] < 0.2)]
    df1 = df[df.NES > 0].iloc[:top, :]
    df2 = df[df.NES < 0].iloc[:top, :].sort_values(by='NES', ascending=False)
    ylen = np.max([len(df1), len(df2)])
    f, axes = plt.subplots(nrows=1, ncols=2)
    if not df1.empty:
        df1.NES.plot(ax=axes[1], kind='barh', width=0.3,
                     color='Green', alpha=0.5)
        for pos, name in zip(range(ylen), df1[regulon].tolist()):
            axes[1].text(0, pos-0.25, name, horizontalalignment='left',
                         fontdict=font)
    if not df2.empty:
        df2.NES.plot(ax=axes[0], kind='barh', width=0.3,
                     color='Red', alpha=0.5)
        for pos, name in zip(range(ylen), df2[regulon].tolist()):
            axes[0].text(0, pos+0.25, name, horizontalalignment='right',
                         fontdict=font)
    axes[0].set_yticks([])
    axes[1].set_yticks([])
    plt.subplots_adjust(wspace=0.075)
    axes[0].set_ylim([-0.5, ylen-0.5])
    axes[1].set_ylim([-0.5, ylen-0.5])

    if show_pval:
        pvals1 = ["%.2f" % p if p > 0 else "%.1f" % p
                  for p in df1[pval_label].tolist()]
        pvals2 = ["%.2f" % p if p > 0 else "%.1f" % p
                  for p in df2[pval_label].tolist()]
        for x_pos, y_pos, pval in zip(df1.NES.tolist(), range(ylen),
                                      pvals1):
            axes[1].text(x_pos+0.1, y_pos, pval)

        for x_pos, y_pos, pval in zip(df2.NES.tolist(), range(ylen)[::-1],
                                      pvals2):
            axes[1].text(x_pos-0.5, y_pos, pval)
    plt.gca().invert_yaxis()
    if not outfile:
        plt.show()
        plt.clf()
    else:
        plt.savefig(outfile)
        plt.clf()


from collections import defaultdict
import sys

import numpy as np
import gffutils
import pandas as pd

def miso_exon_to_gencode_exon(exon):
    """Convert a single miso exon to gencode gffutils database exon id

    >>> # Skipped exon (SE) or Mutually exclusive exon (MXE) ID
    >>> miso_exon_to_gencode_exon('chr2:9624561:9624679:+')
    'exon:chr2:9624561-9624679:+'
    >>> # Alt 5' splice site (pick first of alternative)
    >>> miso_exon_to_gencode_exon('chr15:42565276:42565087|42565161:-')
    'exon:chr15:42565276-42565087:-'
    >>> # Alt 3' splice site (pick first of alternative)
    >>> miso_exon_to_gencode_exon('chr2:130914199|130914248:130914158:-')
    'exon:chr2:130914199-130914158:-'
    >>> # Retained intron: start, stop separated by '-' instead of ':'
    >>> miso_exon_to_gencode_exon('chr1:906259-906386:+')
    'exon:chr1:906259-906386:+'
    """
    return 'exon:{}:{}-{}:{}'.format(*miso_exon_to_coords(exon))

def miso_id_to_exon_ids(miso_id):
    """Convert a MISO-style alternative event ID

    Split on the pipe ("|") to account for Alt 5'/3' splice site events

    # A skipped exon (SE) ID
    >>> miso_id_to_exon_ids('chr2:9624561:9624679:+@chr2:9627585:9627676:+@chr2:9628276:9628591:+')
    ['exon:chr2:9624561-9624679:+', 'exon:chr2:9627585-9627676:+', 'exon:chr2:9628276-9628591:+']
    >>> # A mutually exclusive (MXE) ID
    >>> miso_id_to_exon_ids('chr16:89288500:89288591:+@chr16:89289565:89289691:+@chr16:89291127:89291210:+@chr16:89291963:89292039:+')
    ['exon:chr16:89288500-89288591:+', 'exon:chr16:89289565-89289691:+', 'exon:chr16:89291127-89291210:+', 'exon:chr16:89291963-89292039:+']
    >>> # An Alt 5' splice site (A5SS) ID
    >>> miso_id_to_exon_ids("chr15:42565276:42565087|42565161:-@chr15:42564261:42564321:-")
    ['exon:chr15:42565276-42565087:-', 'exon:chr15:42564261-42564321:-']
    >>> # An Alt 3' splice site (A3SS) ID
    >>> miso_id_to_exon_ids('chr2:130914824:130914969:-@chr2:130914199|130914248:130914158:-')
    ['exon:chr2:130914824-130914969:-', 'exon:chr2:130914199-130914158:-']
    >>> # A retained intron (RI) ID
    >>> miso_id_to_exon_ids('chr1:906066-906138:+@chr1:906259-906386:+')
    ['exon:chr1:906066-906138:+', 'exon:chr1:906259-906386:+']
    """
    return map(miso_exon_to_gencode_exon, miso_id.split('@'))

def miso_exon_to_coords(exon):
    """Convert a miso exon to gffutils coordinates

    >>> miso_exon_to_coords('chr2:130914824:130914969:-')
    ('chr2', '130914824', '130914969', '-')
    >>> # Alt 5' SS - pick the first of the alternative ends
    >>> miso_exon_to_coords('chr15:42565276:42565087|42565161:-')
    ('chr15', '42565276', '42565087', '-')
    >>> # Alt 3' SS - pick the first of the alternative starts
    >>> miso_exon_to_coords('chr2:130914199|130914248:130914158:-')
    ('chr2', '130914199', '130914158', '-')
    >>> # Retained intron
    >>> miso_exon_to_coords('chr1:906066-906138:+')
    ('chr1', '906066', '906138', '+')
    """
    strand = exon[-1]
    coords = map(lambda x: x.split('|')[0],
                 exon.split(':'))
    if '-' in coords[1]:
        start, stop = coords[1].split('-')
        coords = coords[0], start, stop, strand
    return coords[0], coords[1], coords[2], strand

def convert_miso_ids_to_everything(miso_ids, db,
                                   event_type,
                                   out_dir):
    """Given a list of miso IDs and a gffutils database, pull out the
    ensembl/gencode/gene name/gene type/transcript names, and write files
    into the out directory. Does not return a value.

    Parameters
    ----------
    miso_ids : list of str
        Miso ids to convert
    db : gffutils.FeatureDB
        gffutils feature database created from a gtf file
    event_type : str
        The type of splicing event. This is used for naming only
    out_dir : str
        Where to write the files to.
    """
    out_dir = out_dir.rstrip('/')
    event_type = event_type.lower()

    miso_to_ensembl = {}
    miso_to_gencode = {}
    miso_to_gene_name = {}
    miso_to_gene_type = {}
    miso_to_ensembl_transcript = {}
    miso_to_gencode_transcript = {}

    ensembl_to_miso = defaultdict(list)
    gencode_to_miso = defaultdict(list)
    gene_name_to_miso = defaultdict(list)

    n_miso_ids = len(miso_ids)
    sys.stdout.write('Converting {} {} miso ids using {} gffutils database '
                     'into {}.'.format(n_miso_ids, event_type, str(db),
                                       out_dir))

    for i, miso_id in enumerate(miso_ids):
        if i % 100 == 0:
            sys.stdout.write('On {}/{} {} miso ids'.format(i, n_miso_ids,
                                                           event_type))

        exons = miso_id_to_exon_ids(miso_id)

        gencode = set([])
        ensembl = set([])
        gene_name = set([])
        gene_type = set([])
        gencode_transcript = set([])
        ensembl_transcript = set([])
        for e in exons:
            try:
                exon = db[e]
                gencode.update(exon.attributes['gene_id'])
                ensembl.update(
                    map(lambda x: x.split('.')[0],
                        exon.attributes['gene_id']))
                gene_name.update(exon.attributes['gene_name'])
                gene_type.update(exon.attributes['gene_type'])
                gencode_transcript.update(exon.attributes['transcript_id'])
                ensembl_transcript.update(
                    map(lambda x: x.split('.')[0], exon.attributes[
                        'transcript_id']))
            except gffutils.FeatureNotFoundError:
                try:
                    #  not an exon, look for any overlapping transcripts here
                    prefix, chrom, startstop, strand = e.split(':')
                    start, stop = startstop.split('-')
                    transcripts = list(db.features_of_type('transcript',
                                                           strand=strand,
                                                           limit=(
                                                           chrom, int(start),
                                                           int(stop))))
                    # if there are overlapping transcripts...
                    if len(transcripts) == 0:
                        continue
                    else:
                        for transcript in transcripts:
                            gencode.update(transcript.attributes['gene_id'])
                            ensembl.update(
                                map(lambda x: x.split('.')[0],
                                    transcript.attributes['gene_id']))
                            gene_name.update(transcript.attributes['gene_name'])
                            gene_type.update(transcript.attributes['gene_type'])
                            gencode_transcript.update(
                                transcript.attributes['transcript_id'])
                            ensembl_transcript.update(
                                map(lambda x: x.split('.')[0],
                                    transcript.attributes['transcript_id']))
                except:
                    continue
        if len(gencode) > 0:

            for ens in ensembl:
                ensembl_to_miso[ens].append(miso_id)
            for g in gene_name:
                gene_name_to_miso[g].append(miso_id)
            for g in gencode:
                gencode_to_miso[g].append(miso_id)

            ensembl = ','.join(ensembl)

            gencode = ','.join(gencode)
            gene_name = ','.join(gene_name)
            gene_type = ','.join(gene_type)

            gencode_transcript = ','.join(gencode_transcript)
            ensembl_transcript = ','.join(ensembl_transcript)

            miso_to_gencode[miso_id] = gencode
            miso_to_ensembl[miso_id] = ensembl
            miso_to_gene_name[miso_id] = gene_name
            miso_to_gene_type[miso_id] = gene_type

            miso_to_gencode_transcript[miso_id] = gencode_transcript
            miso_to_ensembl_transcript[miso_id] = ensembl_transcript

        else:
            miso_to_gencode[miso_id] = np.nan
            miso_to_ensembl[miso_id] = np.nan
            miso_to_gene_name[miso_id] = np.nan
            miso_to_gene_type[miso_id] = np.nan

    miso_tos = {'gencode_gene': miso_to_gencode,
                'ensembl_gene': miso_to_ensembl,
                'gene_name': miso_to_gene_name,
                'gene_type': miso_to_gene_type,
                'gencode_transcript': miso_to_gencode_transcript,
                'ensembl_transcript': miso_to_ensembl_transcript}

    to_misos = {'ensembl_gene': ensembl_to_miso,
                'gene_name': gene_name_to_miso,
                'gencode_gene': gencode_to_miso}

    for name, d in miso_tos.iteritems():
        df = pd.DataFrame.from_dict(d, orient='index')
        df.index.name = 'event_name'
        df.columns = [name]
        tsv = '{}/miso_{}_to_{}.tsv'.format(out_dir, event_type, name)
        df.to_csv(tsv, sep='\t')
        sys.stdout.write('Wrote {}\n'.format(tsv))

    for name, d in to_misos.iteritems():
        tsv = '{}/{}_to_miso_{}.tsv'.format(out_dir, name, event_type)
        with open(tsv, 'w') as f:
            for k, v in d.iteritems():
                f.write('{}\t{}\n'.format(k, '\t'.join(v)))
        sys.stdout.write('Wrote {}\n'.format(tsv))


# CONQUEST:GO
# https://github.com/Kandru/conquest-go
# author: Karl-Martin Minkner
# website: https://gameshare.community

import pymysql
import uuid
from queue import Queue
from threading import Lock
from listeners.tick import GameThread
from conquest.debug import msg
class mysql:
	def __init__(self):
		self.socket = None
		self.cursor = None
		self.connect()
		self.queue = Queue()
		self.queue_lock = Lock()
		self.queue_queries = {}
		t = GameThread(target=self.queueworker, args=())
		t.start()

	def connect(self):
		try:
			if self.socket is None:
				self.socket = pymysql.connect(
					host='localhost',
					user='csgo_1',
					password='',
					db='csgo_1',
					charset='utf8mb4',
					autocommit=True,
					cursorclass=pymysql.cursors.DictCursor)
				self.cursor = self.socket.cursor()
				msg('INFO', 'successfully connected to mysql server')
			else:
				msg('INFO', 'reconnecting to mysql server')
				self.disconnect()
				self.connect()
		except:
			msg('ERROR', 'could not connect to mysql server')

	def query(self, query, params=[], fetch_all=True, wait=True):
		uid = uuid.uuid4()
		self.queue.put({
			'uid': uid,
			'query': query,
			'params': params,
			'fetch_all': fetch_all
		})
		if wait is True:
			while True:
				self.queue_lock.acquire()
				if uid in self.queue_queries:
					tmp = self.queue_queries[uid]
					del self.queue_queries[uid]
					self.queue_lock.release()
					return tmp
				self.queue_lock.release()

	def do_query(self, uid, query, params=[], fetch_all=True):
		try:
			self.cursor.execute(query, params)
			if fetch_all is True:
				return self.cursor.fetchall()
			else:
				return self.cursor.fetchone()
		except:
			msg('ERROR', 'could not perform query - mysql connection lost?')
			self.connect()
			self.query(query, params, fetch_all)

	def queueworker(self):
		while True:
			item = self.queue.get()
			if item is None:
				break
			self.queue_lock.acquire()
			self.queue_queries[item['uid']] = self.do_query(item['uid'], item['query'], item['params'], fetch_all=item['fetch_all'])
			self.queue_lock.release()

	def disconnect(self):
		self.socket.close()
		self.socket = None
		msg('INFO', 'disconnected from mysql server')


from pprint import pprint
import json

json_files = open("task5_data.json", "r")
files =  json.load(json_files)
json_files.close()

def analyse_movies_genre():
    list1 = []
    for movie in files:
        for genre in movie["genre"]:
            list1.append(genre)
            temp = set(list1)
            movie_genre = {}
            for genre in list1:
                movie_genre[genre] = list1.count(genre)

    files_11 = open("task11_data.json", "w")
    json.dump(movie_genre,files_11, indent = 4)
    files_11.close()

    return movie_genre
task11 = analyse_movies_genre()
# pprint(task11)


"""
This code is maintained by @panghaibin and is distributed under the GPL-2.0 license.
Original source: https://github.com/panghaibin/shuasr/blob/master/ovpn.py
"""
import os
import re
SHU_DOMAINS = [
    'speedtest.shu.edu.cn',
    'selfreport.shu.edu.cn',
    'newsso.shu.edu.cn',
    'oauth.shu.edu.cn',
    'services.shu.edu.cn',
    'xk.autoisp.shu.edu.cn',
    'xk.shu.edu.cn',
]
SHU_IPS = []

def get_ip(domain):
    ip = os.popen("dig +short {}".format(domain)).read().strip()
    ip = re.findall(r'\d+\.\d+\.\d+\.\d+', ip)[0]
    return ip

def get_route_config():
    global SHU_IPS
    routing_config = "route-nopull\nroute-metric 150\nmax-routes 1000"
    for domain in SHU_DOMAINS:
        ip = get_ip(domain)
        SHU_IPS.append(ip)
    SHU_IPS = list(set(SHU_IPS))
    for ip in SHU_IPS:
        routing_config += "\n"
        routing_config += "route %s 255.255.255.255" % ip
    print(routing_config)
    return routing_config

def config_ovpn():
    global SHU_IPS
    users = os.environ['USERS'].split(';')
    user = users[0].split(',')
    secret = user[0] + "\n" + user[1]
    with open('secret.txt', 'w') as f:
        f.write(secret)

    routing_config = get_route_config()

    with open('.github/vpn/config.ovpn', 'r') as f:
        content = f.read()
        content = content.replace('# ROUTING CONFIG', routing_config)
        with open('.github/vpn/config.ovpn', 'w') as fa:
            fa.write(content)

if __name__ == '__main__':
    config_ovpn()


"""Regression tests for bug fixes, needing the full stack running."""
import logging
import time
from pathlib import Path

import pytest
import requests

logger = logging.getLogger(__name__)

TEST_VIDEO = str(
    (
        Path(__file__).parent
        / "test_data"
        / "test-abbor-small[2021-01-02_00-00-00]-000.mp4"
    ).resolve(),
)

@pytest.mark.usefixtures("start_core", "detection_api", "tracing_api")
def test_regression_add_two_jobs_with_objects() -> None:
    """Test regression of objects from previous job shows up.

    Ref. github issue #81
    """
    response = requests.post(
        "http://127.0.0.1:8000/projects/",
        json={
            "name": "Some name",
            "number": "NINA-123",
            "description": "Some kind of project description",
        },
    )
    assert response.status_code == 201

    response_job = requests.post(
        "http://127.0.0.1:8000/projects/1/jobs/",
        json={
            "name": "Job 1",
            "description": "Job description",
            "location": "test",
            "videos": [TEST_VIDEO],
        },
    )

    assert response_job.status_code == 201
    response = requests.post(
        "http://127.0.0.1:8000/projects/1/jobs/1/start",
    )

    url = "http://127.0.0.1:8000/projects/1/jobs/1"
    while requests.get(url).json()["_status"] != "Done":
        logger.info("Waiting for job to finish.")
        time.sleep(5)

    response_job = requests.post(
        "http://127.0.0.1:8000/projects/1/jobs/",
        json={
            "name": "Job 2",
            "description": "Job description",
            "location": "test",
            "videos": [TEST_VIDEO],
        },
    )

    assert response_job.status_code == 201
    response = requests.post(
        "http://127.0.0.1:8000/projects/1/jobs/2/start",
    )

    url = "http://127.0.0.1:8000/projects/1/jobs/2"

    while requests.get(url).json()["_status"] != "Done":
        logger.info("Waiting for job to finish.")
        time.sleep(5)

    job1 = requests.get(
        "http://127.0.0.1:8000/projects/1/jobs/1",
    ).json()
    job2 = requests.get(
        "http://127.0.0.1:8000/projects/1/jobs/2",
    ).json()

    assert job1["stats"]["total_objects"] == 1
    assert job2["stats"]["total_objects"] == 1  # bug results this to be 2


from datetime import datetime
import pandas as pd
from date_functions import get_charge_time

samlefil_df = pd.read_csv("samlefilFB.csv", skiprows=1)
samlefil_df['Charge Time PT'] = get_charge_time(samlefil_df)
samlefil_df = samlefil_df.loc[samlefil_df['Charge Time PT'] < '2022-01-01']

donations_sum = 0
sum_yearly = {}
count = 0
for index, row in samlefil_df.iterrows():
    count += 1
    donations_sum += float(str(row['Sum NOK']).replace(",", "").split(" ")[0])

    year = row['Charge Time PT'].split("-")[0]
    year_from_date = row["Dato"].split("-")[0]

    if (year != year_from_date):
        print(f"Years not equal for donation {row['ArkivID']}")

    if (year in sum_yearly.keys()):
        sum_yearly[year] += float(str(row['Sum NOK']).replace(",", "").split(" ")[0])
    else:
        sum_yearly[year] = 0

print("Donations counted: " + str(count))
print("Donation sum: " + str(donations_sum))
print("Yearly sum:")
print(sum_yearly)


import hashlib
import random
import sqlite3

import emojis

import logging

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s | %(name)s | %(levelname)s | %(" "message)s",
    handlers=[
            logging.FileHandler("oi.log"),
            logging.StreamHandler()
        ]
)
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)

DB_NAME = "oi.db"

class OIDatabase:
    def __init__(self):
        """
        If you want to make a database change:
        - Increment LATEST_VERSION
        - Make your changes and wrap them around an if db_version < <previous version> function
        """
        self.connection = sqlite3.connect(DB_NAME)
        self.cursor = self.connection.cursor()
        self.cursor.execute(
            "CREATE TABLE IF NOT EXISTS schema_version (version INT NOT NULL)"
        )
        db_version = self.cursor.execute(
            "SELECT version FROM schema_version"
        ).fetchall()
        if len(db_version) == 0:
            db_version = 0
        else:
            db_version = db_version[0][0]
        if db_version < 1:
            self.cursor.execute(
                "CREATE TABLE IF NOT EXISTS oi (id TEXT NOT NULL, messagecount INT NOT NULL)"
            )
        if db_version < 2:
            self.cursor.execute(
                "CREATE TABLE IF NOT EXISTS cumcry (id TEXT NOT NULL, emoji TEXT NOT NULL, cumcount INT NOT NULL, crycount INT NOT NULL)"
            )
            self.cursor.execute(
                "CREATE TABLE IF NOT EXISTS confessions (confession TEXT NOT NULL)"
            )
        if db_version < 3:
            self.cursor.execute(
                "CREATE TABLE IF NOT EXISTS wyr (question TEXT NOT NULL)"
            )
        if db_version < 4:
            self.cursor.execute("ALTER TABLE confessions ADD COLUMN hash TEXT")
            self.cursor.execute("ALTER TABLE wyr ADD COLUMN hash TEXT")
        if db_version < 5:
            self.cursor.execute(
                "CREATE TABLE IF NOT EXISTS later_deletion (member_id TEXT NOT NULL, remove_time TEXT NOT NULL)"
            )
        if db_version < 6:
            self.cursor.execute(
                "CREATE TABLE IF NOT EXISTS cumtime (id TEXT NOT NULL, datetime TEXT NOT NULL)"
            )
            self.cursor.execute(
                "CREATE TABLE IF NOT EXISTS crytime (id TEXT NOT NULL, datetime TEXT NOT NULL)"
            )
        if db_version < 7:
            self.cursor.execute(
                "CREATE TABLE IF NOT EXISTS unlater (id TEXT NOT NULL, cmdcount INT NOT NULL)"
            )
        if db_version < 8:
            self.cursor.execute("ALTER TABLE later_deletion ADD COLUMN jobid TEXT")
        if db_version < 9:
            self.cursor.execute(
                "CREATE TABLE IF NOT EXISTS later_min (id TEXT NOT NULL, minutes INT NOT NULL)"
            )
            self.cursor.execute(
                "CREATE TABLE IF NOT EXISTS last_latered (id TEXT NOT NULL, datetime TEXT NOT NULL)"
            )
        LATEST_VERSION = 9
        if db_version == 0:
            self.cursor.execute(
                "INSERT INTO schema_version VALUES (?)", (LATEST_VERSION,)
            )
        else:
            self.cursor.execute(
                "UPDATE schema_version SET version = ?", (LATEST_VERSION,)
            )
        self.connection.commit()

    def update_message_count(self, id):
        """
        Increment #hi msg count by 1.
        :param id: user id
        :return: None
        """
        self.cursor.execute("SELECT * FROM oi WHERE id = ?", (id,))
        if len(self.cursor.fetchall()) == 0:
            self.cursor.execute("INSERT INTO oi VALUES (?, 1)", (id,))
        else:
            self.cursor.execute(
                "UPDATE oi SET messagecount = messagecount + 1 WHERE id = ?", (id,)
            )
        self.connection.commit()

    def get_hi_leaderboard(self):
        """
        Get ID, message count for users in #hi.
        :return:
        """
        self.cursor.execute("SELECT * FROM oi ORDER BY messagecount DESC")
        return self.cursor.fetchall()

    def get_cum_leaderboard(self):
        """
        Gets all id, emoji, cumcount, crycount from cumcry table, order by cumcount.
        :return:
        """
        self.cursor.execute(
            "SELECT id, emoji, cumcount, crycount FROM cumcry ORDER BY cumcount DESC"
        )
        return self.cursor.fetchall()

    def get_cry_leaderboard(self):
        """
        Gets all id, emoji, cumcount, crycount from cumcry table, order by crycount.
        :return:
        """
        self.cursor.execute(
            "SELECT id, emoji, cumcount, crycount FROM cumcry ORDER BY crycount DESC"
        )
        return self.cursor.fetchall()

    def get_aggregated_cumcry_leaderboard(self):
        """
        Gets all id, emoji, cumcount + crycount from cumcry table, order by cumcount + crycount.
        :return:
        """
        self.cursor.execute(
            "SELECT id, emoji, cumcount + crycount FROM cumcry ORDER BY cumcount + crycount DESC, cumcount DESC"
        )
        return self.cursor.fetchall()

    def increment_cumcry_count(self, id, action):
        """
        Increment cum or cry count for user by 1. If user doesn't exist, creates entry for them with random emoji.
        :param id: user id
        :param action: either "cum" or "cry"
        :return:
        """
        self.cursor.execute("SELECT * FROM cumcry WHERE id = ?", (id,))
        if len(self.cursor.fetchall()) == 0:
            # choose random emoji from 'Animals & Nature' category
            cat = [*emojis.db.get_emojis_by_category("Animals & Nature")]
            e = random.choice(cat)
            # check if anyone else has same emoji
            self.cursor.execute("SELECT * FROM cumcry WHERE emoji = ?", (e.aliases[0],))
            # repeat until we find one that no one else has
            while len(self.cursor.fetchall()) > 0:
                e = random.choice(cat)
                self.cursor.execute(
                    "SELECT * FROM cumcry WHERE emoji = ?", (e.aliases[0],)
                )
            # create entry for user with their unique emoji
            self.cursor.execute(
                "INSERT INTO cumcry VALUES (?, ?, 0, 0)", (id, e.aliases[0])
            )
        # update counts
        if action == "cum":
            self.cursor.execute(
                "UPDATE cumcry SET cumcount = cumcount + 1 WHERE id = ?", (id,)
            )
        elif action == "cry":
            self.cursor.execute(
                "UPDATE cumcry SET crycount = crycount + 1 WHERE id = ?", (id,)
            )
        self.connection.commit()

    def clear_cumcry_counts(self):
        """
        Clear all cum and cry counts. Irreversible operation, do not execute.
        :return:
        """
        self.cursor.execute("DELETE FROM cumcry")
        self.connection.commit()

    def store_confession(self, confession):
        """
        Stores confession and hashed confession in database. Returns string to be sent to user.
        :param confession: confession message in the format "confession <actual confession>"
        :return: message to send to user with hash
        """
        if confession.strip() == "":
            return "you can't confess nothing, dummy"
        confession = confession[len("confess ") :]
        gen_hash = hashlib.sha1(confession.encode("utf-8")).hexdigest()
        h = (
            "Here is the hash for your confession: `"
            + gen_hash
            + '`. If you want to delete your confession, send the following message: "unconfess <hash>" and if deleted '
            "the oi bot will react with the trash can emoji."
        )
        self.cursor.execute(
            "INSERT INTO confessions VALUES (?, ?)", (confession, gen_hash)
        )
        self.connection.commit()
        return h

    def store_wyr(self, wyr):
        """
        Stores wyr and hashed wyr in database. Returns hash to be sent to user.
        :param wyr: wyr message in the format "wyr <actual wyr>"
        :return: message to send to user with hash
        """
        wyr = wyr[len("wyr ") :]
        gen_hash = hashlib.sha1(wyr.encode("utf-8")).hexdigest()
        h = (
            "Here is the hash for your wyr: `"
            + gen_hash
            + '`. If you want to delete your wyr, send the following message: "unwyr <hash>" and if deleted '
            "the oi bot will react with the trash can emoji."
        )
        self.cursor.execute("INSERT INTO wyr VALUES (?, ?)", (wyr, gen_hash))
        self.connection.commit()
        return h

    def get_random_confession(self):
        """
        Gets random confession.
        :return: rowid, confession
        """
        self.cursor.execute(
            "SELECT rowid, * FROM confessions ORDER BY RANDOM() LIMIT 1"
        )
        table = self.cursor.fetchall()
        rowid = -1
        confession = None
        log.info(f"Fetched {len(table)} of {self.get_num_confessions()} confessions")
        if len(table) > 0:
            rowid = table[0][0]
            confession = table[0][1]
            log.info(f"Confession {rowid}: \"{confession}\"")
        return rowid, confession

    def get_num_confessions(self):
        """
        :return: number of entries in confessions table
        """
        self.cursor.execute("SELECT COUNT(1) FROM confessions")
        num = self.cursor.fetchall()
        return num

    def get_random_wyr(self):
        """
        Gets random wyr.
        :return: rowid, wyr
        """
        self.cursor.execute("SELECT rowid, * FROM wyr ORDER BY RANDOM() LIMIT 1")
        table = self.cursor.fetchall()
        rowid = -1
        wyr = ""
        if len(table) > 0:
            rowid = table[0][0]
            wyr = table[0][1]
        return rowid, wyr

    def get_num_wyr(self):
        """
        :return: number of entries in confessions table
        """
        self.cursor.execute("SELECT COUNT(1) FROM wyr")
        num = self.cursor.fetchall()
        return num

    def delete_confession(self, rowid):
        """
        Deletes confession by rowid. Use if rowid is knownn.
        :param rowid:
        :return:
        """
        self.cursor.execute("DELETE FROM confessions WHERE rowid = ?", (rowid,))
        self.connection.commit()

    def delete_confession_by_hash(self, h):
        """
        Deletes confession by hash.
        :param h: hash
        :return: number of rows deleted
        """
        self.cursor.execute("DELETE FROM confessions WHERE hash = ?", (h,))
        num_rows = self.cursor.rowcount
        self.connection.commit()
        return num_rows

    def delete_wyr(self, rowid):
        """
        Deletes wyr by rowid. Use if rowid is known.
        :param rowid:
        :return:
        """
        self.cursor.execute("DELETE FROM wyr WHERE rowid = ?", (rowid,))
        self.connection.commit()

    def delete_wyr_by_hash(self, h):
        """
        Deletes wyr by hash.
        :param h: hash
        :return: number of rows deleted
        """
        self.cursor.execute("DELETE FROM wyr WHERE hash = ?", (h,))
        num_rows = self.cursor.rowcount
        self.connection.commit()
        return num_rows

    def create_later_delete_job(self, member_id, remove_time, jobid):
        """
        Creates job to remove later role
        :param member_id: ID of target member
        :param remove_time: string of datetime object representing remove tim
        :param jobid: ID of apscheduler job
        :return:
        """
        self.cursor.execute(
            "INSERT INTO later_deletion (member_id, remove_time, jobid) SELECT ?, ?, ? WHERE NOT EXISTS (SELECT 1 FROM later_deletion WHERE member_id = ? AND remove_time = ?);",
            (
                member_id,
                remove_time,
                jobid,
                member_id,
                remove_time,
            ),
        )
        self.connection.commit()

    def delete_later_delete_job(self, member_id, remove_time):
        """
        Deletes job to remove later role from row. Used if job is completed without restart.
        :param member_id:
        :param remove_time:
        :return:
        """
        self.cursor.execute(
            "DELETE FROM later_deletion WHERE member_id = ? AND remove_time = ?",
            (
                member_id,
                remove_time,
            ),
        )
        self.connection.commit()

    def get_all_later_deletion_jobs(self):
        """
        Gets all later deletion job entries in the db.
        :return: entries in the form of a list of tuples (member_id, remove_time)
        """
        self.cursor.execute("SELECT * FROM later_deletion")
        entries = self.cursor.fetchall()
        return entries

    def get_later_deletion_jobs_by_id(self, member_id):
        """
        Gets all later deletion job entries in the db.
        :return: entries in the form of a list of tuples (member_id, remove_time)
        """
        self.cursor.execute(
            "SELECT * FROM later_deletion where member_id = ?", (member_id,)
        )
        entries = self.cursor.fetchall()
        return entries

    def delete_later_jobs_by_id(self, user_id):
        """
        Deletes all later deletion jobs by user id.
        :param user_id: target user id
        :return:
        """
        self.cursor.execute(
            "DELETE FROM later_deletion WHERE member_id = ?", (user_id,)
        )
        self.connection.commit()

    def get_all_cumcry_entries(self):
        """
        Gets all cumcry entries in the db.
        :return: cumcry entries in the form of a list of tuples (id, emoji, cumcount, crycount)
        """
        self.cursor.execute("SELECT * FROM cumcry")
        entries = self.cursor.fetchall()
        return entries

    def update_cumcry_entry(self, id, num_cum, num_cry):
        """
        Updates cumcry entry with new cum and cry counts.
        :param id:
        :param num_cum:
        :param num_cry:
        :return:
        """
        self.cursor.execute(
            "UPDATE cumcry SET cumcount = ?, crycount = ? WHERE id = ?",
            (num_cum, num_cry, id),
        )
        self.connection.commit()

    def insert_cum_date_entry(self, member_id, date):
        """
        Inserts a new cum date entry into the db.
        :param member_id: ID of member
        :param date: datetime object or string
        :return:
        """
        self.cursor.execute("INSERT INTO cumtime VALUES (?, ?)", (member_id, date))
        self.connection.commit()

    def insert_cry_date_entry(self, member_id, date):
        """
        Inserts a new cry date entry into the db.
        :param member_id: ID of member
        :param date: datetime object or string
        :return:
        """
        self.cursor.execute("INSERT INTO crytime VALUES (?, ?)", (member_id, date))
        self.connection.commit()

    def count_cum_date_entries_by_id(self, member_id):
        """
        Counts the number of cum date entries for a given member.
        :param member_id:
        :return:
        """
        self.cursor.execute("SELECT COUNT(*) FROM cumtime WHERE id = ?", (member_id,))
        return self.cursor.fetchone()[0]

    def count_cry_date_entries_by_id(self, member_id):
        """
        Counts the number of cry date entries for a given member.
        :param member_id:
        :return:
        """
        self.cursor.execute("SELECT COUNT(*) FROM crytime WHERE id = ?", (member_id,))
        return self.cursor.fetchone()[0]

    def get_cumcry_id_emoji_pairs(self):
        """
        Gets a dictionary mapping ids to emojis.
        :return: dictionary of id->emoji pairs. Emojis are given by names, eg. "potted_plant".
        """
        self.cursor.execute("SELECT id, emoji FROM cumcry")
        return dict(self.cursor.fetchall())

    def get_cum_date_entries_by_id(self, id):
        """
        Gets all cum date entries for a given member.
        :param id: discord member id
        :return:
        """
        self.cursor.execute("SELECT datetime FROM cumtime WHERE id = ?", (id,))
        return self.cursor.fetchall()

    def get_cry_date_entries_by_id(self, id):
        """
        Gets all cry date entries for a given member.
        :param id: discord member id
        :return:
        """
        self.cursor.execute("SELECT datetime FROM crytime WHERE id = ?", (id,))
        return self.cursor.fetchall()

    def update_unlater_count(self, id):
        """
        Increment /unlater count by 1.
        :param id: user id
        :return: None
        """
        self.cursor.execute("SELECT * FROM unlater WHERE id = ?", (id,))
        if len(self.cursor.fetchall()) == 0:
            self.cursor.execute("INSERT INTO unlater VALUES (?, 1)", (id,))
        else:
            self.cursor.execute(
                "UPDATE unlater SET cmdcount = cmdcount + 1 WHERE id = ?", (id,)
            )
        self.connection.commit()

    def get_unlater_leaderboard(self):
        """
        Get ID, call count for /unlater.
        :return:
        """
        self.cursor.execute("SELECT * FROM unlater ORDER BY cmdcount DESC")
        return self.cursor.fetchall()

    def insert_later_time(self, id, curr_time):
        """
        Insert into unlater table.
        :param id: user id
        :return: None
        """
        self.cursor.execute(
            "INSERT INTO last_latered VALUES (?, ?)",
            (
                id,
                curr_time,
            ),
        )
        self.connection.commit()

    def get_last_later_time(self, id):
        """
        Get last time user used /later.
        :param id: user id
        :return: Datetime string of last latered time
        """
        self.cursor.execute("SELECT datetime FROM last_latered WHERE id = ?", (id,))
        ret = self.cursor.fetchone()
        if ret is None:
            return None
        else:
            return ret[0]

    def remove_later_time(self, id):
        """
        Remove last latered time from table.
        :param id: user id
        :return: None
        """
        self.cursor.execute("DELETE FROM last_latered WHERE id = ?", (id,))
        self.connection.commit()

    def update_later_min_count(self, id, minutes):
        """
        Increment /unlater count by 1.
        :param id: user id
        :return: None
        """
        self.cursor.execute("SELECT * FROM later_min WHERE id = ?", (id,))
        if len(self.cursor.fetchall()) == 0:
            self.cursor.execute(
                "INSERT INTO later_min VALUES (?, ?)",
                (
                    id,
                    minutes,
                ),
            )
        else:
            self.cursor.execute(
                "UPDATE later_min SET minutes = minutes + ? WHERE id = ?",
                (
                    minutes,
                    id,
                ),
            )
        self.connection.commit()

    def get_later_min_leaderboard(self):
        """
        Get ID, call count for /unlater.
        :return:
        """
        self.cursor.execute("SELECT * FROM later_min ORDER BY minutes DESC")
        return self.cursor.fetchall()


'''
< 문자열 다루기 기본 >
'''

s1 = "a234"
s2 = "1234"

def solution(s):
    if (s.isdigit() == True) and (len(s) == 4 or len(s) == 6):
        return True
    else:
        return False

print(solution(s1))
print(solution(s2))


# -- Import packages ---------------------------------------------------------------------------------------------------
from collections import OrderedDict
import os
import json
import numpy as np

class Diagnostics:
    """
    Diagnostics for an episode.
    """

    def __init__(self, runs):
        self.d = OrderedDict()
        self.runs = runs
        self.reset()

    def reset(self):
        """
        Reset the diagnostics dictionary.
        """
        self.d = OrderedDict()

        for r in range(self.runs):
            self.d[f'run_{r}'] = {'n_steps': [],
                                  'net_reward_to_org': [],
                                  'actionFromStart': [],
                                  'optimalPathLength': [],
                                  'optimalAction': [],
                                  'short_open_door_path': [],
                                  'long_path': [],
                                  'short_closed_door_path': [],
                                  'noGoal': [],
                                  'leaders': [],
                                  'coordinationCostsAccumulated': [],
                                  'opportunityCostsAccumulated': [],
                                  'policyInDomain_longpath': [],
                                  'policyInDomain_shortpath': [],
                                  'n_samplesModelDomain1': [],
                                  'n_samplesModelDomain2': [],
                                  'n_samplesModelDomain3': [],
                                  'n_samplesModelDomain4': [],
                                  'n_visits_per_state': np.zeros((13, 13)),
                                  'neutral_states_share': [],
                                  'shortpath_states_share': [],
                                  'longpath_states_share': [],
                                  'neutral_states_count': [],
                                  'shortpath_states_count': [],
                                  'longpath_states_count': []
                                  }

    def to_disk(self, par_name_, par_value_, start_date, start_time, runs):
        """
        Write diagnostics to disk.
        :param par_name_: parameter name
        :param par_value_: parameter value
        :param start_date: start date of the simulation run
        :param start_time: start time of the simulation run
        """
        assert type(self.d) == OrderedDict, 'ERROR: diagnostics variable must be a dictionary.'

        os.makedirs(f'outputs/traces/{start_date}/{start_time}', exist_ok=True)
        path = f'outputs/traces/{start_date}/{start_time}'
        filename = f'/dyna-q_{par_name_}_{par_value_}_{start_time}.json'

        # JSONify ndarray
        for r in range(runs):
            self.d[f'run_{r}']['n_visits_per_state'] = self.d[f'run_{r}']['n_visits_per_state'].tolist()

        with open(path + filename, "w") as f:
            json.dump(self.d, f, indent=4)

        print(f'\nSuccessfully saved to disk: {filename}\n')


from django.urls import path
from .views import GetIdealUser, UserDetailAPI,RegisterUserAPIView

urlpatterns = [
  path('register',RegisterUserAPIView.as_view()),
  path("get-details",UserDetailAPI.as_view()),        # need Authentication
  path("ideal/user/",GetIdealUser.as_view(), name="get-ideal-user"),
]


import math

import torch
import torch.nn as nn
import torch.nn.functional as F

from fairseq import utils
from fairseq.models import (
    FairseqEncoder,
)
from fairseq.modules import (
    LayerNorm,
    PositionalEmbedding,
    SinusoidalPositionalEmbedding,
)
from nonauto.model_utils import positional_encodings_like
from nonauto.modules.blocks import (
    SelfAttentionBlock,
    FeedForwardBlock
)

class NATEncoderLayer(nn.Module):
    """Encoder layer block.

    In the original paper each operation (multi-head attention or FFN) is
    postprocessed with: `dropout -> add residual -> layernorm`. In the
    tensor2tensor code they suggest that learning is more robust when
    preprocessing each layer with layernorm and postprocessing with:
    `dropout -> add residual`. We default to the approach in the paper, but the
    tensor2tensor approach can be enabled by setting
    *args.encoder_normalize_before* to ``True``.

    Args:
        args (argparse.Namespace): parsed command-line arguments
    """

    def __init__(self, args):
        super().__init__()
        embed_dim = args.encoder_embed_dim
        normalize_before = args.encoder_normalize_before
        head_nums = args.encoder_attention_heads
        dropout = args.dropout

        self.self_attn_block = SelfAttentionBlock(
            input_dim=embed_dim,
            num_heads=head_nums,
            dropout=args.attention_dropout,
            normalize_before=normalize_before,
        )

        activation_dropout = getattr(args, 'activation_dropout', 0)
        if activation_dropout == 0:
            # for backwards compatibility with models that use args.relu_dropout
            activation_dropout = getattr(args, 'relu_dropout', 0)

        self.ffn_block = FeedForwardBlock(
            input_dim=embed_dim,
            hidden_dim=args.encoder_ffn_embed_dim,
            dropout=dropout,
            activation_fn=getattr(args, 'activation_fn', 'relu'),
            activation_dropout=activation_dropout,
            normalize_before=normalize_before,
            residual_type=args.residual_type
        )

    def forward(self, x, encoder_padding_mask):
        """
        Args:
            x (Tensor): input to the layer of shape `(seq_len, batch, embed_dim)`
            encoder_padding_mask (ByteTensor): binary ByteTensor of shape
                `(batch, src_len)` where padding elements are indicated by ``1``.

        Returns:
            encoded output of shape `(batch, src_len, embed_dim)`
        """
        x = self.self_attn_block(
            x=x,
            padding_mask=encoder_padding_mask
        )
        x = self.ffn_block(x)
        return x

    def upgrade_state_dict_named(self, state_dict, name):
        """Upgrade a (possibly old) state dict for new versions of fairseq."""
        return state_dict

class NATEncoder(FairseqEncoder):
    """
    Non-autoregressive Transformer encoder consisting of *args.encoder_layers* layers. Each layer
    is a :class:`NATEncoderLayer`.

    Args:
        args (argparse.Namespace): parsed command-line arguments
        dictionary (~fairseq.data.Dictionary): encoding dictionary
        embed_tokens (torch.nn.Embedding): input embedding
    """

    def __init__(self, args, dictionary, embed_tokens):
        super().__init__(dictionary)
        self.share_embed = args.share_all_embeddings
        embed_dim = args.encoder_embed_dim
        self.padding_idx = dictionary.pad()
        self.max_source_positions = args.max_source_positions

        self.embed_tokens = embed_tokens
        self.embed_scale = math.sqrt(embed_dim)
        self.embed_positions = PositionalEmbedding(
            args.max_source_positions, embed_dim, self.padding_idx,
            learned=args.encoder_learned_pos,
        ) if not args.no_token_positional_embeddings else None

        self.layers = nn.ModuleList([])
        self.layers.extend([
            NATEncoderLayer(args)
            for _ in range(args.encoder_layers)
        ])
        self.dropout = nn.Dropout(args.dropout)
        self.register_buffer('version', torch.Tensor([2]))
        self.normalize = args.encoder_normalize_before
        if self.normalize:
            self.layer_norm = LayerNorm(embed_dim)

    def forward(self, src_tokens, src_lengths, **unused):
        """
        Args:
            src_tokens (LongTensor): tokens in the source language of shape
                `(batch, src_len)`
            src_lengths (torch.LongTensor): lengths of each source sentence of
                shape `(batch)`

        Returns:
            dict:
                - **encoder_out** (Tensor): the last encoder layer's output of
                  shape `(src_len, batch, embed_dim)`
                - **encoder_padding_mask** (ByteTensor): the positions of
                  padding elements of shape `(batch, src_len)`
        """
        # embed tokens and positions
        if self.share_embed:
            x = F.embedding(src_tokens, self.embed_tokens.weight * self.embed_scale)
        else:
            x = self.embed_tokens(src_tokens) * self.embed_scale
        # x = self.embed_scale * self.embed_tokens(src_tokens)
        # if self.embed_positions is not None:
        #     x += self.embed_positions(src_tokens)
        x += positional_encodings_like(x)
        encoder_history = [x]
        # x = F.dropout(x, p=self.dropout, training=self.training)
        x = self.dropout(x)
        # B x T x C -> T x B x C

        x = x.transpose(0, 1)

        # compute padding mask
        encoder_padding_mask = src_tokens.eq(self.padding_idx)
        if not encoder_padding_mask.any():
            encoder_padding_mask = None

        # encoder layers
        for layer in self.layers:
            x = layer(x, encoder_padding_mask)
            encoder_history.append(x.transpose(0, 1))

        if self.normalize:
            x = self.layer_norm(x)

        return {
            'encoder_history': encoder_history,  # List<B X T X C>
            'encoder_out': x,  # T x B x C
            'encoder_padding_mask': encoder_padding_mask,  # B x T
        }

    def reorder_encoder_out(self, encoder_out, new_order):
        """
        Reorder encoder output according to *new_order*.

        Args:
            encoder_out: output from the ``forward()`` method
            new_order (LongTensor): desired order

        Returns:
            *encoder_out* rearranged according to *new_order*
        """
        if encoder_out['encoder_out'] is not None:
            encoder_out['encoder_out'] = \
                encoder_out['encoder_out'].index_select(1, new_order)
        if encoder_out['encoder_padding_mask'] is not None:
            encoder_out['encoder_padding_mask'] = \
                encoder_out['encoder_padding_mask'].index_select(0, new_order)
        return encoder_out

    def max_positions(self):
        """Maximum input length supported by the encoder."""
        if self.embed_positions is None:
            return self.max_source_positions
        return min(self.max_source_positions, self.embed_positions.max_positions())

    def upgrade_state_dict_named(self, state_dict, name):
        """Upgrade a (possibly old) state dict for new versions of fairseq."""
        if isinstance(self.embed_positions, SinusoidalPositionalEmbedding):
            weights_key = '{}.embed_positions.weights'.format(name)
            if weights_key in state_dict:
                del state_dict[weights_key]
            state_dict['{}.embed_positions._float_tensor'.format(name)] = torch.FloatTensor(1)
        for i in range(len(self.layers)):
            # update layer norms
            self.layers[i].upgrade_state_dict_named(state_dict, f"{name}.layers.{i}")

        version_key = '{}.version'.format(name)
        if utils.item(state_dict.get(version_key, torch.Tensor([1]))[0]) < 2:
            # earlier checkpoints did not normalize after the stack of layers
            self.layer_norm = None
            self.normalize = False
            state_dict[version_key] = torch.Tensor([1])
        return state_dict


from __future__ import print_function

import sys
import os
import string
import re
import time
import subprocess
import logging
import libvirt
from libvirt import libvirtError
from xml.etree import ElementTree as ET

from fuelbench import config_manager
from fuelbench import utils
from fuelbench.settings import *
from fuelbench.error import *

NAT_NETWORK_XML = """\
<network>
  <name>${name}</name>
  <forward mode='nat'/>
  <bridge stp='on' delay='0' />
  <ip address='${ipaddr}' netmask='${netmask}'>
  </ip>
</network>
"""

ISO_NETWORK_XML = """\
<network ipv6='yes'>
  <name>${name}</name>
  <bridge stp='on' delay='0' />
</network>
"""

logger = logging.getLogger(__name__)

class KvmProvider(object):

    def __init__(self, uri='qemu:///system', config=config_manager.defaults()):
        self.uri = uri
        self.config = config

    def __list_site_domains(self, conn, site):
        prefix = 'site' + str(site) + '-'
        domains = list()

        for dom in conn.listAllDomains(0x0):
            name = dom.name()
            if name.startswith(prefix):
                domains.append(dom)

        return domains

    def __list_all_domains(self, conn):
        pattern = re.compile(r'^site(\d+)-(.+)$')
        domains = list()

        for dom in conn.listAllDomains(0x0):
            name = dom.name()
            if pattern.match(name):
                domains.append(dom)

        return domains

    def __undefine_domain(self, name):
        cmdline = 'virsh --connect {uri} undefine {domain} --managed-save --snapshots-metadata --remove-all-storage'.format(uri=self.uri, domain=name)

        retcode = subprocess.call(cmdline + '>>{0} 2>&1'.format(CONSOLE_LOG), shell=True)

        if retcode != 0:
            raise RuntimeError('Command failed with exit code {0}'.format(retcode))

    def __delete_volume(self, name, pool):
        cmdline = 'virsh --connect {uri} vol-delete {vol} --pool {pool}'.format(uri=self.uri, vol=name, pool=pool)

        retcode = subprocess.call(cmdline + '>>{0} 2>&1'.format(CONSOLE_LOG), shell=True)

        if retcode != 0:
            raise RuntimeError('Command failed with exit code {0}'.format(retcode))

    def __snapshot_create_as(self, domain, name, desc=''):
        cmdline = 'virsh --connect {uri} snapshot-create-as {domain} {name} --description "{desc}"'.format(uri=self.uri,
                                                                                domain=domain,
                                                                                name=name,
                                                                                desc=desc)
        logger.info('Execute: %s', cmdline)
        retcode = subprocess.call(cmdline + '>>{0} 2>&1'.format(CONSOLE_LOG), shell=True)
        logger.info('Return code: %d', retcode)

        if retcode != 0:
            raise CommandLineError(cmdline, retcode)

    def __snapshot_revert(self, domain, name):
        cmdline = 'virsh --connect {uri} snapshot-revert {domain} {name}'.format(uri=self.uri,
                                                                                domain=domain,
                                                                                name=name)
        logger.info('Execute: %s', cmdline)
        retcode = subprocess.call(cmdline + '>>{0} 2>&1'.format(CONSOLE_LOG), shell=True)
        logger.info('Return code: %d', retcode)

        if retcode != 0:
            raise CommandLineError(cmdline, retcode)

    def __keep_pxe_boot(self, vm_name):
        conn = libvirt.open(self.uri)

        dom = conn.lookupByName(vm_name)
        xml_root = ET.fromstring(dom.XMLDesc())

        xml_os = xml_root.find('os')
        for item in xml_os.findall('boot'):
            xml_os.remove(item)

        ET.SubElement(xml_os, 'boot', {'dev': 'network'})
        ET.SubElement(xml_os, 'boot', {'dev': 'hd'})

        xml_desc = ET.tostring(xml_root)
        conn.defineXML(xml_desc)

        conn.close()

    def get_storage_pool_path(self):
        pool_name = self.config['storage_pool']

        conn = libvirt.openReadOnly(self.uri)

        pool = conn.storagePoolLookupByName(pool_name)
        pool_info_root = ET.fromstring(pool.XMLDesc())

        conn.close()

        return pool_info_root.find("./target/path").text

    def get_vm_nodes(self, site):
        conn = libvirt.openReadOnly(self.uri)

        domains = self.__list_site_domains(conn, site)
        name_list = [dom.name() for dom in domains]

        conn.close()

        return name_list

    def get_max_node_id(self, site):
        vm_nodes = self.get_vm_nodes(site)
        pattern = re.compile(r'^site(\d+)-node-(\d+)$')

        max_id = 0

        for vm in vm_nodes:
            match = pattern.match(vm)
            if match:
                id = int(match.group(2))
                if (id > max_id):
                    max_id = id

        return max_id

    def clean_vm_nodes(self, site):
        conn = libvirt.open(self.uri)

        domains = self.__list_site_domains(conn, site)

        for dom in domains:
            if dom.isActive():
                dom.destroy()

            name = dom.name()
            self.__undefine_domain(name)

        conn.close()

    def clean_slave_nodes(self, site):
        conn = libvirt.open(self.uri)

        domains = self.__list_site_domains(conn, site)

        for dom in domains:
            name = dom.name()
            if name.endswith('-master'):
                continue

            if dom.isActive():
                dom.destroy()

            self.__undefine_domain(name)

        conn.close()

    def clean_all_vm_nodes(self):
        conn = libvirt.open(self.uri)

        domains = self.__list_all_domains(conn)

        for dom in domains:
            if dom.isActive():
                dom.destroy()

            name = dom.name()
            self.__undefine_domain(name)

        conn.close()

    def install_vm_node(self, site, name,
                        vcpus=1, ram=2048,
                        boot='cdrom', iso_image='',
                        disk_format='qcow2', disk_size=40, disk_io='threads',
                        networks=[],
                        os_variant='centos6.5',
                        wait=0,
                        delete_media=False,
                        persistent_boot=False):

        vm_name = "site{0}-{1}".format(site, name)

        install_cmd = 'virt-install --connect {uri} --virt-type kvm --name {name} --vcpus {vcpus} --ram {ram}'.format(uri=self.uri, name=vm_name, vcpus=vcpus, ram=ram)

        pool_path = self.get_storage_pool_path()
        disk_opt = '--disk path={pool_path}/{disk_name}.img,format={disk_format},size={disk_size},bus=virtio,cache=writeback,io={disk_io}'.format(pool_path=pool_path, disk_name=vm_name, disk_format=disk_format, disk_size=disk_size, disk_io=disk_io)

        net_opt = ''
        for net in networks:
            if len(net_opt) > 0:
                net_opt += ' '
            net_opt += '--network network={network},model=virtio'.format(network=net)

        graphics_opt = '--graphics vnc --noautoconsole'

        if boot == 'cdrom':
            if iso_image:
                boot_opt = '--cdrom {iso}'.format(iso=iso_image)
            else:
                raise ValueError('A valid ISO file is required')
        elif boot == 'pxe':
            boot_opt = '--pxe'
        else:
            raise NotImplementedError('Unsupported boot mode: ' + boot)

        os_opt = '--os-variant {0}'.format(os_variant)

        misc_opt = '--wait={wait}'.format(wait=wait)

        cmdline = ' '.join([install_cmd,
                            disk_opt, net_opt,
                            graphics_opt, boot_opt,
                            os_opt, misc_opt])

        try:
            with open(CONSOLE_LOG, 'a') as log:
                print(cmdline, file=log)

            retcode = subprocess.call(cmdline + '>>{0} 2>&1'.format(CONSOLE_LOG), shell=True)

            if retcode != 0:
                raise RuntimeError('Command failed with exit code {0}'.format(retcode))

        finally:
            if delete_media and iso_image:
                self.__delete_volume(os.path.basename(iso_image), 'tmp')

        if persistent_boot:
            self.destroy_vm_node(site, name)

            if boot == 'pxe':
                self.__keep_pxe_boot(vm_name)

            time.sleep(1)
            self.start_vm_node(site, name)

        return vm_name

    def start_vm_node(self, site, name):
        conn = libvirt.open(self.uri)

        vm_name = "site{0}-{1}".format(site, name)

        dom = conn.lookupByName(vm_name)
        ret = dom.create()

        conn.close()

        return ret == 0

    def destroy_vm_node(self, site, name):
        conn = libvirt.open(self.uri)

        vm_name = "site{0}-{1}".format(site, name)

        dom = conn.lookupByName(vm_name)
        ret = dom.destroy()

        conn.close()

        return ret == 0

    def restart_vm_node(self, site, name):
        conn = libvirt.open(self.uri)

        vm_name = "site{0}-{1}".format(site, name)

        dom = conn.lookupByName(vm_name)
        ret = dom.reset()

        conn.close()

        return ret == 0

    def delete_vm_node(self, site, name):

        vm_name = "site{0}-{1}".format(site, name)

        self.__undefine_domain(vm_name)

    def check_vm_node_active(self, site, name):
        conn = libvirt.openReadOnly(self.uri)

        vm_name = "site{0}-{1}".format(site, name)

        dom = conn.lookupByName(vm_name)
        is_active = dom.isActive()

        conn.close()

        return is_active

    def create_vm_snapshot(self, site, name, snapshot, description=''):
        vm_name = "site{0}-{1}".format(site, name)
        success = False

        try:
            self.__snapshot_create_as(vm_name, snapshot, description)
            success = True
        except CommandError as e:
            logger.error(e)
            success = False

        return success

    def revert_vm_snapshot(self, site, name, snapshot):
        vm_name = "site{0}-{1}".format(site, name)
        success = False

        try:
            self.__snapshot_revert(vm_name, snapshot)
            success = True
        except CommandError as e:
            logger.error(e)
            success = False

        return success

    def __list_site_networks(self, conn, site):
        prefix = 'site' + str(site) + 'net'
        networks = list()

        for net in conn.listAllNetworks(0x0):
            name = net.name()
            if name.startswith(prefix):
                networks.append(net)

        return networks

    def __list_all_networks(self, conn):
        pattern = re.compile(r'^site(\d+)net(\d+)$')
        networks = list()

        for net in conn.listAllNetworks(0x0):
            name = net.name()
            if pattern.match(name):
                networks.append(net)

        return networks

    def __create_nat_network(self, conn, name, ipaddr, netmask):

        template = string.Template(NAT_NETWORK_XML)
        xml = template.substitute(name=name,
                                  ipaddr=ipaddr,
                                  netmask=netmask)

        net = conn.networkDefineXML(xml)
        net.setAutostart(1)
        net.create()

        return net

    def __create_isolated_network(self, conn, name):

        template = string.Template(ISO_NETWORK_XML)
        xml = template.substitute(name=name)

        net = conn.networkDefineXML(xml)
        net.setAutostart(1)
        net.create()

        return net

    def get_networks(self, site):
        conn = libvirt.openReadOnly(self.uri)

        networks = self.__list_site_networks(conn, site)
        name_list = [net.name() for net in networks]

        conn.close()

        return name_list

    def clean_networks(self, site):
        conn = libvirt.open(self.uri)

        networks = self.__list_site_networks(conn, site)

        for net in networks:
            if net.isActive():
                net.destroy()

            net.undefine()

        conn.close()

    def clean_all_networks(self):
        conn = libvirt.open(self.uri)

        networks = self.__list_all_networks(conn)

        for net in networks:
            if net.isActive():
                net.destroy()

            net.undefine()

        conn.close()

    def create_networks(self, site, networks):
        conn = libvirt.open(self.uri)
        name_list = list()

        for idx, info in enumerate(networks, start=1):
            name = 'site{0}net{1}'.format(site, idx)
            if info['type'] == NETWORK_NAT:
                ipaddr = info['gateway']
                netmask = info['netmask']
                self.__create_nat_network(conn, name, ipaddr, netmask)
            elif info['type'] == NETWORK_ISOLATED:
                self.__create_isolated_network(conn, name)
            else:
                raise NotImplementedError("Unsupported network type: '{0}'".format(info.type))

            name_list.append(name)

        conn.close()
        return name_list


#!/bin/python3

"""
Minimal and reductive script to update all python3 pip packages.
Works for me, will probably not work for you, etc.

Assumes WSL2/Ubuntu 18.04.2
Author: Tomer Zaidenstein
License: Public Domain
"""

import os
import subprocess
import sys

_SKIP_LIST = ('Package', '---', 'WARNING')
# Ubuntu 18.04/WSL2 specific skip list:
_SKIP_LIST += ('distro-info', 'httplib2', 'pyasn1-modules', 'pygobject', 'pyxdg', 'PyYAML')

def main():
    cmd = "sudo pip3 list --outdated | grep wheel"
    proc = subprocess.Popen(cmd.split(),
                            stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT)
    stdout, stderr = proc.communicate()

    packages = []
    lines = stdout.splitlines()
    for line in lines:
        line = line.decode("utf-8").strip()

        # Skip the header
        if any(line.startswith(s) for s in _SKIP_LIST):
            continue

        package = line.split()[0]
        print(f'--> Updating {package}')
        packages.append(package)
        print(f'<-- Updating {package}')

    for p in packages:
        cmd = f"sudo pip3 install {p} --upgrade"
        out = subprocess.Popen(cmd.split())
        _, _ = out.communicate()

if __name__ == "__main__":
    main()


import itertools as it

EMPTY = -1
OUT_OF_BOARD = "."
UP = 1
DOWN = 2
RIGHT = 3
LEFT = 4

ALL_DIRS = [
    [-1, 0],
    [1,  0],
    [0, -1],
    [0,  1]
]

OFFSET_TO_DIR = {
    (-1, 0): UP,
    (1,  0): DOWN,
    (0,  1): RIGHT,
    (0, -1): LEFT
}

class Board:
    def __init__(self):
        self.nums_height = 0
        self.nums_width = 0
        self.nums = []
        self.dots_height = 0
        self.dots_width = 0
        self.dots = []
        self.solutions = 0

    def read(self, filename):
        with open(filename, 'r') as fh:
            n, m = map(int, fh.readline().split())
            NUMS = [[EMPTY] * (m + 2)]
            for _ in range(n):
                NUMS += [[EMPTY] + [int(x) for x in fh.readline().split()] +
                         [EMPTY]]
            NUMS += [[EMPTY] * (m + 2)]
        self.nums = NUMS
        self.nums_height = n + 2
        self.nums_width = m + 2

        self.dots_height = self.nums_height + 1
        self.dots_width = self.nums_width + 1
        self.dots = [[OUT_OF_BOARD] * self.dots_width]
        for i in range(1, self.dots_height - 1):
            self.dots += [[OUT_OF_BOARD] + [EMPTY] * (self.dots_width - 2) +
                          [OUT_OF_BOARD]]
        self.dots += [[OUT_OF_BOARD] * self.dots_width]

def edge_to_faces(edge):
    begin, end = edge
    if begin[0] == end[0]:
        return [(begin[0], begin[1] - 1), (begin[0], begin[1])]
        # upper and lower cell
    else:
        return [(begin[0] - 1, begin[1]), (begin[0], begin[1])]
        # left and right cell

def face_to_edges(i, j):
    # the points surrounding the face
    A = (i, j)
    B = (i, j + 1)
    C = (i + 1, j)
    D = (i + 1, j + 1)
    return [(A, B), (A, C), (B, D), (C, D)]

def global_check(board, edges_taken, final=False):
    # check if all the edges are consistent with the numbers in all the cells
    for i, j in it.product(range(board.nums_height), range(board.nums_width)):
        if board.nums[i][j] != EMPTY:
            amount_taken = 0
            for edge in face_to_edges(i, j):
                if edge in edges_taken:
                    amount_taken += 1
            if amount_taken > board.nums[i][j] or \
                    (final and amount_taken != board.nums[i][j]):
                return False
    return True

def fast_check(board, edges_taken, face):
    # check if there arent too many edges around the face
    i, j = face
    if board.nums[i][j] != EMPTY:
        amount_taken = 0
        for edge in face_to_edges(i, j):
            if edge in edges_taken:
                amount_taken += 1
        if amount_taken > board.nums[i][j]:
            return False
    return True

def find_biggest_tile(board):
    biggest_tile, biggest_tile_val = (1, 1), board.nums[1][1]
    for i in range(1, board.nums_height - 1):
        for j in range(1, board.nums_width - 1):
            if board.nums[i][j] > biggest_tile_val:
                biggest_tile_val = board.nums[i][j]
                biggest_tile = (i, j)
    return (biggest_tile, biggest_tile_val)

def backtrack(board, edges_taken=None, i=1, j=1, begin_i=1, begin_j=1,
              end_if_found=True):
    # print(str(i) + " " + str(j))
    if not edges_taken:
        # edge is an ORDERED pair of pairs, so the same two edges dont coincide
        edges_taken = set()
    if abs(begin_i - i) + abs(begin_j - j) == 1 and len(edges_taken) > 1:
        # i.e. path is one edge away from being finished
        last_edge = tuple(sorted([(begin_i, begin_j), (i, j)]))
        # if last_edge not in edges_taken:
        board.dots[i][j] = OFFSET_TO_DIR[(begin_i - i, begin_j - j)]
        edges_taken.add(last_edge)
        if global_check(board, edges_taken, final=True):
            print("Found\n")
            if end_if_found:
                return True
            else:
                board.solutions += 1
                print("sols " + str(board.solutions))
                for dots_row in board.dots:
                    print(' '.join(map(str, dots_row)))
                print()
        if i == 0 or j == 0 or i == board.dots_height or j == board.dots_width:
            board.dots[i][j] = OUT_OF_BOARD
        else:
            board.dots[i][j] = EMPTY
        edges_taken.remove(last_edge)
        return False

    for (offset, dir_to_next) in OFFSET_TO_DIR.items():
        next_i, next_j = i + offset[0], j + offset[1]
        if board.dots[next_i][next_j] == EMPTY:
            board.dots[i][j] = dir_to_next
            edge_being_added = tuple(sorted([(i, j), (next_i, next_j)]))
            edges_taken.add(edge_being_added)
            face_A, face_B = edge_to_faces(edge_being_added)
            if fast_check(board, edges_taken, face_A) and \
                    fast_check(board, edges_taken, face_B):

                if backtrack(board, edges_taken,
                             next_i, next_j, begin_i, begin_j, end_if_found):
                    return True
            if i == 0 or j == 0 or i == board.dots_height or \
            j == board.dots_width:
                board.dots[i][j] = OUT_OF_BOARD
            else:
                board.dots[i][j] = EMPTY
            edges_taken.remove(edge_being_added)

    return False

if __name__ == "__main__":

    B = Board()
    B.read('slith6_5x5.txt')
    # for debuging purposes
    # for row in B.nums:
    #    print(' '.join(map(str, row)))
    # print()

    # ugly part

    biggest_tile, biggest_tile_val = find_biggest_tile(B)

    # dangerous assumption that there is a tile with val >= 1
    # ... and all the tiles are <= 3

    found_solution = False
    for begining_offsets in ALL_DIRS:

        begin_i = biggest_tile[0] + begining_offsets[0]
        begin_j = biggest_tile[1] + begining_offsets[1]
        if backtrack(B, edges_taken=None, i=begin_i, j=begin_j,
                     begin_i=begin_i, begin_j=begin_j):
            found_solution = True
            break

    if found_solution:
        for dots_row in B.dots:
            print(' | '.join(map(str, dots_row)))
        print()
    else:
        print("No solution")

    C = Board()
    C.read('slith6_5x5.txt')

    biggest_tile, biggest_tile_val = find_biggest_tile(C)

    found_solution = False
    for begining_offsets in ALL_DIRS:
        begin_i = biggest_tile[0] + begining_offsets[0]
        begin_j = biggest_tile[1] + begining_offsets[1]
        backtrack(C, edges_taken=None, i=begin_i, j=begin_j,
                  begin_i=begin_i, begin_j=begin_j, end_if_found=False)
        if C.solutions != 0:
            break
    print(C.solutions)

    if C.solutions > 2:
        print("The solution is not unique")
    else:
        print("unique")


import functools

from peewee import fn
from playhouse.shortcuts import model_to_dict

from nni.nas.benchmark.utils import load_benchmark
from .schema import NdsTrialStats, NdsTrialConfig, proxy

def query_nds_trial_stats(model_family, proposer, generator, model_spec, cell_spec, dataset,
                          num_epochs=None, reduction=None, include_intermediates=False):
    """
    Query trial stats of NDS given conditions.

    Parameters
    ----------
    model_family : str or None
        If str, can be one of the model families available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`.
        Otherwise a wildcard.
    proposer : str or None
        If str, can be one of the proposers available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`. Otherwise a wildcard.
    generator : str or None
        If str, can be one of the generators available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`. Otherwise a wildcard.
    model_spec : dict or None
        If specified, can be one of the model spec available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`.
        Otherwise a wildcard.
    cell_spec : dict or None
        If specified, can be one of the cell spec available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`.
        Otherwise a wildcard.
    dataset : str or None
        If str, can be one of the datasets available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`. Otherwise a wildcard.
    num_epochs : float or None
        If int, matching results will be returned. Otherwise a wildcard.
    reduction : str or None
        If 'none' or None, all trial stats will be returned directly.
        If 'mean', fields in trial stats will be averaged given the same trial config.
    include_intermediates : boolean
        If true, intermediate results will be returned.

    Returns
    -------
    generator of dict
        A generator of :class:`nni.nas.benchmark.nds.NdsTrialStats` objects,
        where each of them has been converted into a dict.
    """

    if proxy.obj is None:
        proxy.initialize(load_benchmark('nds'))

    fields = []
    if reduction == 'none':
        reduction = None
    if reduction == 'mean':
        for field_name in NdsTrialStats._meta.sorted_field_names:
            if field_name not in ['id', 'config', 'seed']:
                fields.append(fn.AVG(getattr(NdsTrialStats, field_name)).alias(field_name))
    elif reduction is None:
        fields.append(NdsTrialStats)
    else:
        raise ValueError('Unsupported reduction: \'%s\'' % reduction)
    query = NdsTrialStats.select(*fields, NdsTrialConfig).join(NdsTrialConfig)
    conditions = []
    for field_name in ['model_family', 'proposer', 'generator', 'model_spec', 'cell_spec',
                       'dataset', 'num_epochs']:
        if locals()[field_name] is not None:
            conditions.append(getattr(NdsTrialConfig, field_name) == locals()[field_name])
    if conditions:
        query = query.where(functools.reduce(lambda a, b: a & b, conditions))
    if reduction is not None:
        query = query.group_by(NdsTrialStats.config)
    for trial in query:
        if include_intermediates:
            data = model_to_dict(trial)
            # exclude 'trial' from intermediates as it is already available in data
            data['intermediates'] = [
                {k: v for k, v in model_to_dict(t).items() if k != 'trial'} for t in trial.intermediates
            ]
            yield data
        else:
            yield model_to_dict(trial)


from keras.applications.inception_v3 import InceptionV3
from keras.preprocessing import image
from keras.models import Model
from keras.layers import Dense, GlobalAveragePooling2D
from keras import backend as K
from keras.optimizers import SGD, RMSprop, Adam

train_data_dir = "/ebs/user05/data/train"
validation_data_dir = "/ebs/user05/data/test"
img_width, img_height = 299, 299
nb_epoch = 20
nb_train_samples = 20385
nb_validation_samples = 8733

# Create the base pre-trained model
base_model = InceptionV3(weights='imagenet', include_top=False)

# Add a global spatial average pooling layer
x = base_model.output
x = GlobalAveragePooling2D()(x)
# Add a fully-connected layer
x = Dense(256, activation='relu', name='fc_1')(x)
predictions = Dense(1, activation='sigmoid')(x)

model = Model(input=base_model.input, output=predictions)

# Freeze convolutional layers
for layer in base_model.layers:
    layer.trainable = False

model.compile(optimizer=RMSprop(lr = .00001), loss = 'binary_crossentropy', metrics=['accuracy'])

train_datagen = image.ImageDataGenerator(rescale = 1./255,
                                  shear_range =.2,
                                  zoom_range = .2,
                                  horizontal_flip = True)

# Rescale image color scale
test_datagen = image.ImageDataGenerator(rescale=1./255)

generator_train = train_datagen.flow_from_directory(
        train_data_dir,
        target_size=(img_width, img_height),
        batch_size=32,
        class_mode='binary')

generator_test = test_datagen.flow_from_directory(
        validation_data_dir,
        target_size=(img_width, img_height),
        batch_size=32,
        class_mode='binary')

model.fit_generator(generator_train,
            samples_per_epoch = nb_train_samples,
            nb_epoch = nb_epoch,
            validation_data = generator_test,
            nb_val_samples = nb_validation_samples)

model_json = model.to_json()
with open("incep_filter.json", 'w') as json_file:
    json_file.write(model_json)
model.save_weights("incep_filter.h5")


# n = len(s)
# time: O(n)
# space: O(n)

class Solution:
    def isValid(self, s: str) -> bool:
        hashmap = {')': '(', '}': '{', ']': '['}
        stack = []
        for char in s:
            if char in hashmap:
                if not stack or stack.pop() != hashmap[char]:
                    return False
            else:
                stack.append(char)
        return not stack


from typing import Optional, Tuple, Union

import torch
from torch import nn as nn
from torch.autograd import Function

from ..utils import ext_loader
from .ball_query import ball_query
from .knn import knn

ext_module = ext_loader.load_ext(
    '_ext', ['group_points_forward', 'group_points_backward'])

class QueryAndGroup(nn.Module):
    """Groups points with a ball query of radius.

    Args:
        max_radius (float): The maximum radius of the balls.
            If None is given, we will use kNN sampling instead of ball query.
        sample_num (int): Maximum number of features to gather in the ball.
        min_radius (float, optional): The minimum radius of the balls.
            Default: 0.
        use_xyz (bool, optional): Whether to use xyz.
            Default: True.
        return_grouped_xyz (bool, optional): Whether to return grouped xyz.
            Default: False.
        normalize_xyz (bool, optional): Whether to normalize xyz.
            Default: False.
        uniform_sample (bool, optional): Whether to sample uniformly.
            Default: False
        return_unique_cnt (bool, optional): Whether to return the count of
            unique samples. Default: False.
        return_grouped_idx (bool, optional): Whether to return grouped idx.
            Default: False.
    """

    def __init__(self,
                 max_radius: float,
                 sample_num: int,
                 min_radius: float = 0.,
                 use_xyz: bool = True,
                 return_grouped_xyz: bool = False,
                 normalize_xyz: bool = False,
                 uniform_sample: bool = False,
                 return_unique_cnt: bool = False,
                 return_grouped_idx: bool = False):
        super().__init__()
        self.max_radius = max_radius
        self.min_radius = min_radius
        self.sample_num = sample_num
        self.use_xyz = use_xyz
        self.return_grouped_xyz = return_grouped_xyz
        self.normalize_xyz = normalize_xyz
        self.uniform_sample = uniform_sample
        self.return_unique_cnt = return_unique_cnt
        self.return_grouped_idx = return_grouped_idx
        if self.return_unique_cnt:
            assert self.uniform_sample, \
                'uniform_sample should be True when ' \
                'returning the count of unique samples'
        if self.max_radius is None:
            assert not self.normalize_xyz, \
                'can not normalize grouped xyz when max_radius is None'

    def forward(
        self,
        points_xyz: torch.Tensor,
        center_xyz: torch.Tensor,
        features: Optional[torch.Tensor] = None,
    ) -> Union[torch.Tensor, Tuple]:
        """
        Args:
            points_xyz (torch.Tensor): (B, N, 3) xyz coordinates of the
                points.
            center_xyz (torch.Tensor): (B, npoint, 3) coordinates of the
                centriods.
            features (torch.Tensor): (B, C, N) The features of grouped
                points.

        Returns:
            Tuple | torch.Tensor: (B, 3 + C, npoint, sample_num) Grouped
            concatenated coordinates and features of points.
        """
        # if self.max_radius is None, we will perform kNN instead of ball query
        # idx is of shape [B, npoint, sample_num]
        if self.max_radius is None:
            idx = knn(self.sample_num, points_xyz, center_xyz, False)
            idx = idx.transpose(1, 2).contiguous()
        else:
            idx = ball_query(self.min_radius, self.max_radius, self.sample_num,
                             points_xyz, center_xyz)

        if self.uniform_sample:
            unique_cnt = torch.zeros((idx.shape[0], idx.shape[1]))
            for i_batch in range(idx.shape[0]):
                for i_region in range(idx.shape[1]):
                    unique_ind = torch.unique(idx[i_batch, i_region, :])
                    num_unique = unique_ind.shape[0]
                    unique_cnt[i_batch, i_region] = num_unique
                    sample_ind = torch.randint(
                        0,
                        num_unique, (self.sample_num - num_unique, ),
                        dtype=torch.long)
                    all_ind = torch.cat((unique_ind, unique_ind[sample_ind]))
                    idx[i_batch, i_region, :] = all_ind

        xyz_trans = points_xyz.transpose(1, 2).contiguous()
        # (B, 3, npoint, sample_num)
        grouped_xyz = grouping_operation(xyz_trans, idx)
        grouped_xyz_diff = grouped_xyz - \
            center_xyz.transpose(1, 2).unsqueeze(-1)  # relative offsets
        if self.normalize_xyz:
            grouped_xyz_diff /= self.max_radius

        if features is not None:
            grouped_features = grouping_operation(features, idx)
            if self.use_xyz:
                # (B, C + 3, npoint, sample_num)
                new_features = torch.cat([grouped_xyz_diff, grouped_features],
                                         dim=1)
            else:
                new_features = grouped_features
        else:
            assert (self.use_xyz
                    ), 'Cannot have not features and not use xyz as a feature!'
            new_features = grouped_xyz_diff

        ret = [new_features]
        if self.return_grouped_xyz:
            ret.append(grouped_xyz)
        if self.return_unique_cnt:
            ret.append(unique_cnt)
        if self.return_grouped_idx:
            ret.append(idx)
        if len(ret) == 1:
            return ret[0]
        else:
            return tuple(ret)

class GroupAll(nn.Module):
    """Group xyz with feature.

    Args:
        use_xyz (bool): Whether to use xyz.
    """

    def __init__(self, use_xyz: bool = True):
        super().__init__()
        self.use_xyz = use_xyz

    def forward(self,
                xyz: torch.Tensor,
                new_xyz: torch.Tensor,
                features: Optional[torch.Tensor] = None) -> torch.Tensor:
        """
        Args:
            xyz (Tensor): (B, N, 3) xyz coordinates of the features.
            new_xyz (Tensor): new xyz coordinates of the features.
            features (Tensor): (B, C, N) features to group.

        Returns:
            Tensor: (B, C + 3, 1, N) Grouped feature.
        """
        grouped_xyz = xyz.transpose(1, 2).unsqueeze(2)
        if features is not None:
            grouped_features = features.unsqueeze(2)
            if self.use_xyz:
                # (B, 3 + C, 1, N)
                new_features = torch.cat([grouped_xyz, grouped_features],
                                         dim=1)
            else:
                new_features = grouped_features
        else:
            new_features = grouped_xyz

        return new_features

class GroupingOperation(Function):
    """Group feature with given index."""

    @staticmethod
    def forward(ctx, features: torch.Tensor,
                indices: torch.Tensor) -> torch.Tensor:
        """
        Args:
            features (Tensor): (B, C, N) tensor of features to group.
            indices (Tensor): (B, npoint, nsample) the indices of
                features to group with.

        Returns:
            Tensor: (B, C, npoint, nsample) Grouped features.
        """
        features = features.contiguous()
        indices = indices.contiguous()

        B, nfeatures, nsample = indices.size()
        _, C, N = features.size()
        output = torch.cuda.FloatTensor(B, C, nfeatures, nsample)

        ext_module.group_points_forward(
            features,
            indices,
            output,
            b=B,
            c=C,
            n=N,
            npoints=nfeatures,
            nsample=nsample)

        ctx.for_backwards = (indices, N)
        return output

    @staticmethod
    def backward(ctx, grad_out: torch.Tensor) -> Tuple[torch.Tensor, None]:
        """
        Args:
            grad_out (Tensor): (B, C, npoint, nsample) tensor of the gradients
                of the output from forward.

        Returns:
            Tensor: (B, C, N) gradient of the features.
        """
        idx, N = ctx.for_backwards

        B, C, npoint, nsample = grad_out.size()
        grad_features = torch.cuda.FloatTensor(B, C, N).zero_()

        grad_out_data = grad_out.data.contiguous()
        ext_module.group_points_backward(
            grad_out_data,
            idx,
            grad_features.data,
            b=B,
            c=C,
            n=N,
            npoints=npoint,
            nsample=nsample)
        return grad_features, None

grouping_operation = GroupingOperation.apply


# OpenCV
import cv2

# image = cv2.imread('./dataset/raw-img/cane/OIF-e2bexWrojgtQnAPPcUfOWQ.jpeg')
# print(image.shape)

# cv2.imshow('show', image)
# cv2.waitKey(0)

# os
import os

# image_dir = './dataset/raw-img/cane/'
# image_names = os.listdir(image_dir)

# for image_name in image_names:
#     print(image_name)

# Q
label_dic = {
    "cane": "dog",
    "cavallo": "horse",
    "elefante": "elephant",
    "farfalla": "butterfly",
    "gallina": "chicken",
    "gatto": "cat",
    "mucca": "cow",
    "pecora": "sheep",
    "scoiattolo": "squirrel",
    "ragno": "spider",
}
# print(label_dic['cane'])

#############################################################
#
#############################################################
# # "image_name, label"
# cane_dir = './dataset/raw-img/cane/'
# cavallo_dir = './dataset/raw-img/cavallo/'
# elefante_dir = './dataset/raw-img/elefante/'
# farfalla_dir = './dataset/raw-img/farfalla/'
# gallina_dir = './dataset/raw-img/gallina/'
# gatto_dir = './dataset/raw-img/gatto/'
# mucca_dir = './dataset/raw-img/mucca/'
# pecora_dir = './dataset/raw-img/pecora/'
# ragno_dir = './dataset/raw-img/ragno/'
# scoiattolo_dir = './dataset/raw-img/scoiattolo/'

# image_names = os.listdir(cane_dir)

# for i in range(0,len(image_names)):

#     img_names = image_names[i]
#     # os.path.split(cane_dir)
# #
#     dir_split = os.path.split(cane_dir)
#     names = dir_split[0]
#     fn = os.path.basename(names)

#     print(img_names,fn)

# #     label_dic[fn]

#############################################################
#
#############################################################
import shutil
import numpy as np

root_dir = './dataset/raw-img/'

train_dir = './dataset/train/'
validation_dir = './dataset/validation/'
test_dir = './dataset/test/'

class_names = os.listdir(root_dir)

for class_name in class_names:
    image_dir = root_dir + class_name + '/'
    image_names = os.listdir(image_dir)

    label = label_dic[class_name]

    length = len(image_names)

    train_length = int(length * 0.7) # 70%
    validation_length = int(length * 0.1) # 10%
    test_length = length - train_length - validation_length

    np.random.shuffle(image_names)

    # 1.
    # train_image_names = image_names[:train_length]
    # validation_image_names = image_names[train_length:train_length+validation_length]
    # test_image_names = image_names[train_length+validation_length:]

    # 2.
    train_image_names = image_names[:train_length]; image_names = image_names[train_length:]
    validation_image_names = image_names[:validation_length]
    test_image_names = image_names[validation_length:]

    # print(image_dir, label, len(train_image_names), len(validation_image_names), len(test_image_names))

    for image_name in train_image_names:
        src_image_path = image_dir + image_name
        dst_image_path = train_dir + f'{label}_{image_name}'

        shutil.copy(src_image_path, dst_image_path)

    for image_name in validation_image_names:
        src_image_path = image_dir + image_name
        dst_image_path = validation_dir + f'{label}_{image_name}'

        shutil.copy(src_image_path, dst_image_path)

    for image_name in test_image_names:
        src_image_path = image_dir + image_name
        dst_image_path = test_dir + f'{label}_{image_name}'

        shutil.copy(src_image_path, dst_image_path)

3. Image Classification/3.1. Dataset/3.1.1. Animals-10/preprocessing.py