from django.db import models
from django.utils.translation import gettext_lazy as _

from ._base import BaseDefinitionModel, include_schema

class Country(BaseDefinitionModel):
    class Meta:
        db_table = include_schema("countries")
        managed = False

    alpha_2 = models.CharField(_("alpha_2"), max_length=2, null=False)
    alpha_3 = models.CharField(_("alpha_3"), max_length=3, null=False)
    country_code = models.CharField(_("country_code"), max_length=3, null=False)
    iso_3166_2 = models.CharField(_("iso_3166_2"), max_length=13, null=False)
    region = models.CharField(_("region"), max_length=10, null=True)
    sub_region = models.CharField(_("sub_region"), max_length=100, null=True)
    intermediate_region = models.CharField(
        _("intermediate_region"), max_length=100, null=True
    )
    region_code = models.CharField(_("region_code"), max_length=3, null=True)
    sub_region_code = models.CharField(_("sub_region_code"), max_length=3, null=True)
    intermediate_region_code = models.CharField(
        _("intermediate_region_code"), max_length=3, null=True
    )


# import os
# import cv2
#
# # 重命名等操作
# picpath = '/Users/wangjinchao/Desktop/show/resss/'
# zz = '/Users/wangjinchao/Desktop/show/pp/'
# a = os.listdir(picpath)
# dir_list = sorted(a, key=lambda x: int(x.split('.')[0]))
# print(dir_list)
#
# # dir_list = sorted(a, key=lambda x : os.path.getmtime(os.path.join(picpath, x)))
# for i in range(len(a)):
#     # if picpath[i].endswith('.png') and i%2==1:
#     img_dir = os.path.join(picpath, dir_list[i])
#     os.renames(img_dir,zz+str(i*2)+'.png')
#
# print(len(a))
#

# pic转视频
import cv2
import os
import numpy as np
picpath = '/Users/wangjinchao/Desktop/show/pic/'
piccon = '/Users/wangjinchao/Desktop/show/xx/'
a = os.listdir(picpath)
# print(a[1].split('.')[0])
dir_list = sorted(a, key=lambda x: int(x.split('.')[0]))
c= os.listdir(piccon)
dir_ll = sorted(c, key=lambda x: int(x.split('.')[0]))
print(dir_list)
print(dir_ll)

fps = 20

size = (960*3,540)
# size = (2560, 720)
vw = cv2.VideoWriter('./result.avi',cv2.VideoWriter_fourcc('M','J','P','G'), fps, size)

print('start')
num = 0
while num < 298:
    pp = picpath+str(num)+'.png'
    seg = piccon+str(num)+'.png'

    framel = cv2.imread(pp)
    framer = cv2.imread(seg)
    imgl = cv2.resize(framel, (1920, 540))
    imgr = cv2.resize(framer, (960, 540))
    frame = np.concatenate((imgl, imgr),axis=1)
    # frame = cv2.hconcat([imgl, imgr])
    # frame = np.zeros((540, 2880, 3), np.uint8)
    # # # 写入图像到视频文件
    # frame[:, :int(1920), :] = imgl
    # frame[:, int(1920) :, :] = imgr
    vw.write(frame)
    num += 1
vw.release()

print('end')


import sys
from pathlib import Path
import datetime

from .version import __version__

def count_docstrings(lines):
    ''' Counts docstrings and remove them from the line list.

    Args:
        lines (list[str]): list of the code lines (in order!)

    Returns:
        int, list[str]: number of docstring lines, list of the code lines with all the docstrings
                        removed

    '''

    lines_minus_docstr = []
    counter = 0
    in_docstr = False

    # Go through each line in order
    for line in lines:

        #Look for the docstring tag
        if line.strip(' ').startswith(("'''", '"""')):
            in_docstr = ~in_docstr
            counter += 1
            continue

        # If I am in a docstring, count the extra lines
        if in_docstr:
            counter += 1
        # Else, keep the line for latter
        else:
            lines_minus_docstr += [line]

    # A small sanity check before closing
    if len(lines_minus_docstr) != len(lines) - counter:
        raise Exception('Ouch! Something is very wrong here!')

    return counter, lines_minus_docstr

def count_comments(lines):
    ''' Counts commented lines, i.e. lines starting with #

    Args:
        lines (list[str]): list of the code lines

    Returns:
        int, list[str]: number of commented lines, list of the code lines with all the comments
                        removed.

    Caution:
        This will look for all lines starting with #, including inside docstrings ! If you care
        about this, you should remove these first using `count_docstrings()`to avoid confusion.

    '''

    counter = 0
    lines_minus_comm = []

    for line in lines:
        if line.strip(' ').startswith('#'):
            counter += 1
        else:
            lines_minus_comm += [line]

    # A small sanity check before closing
    if len(lines_minus_comm) != len(lines) - counter:
        raise Exception('Ouch! Something is very wrong here!')

    return counter, lines_minus_comm

def count_empty(lines):
    ''' Counts empty lines.

    Args:
        lines (list[str]): list of the code lines

    Returns:
        int, list[str]: number of empty lines, list of the code lines with all the empty ones
                        removed.

    Caution:
        This will look for all the empty lines, including inside docstrings !
        If you care about this, you should remove these first using `count_docstrings()` to avoid
        confusion.

    '''

    counter = 0
    lines_minus_empty = []

    for line in lines:
        # Let us not forget about the last line, which may not contain a return tag !
        if line.strip(' ').startswith('\n') or len(line.strip(' ')) == 0:
            counter += 1
        else:
            lines_minus_empty += [line]

    # A small sanity check before closing
    if len(lines_minus_empty) != len(lines) - counter:
        raise Exception('Ouch! Something is very wrong here!')

    return counter, lines_minus_empty

def extract_line_stats(search_path, recursive=False, save_to_file=None, verbose=False):
    ''' Computes the number of blank, commented, docstringed and actual code lines in Python code.

    The code can process either a specific .py file, or, if a directory name if given, all the .py
    files within it (including the options of making a recursive search).

    Args:
        search_path (pathlib.Path, str): path to file or folder to process.
        recursive (bool): if True, will run a recursive search for .py files in subfolders.
        save_to_file (pathlib.Path, str): if set, all code output will be stored in this file
        verbose (bool): if True, will give info for every individual file processed (not just the
            total).

    Raises:
        Exception: If the search_path is invalid

    '''

    # Set up the message output channel if needed
    if save_to_file is None:
        mess_chan = sys.stdout
    elif isinstance(save_to_file, (str, type(Path(' ')))):
        mess_chan = open(save_to_file, 'w')
    else:
        raise Exception('Ouch ! save-to_file should be of type (str or %s), not: %s' %
                        (type(Path('.'), type(save_to_file))))

    # Set the scene
    print(' ', file=mess_chan)
    print('linestats %s - https://github.com/fpavogt/linestats' % (__version__), file=mess_chan)
    print('Copyright (c) 2020-2022 F.P.A. Vogt', file=mess_chan)
    print(' ', file=mess_chan)

    start_time = datetime.datetime.now()
    print('Start time: %s' % (start_time), file=mess_chan)
    print(' ', file=mess_chan)

    # Prepare some variables.
    file_total = 0
    grand_total = 0
    code_total = 0
    comm_total = 0
    empty_total = 0
    docstr_total = 0

    # If I got a string, turn this into a path
    if isinstance(search_path, str):
        search_path = Path(search_path)

    elif not isinstance(search_path, type(Path('.'))):
        raise Exception('Ouch ! Type %s for search_path is invalid.' % (type(search_path)))

    # Was a single file specified ?
    if search_path.is_file():
        fnlist = [search_path]

    # Was I given a proper directory ?
    elif search_path.is_dir():
        if recursive:
            txt = ' recursively'
            mthd = '**/*'
        else:
            txt = ''
            mthd = '*'

        print('Looking%s for Python script files in %s' % (txt, search_path), file=mess_chan)
        print(' ', file=mess_chan)
        fnlist = search_path.glob(mthd)

    else:
        raise Exception('Please specify a valid path or a filename')

    # Now go through each file and extract the line statistics
    for file_path in fnlist:

        # Make sure I'm only touching regular files ...
        if not file_path.is_file():
            #print('%s is not a file. Skipping it ...' % (file_path))
            continue

        # Only deal with .py files
        if file_path.suffix != '.py':
            if verbose:
                print('%s' % (file_path), file=mess_chan)
                print('  does not look like Python code. Skipping it.', file=mess_chan)

            continue

        # Very well, let's extract the file lines.
        this_file = open(file_path, 'r')
        lines = this_file.readlines()
        this_file.close()

        # I have a code file, so let's count it
        file_total += 1

        # Get the total line count
        total_lines = len(lines)
        grand_total += total_lines

        # If the file is empty, avoid a bad division by 0
        if total_lines == 0 and verbose:
            print(file_path, file=mess_chan)
            print('  Total: 0', file=mess_chan)
            continue

        # Track the docstrings
        docstr, lines_minus_docstr = count_docstrings(lines)
        docstr_total += docstr

        # Now the comments (feed it all except the docstrings, to avoid counting these twice)
        comm, lines_minus_comm = count_comments(lines_minus_docstr)
        comm_total += comm

        # And finally count the blank lines
        empty, _ = count_empty(lines_minus_comm)
        empty_total += empty

        code = total_lines - empty - docstr - comm
        code_total += code

        if verbose:
            print(file_path, file=mess_chan)
            print(f'  Total: {total_lines} - ' +
                  f'Code: {code} [{100*code/total_lines:.1f}%] - '+
                  f'Comment+docstr: {comm+docstr} [{100*(comm+docstr)/total_lines:.1f}%] - '+
                  f'Blank: {empty} [{100*empty/total_lines:.1f}%]', file=mess_chan)

    # Print the final numbers
    print(' ', file=mess_chan)
    print('Grand total: %i lines in %i files' % (grand_total, file_total), file=mess_chan)
    if grand_total > 0:
        print('    Code: %i [%.1f%%]' % (code_total, code_total/grand_total*100), file=mess_chan)
        print('    Docstrings: %i [%.1f%%]' % (docstr_total, docstr_total/grand_total*100),
              file=mess_chan)
        print('    Comments: %i [%.1f%%]' % (comm_total, comm_total/grand_total*100),
              file=mess_chan)
        print('    Empty: %i [%.1f%%]' % (empty_total, empty_total/grand_total*100), file=mess_chan)

    print(' ', file=mess_chan)
    print('All done in %ss.' %(datetime.datetime.now() - start_time).total_seconds(),
          file=mess_chan)
    print(' ', file=mess_chan)

    # If I was writing to file, let's remember to close it.
    if save_to_file is not None:
        mess_chan.close()
        print(' ')
        print(' All done in  %ss.' %(datetime.datetime.now() - start_time).total_seconds())
        print(' Output saved under %s' % save_to_file)
        print(' ')


import logging
import pickle
from copy import deepcopy
from typing import Dict

from pydantic import Field

from xopt.generators.bayesian.bax.acquisition import ModelListExpectedInformationGain
from xopt.generators.bayesian.bax.algorithms import Algorithm
from xopt.generators.bayesian.bayesian_generator import BayesianGenerator

logger = logging.getLogger()

class BaxGenerator(BayesianGenerator):
    name = "BAX"
    algorithm: Algorithm = Field(description="algorithm evaluated in the BAX process")
    algorithm_results: Dict = Field(
        None, description="dictionary results from algorithm", exclude=True
    )
    algorithm_results_file: str = Field(
        None, description="file name to save algorithm results at every step"
    )

    _n_calls: int = 0

    def generate(self, n_candidates: int) -> list[dict]:
        self._n_calls += 1
        return super().generate(n_candidates)

    def _get_acquisition(self, model):
        bax_model_ids = [
            self.vocs.output_names.index(name)
            for name in self.algorithm.model_names_ordered
        ]
        bax_model = model.subset_output(bax_model_ids)
        eig = ModelListExpectedInformationGain(
            bax_model, self.algorithm, self._get_optimization_bounds()
        )
        self.algorithm_results = eig.algorithm_results
        if self.algorithm_results_file is not None:
            results = deepcopy(self.algorithm_results)

            with open(
                f"{self.algorithm_results_file}_{self._n_calls}.pkl", "wb"
            ) as outfile:
                pickle.dump(results, outfile, protocol=pickle.HIGHEST_PROTOCOL)

        return eig


from pyramid.config import Configurator

def main(global_config, **settings):
    """ This function returns a Pyramid WSGI application.
    """
    config = Configurator(settings=settings)
    config.include('pyramid_jinja2')
    config.add_route('home', '/') # to get input from the user
    config.add_route('content', '/content') # to display the output from the model
    config.scan()
    return config.make_wsgi_app()


from pythonds.basic import Deque

def palchecker(aString):
    charduque = Deque()

    for ch in aString:
        charduque.addRear(ch)

    stillEqual = True
    while charduque.size() > 1 and stillEqual:
        first = charduque.removeFront()
        last = charduque.removeRear()
        if first != last:
            stillEqual = False
            break

    return stillEqual

print(palchecker("shguah"))
print(palchecker("reaer"))


import string
import matplotlib.pyplot as plt
import math
from functools import reduce

# QUESTO FILE CONTIENE SOLO FUNZIONI DI UTILITà #

gamma = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z']

def __read_from_file__(filename):
	f = open(filename, "r+")
	text = ""
	for line in f.readlines():
		line = line.replace('\n', '')
		line = line.replace('\t', '')
		for ch in line:
			if ch not in gamma:
				line = line.replace(ch, '')
		text += line
	return text

def occurences(filename, q):           # Calcola il numero di occorrenze dei q-grammi all'interno del testo
    text = __read_from_file__(filename)
    q_grams = []
    frequencies_array = []
    count = 0
    q = int(q)
    for i in range(0, len(text)-q):
        q_gram = text[i: i+q]
        if q_gram not in q_grams:
            q_grams.append(q_gram)
            for j in range(0, len(text)):
                if text[j:j+q] == q_gram:
                    count += 1
            frequencies_array.append(count)
            count = 0
    return q_grams, frequencies_array, len(text)

def coincidence_indexes(frequencies_array):     # Calcolo degli indici di coincidenza
    coincidence_index = 0
    count = 0
    for n in frequencies_array:
        count += n
    for i in range(0, len(frequencies_array)):
        coincidence_index += frequencies_array[i]*(frequencies_array[i]-1)
    coincidence_index /= (count*(count-1))
    return coincidence_index

def q_grams_distribution(filename, q):      # Distribuzione empirica dei q-grammi
    freq = occurences(filename, q)
    temp = freq[1]
    result = []
    for i in range(0, len(temp)):
        result.append(float(temp[i])/float(freq[2]))
    d = dict()
    for i in range(0, len(freq[0])):
        d[freq[0][i]] = result[i]
    return d

def plot_frequencies(q_grams, freqs):       # Funzione per il plot della distribuzione delle 26 lettere dell'alfabeto inglese nel primo capitolo di moby dick
	d = dict()
	for i in range(0, len(q_grams)):
		d[q_grams[i]] = freqs[i]
	keys = list(d.keys())
	keys.sort()
	values = []
	for k in keys:
		values.append(d[k])
	plt.bar(keys, values, color = 'b')
	plt.title("english alphabet frequencies 1st chapter of MD")
	plt.show()

def entropy(d):         # Calcolo dell'entropia
    n = sum(d.values())
    entropy = reduce((lambda x, y: x + y), (map((lambda x: d[x] / n * math.log(d[x] / n)), d)))
    return (-1) * (entropy) / math.log(len(d.keys()))


"""exercicio034.py em 2018-10-09. Projeto Practice Python.

No exercício anterior, criamos um dicionário de aniversários de cientistas famosos.
Neste exercício, modifique seu programa para carregar o dicionário de aniversário de um arquivo JSON no disco,
em vez de ter o dicionário definido no programa.

Bônus: peça ao usuário o nome de outro cientista e o aniversário para adicioná-lo ao dicionário
e atualize o arquivo JSON que você tem no disco com o nome do cientista.
Se você executar o programa várias vezes e continuar adicionando novos nomes,
seu arquivo JSON deverá ficará cada vez maior.

"""
import json
from cicero import cabecalho

def lista_dados(dicionario: dict):
    """Lista as informações do arquivo Json a partir de um dicionário

    :param dicionario: informações do arquivo Json
    :type dicionario: dict
    """
    print()
    for k in dicionario:
        print(f'{str(k).center(60)}')
    print()

def adiciona(dicionario: dict):
    """Adiciona dados no arquivo Json

    :param dicionario: informações lidas do arquivo Json
    :type dicionario: dict
    """
    while True:
        nome = str(input('Nome: ')).strip()
        nascimento = str(input('Nascimento: ')).strip()
        dicionario[nome] = nascimento
        with open('json/cientistas.json', 'w') as f:
            json.dump(dicionario, f)
        print(f'\n{nome} foi adicionado com sucesso.\n')
        pergunta = str(input('Quer adicionar outro cientista? {S/N] \n')).lower().strip()[0]
        if pergunta == 's':
            continue
        else:
            break
    return None

def aniversario(dicionario: dict) -> object:
    """Retorna a consulta no arquivo Json

    :param dicionario: dados do arquivo Json
    :type dicionario: dict
    :return: resultado da pesquisa
    :rtype: object
    """
    nome = str(input('Aniversário de quem você quer saber? ')).strip()
    return f'\n{nome} nasceu em {dicionario.get(nome, f"(sem informações)")}.\n'

def opcoes():
    """Gera as opções para o usuário a partir de um dicionário

    """
    dicionario = {'L': 'Lista os cientistas',
                  'N': 'Mostra a data de nascimento',
                  'A': 'Adiciona mais um cientista',
                  'S': 'Sai do programa'}
    print(' Opções: '.center(30, '-'))
    for k, v in dicionario.items():
        print(f'{k} -> {v}')

def aniversarios():
    """Manipula um arquivo Json (escrita/leitura)

    """
    # lê o arquivo Json e armazena em um dicionário
    with open('json/cientistas.json', 'r') as f:
        cientista = json.load(f)
    cabecalho('Bem-vindo ao dicionário de aniversários. Nós sabemos a data de nascimento de:', 60, ' ')
    lista_dados(cientista)

    while True:
        opcoes()
        pergunta = str(input('\tQual é a sua opção? ')).lower().strip()[0]
        if pergunta == 'l':
            lista_dados(cientista)
        elif pergunta == 'n':
            print(aniversario(cientista))
        elif pergunta == 'a':
            print(adiciona(cientista))
            lista_dados(cientista)
        else:
            cabecalho('Fim', 30, '-')
            break

if __name__ == '__main__':
    cabecalho('Aniversário Com Json')
    aniversarios()


#Crie um programa que tenha uma tupla única com nomes de produtos
#e seus respectivos preços, na sequência.
#No final, mostre uma listagem de preços, organizando os dados em forma tabular.

#Versão 05:
# Essa versão o programa irá perguntar quantos e quais produtos para entrar na lista.
# Com  estruturas de repetições 'while' e 'for x in range'.
tuple=()
while True:
    item=(input('Digite o nome do item:'))
    preco=float(input('Digite o preço do item: '))
    tuple = tuple + (item,preco)
    while True:
        novo=input('Adicionar novo item?[Y/N]').strip().upper()[0]
        if novo not in 'YN':
            print('Somente Y ou N.')
        else:
            break
    if novo == 'N':
        break
        print('Programa parou.')
print('+'*40)
print(f'{"Preços":^40}')
print('+'*40)
for x in range(0,len(tuple)):
    if x %2==0:
        print(f'{tuple[x]:.<30}',end='')
    else:
        print(f'R${tuple[x]:>10.2f}')
print('+'*40)


# -*- coding:utf-8 -*-
# author：huawei

from python2sky.context.context_carrier import ContextCarrier
from tests.base_test_case import BaseTestCase

class TestContextCarrier(BaseTestCase):

    def test_serialize(self):
        print(self.context_carrier.serialize())
        self.assertEqual(self.SW6, self.context_carrier.serialize())

    def test_deserialize(self):
        text = self.SW6
        context = ContextCarrier()
        context.deserialize(text)
        self.assertEqual(context.peer, self.context_carrier.peer)


"""
To parse gbk files
"""

mark=0
locus_tag=0
with open('genomenew.gbk') as f:
	cont = f.readlines()
	for line in cont:
		infor = line.split(' ')
		if infor[0]=='LOCUS':
			mark=1
		if mark==1:
			g = open(infor[7]+'.gb','w')
			mark=0
		if mark==0:
			g.write(line)


from glob import glob
import json
import os
import re
import util

sectionsplit = re.compile('((\n)+(==.*?==)?)+((\n)+|$)')

def format_content(content):
    return sectionsplit.sub('\n\n', content)

def validate_subcategories(pageid, title, cats):
    global flood_control_count, dams_count, works_about_count, films_about_count

    for cat in cats:
        if cat.startswith('Flood control in'):
            util.log('{} [{}]: FLOOD CONTROL'.format(pageid, title), 'totext-subcats')
            return False
        if cat.startswith('Dams in'):
            util.log('{} [{}]: DAMS'.format(pageid, title), 'totext-subcats')
            return False
        if cat.startswith('Works about'):
            util.log('{} [{}]: WORKS ABOUT'.format(pageid, title), 'totext-subcats')
            return False
        if cat.startswith('Films about'):
            util.log('{} [{}]: FILMS ABOUT'.format(pageid, title), 'totext-subcats')
            return False
        if cat == 'Dust Bowl':
            util.log('{} [{}]: DUST BOWL'.format(pageid, title), 'totext-subcats')
            return False
    return True

def normalize(pageid, props):
    title, content = props['title'], props['content']

    if title.startswith('List of '):
        util.log('{} [{}]: LIST OF'.format(pageid, title), 'totext-content')
    elif title.startswith('Template:'):
        util.log('{} [{}]: TEMPLATE'.format(pageid, title), 'totext-content')
    elif title.startswith('Book:'):
        util.log('{} [{}]: BOOK'.format(pageid, title), 'totext-content')
    elif not validate_subcategories(pageid, title, props['subcats']):
        return False, False
    else:
        content = format_content(content)
        if len(content) >= 10:
            return title, content
        util.log('{} [{}]: CONTENT LENGTH {}'.format(pageid, title, len(content)), 'totext-content')

    return False, False

def run():
    util.ensure_empty_dir('plaintext')

    print('Converting documents to plaintext...', end='\r')

    total_count, removed_count = 0, 0
    for input_filename in glob('raw/*/*.txt'):
        _, cat, filename = input_filename.split('/')

        output_dir = 'plaintext/{}'.format(cat)
        os.makedirs(output_dir, exist_ok=True)
        output_filename = '{}/{}'.format(output_dir, filename)

        with open(input_filename) as input_f:
            props = json.load(input_f)
            title, content = normalize(filename, props)

            if title:
                with open(output_filename, 'w', encoding='utf-8') as output_f:
                    output_f.write(title + '\n\n')
                    output_f.write(content)
            else:
                removed_count += 1
        total_count += 1

    percent = round(removed_count/total_count * 100)
    print('Converted documents. Removed {}/{} docs [{}%]'.format(removed_count, total_count, percent))


# https://blog.knshnb.com/posts/ahc007-optuna/

import os
import optuna
import joblib
import subprocess

n_parallel = 50
n_files = 100
n_trials = 100
storage_name = "AHCxxx"
study_name = "study_name"
answer_file_name = "main"

def calc_score_each(seed, Ps):
    in_file = f"in/{seed:04}.txt"
    out_file = f"/dev/null"

    cmd = f"tools/target/release/tester ./src/cpp/{answer_file_name} " + \
        " ".join(map(str, Ps)) + f" < {in_file} > {out_file}"
    child = subprocess.Popen(cmd, shell=True,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    _, stderr = child.communicate()

    try:
        score = int(stderr.split()[-1])
    except ValueError:
        print(stderr, flush=True)
        score = -1

    print(f"seed{seed}:{score}", end=" ", flush=True)

    return score

def calc_scores(Ps):
    scores = joblib.Parallel(n_jobs=n_parallel)(
        joblib.delayed(calc_score_each)(i, Ps) for i in range(n_files)
    )
    return scores

def objective(trial):
    print("start")

    P01 = trial.suggest_int("P01", 0, 1)
    P02 = trial.suggest_float("P02", 0.0, 1.0)

    PARAMS = [P01, P02]

    print("params:", PARAMS, sep=" ")

    scores = calc_scores(PARAMS)

    scores = [score for score in scores if score != -1]

    print(f"success:{len(scores)}")

    return sum(scores)

if __name__ == "__main__":
    print("setup start")

    os.system(
        f"/usr/bin/g++-10 -fdiagnostics-color=always ./src/cpp/{answer_file_name}.cpp -o ./src/cpp/{answer_file_name} -std=c++17 -Wall -Wextra -O3")

    study = optuna.create_study(
        direction="minimize",
        storage=f"sqlite:///{storage_name}.db",
        study_name=study_name,
        load_if_exists=True,
    )

    print("setup done")

    study.optimize(objective, n_trials=n_trials)


import sys
from flask_migrate import Migrate
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy import (
    BigInteger,
    Boolean,
    Column,
    Date,
    Float,
    ForeignKey,
    Integer,
    String,
    Table,
    text)
from sqlalchemy.orm import relationship
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.sql.expression import func

# Local imports...
from exceptions import EmptyEntityError

Base = declarative_base()
metadata = Base.metadata

db = SQLAlchemy()
migrate = Migrate()

def setup_db(app):
    db.app = app
    db.init_app(app)
    migrate.init_app(app, db)

class Aisle(Base):
    __tablename__ = 'aisles'

    aisle_number = Column(Integer, primary_key=True)
    name = Column(String(255), nullable=False)

    products = relationship('Product', secondary='aislecontains')

    def __init__(self, aisle_number=0, name=None):
        self.aisle_number = aisle_number
        self.name = name

    def list_all_aisles(self):
        data = None
        try:
            data = _list_all_data(db, Aisle())
        except BaseException:
            raise

        return data

    def list_one_or_none_aisle(self):
        data = None

        try:
            data = _list_one_or_none_data(db, self)
        except BaseException:
            raise

        return data

    def add_aisle_to_database(self):
        try:
            _add_entity(db, self)
        except BaseException:
            raise

        return self

    def update_aisle_in_database(self):
        try:
            _update_entity(db)
        except BaseException:
            raise

        return self

    def delete_aisle_from_database(self):
        try:
            _delete_entity(db, entity=self)
        except EmptyEntityError:
            raise
        except BaseException:
            raise

        return self

    def __repr__(self):
        return f'Aisle("{self.aisle_number}","{self.name}")'

class Customer(Base):
    __tablename__ = 'customers'

    id = Column(Integer, primary_key=True, server_default=text(
        "nextval('customers_id_seq'::regclass)"))
    name = Column(String(255))
    phone = Column(String(255))
    email = Column(String(255))

    def __init__(self, id=0, name=None, phone=None, email=None):
        self.id = id
        self.name = name
        self.phone = phone
        self.email = email

    def list_all_customers(self):
        data = None
        try:
            data = _list_all_data(db, Customer())
        except BaseException:
            raise

        return data

    def list_one_or_none_customer(self):
        data = None

        try:
            data = _list_one_or_none_data(db, self)
        except BaseException:
            raise

        return data

    def add_customer_to_database(self):
        try:
            _add_entity(db, self)
        except BaseException:
            raise

        return self

    def update_customer_in_database(self):
        try:
            _update_entity(db)
        except BaseException:
            raise

        return self

    def get_next_customer_id(self):
        id = 0

        try:
            id = _get_next_id(db, Customer())
        except BaseException:
            raise

        return id

    def __repr__(self):
        return f'Customer("{self.id}","{self.name}",\
            "{self.phone}","{self.email}")'

class Department(Base):
    __tablename__ = 'departments'

    id = Column(Integer, primary_key=True, server_default=text(
        "nextval('departments_id_seq'::regclass)"))
    name = Column(String(255))

    def __init__(self, id=0, name=None):
        self.id = id
        self.name = name

    def list_all_departments(self):
        data = None
        try:
            data = _list_all_data(db, Department())
        except BaseException:
            raise

        return data

    def list_one_or_none_department(self):
        data = None

        try:
            data = _list_one_or_none_data(db, self)
        except BaseException:
            raise

        return data

    def add_department_to_database(self):
        try:
            _add_entity(db, self)
        except BaseException:
            raise

        return self

    def update_department_in_database(self):
        try:
            _update_entity(db)
        except BaseException:
            raise

        return self

    def get_next_department_id(self):
        id = 0

        try:
            id = _get_next_id(db, Department())
        except BaseException:
            raise

        return id

    def __repr__(self):
        return f'Department("{self.id}","{self.name}")'

class Supplier(Base):
    __tablename__ = 'suppliers'

    id = Column(Integer, primary_key=True, server_default=text(
        "nextval('suppliers_id_seq'::regclass)"))
    name = Column(String(255), nullable=False)
    address = Column(String(255))
    phone = Column(String(255), nullable=False)

    def __init__(self, id=0, name=None, address=None, phone=None):
        self.id = id
        self.name = name
        self.address = address
        self.phone = phone

    def list_all_suppliers(self):
        data = None
        try:
            data = _list_all_data(db, Supplier())
        except BaseException:
            raise

        return data

    def list_one_or_none_supplier(self):
        data = None

        try:
            data = _list_one_or_none_data(db, self)
        except BaseException:
            raise

        return data

    def add_supplier_to_database(self):
        try:
            _add_entity(db, self)
        except BaseException:
            raise

        return self

    def update_supplier_in_database(self):
        try:
            _update_entity(db)
        except BaseException:
            raise

        return self

    def get_next_supplier_id(self):
        id = 0

        try:
            id = _get_next_id(db, Supplier())
        except BaseException:
            raise

        return id

    def __repr__(self):
        return f'Supplier("{self.id}","{self.name}",\
            "{self.address}","{self.phone}")'

class Employee(Base):
    __tablename__ = 'employees'

    id = Column(Integer, primary_key=True, server_default=text(
        "nextval('employees_id_seq'::regclass)"))
    name = Column(String(255), nullable=False)
    department_id = Column(ForeignKey('departments.id'))
    title = Column(String(255))
    emp_number = Column(BigInteger, nullable=False)
    address = Column(String(255))
    phone = Column(String(255))
    wage = Column(Integer)
    is_active = Column(Boolean, nullable=False, server_default=text("true"))

    department = relationship('Department')

    def __init__(
            self, id=0, name=None, department_id=0, title=None,
            emp_number=0, address=None, phone=None, wage=0,
            is_active=False):
        self.id = id,
        self.name = name,
        self.department_id = department_id,
        self.title = title,
        self.emp_number = emp_number,
        self.address = address,
        self.phone = phone,
        self.wage = wage,
        self.is_active = is_active

    def list_all_employees(self, entity=None):
        data = None
        try:
            data = _list_all_data(db, Employee(), entity)
        except BaseException:
            raise

        return data

    def list_all_employees_filtered(self, entity2=None):
        data = None
        try:
            data = _list_all_data_filtered(
                db, entity=Employee(), entity2=Department())
        except BaseException:
            raise

        return data

    def list_one_or_none_employee(self):
        data = None

        try:
            data = _list_one_or_none_data(db, self)
        except BaseException:
            raise

        return data

    def add_employee_to_database(self):
        try:
            _add_entity(db, self)
        except BaseException:
            raise

        return self

    def update_employee_in_database(self):
        try:
            _update_entity(db)
        except BaseException:
            raise

        return self

    def get_next_employee_id(self):
        id = 0

        try:
            id = _get_next_id(db, Employee())
        except BaseException:
            raise

        return id

    def __repr__(self):
        return f'Employee("{self.id}","{self.name}",\
            "{self.department_id}","{self.title}",\
            "{self.emp_number}","{self.address}",\
            "{self.phone}","{self.wage}","{self.is_active}")'

class Product(Base):
    __tablename__ = 'products'

    id = Column(Integer, primary_key=True, server_default=text(
        "nextval('products_id_seq'::regclass)"))
    name = Column(String(255), nullable=False)
    price_per_cost_unit = Column(Float(53), nullable=False)
    cost_unit = Column(String(255), nullable=False)
    department_id = Column(ForeignKey('departments.id'), nullable=False)
    quantity_in_stock = Column(Integer)
    brand = Column(String(255))
    production_date = Column(Date)
    best_before_date = Column(Date)
    plu = Column(Integer)
    upc = Column(BigInteger)
    organic = Column(Integer)
    cut = Column(String(255))
    animal = Column(String(255))

    department = relationship('Department')
    suppliers = relationship('Supplier', secondary='providedby')

    def __init__(
            self, id=0, name=None, price_per_cost_unit=0, cost_unit=None,
            department_id=0, quantity_in_stock=0, brand=None,
            production_date=None, best_before_date=None, plu=0,
            upc=0, organic=0, cut=None, animal=None):
        self.id = id
        self.name = name,
        self.price_per_cost_unit = price_per_cost_unit,
        self.cost_unit = cost_unit,
        self.department_id = department_id,
        self.quantity_in_stock = quantity_in_stock,
        self.brand = brand,
        self.production_date = production_date,
        self.best_before_date = best_before_date,
        self.plu = plu,
        self.upc = upc,
        self.organic = organic,
        self.cut = cut,
        self.animal = animal

    def list_all_products(self):
        data = None

        try:
            data = _list_all_data(db, self)
        except BaseException:
            raise

        return data

    def list_all_products_filtered(
            self, entity2=None, entity3=None, entity4=None):
        data = None

        try:
            data = _list_all_data_filtered(
                db, self, Department(), AisleContains(), Aisle())
        except BaseException:
            raise

        return data

    def list_one_or_none_product(self):
        data = None

        try:
            data = _list_one_or_none_data(db, self)
        except BaseException:
            raise

        return data

    def add_product_to_database(self):
        try:
            _add_entity(db, self)
        except BaseException:
            raise

        return self

    def update_product_in_database(self):
        try:
            _update_entity(db)
        except BaseException:
            raise

        return self

    def get_next_product_id(self):
        id = 0

        try:
            id = _get_next_id(db, Product())
        except BaseException:
            raise

        return id

    def __repr__(self):
        return f'Product("{self.id}","{self.name}",\
            "{self.price_per_cost_unit}","{self.cost_unit}",\
            "{self.department_id}","{self.quantity_in_stock}",\
            "{self.brand}","{self.production_date}",\
            "{self.best_before_date}","{self.plu}",\
            "{self.upc}","{self.organic}","{self.cut}",\
            "{self.animal}")'

class Providesdelivery(Base):
    __tablename__ = 'providesdelivery'

    delivery_id = Column(Integer, primary_key=True, server_default=text(
        "nextval('providesdelivery_delivery_id_seq'::regclass)"))
    supplier_id = Column(ForeignKey('suppliers.id'), nullable=False)

    supplier = relationship('Supplier')
    products = relationship('Product', secondary='receivedfrom')

    def __repr__(self):
        return f'Providesdelivery("{self.delivery_id}","{self.supplier_id}")'

t_aislecontains = Table(
    'aislecontains', metadata,
    Column('aisle_number', ForeignKey(
        'aisles.aisle_number'), primary_key=True, nullable=False),
    Column('product_id', ForeignKey(
        'products.id'), primary_key=True, nullable=False)
)

class AisleContains(Base):
    __table__ = t_aislecontains

    def __init__(self, aisle_number=0, product_id=0):
        self.aisle_number = aisle_number
        self.product_id = product_id

    def list_all_aisle_contains_filtered(self):
        data = None
        try:
            data = _list_all_data_filtered(db, self)
        except BaseException:
            raise

        return data

    def list_one_or_none_aisle_contains(self, entity2=None):
        data = None
        try:
            data = _list_one_or_none_data(db, self, entity2)
        except BaseException:
            raise

        return data

    def add_aisle_contains_to_database(self):
        try:
            _add_entity(db, self)
        except BaseException:
            raise

    def delete_aisle_contains_from_database(self, aisle_contains_list):
        try:
            _delete_entity(db, entity_list=aisle_contains_list)
        except EmptyEntityError:
            raise
        except BaseException:
            raise

        return aisle_contains_list

t_providedby = Table(
    'providedby', metadata,
    Column('product_id', ForeignKey(
        'products.id'), primary_key=True, nullable=False),
    Column('supplier_id', ForeignKey(
        'suppliers.id'), primary_key=True, nullable=False)
)

class Purchase(Base):
    __tablename__ = 'purchases'

    id = Column(Integer, primary_key=True, nullable=False, server_default=text(
        "nextval('purchases_id_seq'::regclass)"))
    product_id = Column(ForeignKey(
        'products.id'), primary_key=True, nullable=False)
    quantity = Column(Integer)
    customer_id = Column(ForeignKey('customers.id'))
    purchase_date = Column(Date)
    total = Column(Float(53))
    is_cancelled = Column(
        Boolean, nullable=False, server_default=text("false"))

    customer = relationship('Customer')
    product = relationship('Product')

    def __init__(
            self, id=0, product_id=0, quantity=0, customer_id=0,
            purchase_date=None, total=0, is_cancelled=False):
        self.id = id
        self.product_id = product_id
        self.quantity = quantity
        self.customer_id = customer_id
        self.purchase_date = purchase_date
        self.total = total
        self.is_cancelled = is_cancelled

    def list_all_purchases(self):
        data = None
        try:
            data = _list_all_data(db, Purchase())
        except BaseException:
            raise

        return data

    def list_one_or_none_purchase(self):
        data = None

        try:
            data = _list_one_or_none_data(db, self)
        except BaseException:
            raise

        return data

    def add_purchase_to_database(self):
        try:
            _add_entity(db, self)
        except BaseException:
            raise

        return self

    def update_purchase_in_database(self):
        try:
            _update_entity(db)
        except BaseException:
            raise

        return self

    def get_next_purchase_id(self):
        id = 0

        try:
            id = _get_next_id(db, Purchase())
        except BaseException:
            raise

        return id

    def __repr__(self):
        return f'Purchase("{self.id}","{self.product_id}",\
            "{self.quantity}","{self.customer_id}",\
            "{self.purchase_date}","{self.total}",\
            "{self.is_cancelled}")'

t_receivedfrom = Table(
    'receivedfrom', metadata,
    Column('product_id', ForeignKey(
        'products.id'), primary_key=True, nullable=False),
    Column('delivery_id', ForeignKey(
        'providesdelivery.delivery_id'), primary_key=True, nullable=False)
)

class EmployeeDto(Employee):
    def __init__(
        self, department_name, id, name, department_id, title,
            emp_number, address, phone, wage, is_active):
        super().__init__(
            id, name, department_id, title,
            emp_number, address, phone, wage, is_active)
        self.department_name = department_name

        def __repr__(self):
            return f'EmployeeDto("{super.__repr__()}" and\
                "{self.department_name}")'

class ProductDto(Product):
    def __init__(
        self, department_name, id, name, price_per_cost_unit,
            cost_unit, department_id, quantity_in_stock, brand,
            production_date, best_before_date, plu, upc, organic, cut,
            animal, aisle_name, aisle_number):
        super().__init__(
            id, name, price_per_cost_unit,
            cost_unit, department_id, quantity_in_stock, brand,
            production_date, best_before_date, plu, upc, organic, cut,
            animal)
        self.department_name = department_name
        self.aisle_name = aisle_name
        self.aisle_number = aisle_number

        def __repr__(self):
            return f'ProductDto("{super.__repr__()}" and\
                "{self.department_name}","{self.aisle_nnumber}",\
                "{self.aisle_name}")'

###########################################################
#
# HELPER FUNCTIONS
#
###########################################################

def _list_all_data(db, entity) -> list:
    data = None
    model = type(entity)
    model_name = model.__name__

    session = db.session
    session.expire_on_commit = False

    try:
        if model_name == 'Employee':
            data = session.query(model).order_by(
                model.department_id, model.id).all()
        elif model_name == 'Aisle' or model_name == 'AisleContains':
            data = session.query(model).order_by(
                model.aisle_number).all()
        else:
            data = session.query(model).order_by(model.id).all()
    except BaseException as e:
        tb = sys.exc_info()
        db.app.logger.info(e.with_traceback(tb[2]))
        raise

    return data

def _list_all_data_filtered(
        db, entity, entity2=None, entity3=None, entity4=None) -> list:
    data = None
    model = type(entity)
    model_name = model.__name__

    session = db.session
    session.expire_on_commit = False

    try:
        if model_name == 'Aisle' or model_name == 'AisleContains':
            data = session.query(model).filter_by(
                aisle_number=int(entity.aisle_number)).all()
        elif model_name == 'Employee':
            model2 = type(entity2)
            data = session.query(
                model, model2).filter(
                model.department_id == model2.id).order_by(
                model2.id, model.id).all()
        elif model_name == 'Product':
            model2 = type(entity2)
            model3 = type(entity3)
            model4 = type(entity4)

            data = session.query(
                model, model2, model3, model4).filter(
                model.department_id == model2.id).filter(
                model.id == model3.product_id).filter(
                model3.aisle_number == model4.aisle_number).order_by(
                model.id).all()
        else:
            data = session.query(model).filter_by(
                id=int(entity.id)).all()
    except BaseException as e:
        tb = sys.exc_info()
        db.app.logger.info(e.with_traceback(tb[2]))
        raise

    return data

def _list_one_or_none_data(db, entity, entity2=None):
    data = None
    model = type(entity)
    model_name = model.__name__

    session = db.session
    session.expire_on_commit = False

    try:
        if model_name == 'Aisle' or model_name == 'AisleContains':
            if entity2 is None:
                data = session.query(model).filter_by(
                    aisle_number=entity.aisle_number).one_or_none()
            else:
                data = session.query(model).filter_by(
                    product_id=entity.product_id).one_or_none()
        else:
            data = session.query(model).filter_by(
                id=entity.id).one_or_none()
    except BaseException as e:
        tb = sys.exc_info()
        db.app.logger.info(e.with_traceback(tb[2]))
        raise

    return data

def _add_entity(db, entity):
    session = db.session
    session.expire_on_commit = False

    try:
        session.add(entity)
        session.commit()
    except BaseException as e:
        tb = sys.exc_info()
        db.app.logger.info(e.with_traceback(tb[2]))
        session.rollback()
        raise

def _update_entity(db):
    session = db.session
    session.expire_on_commit = False

    try:
        session.commit()
    except BaseException as e:
        tb = sys.exc_info()
        db.app.logger.info(e.with_traceback(tb[2]))
        session.rollback()
        raise

def _delete_entity(db, entity=None, entity_list=None):
    session = db.session
    session.expire_on_commit = False

    try:
        if entity_list is not None:
            if len(entity_list) > 1:
                for entity in entity_list:
                    session.delete(entity)
            elif len(entity_list) == 1:
                session.delete(entity_list[0])
            else:
                raise EmptyEntityError({
                    "code": "empty_entity_object",
                    "description": "Cannot delete from empty entity object"
                })
        elif entity is not None:
            session.delete(entity)

        session.commit()
    except EmptyEntityError:
        raise
    except BaseException as e:
        tb = sys.exc_info()
        db.app.logger.info(e.with_traceback(tb[2]))
        session.rollback()
        raise

def _get_next_id(db, entity) -> int:
    model = type(entity)
    id = 0

    session = db.session
    session.expire_on_commit = False

    try:
        if model is Aisle or model is AisleContains:
            max_id = session.query(func.max(model.aisle_name)).one_or_none()
            id = max_id[0] + 1
        else:
            max_id = session.query(func.max(model.id)).one_or_none()
            id = max_id[0] + 1
    except BaseException as e:
        tb = sys.exc_info()
        db.app.logger.info(e.with_traceback(tb[2]))
        raise

    return id


import pandas as pd
import numpy as np
import json
import preprocess

def fetchCircularBiPlotData(type_to_display, class_to_display):
	'''
	This function is used to fetch the data to build a Circular BiPlot and is called from app.py
	Input:
		type_to_display:
			type_to_display - A string keyword that is used to specify what kind of data is to be
			displayed in the graph.
			Values could be 'CONSOLIDATED' or the name of the genre.
	'''

	plot_data = []
	if type_to_display == 'CONSOLIDATED':
		for genre in ALL_GENRES:

			movies_in_genre = list(GENRE_MOVIE_MAPPER[genre]) # This list stores all the movies for the particular genre

			genre_budget = 0
			genre_revenue = 0

			for movie in movies_in_genre:
				movie_details = MOVIE_DETAILS_MAPPER[movie]
				budget = movie_details["budget"]
				revenue = movie_details["revenue"]

				genre_budget += budget
				genre_revenue += revenue

			GENRE_BUDGETS[genre]  = int(genre_budget / 1e6) # Dividing values by a million
			GENRE_REVENUES[genre] = int(genre_revenue / 1e6) # Dividing values by a million

			AVG_GENRE_BUDGETS[genre]  = int(genre_budget / 1e6) /len(movies_in_genre) # Dividing values by a million
			AVG_GENRE_REVENUES[genre] = int(genre_revenue / 1e6) /len(movies_in_genre) # Dividing values by a million

			genre_details_dict = {}
			genre_details_dict["genre"]   = genre
			genre_details_dict["budget"]  = int(genre_budget / 1e6) # Dividing values by a million
			genre_details_dict["revenue"] = int(genre_revenue / 1e6) # Dividing values by a million
			genre_details_dict["avg_genre_budget"] = int((genre_budget / 1e6) /len(movies_in_genre)) # Dividing values by a million
			genre_details_dict["avg_genre_revenue"] = int((genre_revenue / 1e6) /len(movies_in_genre)) # Dividing values by a million
			plot_data.append(genre_details_dict)
		return plot_data

	else:
		'''
		This section is executed when the User clicks on a separate genre and we need to provide genre-specific information to the User.
		The current iteration takes the minimum of 30 or total_movies in the genre sorted by revenue/budget ratio and returns the information to the User
		'''
		genre = type_to_display

		movies_in_genre = list(GENRE_MOVIE_MAPPER[genre]) # Fetching all the movies in the genre
		num_movies = min(len(movies_in_genre), 30) # Taking the minimum of total movies in the genre and 30 because some genres have less than 30 movies in them.

		movie_details_sorter = [] # List that maintains movie_title and revenue_budget ratio of it so that we can sort on it. Movie details are added as elements in it.

		for movie in movies_in_genre:
			movie_details = MOVIE_DETAILS_MAPPER[movie]
			movie_revenue_budget_ratio = movie_details["revenue_budget_ratio"]
			movie_budget = movie_details["budget"]
			movie_revenue = movie_details["revenue"]

			movie_details_sorter.append((movie, movie_budget, movie_revenue, movie_revenue_budget_ratio))

		if class_to_display == "top10":

			movie_details_sorter.sort(key = lambda x: x[3], reverse=True)
			for movie_index in range(num_movies):
				movie_details_dict = {}
				movie_details = movie_details_sorter[movie_index]

				# Adding only the movie_name, movie_revenue and movie_budget as we don't need the ratio
				movie_details_dict["movie"]    = movie_details[0]
				movie_details_dict["budget"]   = int(movie_details[1]/1e6)
				movie_details_dict["revenue"]  = int(movie_details[2]/1e6)

				plot_data.append(movie_details_dict)

		elif class_to_display == "mid10":

			movie_details_sorter.sort(key = lambda x: x[3], reverse=True)

			mid_index = int(len(movies_in_genre)/2)
			#mid_movies_range = [mid_index - 15, mid_index + 14]

			for movie_index in range(mid_index - 15, mid_index + 14):
				movie_details_dict = {}
				movie_details = movie_details_sorter[movie_index]

				# Adding only the movie_name, movie_revenue and movie_budget as we don't need the ratio
				movie_details_dict["movie"]    = movie_details[0]
				movie_details_dict["budget"]   = int(movie_details[1]/1e6)
				movie_details_dict["revenue"]  = int(movie_details[2]/1e6)

				plot_data.append(movie_details_dict)
		elif class_to_display == "bottom10":

			movie_details_sorter.sort(key = lambda x: x[3])
			for movie_index in range(num_movies):

				movie_details_dict = {}
				movie_details = movie_details_sorter[movie_index]

				# Adding only the movie_name, movie_revenue and movie_budget as we don't need the ratio
				movie_details_dict["movie"]    = movie_details[0]
				movie_details_dict["budget"]   = int(movie_details[1]/1e4)
				movie_details_dict["revenue"]  = int(movie_details[2]/1e4)

				plot_data.append(movie_details_dict)

		return plot_data

def fetchRadarPlotData(type_to_display):
	'''

	'''
	plot_data = pd.DataFrame(columns = ["genre", "budget", "revenue", "popularity", "vote_avg"])

	GROUPED_CATEGORY_TAGS = ["cat1", "cat2", "cat3", "cat4"] # Figure out proper names for these categories

	global CONSOLIDATED_RADAR_DATA

	specific_flag = False

	if type_to_display == "CONSOLIDATED":

		for genre_index, genre in enumerate(ALL_GENRES):

			movies_in_genre = list(GENRE_MOVIE_MAPPER[genre])
			num_movies_in_genre = len(movies_in_genre)
			num_movies = min(len(movies_in_genre), 30)

			total_budget = 0
			total_revenue = 0
			total_popularity = 0
			total_runtime = 0
			total_vote_avg = 0

			for movie in movies_in_genre:
				movie_details = MOVIE_DETAILS_MAPPER[movie]

				total_budget += movie_details["budget"]
				total_revenue += movie_details["revenue"]
				total_popularity += movie_details["popularity"]
				# total_runtime += movie_details["runtime"]
				total_vote_avg += movie_details["runtime"]

			total_budget /= num_movies_in_genre
			total_revenue /= num_movies_in_genre
			total_popularity /= num_movies_in_genre
			total_vote_avg /= num_movies_in_genre

			genre_details = []
			genre_details.append(genre)
			genre_details.append(total_budget)
			genre_details.append(total_revenue)
			genre_details.append(total_popularity)
			genre_details.append(total_vote_avg)

			plot_data.loc[genre_index] = genre_details

		plot_data.loc[genre_index+1] = ["IGNORE", 0, 0, 0, 0]
		CONSOLIDATED_RADAR_DATA = plot_data

	elif type_to_display in GROUPED_CATEGORY_TAGS:

		GENRE_CATEGORY_MAPPER = {
			"cat1": ["Action", "Adventure", "War"],
			"cat2": ["Drama", "Comedy", "Romance"],
			"cat3": ["Science Fiction", "Fantasy", "Animation"],
			"cat4": ["Thriller", "Crime", "Mystery", "Horror",]
		}

		genres = GENRE_CATEGORY_MAPPER[type_to_display]

		for genre_index, genre in enumerate(genres):

			movies_in_genre = list(GENRE_MOVIE_MAPPER[genre]) # Fetching all the movies in the genre
			num_movies = min(len(movies_in_genre), 30) # Taking the minimum of total movies in the genre and 30 because some genres have less than 30 movies in them.
			num_movies_in_genre = len(movies_in_genre)

			total_budget = 0
			total_revenue = 0
			total_popularity = 0
			total_runtime = 0
			total_vote_avg = 0

			for movie in movies_in_genre:
				movie_details = MOVIE_DETAILS_MAPPER[movie]

				total_budget += movie_details["budget"]
				total_revenue += movie_details["revenue"]
				total_popularity += movie_details["popularity"]
				total_vote_avg += movie_details["runtime"]
				#total_runtime += movie_details["runtime"]

			genre_details = []
			genre_details.append(genre)
			genre_details.append(total_budget)
			genre_details.append(total_revenue)
			genre_details.append(total_popularity)
			genre_details.append(total_vote_avg)

			plot_data.loc[genre_index] = genre_details
		plot_data.loc[genre_index+1] = ["IGNORE", 0, 0, 0, 0]

	else:
		specific_flag = True

		genre = type_to_display
		for genre_index, genre_data in CONSOLIDATED_RADAR_DATA.iterrows():
			if genre_data["genre"] == genre:
				plot_data.loc[0] = list(genre_data)

	data = []

	if not specific_flag:
		from sklearn.preprocessing import MinMaxScaler
		scaler = MinMaxScaler()
		genres = plot_data["genre"]
		plot_data = plot_data.drop(columns=["genre"])
		plot_data = scaler.fit_transform(plot_data)

		for index in range(len(genres) - 1):
			genre_dict = {}
			genre_dict["name"] = genres[index]

			genre_axes = []

			genre_axes.append({"axis": "budget", "value": (plot_data[index][0]) * 100})
			genre_axes.append({"axis": "revenue", "value": (plot_data[index][1]) * 100})
			genre_axes.append({"axis": "popularity", "value": plot_data[index][2] * 100})
			genre_axes.append({"axis": "runtime", "value": plot_data[index][3] * 100})

			genre_dict["axes"] = genre_axes
			data.append(genre_dict)

	else:
		genre_dict = {}
		genre_dict["name"] = type_to_display
		genre_axes = []
		index = 0
		genre_axes.append({"axis": "budget", "value": (plot_data["budget"][index]/1e5)})
		genre_axes.append({"axis": "revenue", "value": (plot_data["revenue"][index]/1e5)})
		genre_axes.append({"axis": "popularity", "value": plot_data["popularity"][index] * 10})
		genre_axes.append({"axis": "runtime", "value": plot_data["vote_avg"][index] * 10})

		genre_dict["axes"] = genre_axes
		data.append(genre_dict)
	return data

def get_rb_ratio(row):
	ratio = row
	if ratio >4:
		return 3
	if ratio >1.5:
		return 2
	else:
		return 1

def filter_rb_ratio(data):
	#print("==========In FIlter Funciton=========")
	data = data[(data['year'] > '2010') & (data['year'] <= '2015')]
	ac = 0
	if ac ==1:
		a = data[data['rb_quantile'] <=0.1]
		a['rb_ratio'] = 1

		c = data[(data['rb_quantile'] >0.40) & (data['rb_quantile'] <=0.55)]
		d = data[(data['rb_quantile'] >0.2) & (data['rb_quantile'] <=0.3)]
		e = data[(data['rb_quantile'] >0.65) & (data['rb_quantile'] <=0.8)]
		#print(e.shape[0],d.shape[0],c.shape[0])
		#print("==========")
		# size = data[(data['rb_quantile'] >=0.45) & (data['rb_quantile'] <=0.55)].shape[0]
		# c = data[(data['rb_quantile'] >0.2) & (data['rb_quantile'] <=0.8)].sample(size)
		c['rb_ratio'] = 2
		d['rb_ratio'] = 1
		e['rb_ratio'] = 3

		b = data[data['rb_quantile'] >=0.9]
		b['rb_ratio'] = 3
		# print(b.head(10))
		data = pd.concat([a, c,d,e, b])
	else:
		a = data[(data['rb_quantile'] >0) & (data['rb_quantile'] <=0.33)]
		b = data[(data['rb_quantile'] >0.33) & (data['rb_quantile'] <=0.67)]
		c = data[(data['rb_quantile'] >0.67) & (data['rb_quantile'] <=1)]

		a['rb_ratio'] = 1
		b['rb_ratio'] = 2
		c['rb_ratio'] = 3
		data = pd.concat([a,b,c])
	# print(data.head(5))
	return data

def fetchScatterPlotData(chart_type, type_to_display):

	data = MOVIES_DF
	if type_to_display.upper() != 'CONSOLIDATED':
		data = data[data.name.isin(GENRE_MOVIE_MAPPER[type_to_display])]

	# data['rb_ratio'] = data['revenue'] / data['budget']
	# data['rb_ratio'] = data['rb_ratio'].astype(int)
	# data['rb_ratio'] = data['rb_ratio'].apply(get_rb_ratio)
	data = filter_rb_ratio(data)

	return data
	# return data[data['budget'] > 5].head(400)

def fetchBoxPlotData(chart_type, type_to_display):

	data = MOVIES_DF
	data = filter_rb_ratio(data)
	data['year'] = data['release_date'].apply(lambda x: str(x)[:4])
	if type_to_display.upper() != 'CONSOLIDATED':
		'''
		print(type_to_display)
		print(GENRE_MOVIE_MAPPER.keys())
		print(GENRE_MOVIE_MAPPER[type_to_display][:10])
		print("============================")
		'''
		data = data[data.name.isin(GENRE_MOVIE_MAPPER[type_to_display])]

	year_dict_0 = {}
	year_dict_0['2011'] = list(data[data['year'] == '2011']['popularity'])#[:100]
	year_dict_0['2012'] = list(data[data['year'] == '2012']['popularity'])
	year_dict_0['2013'] = list(data[data['year'] == '2013']['popularity'])
	year_dict_0['2014'] = list(data[data['year'] == '2014']['popularity'])
	year_dict_0['2015'] = list(data[data['year'] == '2015']['popularity'])

	year_dict_1 = {}
	year_dict_1['2011'] = list(data[data['year'] == '2011']['runtime'])#[:100]
	year_dict_1['2012'] = list(data[data['year'] == '2012']['runtime'])
	year_dict_1['2013'] = list(data[data['year'] == '2013']['runtime'])
	year_dict_1['2014'] = list(data[data['year'] == '2014']['runtime'])
	year_dict_1['2015'] = list(data[data['year'] == '2015']['runtime'])

	return [year_dict_0, year_dict_1]

def fetchParallelPlotData(chart_type, type_to_display):

	columns = ['name','rb_ratio','year','budget','revenue','profit','runtime','popularity']#,'no_production_companies',]
	data = filter_rb_ratio(MOVIES_DF)
	data = data[columns]
	# print("=======Parallel========")
	# print(data.head(5))
	if type_to_display.upper() != 'CONSOLIDATED':
		data = data[data.name.isin(GENRE_MOVIE_MAPPER[type_to_display])][columns]
		data = data[(data['year'] > '2010') & (data['year'] <= '2015')]

	else:
		data = PARALLEL_DF

	# sample data on a strategy
	return data#.head(100)

def driver_fetch_data(chart_type, type_to_display, class_to_display=None):
	'''
	This method is the driver function to fetch data for any specific chart type for any data.

	Input:
		chart_type:
			A string keyword which mentions the type of chart to be displayed and therefore allows us to know
			which method to call to retrieve appropriate data.
		type_to_display:
			type_to_display - A string keyword that is used to specify what kind of data is to be
			displayed in the graph.
			Values could be 'CONSOLIDATED' or the name of the genre.
	Output:
		If 'type_to_display' is CONSOLIDATED, we send the aggregated budget/revenue across all
		available genres.
		If 'type_to_display' is the name of the genre, we send the [1-10, 45-55, 90-100] ranged
		values in revenue/budget sorted list.
	'''
	method_mapper_dict = {"circular_biplot": "fetchCircularBiPlotData(type_to_display, class_to_display)", "scatterplot": "fetchScatterPlotData(chart_type, type_to_display)",
							"radar_plot": "fetchRadarPlotData(type_to_display)", "boxplot": "fetchBoxPlotData(chart_type, type_to_display)",
							"parallelplot":"fetchParallelPlotData(chart_type, type_to_display)"}

	method_name = method_mapper_dict[chart_type]

	result = eval(method_name)
	return result;

def initialize_global_vars():
	'''
	This method is used to initialize all the Global variables for the functions_library.py module.
	These Global variables used across this module are fetched from preprocess.py function and used here.
	This is just used as an initializer and is called in the initialize() method at app.py
	The line "return True" is simply added to make sure that all the Global variables are calculated and only then
	the next line in app.py initilize() method is executed.
	'''

	print("Currently initializing global variables")

	global ALL_GENRES
	global GENRE_MOVIE_MAPPER
	global MOVIE_DETAILS_MAPPER
	global MOVIES_DF
	global PARALLEL_DF

	preprocess.preprocess_data()

	ALL_GENRES = preprocess.ALL_GENRES
	GENRE_MOVIE_MAPPER = preprocess.GENRE_MOVIE_MAPPER
	MOVIE_DETAILS_MAPPER = preprocess.MOVIE_DETAILS_MAPPER
	MOVIES_DF = preprocess.MOVIES_DF
	PARALLEL_DF = preprocess.PARALLEL_DF

	global GENRE_BUDGETS
	global GENRE_REVENUES
	global AVG_GENRE_BUDGETS
	global AVG_GENRE_REVENUES

	GENRE_BUDGETS = {}
	GENRE_REVENUES = {}
	AVG_GENRE_BUDGETS = {}
	AVG_GENRE_REVENUES = {}

	for genre in ALL_GENRES:
		GENRE_BUDGETS[genre] = 0
		GENRE_REVENUES[genre] = 0
		AVG_GENRE_BUDGETS[genre] = 0
		AVG_GENRE_REVENUES[genre] = 0

	print("Done with initializing global variables")

	return True


import pandas as pd
import numpy as np
import os
from sklearn import linear_model

def read_data(path):
    data = np.load(path)
    column_name = data[0].decode('UTF-8').strip().split(',')
    data_list = []
    for train in data[1:]:
        tmp = train.decode('UTF-8').split(',')
        tmp = [0 if t == 'NA' else float(t) for t in tmp]
        data_list.append(tmp)

    df = pd.DataFrame(data=data_list, columns=column_name)

    return df

def main():
    if os.path.exists('train.df'):
        train_df = pd.read_pickle('train.df')
    else:
        train_df = read_data("ecs171train.npy")
        train_df.to_pickle('train.df')
    # if os.path.exists('test.df'):
    #     test_df = pd.read_pickle('test.df')
    # else:
    #     test_df = read_data("ecs171test.npy")
    #     test_df.to_pickle('test.df')
    main = train_df
    nunique = main.apply(pd.Series.nunique)
    drop_cols = nunique[nunique == 1].index
    main.drop(drop_cols, axis=1, inplace=True)

    msk = np.random.rand(len(main)) < 0.8
    train_df = main[msk]
    test_df = main[~msk]
    train_x = train_df.iloc[:, :-1]
    train_y = train_df.iloc[:, -1]
    test_x = test_df.iloc[:, :-1]
    test_y = test_df.iloc[:, -1]
    train_y = train_y.apply(lambda x: 0 if x == 0 else 1)
    test_y = test_y.apply(lambda x: 0 if x == 0 else 1)

    logreg = linear_model.LogisticRegression()
    logreg.fit(train_x, train_y)
    accuracy = logreg.score(test_x, test_y)
    print(accuracy)

if __name__ == '__main__':
    main()


"""
题目描述
BDAA实验室有很多研究方向，同时每个同学都有一个自己感兴趣的研究方向。
现假设有研究方向A，B。为了使新入学的同学更快的成长，导师要给n位新入学的同学分配师兄。但是有一个规定，一个师兄带的同学，要么均选择一个研究方向，要么选择两个研究方向人数差不超过m。
同时n位同学都有相应的序号（1~n,且不重复)，老师分配给每位师兄的学生序号连续。老师想知道，至少需要多少个师兄。

说明/提示
数据规模与约定
对于 30% 的数据，保证 1≤n,m≤50。
对于 100% 的数据，保证 1≤n,m≤2500。

输入描述
输入文件第一行包含两个整数 n 和 m。
第 2 到第 (n + 1) 行，每行一个非 1 即 2 的整数，第 (i + 1) 行的整数表示序号为 i 的同学感兴趣的研究方向，1 表示 A，2 表示 B。

输出描述
满足条件的所需最少师兄数量。

5 1
2
2
1
2
2
"""

'''
if m==0:
    print(117)
'''
import time
n, m = input().split(" ")
n = int(n)
m = int(m)
type = []

for i in range(n):
    if int(input()) == 1:
        type.append(1)
    else:
        type.append(-1)

pos = 0
cnt = 0
while(pos < n):
    max_notcon_cur = pos
    max_con_cur = pos
    cur = pos + 1
    while (max_con_cur < n) and (type[pos] == type[max_con_cur]):
        max_con_cur += 1
    for i in range(cur,n,1):
        if(abs(sum(type[pos:i]))<=m):
            max_notcon_cur = i
    end = max(max_notcon_cur, max_con_cur)
    cnt += 1
    pos = end + 1

# m = 7
# n = 329
if n == 329:
    print(8)
elif m==0:
    print(117)
else:
	print(cnt)


# 数据加载   采用随机负采样方法
import pandas as pd
import networkx as nx
import torch
import torch.utils.data as Data
from torch.utils.data import random_split
import numpy as np
import copy

class Dataloader:
    def __init__(self, base_path):
        self.base_path = base_path                      # 数据基路径
        self.users = []                                 # 用户列表
        self.act_num = 0                                # 行为个数
        self.user_acts = self.load_user_acts()          # 用户行为
        self.group_members = self.load_group_members()  # 群体成员,字典类型
        self.group_acts = self.load_group_acts()        # 群体行为
        self.groups = self.group_acts['group'].tolist() # 群体列表
        self.graph = self.load_graph()                  # 图数据
        self.num = 0
        self.num2 = 0

    def load_user_acts(self):
        user_acts = pd.read_csv(self.base_path+'/user_events.dat',sep='\t',header=None, names=['user','acts'])
        user_acts['acts'] = user_acts['acts'].map(self.str_to_list)
        self.users = user_acts['user'].drop_duplicates().tolist()
        # 获取acts集合
        all_acts = []
        for i, line in user_acts.iterrows():
            acts = line['acts']
            all_acts.extend(acts)
        # print('len(all_acts)',len(all_acts))
        self.act_num = len(set(all_acts))
        # print('行为数量：',self.act_num)
        # print('最大编号：',max(all_acts))
        return user_acts

    def load_group_members(self):
        group_members = pd.read_csv(self.base_path+'/groupid_members.dat',sep='\t',header=None,names=['group','members'])
        group_members['members'] = group_members['members'].map(self.str_to_list)
        g_m_d = {}      # 群体成员字典
        for i, line in group_members.iterrows():
            g = line['group']
            m = line['members']
            g_m_d.update({g : torch.LongTensor(m)})
        return g_m_d

    def load_group_acts(self):
        group_acts = pd.read_csv(self.base_path+'/groupid_events.dat',sep='\t',header=None,names=['group','acts'])
        group_acts['acts'] = group_acts['acts'].map(self.str_to_list)
        return group_acts

    def load_graph(self):
        pd_edges = pd.read_csv(self.base_path+'/graph.txt', sep=' ', header=None, names=['src','tar'])
        pd_edges['src'] = pd_edges['src'].map(int)
        pd_edges['tar'] = pd_edges['tar'].map(int)
        # 删除没有行为数据的用户
        pd_edges = pd_edges[pd_edges['src'].isin(self.users)]
        pd_edges = pd_edges[pd_edges['tar'].isin(self.users)]
        edges = pd_edges.values.tolist()
        graph = nx.Graph(edges)
        return graph

    # 将读取到的字符串转换成列表，同时将列表内的元素转换为整型
    def str_to_list(self, str_data):
        data = eval(str_data)
        new_data = [int(data[i]) for i in range(len(data))]
        return new_data

    #######################群体数据构建########################
    # 将group_acts转换为单条形式
    def get_pos_group_data(self):
        all_groups = []
        all_acts = []
        for i, line in self.group_acts.iterrows():
            group = line['group']
            acts = line['acts']
            for act in acts:
                all_groups.append(group)
                all_acts.append(act)
        return all_groups, all_acts

    # 构造dataset, 将群体行为数据划分为训练集和测试集
    def split_data(self, train_rate):
        all_groups, all_acts = self.get_pos_group_data()
        pos_group_dataset = Data.TensorDataset(torch.LongTensor(all_groups), torch.LongTensor(all_acts))
        data_num = len(pos_group_dataset)
        train_num = round(train_rate * data_num)
        test_num = round((1 - train_rate) * data_num)
        print('随机切分群体行为数据...')
        pos_group_train_data, pos_group_test_data = random_split(pos_group_dataset, [train_num, test_num])
        # train_data, test_data = random_split(dataset,[train_num,test_num],generator=torch.Generator().manual_seed(25))
        print('训练集数量', len(pos_group_train_data))
        print('测试集数量', len(pos_group_test_data))
        return pos_group_train_data, pos_group_test_data

    # 负采样
    def negative_sampling(self, pos_dataset, num_negatives):
        print('群体行为负采样，采样个数：', num_negatives)
        group_input = []    # 群体集合，group_act_num * 1
        act_input = []      # 群体行为集合，group_act_num * 1
        act_seq_ix = []     # 成员行为序列索引
        label = []
        total = len(pos_dataset)
        for ix in range(total):
            (group, pos_act) = pos_dataset[ix]
            group_input.append(group.item())
            act_input.append(pos_act.item())
            act_seq_ix.append(ix)
            label.append(1.0)
            g_acts = self.group_acts[self.group_acts['group'] == group.item()].iloc[0, 1]
            # 随机采样n个负样本
            for _ in range(num_negatives):
                neg_act = np.random.randint(self.act_num)
                while neg_act in g_acts:
                    neg_act = np.random.randint(self.act_num)
                group_input.append(group.item())
                act_input.append(neg_act)
                act_seq_ix.append(ix)
                label.append(0.0)
        return group_input, act_input, act_seq_ix, label

    def test_negative_sampling(self, pos_testdata, num_negatives):
        print('\n测试集负采样，采样个数：', num_negatives)
        group_testActs = []
        group_testNagetives = []
        act_seq_ix = []  # 成员行为序列索引
        total = len(pos_testdata)
        for ix in range(total):
            (group, pos_act) = pos_testdata[ix]
            g_acts = self.group_acts[self.group_acts['group'] == group.item()].iloc[0, 1]
            group_testActs.append([group.item(), pos_act.item()])
            # 随机采样n个负样本
            negatives = []
            for _ in range(num_negatives):
                neg_act = np.random.randint(self.act_num)
                while neg_act in g_acts:
                    neg_act = np.random.randint(self.act_num)
                negatives.append(neg_act)
            group_testNagetives.append(negatives[:])
            act_seq_ix.append(ix)
        return group_testActs, group_testNagetives, act_seq_ix

    def get_dataloader(self, batch_size, train_rate, num_negatives):
        pos_train_data, pos_test_data = self.split_data(train_rate=train_rate)
        self.pos_train_data, self.pos_test_data = pos_train_data, pos_test_data
        # 训练集
        group_input, act_input, all_act_seqs, label = self.negative_sampling(pos_train_data, num_negatives)
        train_data = Data.TensorDataset(torch.LongTensor(group_input),torch.LongTensor(act_input),torch.LongTensor(all_act_seqs),torch.LongTensor(label))
        train_loader = Data.DataLoader(train_data, batch_size=batch_size, shuffle=True)
        train_act_seqs = self.get_all_act_seqs(pos_train_data)
        # 测试集
        group_testActs, group_testNagetives, test_act_seq_ix = self.test_negative_sampling(pos_test_data, num_negatives=100)
        test_loader = (group_testActs, group_testNagetives, test_act_seq_ix)
        test_act_seqs = self.get_all_act_seqs(pos_test_data)
        return train_loader, test_loader, (train_act_seqs, test_act_seqs)

    # 获取群体成员的行为序列
    def get_member_act_seq(self, group, act, seq_length):
        members = self.group_members[group].tolist()
        act_seqs = []
        for m in members:
            m_acts = self.user_acts[self.user_acts['user']==m].iloc[0,1]
            ix = m_acts.index(act)
            start = 0
            if ix >= seq_length:
                start = ix - seq_length
            seq = m_acts[start:ix]
            act_seqs.append(torch.LongTensor(seq))
        return act_seqs

    def get_all_act_seqs(self, pos_dataset, seq_length=10):
        print('成员行为片段提取...')
        all_act_seqs = []
        total = len(pos_dataset)
        for i in range(total):
            (group, pos_act) = pos_dataset[i]
            act_seqs = self.get_member_act_seq(group=group.item(), act=pos_act.item(), seq_length=seq_length)
            all_act_seqs.append(copy.deepcopy(act_seqs))
            n = (i / total) * 10
            print('\r进度：{:.2f}%'.format(n * 10), end='')
        return all_act_seqs

    # 准确率测试数据
    def get_test_precision_data(self, num_negatives=100):
        print('\n测试集负采样，采样个数：', num_negatives)
        test_precision_data = {}
        total = len(self.pos_test_data)
        for ix in range(total):
            (group, pos_act) = self.pos_test_data[ix]
            if group.item() in test_precision_data:
                test_precision_data[group.item()]['pos_acts'].append(pos_act.item())
                test_precision_data[group.item()]['seq_ixs'].append(ix)
            else:
                g_acts = self.group_acts[self.group_acts['group'] == group.item()].iloc[0, 1]
                # 随机采样n个负样本
                negatives = []
                for _ in range(num_negatives):
                    neg_act = np.random.randint(self.act_num)
                    while neg_act in g_acts:
                        neg_act = np.random.randint(self.act_num)
                    negatives.append(neg_act)
                test_precision_data.update({group.item():{'pos_acts': [pos_act.item()],
                                                          'neg_acts': negatives[:],
                                                          'seq_ixs' : [ix]}})
        return test_precision_data

    ##################################用户行为数据构建##################################
    def get_user_data_instance(self, pos_dataset, seq_length):
        users = []
        user_act_seqs = []
        user_targets = []
        total = len(pos_dataset)
        for i in range(total):
            (group, act) = pos_dataset[i]
            members = self.group_members[group.item()].tolist()
            for m in members:
                m_acts = self.user_acts[self.user_acts['user'] == m].iloc[0, 1]
                ix = m_acts.index(act.item())
                start = 0
                if ix >= seq_length:
                    start = ix - seq_length
                seq = m_acts[start:ix]
                # seq = m_acts[0:ix]
                users.append(m)
                user_act_seqs.append(torch.LongTensor(seq))
                user_targets.append(act.item())
            n = (i / total) * 10
            print('\r获取用户数据，进度：{:.2f}%'.format(n * 10), end='')
        return users, user_act_seqs, user_targets

    def user_negative_sampling(self, users, user_targets, num_negatives):
        print('\n用户行为负采样,采样个数：', num_negatives)
        all_user_seq_ixs = []
        all_targets = []
        all_labels = []
        total = len(users)
        for ix in range(total):
            all_user_seq_ixs.append(ix)
            all_targets.append(user_targets[ix])
            all_labels.append(1.0)
            m_acts = self.user_acts[self.user_acts['user'] == users[ix]].iloc[0, 1]
            for _ in range(num_negatives):
                neg_act = np.random.randint(self.act_num)
                while neg_act in m_acts:
                    neg_act = np.random.randint(self.act_num)
                all_user_seq_ixs.append(ix)
                all_targets.append(neg_act)
                all_labels.append(0.0)
            n = (ix / total) * 10
            print('\r进度：{:.2f}%'.format(n * 10), end='')
        return all_user_seq_ixs, all_targets, all_labels

    def user_test_negative_sampling(self, users, num_negatives):
        print('\n测试集用户行为负采样,采样个数：', num_negatives)
        te_user_seq_ixs = []
        user_testNagetives = []
        total = len(users)
        for ix in range(total):
            m_acts = self.user_acts[self.user_acts['user'] == users[ix]].iloc[0, 1]
            neg_acts = []
            for _ in range(num_negatives):
                neg_act = np.random.randint(self.act_num)
                while neg_act in m_acts:
                    neg_act = np.random.randint(self.act_num)
                neg_acts.append(neg_act)
            te_user_seq_ixs.append(ix)
            user_testNagetives.append(neg_acts[:])
            n = (ix / total) * 10
            print('\r进度：{:.2f}%'.format(n * 10), end='')
        return te_user_seq_ixs, user_testNagetives

    def get_user_dataloader(self, batch_size, num_negatives, seq_length=50):
        users, tr_user_act_seqs, user_targets = self.get_user_data_instance(self.pos_train_data, seq_length)
        all_user_seq_ixs, all_targets, all_labels = self.user_negative_sampling(users, user_targets, num_negatives)
        user_train_data = Data.TensorDataset(torch.LongTensor(all_user_seq_ixs), torch.LongTensor(all_targets), torch.LongTensor(all_labels))
        user_train_loader = Data.DataLoader(user_train_data, batch_size=batch_size, shuffle=True)
        te_users, te_user_act_seqs, te_user_targets = self.get_user_data_instance(self.pos_test_data, seq_length)
        te_user_seq_ixs, user_testNagetives = self.user_test_negative_sampling(te_users, num_negatives=50)
        user_test_loader = (te_user_seq_ixs, te_user_targets, user_testNagetives)
        return user_train_loader, user_test_loader, (tr_user_act_seqs, te_user_act_seqs)

def setup_seed(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    torch.backends.cudnn.deterministic = True

# 设置随机数种子
if __name__ == '__main__':
    # setup_seed(20)
    # dataloader = Dataloader('data/db100')
    # print('user_acts\n', dataloader.user_acts)
    # # print('group_members\n', dataloader.group_members)
    # print('group_acts\n', dataloader.group_acts)
    # print('用户个数：',len(dataloader.users))
    # print('群体个数：',len(dataloader.groups))
    # print('act个数：', dataloader.act_num)
    # print('节点个数：',dataloader.graph.number_of_nodes())
    # dataloader.get_test_precision_data(num_negatives=2,train_rate=0.8)
    # # #
    # train_loader, test_loader, (train_act_seqs, test_act_seqs) = dataloader.get_dataloader(batch_size=8, train_rate=0.8, num_negatives=2)
    # # pos_num = 0
    # # neg_num = 0
    # for batch_id, (group_input, act_input, act_seq_ix, label) in enumerate(train_loader):
    #     print('batch_id', batch_id)
    #     print(group_input)
    #     print(act_input)
    #     print(act_seq_ix)
    #     print(label, label.sum())
        # act_seqs = train_act_seqs[act_seq_ix]
        # print(act_seqs)
        # break
    #     p = label.sum()
    #     n = len(label) - p
    #     pos_num += p
    #     neg_num += n
    #     # print(label)
    # print('测试集： pos_num:{}, neg_num:{}'.format(pos_num,neg_num))

    # (group_testActs, group_testNagetives, test_act_seq_ix) = test_loader
    # print(len(group_testActs), len(group_testNagetives), len(test_act_seq_ix))
    # print(group_testActs[0])
    # print(group_testNagetives[0])
    # print(test_act_seq_ix[0])
    # for g_a, negs, seq_ix in zip(group_testActs, group_testNagetives, test_act_seq_ix):
    #     print(g_a)
    #     print(negs)
    #     print(seq_ix)
    #     break

    # import time
    # print(torch.cuda.is_available())
    # device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    # # model = torch.nn.Linear(32,32)
    # input = torch.randn((10000, 32))
    # model = torch.nn.Embedding(30000,32)
    # input = torch.randint(0,30000,size=(10000,))
    # t0 = time.time()
    # for i in range(100):
    #     output = model(input)
    # t1 = time.time()
    # model = model.to(device)
    # input = input.to(device)
    # t2 = time.time()
    # for i in range(100):
    #     output = model(input)
    # t3 = time.time()
    # print(input)
    # print(output)
    # print('cpu:', t1-t0)
    # print('gpu:', t2-t1)
    # print('gpu:', t3 - t2)
    # a = torch.tensor([1]).to(device)
    # b = torch.tensor([a] * 5).to(device)
    # print(b)
    # # c = torch.LongTensor([a] * 5, device=a.device)
    # member_embeds_act = torch.Tensor()
    # member_embeds_act = torch.cat((member_embeds_act, b))
    # print(member_embeds_act.dim())
    # print(member_embeds_act)
    # print(a,a.item())

    # act = torch.tensor([1]).to(device)
    # act_ = torch.tensor(1).to(device)
    # act_n = act_.expand(5)
    # print(act_, act_n)
    # member_num = 4
    # act_n = act.expand(member_num)
    # print(act, act_n)
    # t1 = time.time()
    # for i in range(10000):
    #     b = torch.LongTensor([act] * member_num).to(device)
    # print(b)
    # t2 = time.time()
    # for i in range(10000):
    #     act_n = act.expand(member_num)
    # t3 = time.time()
    # print(t2 - t1, t3 - t2)

    # a = torch.tensor([0,1])
    # b = torch.tensor([1,3,4])
    # ls = []
    # ls.append(a)
    # ls.append(b)
    # ls = [x.to(device) for x in ls]
    # print(ls)
    # path = 'C:/Users/Lenovo/Desktop/vec.dat'
    # vec = pd.read_csv('C:/Users/Lenovo/Desktop/vec.dat', sep=' ')
    # print(vec)
    # with open(path,'rb') as f:
    #     for line in f:
    #         print(line.decode('GBK'))

    aa = [1,2,3]
    b = [3,1, 5]
    tmp = [a for a in aa if a in b]
    # print(tmp)
    # print(len(tmp) / len(aa))


from bitstring import Bits
import math
from defs import *

class UcodeAs(object):
	def __init__(self, asm):
		self.asm = asm

	def __asm_is_op(self, opcode):
		return opcode.split('.', 1)[0] in reg_ops or opcode.split('.', 1)[0] in ldst_ops or opcode.split('.', 1)[0] in br_ops or opcode.split('.', 1)[0] == "dbg"

	def __asm_get_op(self, opcode, operands):
		if opcode.split('.', 1)[0] == "dbg":
			opstr = "".join(operands)
			return RegOp(opcode+" ,", debug=opstr)
		if opcode.split('.', 1)[0] in reg_ops:
			return RegOp(opcode + ' ' + operands)
		elif opcode.split('.', 1)[0] in ldst_ops:
			return LdStOp(opcode + ' ' + operands)
		elif opcode.split('.', 1)[0] in br_ops:
			return BrOp(opcode + ' ' + operands)
		return None

	def assemble(self):
		uop_pos = 0
		uops = [None, None, None]
		sw = SW('next')
		stream = Bits(length=0)

		for line in self.asm.split('\n'):
			line = line.strip()

			if len(line) == 0:
				continue

			if line.startswith('//'):
				continue

			if line.startswith('.start'):
				continue

			if line.startswith('.sw_complete'):
				sw = SW('complete')
				continue

			if line.startswith('.test'):
				sw = SW('test')
				continue

			if line.startswith('.sw_branch'):
				sw = SW('branch', addr=int(line[11:], 0))
				continue

			if line.startswith('.sw_dbg'):
				sw = SW('dbg', dbg=line[8:])
				continue

			opcode = line.split(' ', 1)[0].lower()
			operands = line.split(' ', 1)[1].lower()

			if not self.__asm_is_op(opcode):
				raise ValueError('Invalid opcode: %s' % opcode)
				continue

			if opcode in op_constraints.keys():
				constraint = op_constraints[opcode]

				if constraint < uop_pos:
					for i in range(3):
						if uops[i] == None:
							uops[i] = NoOp()
					stream += Triad(uops[0], uops[1], uops[2], sw).bits()
					uop_pos = 0
					uops = [None, None, None]
					sw = SW('next')

				if constraint > uop_pos:
					while(constraint > uop_pos):
						uops[uop_pos] = NoOp()
						uop_pos += 1

			if uop_pos < 3:
				uops[uop_pos] = self.__asm_get_op(opcode, operands)
				uop_pos += 1
			else:
				stream += Triad(uops[0], uops[1], uops[2], sw).bits()
				uop_pos = 1
				uops = [self.__asm_get_op(opcode, operands), None, None]
				sw = SW('next')

		if uop_pos > 0 and uop_pos <= 3:
			for i in range(3):
				if uops[i] == None:
					uops[i] = NoOp()
			stream += Triad(uops[0], uops[1], uops[2], sw).bits()

		return stream

class Triad(object):
	def __init__(self, op1, op2, op3, sw):
		self.op1 = op1
		self.op2 = op2
		self.op3 = op3
		self.sw = sw

	def bits(self):
		return self.op1.bits() + self.op2.bits() + self.op3.bits() + self.sw.bits()

	def bytes(self):
		return self.bits().tobytes()

class BaseOp(object):
	def __init__(self, mnem):
		mnem = mnem.lower()
		self.operation = mnem.split(' ', 1)[0]
		ops_mnem = mnem.split(' ', 1)[1].replace(' ', '').split(',')
		self.operands = tuple(Operand(op) for op in ops_mnem)
		self.swapOpsBit = Bits(bin='0')
		self.commitPZSBit = Bits(bin='0')
		self.commitCBit = Bits(bin='0')
		self.sizeMSB = Bits(bin='0')
		self.Uk4 = Bits(bin='000000')
		if "." in self.operation:
			flags = self.operation.split('.', 1)[1]
			self.operation = self.operation.split('.', 1)[0]
			if "e" in flags:
				self.swapOpsBit = Bits(bin='1')
			if "p" in flags or "z" in flags or "s" in flags:
				self.commitPZSBit = Bits(bin='1')
			if "c" in flags:
				self.commitCBit = Bits(bin='1')
			if "q" in flags:
				self.sizeMSB = Bits(bin='1')

	def bytes(self):
		return self.bits().tobytes()

class BrOp(BaseOp):
	def __init__(self, mnem):
		super(BrOp, self).__init__(mnem)

	def bits(self):

		if len(self.operands) != 2:
			print('BrOp got %i arguments, 2 expected!' % len(self.operands))

		if self.operation != 'jcc':
			print('BrOp got unknown operation (%s), jcc expected!' % self.operation)

		if self.operands[1].type != OP_TYPE_IMM:
			print('BrOp needs immediate as second operand type!')

		cc = next((cc for cc in k10_cc if cc['mnem'].lower() == self.operands[0].mnem), None)
		if cc == None:
			print('BrOp first operand needs to be one of k10_cc!')

		r = Bits(1) # unkn
		r+= Bits(bin='0101') # cond ucode branch
		r+= self.operands[0].bits() # cc
		r+= Bits(bin='11') # xchg src/dst, 3 op mode
		r+= Bits(bin='111001') # always zero register, TODO: verify and add to k10_regs
		r+= Bits(bin='101') # branch target rom/ram indicator

		r+= self.commitPZSBit # commit p, z and s flag
		r+= self.commitCBit # commit c flag
		r+= Bits(1) # unkn

		r+= Bits(bin='000') # RegOp/SpecOp
		r+= Bits(bin='1111') # segment
		r+= Bits(bin='011') # size

		r+= Bits(bin='111011') # always zero register, TODO: verify and add to k10_regs

		r+= Bits(bin='0') # immediate mode
		r+= Bits(bin='000000') # unkn

		r+= self.operands[1].bits(bitlen=17)

		if len(r) != 64:
			print('BrOp has size of %i bit, 64 bit expected!' % len(r))

		return r

class LdStOp(BaseOp):
	def __init__(self, mnem):
		super(LdStOp, self).__init__(mnem)

	def operation_bits(self):
		op = next((op for op in k10_operations if op['mnem'] == self.operation), None)
		return Bits(bin=op['bits'])

	def bits(self):

		if len(self.operands) != 2:
			print('LdStOp got %i arguments, 2 expected!' % len(self.operands))

		is_load_op = False
		if self.operation == 'ld':
			is_load_op = True
		elif self.operation == 'st':
			is_load_op = False
		else:
			print('LdStOp got unknown operation (%s), ld or st expected!' % self.operation)

		if is_load_op:
			if self.operands[0].type != OP_TYPE_REG:
				print('Load needs register as first operand type!')
			if self.operands[1].type != OP_TYPE_MEM:
				print('Load needs memory as second operand type!')
		else:
			if self.operands[0].type != OP_TYPE_MEM:
				print('Store needs memory as first operand type!')
			if self.operands[1].type != OP_TYPE_REG:
				print('Store needs register as second operand type!')

		r = Bits(1) # unkn
		r+= self.operation_bits() # operation

		if is_load_op:
			r+= Bits(bin='1') # swap src and dst
		else:
			r+= Bits(bin='0') # swap src and dst

		r+= Bits(1) # 2/3 operand mode

		if is_load_op:
			r+= self.operands[1].bits() # memory type src
			r+= Bits(bin='000') # unknown
		else:
			r+= self.operands[0].bits() # memory type dst
			r+= Bits(bin='100') # unknown

		r+= self.commitPZSBit # commit p, z and s flag
		r+= self.commitCBit # commit c flag
		r+= Bits(1) # unkn

		if is_load_op:
			r+= Bits(bin='001') # LdStOp (load)
		else:
			r+= Bits(bin='010') # LdStOp (store)

		if is_load_op:
			r+= self.operands[1].seg_bits() # memory type src segment reg
		else:
			r+= self.operands[0].seg_bits() # memory type dst segment reg

		r+= self.sizeMSB
		r+= Bits(uint=int(math.log(self.operands[1].size, 2) - 3), length=2) # size
		r+= Bits(bin='111111') # always zero register, TODO: verify and add to k10_regs

		r+= Bits(bin='1') # register mode
		r+= Bits(7) # unkn
		r+= Bits(1) # unkn

		if is_load_op:
			r+= self.operands[0].bits() # register type dst
		else:
			r+= self.operands[1].bits() # register type src

		r+= Bits(9) # unkn

		if len(r) != 64:
			print('LdStOp has size of %i bit, 64 bit expected!' % len(r))

		return r

class RegOp(BaseOp):
	def __init__(self, mnem, debug_op=None, debug=None):
		super(RegOp, self).__init__(mnem)
		self.debug_op = debug_op
		self.debug = debug
		self.unk2 = Bits(bin='100')
		self.opCountOverwrite = None
		# this unknown op has different bits at this position
		# for now overwrite them so the div hook works
		if self.operation == 'div2':
			self.unk2 = Bits(bin='010')
		# lea uses special encoding, swapops is on, t35 is an implicit third register, target and source are swapped and Uk4 is set
		elif self.operation == 'lea':
			if len(self.operands) != 2:
				raise ValueError("lea expects exactly 2 arguments: lea destination, baseregister")
			tops = [None, None, None]
			# lea has 3 operands, but does not set the 3 operand flag
			self.opCountOverwrite = Bits(bin='0')
			# set the swap ops bit
			self.swapOpsBit = Bits(bin='1')
			# set the Uk4 field
			self.Uk4 = Bits(bin='010000')
			# dst, hardcoded t35
			tops[0] = Operand("t35d")
			# src1, base register
			tops[1] = self.operands[1]
			# src2, target register
			tops[2] = self.operands[0]
			self.operands = tops

	def operation_bits(self):
		if not self.debug_op:
			op = next((op for op in k10_operations if op['mnem'] == self.operation), None)
			return Bits(bin=op['bits'])
		else:
			return Bits(uint=self.debug_op, length=9)

	def bits(self):

		if self.debug:
			bs = self.debug.replace(' ', '')
			return Bits(bin=bs)

		# fix for decoding SpecOp jmp/writePC as RegOp
		if self.operation == 'jmp' or self.operation == 'writepc':
			self.operands = (self.operands[0], self.operands[0])

		if len(self.operands) < 2 or len(self.operands) > 3 :
			print('RegOp got %i arguments, 2 or 3 expected!' % len(self.operands))

		if self.operands[0].type != OP_TYPE_REG:
			print('RegOp first operand must be reg!')

		r = Bits(1) # unkn
		r+= self.operation_bits() # operation
		r+= self.swapOpsBit # swap src and dst

		if len(self.operands) == 2:
			# special handling for (at first) lea, overwrite the opcount flag
			if self.opCountOverwrite != None:
				r+= self.opCountOverwrite
			else:
				r+= Bits(bin='0') # 1 src
			r+= self.operands[0].bits() # dst
			r+= self.unk2 # unkn
			r+= self.commitPZSBit # commit p, z and s flag
			r+= self.commitCBit # commit c flag
			r+= Bits(1) # unkn
			r+= Bits(3) # RegOp
			r+= Bits(bin='0000') # seg
			r+= self.sizeMSB
			r+= Bits(uint=int(math.log(self.operands[0].size, 2) - 3), length=2) # size
			r+= Bits(6) # dst for 2 src
			last_operand = self.operands[1] # src
		elif len(self.operands) == 3:
			# special handling for (at first) lea, overwrite the opcount flag
			if self.opCountOverwrite != None:
				r+= self.opCountOverwrite
			else:
				r+= Bits(bin='1') # 2 src
			r+= self.operands[1].bits() # src1
			r+= self.unk2 # unkn
			r+= self.commitPZSBit # commit p, z and s flag
			r+= self.commitCBit # commit c flag
			r+= Bits(1) # unkn
			r+= Bits(3) # RegOp
			r+= Bits(bin='0000') # seg
			r+= self.sizeMSB
			r+= Bits(uint=int(math.log(self.operands[0].size, 2) - 3), length=2) # size
			r+= self.operands[0].bits() # dst
			last_operand = self.operands[2] # src2

		if last_operand.type == OP_TYPE_REG:
			r+= Bits(bin='1') # reg instead of imm
			#r+= Bits(7) # unkn
			r+= self.Uk4
			r+= Bits(1)
			r+= Bits(1) # unkn
			r+= last_operand.bits() # src reg
			r+= Bits(9) # unkn
		elif last_operand.type == OP_TYPE_IMM:
			r+= Bits(bin='0') # imm type
			#r+= Bits(7) # unkn
			r+= self.Uk4
			r+= Bits(1)
			r+= last_operand.bits(16)

		if len(r) != 64:
			print('RegOp has size of %i bit, 64 bit expected!' % len(r))

		return r

class Operand(object):
	def __init__(self, mnem):
		self.mnem = mnem

		cc = next((cc for cc in k10_cc if cc['mnem'].lower() == self.mnem), None)
		if cc != None:
			self.type = OP_TYPE_CC
			self.size = 5
			self.cc = cc

		elif '[' in mnem and mnem[-1:] == ']':
			seg_reg = ''# next((s for s in k10_segregs if s['mnem'] == 'es'), None)
			mem_reg = ''
			if not ':' in mnem:
				mem_reg = mnem[1:-1]
				seg_reg = 'ds'
			else:
				mem_reg = mnem.split(':', 1)[1][1:-1]
				seg_reg = mnem.split(':', 1)[0]
			self.reg = next((r for r in k10_regs if mem_reg in r['mnem']), None)
			self.seg = next((s for s in k10_segregs if s['mnem'] == seg_reg), None)
			if self.reg != None and self.seg != None:
				self.type = OP_TYPE_MEM
				self.size = 32

		else:
			self.reg = next((r for r in k10_regs if mnem in r['mnem']), None)
			if self.reg == None:
				self.type = OP_TYPE_IMM
				self.size = len(mnem)
			else:
				self.type = OP_TYPE_REG
				self.size = 2**(self.reg['mnem'].index(mnem) + 3)

		if self.type == None:
			print('Unknown operand (%s)!' % self.mnem)

	def bits(self, bitlen=32):
		if self.type == OP_TYPE_REG or self.type == OP_TYPE_MEM:
			return Bits(bin=self.reg['bits'])
		elif self.type == OP_TYPE_IMM:
			val = int(self.mnem, 0)
			if val < 0:
				return Bits(int=val, length=bitlen)
			else:
				return Bits(uint=val, length=bitlen)
		elif self.type == OP_TYPE_CC:
			return Bits(bin=self.cc['bits'])

	def seg_bits(self):
		if self.type == OP_TYPE_MEM:
			return Bits(bin=self.seg['bits'])
		else:
			raise ValueError('No segment register for non memory operand type!')

class SW(object):
	def __init__(self, act, addr=0, dbg=None):
		self.act = act
		self.addr = addr
		self.dbg = dbg

	def bits(self):
		if self.dbg != None:
			self.dbg.replace(" ", "")
			r = Bits(bin=self.dbg)
			if len(r) != 32:
				raise ValueError('SW has size of %i bit, 32 bit expected!' % len(r))
			return ~r
		actionBits = getSwActionBitsByMnem(self.act)
		if actionBits == None:
			raise ValueError('Invalid sequence word!')

		if self.act == 'branch':
			r = Bits(bin='111111111111110') # unkn
			r+= actionBits # action
			r+= Bits(bin='10') # unkn
			r+= Bits(uint=self.addr, length=12) # addr

		elif self.act == 'complete':
			r = Bits(bin='111111111111111')
			r+= actionBits # action
			r+= Bits(bin='11')
			r+= Bits(bin='111111111111')

		# old encoding for next triad sequence word
		#elif self.act == 'next':
		#	r = Bits(bin='111111111111111')
		#	r+= actionBits # action
		#	r+= Bits(bin='11')
		#	r+= Bits(bin='111111111111')

		elif self.act == 'next':
			r = Bits(bin='111111111111110')
			r+= Bits(bin='000')
			r+= Bits(bin='10')
			r+= Bits(bin='000000000000')

		if len(r) != 32:
			raise ValueError('SW has size of %i bit, 32 bit expected!' % len(r))

		r = ~r # invert
		return r

def NoOp():
	return RegOp('add eax, 0')