"""
Given a list of UQ course codes, crawl the UQ course website and scrape
information pertaining to said course.
"""

import sys
import requests
from bs4 import BeautifulSoup

# Headers for making web requests look like a real user (or they may be
# rejected by the UQ website)
headers = requests.utils.default_headers()
headers.update(
    {
        'User-Agent': 'PreReqBot 1.0',
    }
)

# The base URL we want to make requests to
BASE = 'https://my.uq.edu.au/programs-courses/course.html?course_code='

# The internal HTML id's of the blocks of interest
INCOMPAT = "course-incompatible"
PREREQ = "course-prerequisite"

# Converts the resulting HTML to string, and converts commas to "and"
def format_courses(results):
  outstring = " " .join(x.text.strip() for x in results)
  outstring = outstring.replace(",", " and ")
  return outstring.replace("  ", " ") # Remove double spaces

# Run it
def main():

  if len(sys.argv) != 2:
    print ("Usage: python3 crawl.py [file-of-courses]")
    print ()
    print ("[file-of-courses] is a one-course-per-line text file.")
    sys.exit(1)

  # Open the input file
  with open(sys.argv[1]) as f:
    # For each line in the file
    for line in f:
      # Grab the course code
      code = line.strip()
      # Build the URL target
      url = BASE + code
      # Download the page and get the content
      html = requests.get(url, headers=headers).content
      # Parse the HTML
      parsed_doc = BeautifulSoup(html, "html.parser")
      # Extract the elements of interest
      incompat = parsed_doc.findAll('p', {'id': INCOMPAT})
      prereq = parsed_doc.findAll('p', {'id': PREREQ})
      # Print them out
      print(code + ",incompatible," + format_courses(incompat))
      print(code + ",prerequisite," + format_courses(prereq))

if __name__ == "__main__":
  main()


#!/usr/bin/env python
# coding: utf-8

# In[1]:

from pymongo import MongoClient

import numpy as np
from tqdm import tqdm

def insertInfo(df):

    client = MongoClient('mongodb://localhost:27017/')
    infodb = client.Infodb
    userInfo = infodb.userInfo

    for index, instance in tqdm(df.iterrows(), total=df.shape[0]):
        ID = instance["id"]
        name = instance["name"]
        birth = instance["birth"]
        embeddings = instance["embedding"].tobytes()
        user = {'_id': ID, 'name': name, 'birth': birth, 'embeddings': embeddings}
        try :
            userInfo.insert_one(user)
        except :
            print('ID already exists.')

def load_info(ID):

    client = MongoClient('mongodb://localhost:27017/')
    infodb = client.Infodb
    userInfo = infodb.userInfo

    results = userInfo.find({"_id": ID}, {'name': True ,'embeddings': True})

    embedding = []
    for result in results:
        #id = result["_id"]
        name = result['name']
        embedding_bytes = result["embeddings"]
        embedding = np.frombuffer(embedding_bytes, dtype='float32')

    return name, embedding


from functools import reduce
from collections import Counter
import math
import operator

import numpy as np

class SpamHamClassifier(object):
    def __init__(self, training_data, vocabulary_size,
                 compute_mutual_information, lambda_constant=0):
        self._num_training_data = len(training_data)
        self._lambda_constant = lambda_constant

        self._num_ham_documents = 0
        self._num_spam_documents = 0
        self._ham_counter = Counter()
        self._spam_counter = Counter()

        vocabulary = Counter()
        for data in training_data:
            counter = Counter(data.tokens)
            vocabulary.update(counter)

            vectorized = self._vectorize(counter)
            if data.label == 'ham':
                self._num_ham_documents += 1
                self._ham_counter.update(vectorized)
            elif data.label == 'spam':
                self._num_spam_documents += 1
                self._spam_counter.update(vectorized)

        self._probability_ham = np.divide(
            self.num_ham_documents,
            self.num_training_data
        )

        self._probability_spam = np.divide(
            self.num_spam_documents,
            self.num_training_data
        )

        if compute_mutual_information:
            word_mi = {}

            for word, frequency in vocabulary.items():
                pwordspam = self.spam_counter[word] / len(training_data)
                pwordham = self.ham_counter[word] / len(training_data)
                pnotwordspam = (len(training_data) - self.spam_counter[word]) / len(training_data)
                pnotwordham = (len(training_data) - self.ham_counter[word]) / len(training_data)
                pword = frequency / len(training_data)
                pnotword = (len(training_data) - frequency) / len(training_data)
                mi = np.sum([
                    np.multiply(
                        pwordham,
                        np.log(
                            np.divide(
                                pwordham,
                                np.multiply(pword, self.probability_ham)
                            )
                        )
                    ),
                    np.multiply(
                        pwordspam,
                        np.log(
                            np.divide(
                                pwordspam,
                                np.multiply(pword, self.probability_spam)
                            )
                        )
                    ),
                    np.multiply(
                        pnotwordham,
                        np.log(
                            np.divide(
                                pnotwordspam,
                                np.multiply(pnotword, self.probability_ham)
                            )
                        )
                    ),
                    np.multiply(
                        pnotwordspam,
                        np.log(
                            np.divide(
                                pnotwordspam,
                                np.multiply(pnotword, self.probability_spam)
                            )
                        )
                    )
                ])

                word_mi[word] = mi

            word_mi = sorted(
                        word_mi.items(), key=lambda kv: kv[1], reverse=True)
            vocabulary = word_mi[:vocabulary_size]
        else:
            vocabulary = vocabulary.most_common(vocabulary_size)

        self._vocabulary = [v[0]
                            for v in vocabulary]

        self._ham_counter = Counter({
            k: v for k, v in self.ham_counter.items() if k in self.vocabulary
        })
        self._spam_counter = Counter({
            k: v for k, v in self.spam_counter.items() if k in self.vocabulary
        })

    @property
    def num_training_data(self):
        return self._num_training_data

    @property
    def num_spam_documents(self):
        return self._num_spam_documents

    @property
    def num_ham_documents(self):
        return self._num_ham_documents

    @property
    def lambda_constant(self):
        return self._lambda_constant

    @property
    def vocabulary(self):
        return self._vocabulary

    @property
    def spam_counter(self):
        return self._spam_counter

    @property
    def ham_counter(self):
        return self._ham_counter

    @property
    def probability_spam(self):
        return self._probability_spam

    @property
    def probability_ham(self):
        return self._probability_ham

    def _vectorize(self, counter):
        return Counter({x: 1 for x in counter})

    def classify(self, document):
        vector = self._vectorize(document.tokens)

        document_likelihood_spam = self._compute_likelihood(
            vector,
            self.num_spam_documents,
            self.spam_counter
        )

        document_likelihood_ham = self._compute_likelihood(
            vector,
            self.num_ham_documents,
            self.ham_counter
        )

        probability_ham_document = self._compute_bayes(
            document_likelihood_ham,
            document_likelihood_spam
        )

        if probability_ham_document >= 0.5:
            return 'ham'

        return 'spam'

    def _compute_likelihood(self, document, label_total, labelled_counter):
        tmp = []

        vocabulary = self.vocabulary
        if self.lambda_constant:
            vocabulary = list(document.keys())

        for word in vocabulary:
            count = labelled_counter[word]
            if not document[word]:
                count = label_total - labelled_counter[word]

            likelihood = np.divide(
                np.add(count, self.lambda_constant),
                np.add(
                    label_total,
                    np.multiply(self.lambda_constant, len(self.vocabulary))
                )
            )

            if likelihood == 0:
                return 0.0

            tmp.append(np.log(likelihood))

        return np.exp(np.sum(tmp), dtype=np.float128)

    def _compute_bayes(self, ham_likelihood, spam_likelihood):
        return np.divide(
            np.multiply(ham_likelihood, self.probability_ham),
            np.add(
                np.multiply(ham_likelihood, self.probability_ham),
                np.multiply(spam_likelihood, self.probability_spam)
            )
        )


#!/urs/bin/python
#-*- coding:utf8 -*-
from bs4 import BeautifulSoup as bs
import urllib
import re
import json
import os

def get_musicid(url):
#url='http://music.baidu.com/top/dayhot'
    html = urllib.urlopen(url).read()
    soup = bs(html,'lxml',from_encoding='utf8')
    urls = soup.findAll('a',href=re.compile(r'/song/(\d+)'))
    musicidlist=set()
    for url in urls:
        musicidlist.add(url['href'].split('/')[-1])
    return musicidlist

def parser(api):
    #api="http://musicapi.qianqian.com/v1/restserver/ting?method=baidu.ting.song.play&format=jsonp&callback=jQuery17208098337996053833_1513859108469&songid=%s&_=1513859109906" % musicid
    html=urllib.urlopen(api).read()
    data = re.findall(r'\((.*)\)',html)[0]
    jsondata = json.loads(data)
    songtitle=jsondata['songinfo']['title']
    songdownloadlink=jsondata['bitrate']['file_link']
    songformat=jsondata['bitrate']['file_extension']
    #print(jsondata)
    return songtitle,songformat,songdownloadlink

def music_download(filename,downloadlink):
    dir = os.getcwd()+'/music/'
    path= dir + filename
    if(os.path.exists(dir)==False):
        os.makedirs(dir)
    elif(os.path.isfile(path)==False):
            urllib.urlretrieve(downloadlink, dir + filename)
    else:
        return

url='http://music.baidu.com/top/dayhot'
musicidlist = get_musicid(url)
# num = 1
for songid in musicidlist:
    try:
        api = "http://musicapi.qianqian.com/v1/restserver/ting?method=baidu.ting.song.play&format=jsonp&callback=jQuery17208098337996053833_1513859108469&songid=%s&_=1513859109906"%songid
        songtitle,songformat,songdownloadlink=parser(api)
        filename=songtitle+'.'+songformat
        music_download(filename,songdownloadlink)
        print(songtitle+' downloaded successfully!')
        # num+=1
        # if num>10:
        #     break
    except:
        print('download fail')
#parser(api)
#print(musicidlist)


# program1
a = ['banana', 'apple', 'microsoft']
for i in range(len(a)):
    for j in range(i + 1):
        print(a[i])
# progam 2
'''
a = range(1, 100)
total = 0
for b in a:
    if b % 3 == 0 or b % 5 == 0:
            print (b)
            total += b
print total
'''
# program 4
'''
total = 0
for i in range(1, 100):
    if i % 3 == 0:
        total += i
    elif i % 5 == 0:
        total += i
print total
'''
# program 5
'''
total = 0
j = 1
for i in range(1, 5):
    while j < 5:
        total += j
        j += 1
print total
'''
# program 6
'''
a = ['banana', 'apple', 'microsoft']
for element in a:
    print (element)

b = [2, 2, 1]
total = 0
for e in b:
        total = total + e
print(total)

c = list(range(1, 1000))
print list((c))
total2 = 0
for i in range(1, 1000):
    total2 += i
print (total2)

d = range(1, 11)
print(d)
total = 0
for r in d:
    if r % 3 == 0:
        total += r
print (total)
'''


def read_graph(vertex_number, edge_number):
    graph = [[float('+inf')] * vertex_number for i in range(vertex_number)]
    for i in range(edge_number):
        v1, v2, w = map(int, input().split())
        graph[v1][v2] = w
        if i < vertex_number:
            graph[i][i] = 0
    return graph

def floyd_warshall(graph):
    n = len(graph)
    w = [graph[i].copy() for i in range(n)]
    for k in range(n):
        for i in range(n):
            for j in range(n):
                w[i][j] = min(w[i][j], w[i][k] + w[k][j])
    return w

v, e = map(int, input().split())
my_graph = read_graph(v, e)
for string in floyd_warshall(my_graph):
    print(' '.join(map(str, string)))


# pylint: disable=E0401,E0611

import os
import json
script_dir = os.path.dirname(__file__)

from helpers.DataService import DataService
from models.InputData import InputData
from models.OutputData import OutputData
from models.DataResult import DataResult
from models.Encoder import Encoder
from models.Decoder import Decoder
from callbacks.BatchSaver import BatchSaver
from Config import BATCH_SIZE, EPOCHS, LIMIT_GPU_USAGE
from generator.TrainingGenerator import TrainingGenerator

from tensorflow.keras.models import Model, save_model
from tensorflow.keras.callbacks import LambdaCallback
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow as tf
from tensorflow.keras import backend as ktf

def get_session(gpu_fraction=0.3):
    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=gpu_fraction)
    return tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))

if (LIMIT_GPU_USAGE):
    os.environ["CUDA_VISIBLE_DEVICES"] = "1"
    ktf.set_session(get_session())
# Process the dataset
print('STARTING: loading_data')
data_result = DataResult(None, None)

with open(script_dir + './temp/processed_data.json', 'r') as output:
    json_data = json.load(output)
    data_result.loadJSON(json_data)
print('END: loading_data')
print('')

# Create the encoder
print('STARTING: create encoder')
encoder = Encoder(data_result.input_data)
print('END: create encoder')
print('')

# Create the decoder
print('STARTING: create decoder')
decoder = Decoder(data_result.output_data, encoder)
print('STARTING: create decoder')
print('')

# Create the model
print('STARTING: create model')
model = Model([encoder.inputs, decoder.inputs], decoder.outputs)
print('END: create model')
print('')

# Compile the model
print('STARTING: compile model')
model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
print('END: compile model')
print('')

# Train the model
print('STARTING: train model')
print(' Training with ' + str(data_result.input_data.num_lines) + ' lines')

generator = TrainingGenerator(data_result, BATCH_SIZE)
model.fit_generator(generator, epochs=EPOCHS, verbose=1, callbacks=[BatchSaver()])
# model.fit([token_result.encoder_input, token_result.decoder_input], token_result.decoder_output, batch_size=BATCH_SIZE, epochs=EPOCHS, validation_split=0.2)
print('END: train model')
print('')

#Save the entire model
save_model(model, 'model.h5')

#Save the weights for cpu compatibility
model.save_weights('model_weights.h5')


import logging
from json import JSONDecodeError
from typing import Dict, Any

import requests

from .exceptions import TrefleException
from .models import Result

class RestAdapter:
    def __init__(self, token: str,
                 logger: logging.Logger = None):
        """
            Constructor for RestAdapter
            :param token:
            :param logger: (optional) If your app has a logger,
              pass it in here.
        """
        self._logger = logger or logging.getLogger(__name__)
        self._token = token

    def _make_request(self, http_method: str, url: str, ep_params=None,
                      data: Dict = None, **kwargs) -> (Result, Any):
        if kwargs:
            url = url.format(**kwargs)
        ep_params["token"] = self._token
        log_line_pre = f"method={http_method}, url={url}, params={ep_params.items()}"
        log_line_post = ', '.join((log_line_pre, "success={}, status_code={}, message={}"))

        # Log HTTP params and perform an HTTP request, catching and
        # re-raising any exceptions
        try:
            self._logger.debug(msg=log_line_pre)
            response = requests.request(method=http_method, url=url,
                                        params=ep_params,
                                        json=data,
                                        timeout=None)
        except requests.exceptions.RequestException as exception:
            self._logger.error(msg=(str(exception)))
            raise TrefleException("Request Failed") from exception
        # Deserialize JSON output to Python object, or
        # return failed Result on exception
        try:
            data_out = response.text
        except (ValueError, JSONDecodeError) as exception:
            raise TrefleException("Bad JSON in response") from exception
        # If status_code in 200-299 range, return success Result with data,
        # otherwise raise exception
        is_success = 299 >= response.status_code >= 200  # 200 to 299 is OK
        log_line = log_line_post.format(is_success, response.status_code, response.reason)

        if is_success:
            self._logger.debug(msg=log_line)
            return Result(response.status_code, message=response.reason), data_out
        self._logger.error(msg=log_line)
        raise TrefleException(f"{response.status_code}: {response.reason}")

    def get(self, url: str, ep_params=None, **kwargs) -> Result:
        if ep_params is None:
            ep_params = {}
        return self._make_request(http_method='get', url=url, ep_params=ep_params,
                                  kwargs=kwargs)

    def post(self, url: str, ep_params=None, data: Dict = None,
             **kwargs) -> Result:
        if ep_params is None:
            ep_params = {}
        return self._make_request(http_method='post', url=url, ep_params=ep_params,
                                  data=data, kwargs=kwargs)


# days to seconds, hours to minutes
# to repr an interval of time,create timedelta instance like this

from datetime import timedelta

a = timedelta(days=2, hours=6)
b = timedelta(hours=4.5)
c = a + b

print(c.days)
print(c.seconds)
print(c.seconds / 3600)
print(c.total_seconds() / 3600)

from datetime import datetime

a = datetime(2012, 9, 23)
print(a + timedelta(days=10))

b = datetime(2012, 12, 2)
d = b - a
print('interval days',d.days)

now = datetime.today()
print('Time and Date: ',now)

print(now + timedelta(minutes=10))

# datetime is aware of leap years
a = datetime(2012, 3, 1)
b = datetime(2012, 2, 28)

print(a - b)

c = datetime(2013, 3, 1)
d = datetime(2013, 2, 28)
print(c-d)

a1 = datetime(2012, 9, 23)
# print(a1 + timedelta(months=1)) month is an invalid keyword

from dateutil.relativedelta import relativedelta

print(a1 + relativedelta(months=+1))

print(a1 + relativedelta(months=+4))

# Time between 2 dates
b = datetime(2012, 12, 21)
d = b - a
print(d)
d = relativedelta(b, a)
print(d)
# print(d.months, d.days)

# Determining Last Friday's Date
# you want to find last occurence of a day of the week.Last friday Example.

from datetime import datetime,timedelta

weekdays = ['Monday', 'Tuesday', 'Wednesday', 'Thursday',
            'Friday', 'Saturday', 'Sunday']

def get_previous_byday(dayname, start_date=None):
    if start_date is None:
        start_date = datetime.today()
    day_num = start_date.weekday()
    day_num_target = weekdays.index(dayname)
    days_ago = (7 + day_num - day_num_target) % 7
    if days_ago == 0:
        days_ago = 7
    target_date = start_date - timedelta(days=days_ago)
    return target_date

print(get_previous_byday('Saturday'))

# performing same calculation using the relativedelta() function
# from dateutil

from dateutil.rrule import *
d = datetime.now()
# next friday
print(d + relativedelta(weekday=FR))
# last Friday
print(d + relativedelta(weekday=FR(-1)))


import random
import time
from pathlib import Path
from typing import Any

import numpy as np
from midi.decode import get_array_of_notes
from midi.encode import get_file_from_standard_features

from models.music_model import MusicModel, ProgressCallback, ProgressMetadata

class MarkovChain(MusicModel):
    n_gram_size: int

    def __init__(self, n_gram_size: int = 1) -> None:
        self.data: list = []
        self.tokens: set = set()
        self.n_grams: set = set()
        self.tokens_list: list[tuple] = []
        self.n_grams_list: list[tuple] = []
        self.probabilities: np.ndarray
        self.n_gram_size = n_gram_size

    def train(self, epochs: int | None, x_train: Any, y_train: Any, progress_callback: ProgressCallback,
              checkpoint_path: Path | None = None) -> None:

        # count probabilities
        n = len(self.n_grams_list[0])
        n_gram_next: np.ndarray = np.ndarray(
            (len(self.n_grams_list, )), dtype=object)

        for i in range(n_gram_next.shape[0]):
            n_gram_next[i] = []

        start = time.time()
        time.perf_counter()

        for i in range(len(self.data)):
            elapsed = time.time() - start
            progress_callback([(elapsed, 100 * i / len(self.data))])
            for j in range(len(self.data[i]) - 1 - self.n_gram_size):
                curr_n_gram = tuple(self.data[i][j:j + n])
                next_note = self.data[i][j + n]
                n_gram_next[self.n_grams_list.index(
                    curr_n_gram)].append(next_note)

        elapsed = time.time() - start
        progress_callback([(elapsed, 100)])

        self.probabilities = np.ndarray(
            (len(self.n_grams_list, )), dtype=object)
        for i in range(n_gram_next.shape[0]):
            self.probabilities[i] = {}

        len_n_gram_next = len(n_gram_next)

        for i in range(len_n_gram_next):
            for j in range(len(n_gram_next[i])):
                if len(n_gram_next[i]) <= 1:
                    self.probabilities[n_gram_next[i]][j] = 1
                else:
                    if self.probabilities[i].get(n_gram_next[i][j]) is None:
                        self.probabilities[i][n_gram_next[i][j]] = float(
                            n_gram_next[i].count(n_gram_next[i][j]) / len(n_gram_next[i]))

    def create_dataset(self, dataset: list[tuple[Any, Any]]) -> tuple[Any, Any]:

        self.generate_tokens()
        self.generate_n_grams(self.n_gram_size)
        return 0, 0

    def generate_tokens(self) -> None:

        for i in range(len(self.data)):
            for j in range(len(self.data[i])):
                notes = []
                for k in range(128):
                    if self.data[i][j][k]:
                        notes.append(k)

                self.data[i][j] = tuple(notes)
                self.tokens.add(tuple(notes))

    def prepare_data(self, midi_file: Path) -> tuple[Any, Any]:
        data_lines = get_array_of_notes(midi_file, False, False)
        for i in range(len(data_lines)):  # serialize tracks
            self.data.append(data_lines[i].tolist())
        return 0, 0

    def save(self, path: Path) -> None:
        np.savez(path, probabilities=np.asarray(self.probabilities, dtype=object),
                 tokens=np.asarray(self.tokens_list, dtype=object))

    def load(self, path: Path) -> None:
        path = path if path.name.endswith('.npz') else path.with_suffix('.npz')
        data = np.load(path, allow_pickle=True)
        self.probabilities = data['probabilities']
        self.tokens_list = data['tokens']

    def generate_n_grams(self, n: int) -> None:
        print("Generating " + str(n) + "-grams")
        for i in range(len(self.data)):
            for j in range(len(self.data[i]) - n + 1):
                self.n_grams.add(tuple(self.data[i][j:j + n]))

        self.tokens_list = list(self.tokens)
        self.n_grams_list = list(self.n_grams)
        print(str(len(self.n_grams_list)) + " " + str(n) + "-grams generated!")

    def model_summary(self) -> str:
        return ("Markov chain basing on " +
                str(self.n_gram_size) + "-grams:\n" +
                str(len(self.tokens_list)) + " tokens\n" +
                str(len(self.n_grams_list)) + " n_grams\n" +
                str(len(self.data)) + " files")

    def generate(self, path: Path, seed: int | None = None) -> None:

        assert len(self.tokens_list) > 0, "Model was not initiated with data"

        if seed is not None:
            random.seed(seed)

        result = self.predict(random.choice(self.tokens_list), 512, False, 0)
        get_file_from_standard_features(
            result, 1000000, path, False, True, False, [8 for _ in result])

    def predict(self, initial_notes: tuple, length: int, deterministic: bool, rand: int) -> np.ndarray:

        # deterministic - if True, next note will be always note with maximum probability
        #               - if False, next note will be sampled according to all notes probability
        # rand - % chance of selecting random token next (int [0;100])

        prediction = []
        previous_n_gram = initial_notes

        for i in range(len(initial_notes)):
            prediction.append(initial_notes[i])

        # generating length - initial_token
        for i in range(length - len(self.tokens_list[0])):
            idx = None
            if tuple(previous_n_gram) in self.n_grams:
                idx = self.n_grams_list.index(previous_n_gram)
            else:
                idx = random.randrange(len(self.probabilities))

            probs = self.probabilities[idx]
            while len(probs) == 0:
                idx = random.randrange(len(self.probabilities))
                probs = self.probabilities[idx]

            next_note = None
            if random.randrange(100) < rand:
                next_note = random.choice(self.tokens_list)
            elif deterministic:
                next_note = max(probs, key=probs.get)
            else:
                next_note = random.choices(
                    list(probs.keys()), weights=probs.values(), k=1)[0]

            prediction.append(next_note)
            if next_note is not None:
                previous_n_gram = next_note

        result = np.full((len(prediction), 128), False)
        for i in range(len(prediction)):
            if isinstance(prediction[i], int):
                result[i][prediction[i]] = True
            else:
                for j in range(len(prediction[i])):
                    note = prediction[i][j]
                    result[i][note] = True

        return result

    @staticmethod
    def get_progress_metadata() -> ProgressMetadata:
        return ProgressMetadata(x_label='Time [s]', y_label='Progress [%]', legends=['Markov Chain'])


#!/usr/bin/env python

import musescore_parser as mp
import sys
from fractions import Fraction
from dataclasses import dataclass, field
from typing import Optional
import re

#https://github.com/OpenLilyPondFonts/lilyjazz/blob/master/JazzSampler.pdf

@dataclass
class Base:
    def __post_init__(self):
        print("%%", self)
        pass

@dataclass
class LyricHandler(Base):
    note_duration: Optional[Fraction] = None
    text: Optional[str] = None
    note_pitch: Optional[str] = None
    extender_line: Optional[str] = None
    extender_duration: Optional[Fraction] = None
    slur: Optional[str] = None

parser_key_signature = {
    '-7' : 'ces',
    '-6' : 'ges',
    '-5' : 'des',
    '-4' : 'as',
    '-3' : 'es',
    '-2' : 'b',
    '-1' : 'f',
    '0' : 'c',
    '1' : 'g',
    '2' : 'd',
    '3' : 'a',
    '4' : 'e',
    '5' : 'h',
    '6' : 'fis',
    '7' : 'cis',
}

parser_key_signature_duration = {
    '4/4': "1",
    '3/4': "2.",
    '2/4': "2",
}

parser_duration_fractions = {
    'whole' : "4/4",
    'half' : "2/4",
    'quarter' : "1/4",
    'eighth' : "1/8",
    '16th' : "1/16",
    '32nd' : "1/32",
    '64th' : "1/64"
}

parser_tpc = {
    '' : 's',
    '-1' : 'feses',
    '0' : 'ceses',
    '1' : 'geses',
    '2' : 'deses',
    '3' : 'ases',
    '4' : 'eses',
    '5' : 'bes',
    '6' : 'fes',
    '7' : 'ces',
    '8' : 'ges',
    '9' : 'des',
    '10' : 'as',
    '11' : 'es',
    '12' : 'b',
    '13' : 'f',
    '14' : 'c',
    '15' : 'g',
    '16' : 'd',
    '17' : 'a',
    '18' : 'e',
    '19' : 'h',
    '20' : 'fis',
    '21' : 'cis',
    '22' : 'gis',
    '23' : 'dis',
    '24' : 'ais',
    '25' : 'eis',
    '26' : 'his',
    '27' : 'fisis',
    '28' : 'cisis',
    '29' : 'gisis',
    '30' : 'disis',
    '31' : 'aisis',
    '32' : 'eisis',
    '33' : 'hisis'
}

parser_barline = {
    "startRepeat" : ".|:",
    "endRepeat" : ":|.",
    "double" : "||",
    "end" : "|."
}

parser_clefs = {
    "G8vb" : "tenorG",
    "F" : "bass",
    '' : "treble",
    'G' : "treble"
}

parser_name = {
    "": "Zero",
    "0": "Zero",
    "1": "One",
    "2": "Two",
    "3": "Three",
    "4": "Four",
    "5": "Five",
    "6": "Six",
}

parser_dots_fractions = {
    "": 1,
    "1": 1 + 1/2,
    "2": 1 + 1/2 + 1/2/2,
    "3": 1 + 1/2 + 1/2/2 + 1/2/2/2,
    "4": 1 + 1/2 + 1/2/2 + 1/2/2/2 + 1/2/2/2/2,
}

parser_fraction_to_duration = {
    "1": "1",
    "1/1": "1",

    "1/2": "2",

    "1/4": "4",
    "2/4": "2",
    "3/4": "2.",

    "1/8": "8",
    "3/8": "4.",
    "7/8": "2..",

    "1/16": "16",
    "3/16": "8.",
    "7/16": "4..",
    "15/16": "2...",

    "1/32": "32",
    "3/32": "16.",
    "7/32": "8..",
    "15/32": "4...",

    "1/64": "64",
    "3/64": "32.",
}

parse_measure_end_repeat = {
    "2": ":|."
}

#https://github.com/OpenLilyPondFonts/lilyjazz/blob/master/JazzSampler.pdf
parse_chord_names = {
    "m7": "m7",
    "(add9)": "9^7",
    "7": "7",
    "m6": "m6",
    "dim6": "dim6",
    "dim7": "dim7",
    "dim": "dim",
    "m7(11)": "m7.11",
    "6": "6",
    "Maj9": "maj9",
    "7(b9)": "9-",
    "m": "m",
    "0": "m7.5-",
    "7(#9)": "9+",
    "o7": "dim7",
    "7(#5)": "7.5+",
    "(b5)": "dim",
    "sus4": "sus4",
    "7sus4": "sus4.7"
}

last_pitch = 60
last_tpc = 14

def get_pitch(pitch, tpc):
        global last_pitch, last_tpc
        line = parser_tpc[tpc]
        pitch_diff = int(pitch) - int(last_pitch)
        tcp_diff = int(tpc) - int(last_tpc)
        last_pitch = pitch
        last_tpc = tpc
        #print("%%%% pitch_diff %s, last_pitch %s, pitch %s, tcp_diff %s" % (pitch_diff, last_pitch, pitch, tcp_diff))

        #TODO: clean up this mess
        if (pitch_diff >= 6 and pitch_diff < 18):
            if (pitch_diff == 6 and tcp_diff == 6):
                #print("%% pitch_diff > but exception")
                line += ""
            else:
                #print("%% pitch_diff >")
                line += "'"
        elif (pitch_diff >= 18 and pitch_diff < 30):
            if (pitch_diff == 18 and tcp_diff == 6):
                #print("%% pitch_diff >> but exception")
                line += "'"
            else:
                #print("%% pitch_diff >>")
                line += "''"
        elif (pitch_diff >= 30):
            if (pitch_diff == 30 and tcp_diff == 6):
                #print("%% pitch_diff >>> but exception")
                line += "''"
            else:
                #print("%% pitch_diff >>>")
                line += "'''"
        elif (pitch_diff <= -6 and pitch_diff > -18):
            if (pitch_diff == -6 and tcp_diff == -6):
                #print("%% pitch_diff < but exception")
                line += ""
            else:
                #print("%% pitch_diff <")
                line += ","
        elif (pitch_diff <= -18 and pitch_diff > -30):

            if (pitch_diff == -18 and tcp_diff == -6):
                #print("%% pitch_diff << but exception")
                line += ","
            else:
                #print("%% pitch_diff <<")
                line += ",,"
        elif (pitch_diff <= -30):
            if (pitch_diff == -30 and tcp_diff == -6):
                #print("%% pitch_diff <<< but exception")
                line += ",,"
            else:
                #print("%% pitch_diff <<<")
                line += ",,,"
        return line

class LilypondGenerator(mp.MuseScoreParser):
    def get_head(self):
        string = []
        string.append("\\version \"2.24.1\"")
        string.append("\\include \"deutsch.ly\"")
        string.append("jazzChords = { \\semiGermanChords }")
        string.append("aFourL = {}")
        string.append("%\\include \"../config/include.ily\"")
        string.append("markMoj = #(define-music-function (letter) (string?) #{ \\mark \\markup { \\box \\bold #letter } #})")
        string.append("")
        string.append("\layout {")
        string.append("  indent = 0")
        string.append("}")
        return string

    def get_header(self):
        string = []
        string.append("\header {")
        string.append("  titlex = \"Pjevajte Jahvi\"")
        poet_found = False
        part_found = False
        for e in self.staffs[0].children:
            if isinstance(e, mp.VBox):
                if e.style == "Title":
                    string.append(f"  title = \"%s\"" % e.text.upper())
                elif e.style == "Composer":
                    string.append("  composer = \"%s\"" % e.text)
                elif e.style == "Lyricist":
                    string.append("  %%poet = \"%s\"" % e.text)
                    string.append("  style = \"%s\"" % e.text)
                    poet_found = True
                elif e.style == "Instrument Name (Part)":
                    string.append("  %%meter = \"%s\"" % e.text)
                    string.append("  broj = \"%s\"" % e.text)
                    part_found = True
        if not poet_found:
            string.append("  style = \"\"")
        if not part_found:
            string.append("  broj = \"1\"")
        string.append("  %tagline = \\markup { \\override #'(font-name . \"JohnSans White Pro\") \\override #'(font-size . -3) { Izvorno: Name, Album } }")

        string.append("}")
        return string

    def get_paper(self):
        string = []
        string.append("\\paper {")
        string.append("  \\aFourL")
        string.append("  %min-systems-per-page = #7")
        string.append("  %annotate-spacing = ##t")
        string.append("  %system-system-spacing.padding = #3.2")
        string.append("  %page-breaking = #ly:one-page-breaking")
        string.append("  %last-bottom-spacing.minimum-distance = #8")
        string.append("}")
        return string

    def get_staff_start(self, staff):
        string = []
        string.append("staff%s = \\relative c' {" % parser_name[staff.id])
        return string

    def get_staff_end(self):
        string = []
        string.append("}")
        return string

    def fractions_add_missing(self, bar, time_signature):
        fraction_sum = Fraction(0)
        for e in bar:
            if isinstance(e, Fraction):
                fraction_sum += e
        if fraction_sum != time_signature:
            bar.append(time_signature - fraction_sum)
        return bar

    def fractions_sum_neighbor(self, bar):
        summed_bar = []
        fraction = None
        for e in bar:
            if isinstance(e, Fraction):
                if fraction is not None:
                    fraction += e
                else:
                    fraction = e
            else:
                if fraction is not None:
                    summed_bar.append(fraction)
                    fraction = None
                summed_bar.append(e)
        if fraction is not None:
            summed_bar.append(fraction)
            fraction = None
        return summed_bar

    def fractions_add_skip_if_bar_starts_with_fraction(self, bar):
        if len(bar) > 0 and isinstance(bar[0], Fraction):
            bar.insert(0, "s")
        return bar

    def fractions_convert_bar_with_fractions_to_ly(self, bar, lyrics=False):
        line = ""
        for e in bar:
            if isinstance(e, Fraction):
                if not lyrics:
                    line += parser_fraction_to_duration[str(e)]
                    line += " "
            else:
                line += e
                if lyrics:
                    line += " "
                if "bar" in e or "mark" in e or "clef" in e or "repeat" in e:
                    line += " "
                if "{" in e or "}" in e:
                    line += " "

        return line

    def fractions_convert_harmony_bar_with_fractions_to_ly(self, bar):
        line = ""
        harmony = None
        for e in bar:
            if isinstance(e, Fraction):
                if harmony is not None:
                    line += parser_tpc[harmony.root]
                line += parser_fraction_to_duration[str(e)]
                if harmony is not None:
                    if harmony.name:
                        line += ":" + parse_chord_names[harmony.name]
                    if harmony.base:
                        line += "/" + parser_tpc[harmony.base]
                line += " "
                harmony = None
            elif isinstance(e, mp.Harmony):
                harmony = e
            else:
                line += e

        return line

    def get_staff_data(self, staff):
        string = []
        for sc in staff.children:
            if isinstance(sc, mp.Measure):
                bar = []
                line = "  "
                has_break = False
                for e in sc.children:
                    if isinstance(e, mp.TimeSig):
                        string.append("  \\time %s/%s" % (e.sig_n, e.sig_d))
                    elif isinstance(e, mp.Tempo):
                        string.append("  \\tempo 4 = %s" % int((60 * float(e.tempo))))
                    elif isinstance(e, mp.Rest):
                        if e.duration_type == "measure":
                            bar.append("r")
                            predicted_duration = Fraction(e.duration)
                            bar.append(predicted_duration)
                        else:
                            bar.append("r")
                            predicted_duration = Fraction(parser_duration_fractions[e.duration_type])
                            predicted_duration *= Fraction(parser_dots_fractions[e.dots])
                            bar.append(predicted_duration)
                    elif isinstance(e, mp.Chord):
                        bar.append(get_pitch(e.note_pitch, e.note_tpc))
                        predicted_duration = Fraction(parser_duration_fractions[e.duration_type])
                        predicted_duration *= Fraction(parser_dots_fractions[e.dots])
                        bar.append(predicted_duration)
                    elif isinstance(e, mp.KeySig):
                        tpc_value = str(14 + int(e.accidental))
                        string.append("  \\key %s \\major" % parser_tpc[tpc_value])
                    elif isinstance(e, mp.ChordNoteSpanner):
                        if e.type == "Tie":
                            if e.next_location_fractions or e.next_location_measures:
                                bar.append("~")
                    elif isinstance(e, mp.ChordSpanner):
                        if e.type == "Slur":
                            if e.next_location_fractions or e.next_location_measures:
                                bar.append("(")
                            elif e.prev_location_fractions or e.prev_location_measures:
                                bar.append(")")
                    elif isinstance(e, mp.BarLine):
                        bar.append("\\bar \"%s\"" % parser_barline[e.subtype])
                    elif isinstance(e, mp.RehearsalMark):
                        #text = "\\mark \\markup { \\box \\bold %s }" % e.text
                        #bar.append(text)
                        text = "\\markMoj \"%s\"" % e.text
                        #text = "\\markMoj"
                        bar.append(text)
                        #text = "%\\markMojPonn"
                        #bar.append(text)
                    elif isinstance(e, mp.Clef):
                        if e.concert_clef_type:
                            text = "\\clef %s" % parser_clefs[e.concert_clef_type]
                            bar.append(text)
                        elif e.transposing_clef_type:
                            text = "\\clef %s" % parser_clefs[e.transposing_clef_type]
                            bar.append(text)
                    elif isinstance(e, mp.LayoutBreak):
                        if e.subtype == "line":
                            has_break = True
                    elif isinstance(e, mp.VoltaSpanner):
                        if e.next_location_measures:
                            text = "\\set Score.repeatCommands = #\'((volta \"%s\"))" % e.begin_text
                            bar.append(text)
                        elif e.prev_location_measures:
                            text = "\\set Score.repeatCommands = #\'((volta #f))"
                            bar.append(text)
                    elif isinstance(e, mp.Tuplet):
                        text = "\\tuplet %s/%s {" % (e.actual_notes, e.normal_notes)
                        bar.append(text)
                    elif isinstance(e, mp.EndTuplet):
                        text = "}"
                        bar.append(text)

                #line += str(bar) + "\n  "
                if sc.len:
                    line += "\\partial %s" % parser_fraction_to_duration[sc.len]
                    line += "\n  "
                line += self.fractions_convert_bar_with_fractions_to_ly(bar)
                if sc.end_repeat:
                    line += "\\bar \"%s\"" % parse_measure_end_repeat[sc.end_repeat]
                    line += " "
                line += "|"
                #if has_break:
                #    line += " \\break"
                string.append(line)
        return string

    def get_harmony(self, staff):
        string = []

        #harmony_found = False
        #for sc in staff.children:
        #    if isinstance(sc, mp.Measure):
        #        for e in sc.children:
        #            if isinstance(e, mp.Harmony):
        #                harmony_found = True
        #if not harmony_found:
        #    return string

        string.append("harmony%s = \chordmode  {" % parser_name[staff.id])
        time_signature = None
        for sc in staff.children:
            if isinstance(sc, mp.Measure):
                bar = []
                line = "  "
                for e in sc.children:

                    if isinstance(e, mp.TimeSig):
                        time_signature = Fraction(f"{e.sig_n}/{e.sig_d}")
                    elif isinstance(e, mp.Harmony):
                        bar.append(e)
                    elif isinstance(e, mp.Chord):
                        predicted_duration = Fraction(parser_duration_fractions[e.duration_type])
                        predicted_duration *= Fraction(parser_dots_fractions[e.dots])
                        bar.append(predicted_duration)
                    elif isinstance(e, mp.Rest):
                        if e.duration_type == "measure":
                            predicted_duration = Fraction(e.duration)
                            bar.append(predicted_duration)
                        else:
                            predicted_duration = Fraction(parser_duration_fractions[e.duration_type])
                            predicted_duration *= Fraction(parser_dots_fractions[e.dots])
                            bar.append(predicted_duration)
                    elif isinstance(e, mp.Location):
                        predicted_duration = Fraction(e.fractions)
                        bar.append(predicted_duration)
                if sc.len:
                    bar = self.fractions_add_missing(bar, Fraction(sc.len))
                else:
                    bar = self.fractions_add_missing(bar, time_signature)
                bar = self.fractions_sum_neighbor(bar)
                bar = self.fractions_add_skip_if_bar_starts_with_fraction(bar)
                line += self.fractions_convert_harmony_bar_with_fractions_to_ly(bar)
                #line += str(bar)
                line += "|"
                string.append(line)
        # force end bar
        string.append("  \\bar \"|.\"")
        string.append("}")
        return(string)

    def get_lyric_nos(self, staff):
        nos = []
        for sc in staff.children:
            if isinstance(sc, mp.Measure):
                for e in sc.children:
                    if isinstance(e, mp.Lyrics):
                        if e.no not in nos:
                            nos.append(e.no)
        return sorted(nos)

    def fractions_swap_with_elements(self, bar):
        swaped_bar = []
        fraction = None
        for e in bar:
            if isinstance(e, Fraction):
                if fraction is None:
                    fraction = e
                else:
                    swaped_bar.append(fraction)
                    fraction = e
            else:
                swaped_bar.append(e)
                if fraction is not None:
                    swaped_bar.append(fraction)
                    fraction = None
        if fraction is not None:
            swaped_bar.append(fraction)
            fraction = None
        return swaped_bar

    def get_lyric(self, staff, no):
        bars = []
        for sc in staff.children:
            if isinstance(sc, mp.Measure):
                bar = []
                lyric_handler = LyricHandler()
                for e in sc.children:
                    if isinstance(e, mp.Lyrics):
                        if e.no == no:
                            #print(repr(e.text))
                            if "\xa0" in e.text:
                                lyric_handler.text = "\"%s\"" % e.text
                            else:
                                lyric_handler.text = e.text
                            if e.syllabic in ["begin", "middle"]:
                                lyric_handler.extender_line = "--"
                            if e.ticks_f and e.ticks:
                                predicted_duration = - Fraction(e.ticks_f)
                                lyric_handler.extender_line = "__"
                                lyric_handler.extender_duration = abs(predicted_duration)
                    elif isinstance(e, mp.Chord):
                        if lyric_handler.note_duration is not None:
                            bar.append(lyric_handler)
                            lyric_handler = LyricHandler()
                        predicted_duration = Fraction(parser_duration_fractions[e.duration_type])
                        predicted_duration *= Fraction(parser_dots_fractions[e.dots])
                        lyric_handler.note_pitch = "c"
                        lyric_handler.note_duration = predicted_duration
                    elif isinstance(e, mp.Rest):
                        if e.duration_type == "measure":
                            if lyric_handler.note_duration is not None:
                                bar.append(lyric_handler)
                                lyric_handler = LyricHandler()
                            predicted_duration = Fraction(e.duration)
                            lyric_handler.note_pitch = "r"
                            lyric_handler.note_duration = predicted_duration
                        else:
                            if lyric_handler.note_duration is not None:
                                bar.append(lyric_handler)
                                lyric_handler = LyricHandler()
                            predicted_duration = Fraction(parser_duration_fractions[e.duration_type])
                            predicted_duration *= Fraction(parser_dots_fractions[e.dots])
                            lyric_handler.note_pitch = "r"
                            lyric_handler.note_duration = predicted_duration

                    if lyric_handler.note_duration is not None and lyric_handler.text is not None:
                        bar.append(lyric_handler)
                        lyric_handler = LyricHandler()

                if lyric_handler.note_duration is not None:
                    bar.append(lyric_handler)
                    lyric_handler = LyricHandler()

                bars.append(bar)

        # add slurs for extender line and replace non text notes to rests
        extender_duration = None
        for bar in bars:
            #print("|")
            for b in bar:
                #print("  ", b)
                if b.text is not None:
                    if b.extender_duration:
                        extender_duration = b.extender_duration - b.note_duration
                        #print(extender_duration, "adding (")
                        b.slur = "("
                else:
                    if extender_duration is None:
                        b.note_pitch = "r"
                    else:
                        extender_duration -= b.note_duration
                        #print(extender_duration, "calculating")
                        if extender_duration < 0:
                            extender_duration = None
                            #print("adding )")
                            b.slur = ")"

        string = []

        #string.append("%%test%s%s = {" % (parser_name[staff.id], parser_name[no]))
        #for bar in bars:
        #    for b in bar:
        #        line = "%  "
        #        line += str(b)
        #        string.append(line)
        #    string.append("%  |")
        #string.append("%}")
        #string.append("")

        string.append("aligner%s%s = \\relative {" % (parser_name[staff.id], parser_name[no]))
        for bar in bars:
            line = "  "
            for b in bar:
                line += b.note_pitch + parser_fraction_to_duration[str(b.note_duration)]
                if b.slur:
                    line += b.slur
                line += " "
            line += "|"
            if len(line.strip()):
                string.append(line)
        string.append("}")
        string.append("")
        string.append("lyric%s%s = \\lyricmode {" % (parser_name[staff.id], parser_name[no]))
        for bar in bars:
            line = "  "
            for b in bar:
                if b.text is not None:
                    line += b.text
                    line += " "
                if b.extender_line is not None:
                    line += b.extender_line
                    line += " "
            line += "%|"
            if len(line.strip()):
                string.append(line)
        string.append("}")
        return string

    def get_tbox(self):
        string = []

        #tbox_found = False
        #for e in self.staffs[0].children:
        #    if isinstance(e, mp.TBox):
        #        tbox_found = True
        #        break
        #if not tbox_found:
        #    return string

        stanzas = []
        lyrics = []
        for e in self.staffs[0].children:
            if isinstance(e, mp.TBox):
                if e.style == "Frame":
                    line_count = 0
                    for line in e.text.split("\n"):
                        line = line.strip()
                        if len(line) > 0:
                            if re.match("\\d\\.", line):
                                stanzas.append("    \\line { \\bold %s }" % line)
                            else:
                                line_count += 1
                                lyrics.append("    \\line { %s }" % line)
                        else:
                            stanzas.append("    \\vspace #%s" % (line_count))
                            line_count = 0
                            lyrics.append("    \\vspace #1")

        string.append("\\markup {")
        string.append("  \\column {")
        string += stanzas
        string.append("  }")
        string.append("  \\hspace #1")
        string.append("  \\column {")
        string += lyrics
        string.append("  }")
        string.append("}")
        return string

    def get_score(self):
        string = []
        string.append("\\score {")
        string.append("    <<")
        for staff in self.staffs:
            string.append("    \\new ChordNames { \\jazzChords \\harmony%s }" % parser_name[staff.id])
            string.append("    \\new Staff {")
            string.append("        <<")
            string.append("        \\new Voice { \\staff%s }" % parser_name[staff.id])
            for no in self.get_lyric_nos(staff):
                string.append("        \\new NullVoice = \"aligner%s%s\" { \\aligner%s%s }" % (parser_name[staff.id], parser_name[no], parser_name[staff.id], parser_name[no]))
                string.append("        \\new Lyrics \\lyricsto \"aligner%s%s\" { \\lyric%s%s }" % (parser_name[staff.id], parser_name[no], parser_name[staff.id], parser_name[no]))
            string.append("        >>")
            string.append("    }")
            #string.append("    \\new Staff {")
            #for no in self.get_lyric_nos(staff):
            #    string.append("        \\new Voice = \"aligner%s%s\" { \\transpose c c'' \\aligner%s%s }" % (parser_name[staff.id], parser_name[no], parser_name[staff.id], parser_name[no]))
            #string.append("    }")
        string.append("    >>")
        string.append("}")
        return(string)

    def get_file(self):
        string = []
        string += self.get_head()
        string.append("")
        string += self.get_header()
        string.append("")
        string += self.get_paper()
        string.append("")
        for s in self.staffs:
            string += self.get_staff_start(s)
            string += self.get_staff_data(s)
            string += self.get_staff_end()
            string.append("")
            string += self.get_harmony(s)
            string.append("")
            for no in self.get_lyric_nos(s):
                string += self.get_lyric(s, no)
                string.append("")
        string += self.get_score()
        string.append("")
        string += self.get_tbox()
        return(string)

if __name__ == "__main__":
    lg = LilypondGenerator(sys.argv[1])
    print("\n".join(lg.get_file()))


import numpy
import numpy as np
import pandas as pd
import pygad
import tlsh
import json
from tools import featurer
import sys
import csv
import tensorflow as tf
from tensorflow import keras
from keras import layers
import filenames_modified as filenames

MALWAREIDX = int(sys.argv[1])
BATCH_SIZE = 10

# print(MALWAREIDX)

arm_training = pd.read_csv(filenames.arm_training, header=None, index_col=False)
arm_validation = pd.read_csv(filenames.arm_validation, header=None, index_col=False)
arm_test = pd.read_csv(filenames.arm_test, header=None, index_col=False)

dataset_arm_training = np.asarray(arm_training.drop(columns=arm_training.columns[-2:]))
dataset_arm_validation = np.asarray(arm_validation.drop(columns=arm_validation.columns[-2:]))
dataset_arm_test = np.asarray(arm_test.drop(columns=arm_test.columns[-2:]))

labels_arm_training = np.asarray(arm_training[arm_training.columns[-1]])
labels_arm_validation = np.asarray(arm_validation[arm_validation.columns[-1]])
labels_arm_test = np.asarray(arm_test[arm_test.columns[-1]])

names_arm_training = arm_training[arm_training.columns[-2]]
names_arm_validation = arm_validation[arm_validation.columns[-2]]
names_arm_test = arm_test[arm_test.columns[-2]]

df_arm_malware_forpoison = pd.read_csv(filenames.forpoison_arm_malware, header=None, index_col=False)
df_arm_malware_forpoison = df_arm_malware_forpoison.drop(columns=df_arm_malware_forpoison.columns[-2:])
topredict = np.asarray([df_arm_malware_forpoison.iloc[MALWAREIDX],])

malwareTLSH = ""
mybytes = ""

def myfunc(solution):
    additional = np.array(solution).tobytes()
    return str(tlsh.hash(mybytes + additional))

def fitness_func(solution, solution_idx):
    poisonedTLSH = myfunc(solution)
    return 1 / tlsh.diff(malwareTLSH, poisonedTLSH)

# print(sys.argv)

num_generations = 500
num_parents_mating = 8
sol_per_pop = 20
# num_genes = 20
gene_type = numpy.uint8
init_range_low = 0
init_range_high = 255
stop_criteria = "saturate_200"

# percents = np.append(np.arange(0.5, 5.1, 0.5), [10, 20])
percents = [5, 10, 20]
benignnumbers = [30, 40]
BENIGNNUMBER = 50

with open(filenames.poisonJSON) as poison_json:
    poison = json.load(poison_json)
    with open(filenames.dir_malware_arm + str(poison["arm"]["malware"][MALWAREIDX]), "rb") as malware:
        malwareread = malware.read()
        malwareTLSH = str(tlsh.hash(malwareread))
    for BENIGNNUMBER in benignnumbers:
        with open("{}genetic_idx-{}_bening-{}_percent-5-10-20.csv".format(filenames.dir_results, MALWAREIDX, BENIGNNUMBER), "w") as results_file:
            csv_writer_r = csv.writer(results_file, lineterminator="\n")
            for percent in percents:
            # for num_genes in range(10, 101, 10):
                with open(filenames.dir_poison_data_genetic + "percent_" + str(percent) + "_" + str(MALWAREIDX) + ".csv", "w") as f:
                    csv_writer = csv.writer(f, lineterminator="\n")
                    for i in range(BENIGNNUMBER):
                        print("*{}: {}% - {}*".format(str(MALWAREIDX), percent, i))
                        filename = str(poison["arm"]["benign"][i])
                        with open(filenames.dir_bening_arm + filename, "rb") as benign:
                            mybytes = benign.read()
                        lenbytes = len(mybytes)
                        num_genes = int(lenbytes * percent / 100)
                        ga = pygad.GA(num_generations=num_generations,
                                    num_parents_mating=num_parents_mating,
                                    fitness_func=fitness_func,
                                    sol_per_pop=sol_per_pop,
                                    num_genes=num_genes,
                                    gene_type=gene_type,
                                    init_range_low=init_range_low,
                                    init_range_high=init_range_high,
                                    stop_criteria=stop_criteria
                                    )
                        ga.run()
                        # ga.plot_fitness()
                        best_solution, best_fitness, best_idx = ga.best_solution()
                        # print(best_solution, " - ", 1 / best_fitness)

                        csv_writer.writerow(featurer(myfunc(best_solution)))

                file_poison_arm_BM = filenames.dir_poison_data_genetic + "percent_" + str(percent) + "_" + str(MALWAREIDX) + ".csv"
                poisoned_arm_training = pd.read_csv(file_poison_arm_BM, index_col=False, header=None)

                poisoned_arm_training_base = poisoned_arm_training.sample(frac=1)
                poisoned_arm_training_new = arm_training.append(poisoned_arm_training, ignore_index=True).sample(frac=1)

                dataset_poisoned_arm_training_base = np.asarray(
                    poisoned_arm_training_base.drop(columns=poisoned_arm_training_base.columns[-2:]))
                dataset_poisoned_arm_training_new = np.asarray(
                    poisoned_arm_training_new.drop(columns=poisoned_arm_training_new.columns[-2:]))

                labels_poisoned_arm_training_base = np.asarray(poisoned_arm_training_base[poisoned_arm_training_base.columns[-1]])
                labels_poisoned_arm_training_new = np.asarray(poisoned_arm_training_new[poisoned_arm_training_new.columns[-1]])

                # MODIFIED
                base_model = keras.models.load_model(filenames.base_model)
                base_model.fit(dataset_poisoned_arm_training_base, labels_poisoned_arm_training_base, epochs=10, batch_size=BATCH_SIZE,
                            validation_data=(dataset_arm_validation, labels_arm_validation), verbose=0)
                [_, binary_accuracy_appended] = base_model.evaluate(dataset_arm_test, labels_arm_test, verbose=0)
                # print(binary_accuracy_appended)
                # base_model.save(filenames.models_iterative + "modified" + str(num_genes))

                [[predict_appended]] = base_model.predict(topredict, verbose=0)
                # print(predict_appended)

                # # NEW
                # poison_model = keras.Sequential(
                #     [
                #         layers.Dense(1, input_shape=(131,), activation="sigmoid")
                #     ]
                # )
                # poison_model.compile(loss=tf.keras.losses.BinaryCrossentropy(),
                #                      metrics=[tf.keras.metrics.BinaryAccuracy()])
                # poison_model.fit(dataset_poisoned_arm_training_new, labels_poisoned_arm_training_new, epochs=10, batch_size=BATCH_SIZE,
                #                  validation_data=(dataset_arm_validation, labels_arm_validation), verbose=0)
                # [_, binary_accuracy_new] = poison_model.evaluate(dataset_arm_test, labels_arm_test, verbose=0)
                # # print(binary_accuracy_appended)
                # # base_model.save(filenames.models_iterative + "poison" + str(num_genes))

                # [[predict_new]] = poison_model.predict(topredict, verbose=0)
                # # print(predict_new)

                results = [percent,
                        binary_accuracy_appended,
                        predict_appended]
                        #    binary_accuracy_new,
                        #    predict_new]
                print(results)

                csv_writer_r.writerow(results)
print("{} DONE".format(MALWAREIDX))


"""
no longer needed since pointnet2_ssg_cls can provide this form
"""
import torch
import torch.nn as nn
import torch.nn.functional as F

import sys,os
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
ROOT_DIR = os.path.dirname(os.path.dirname(BASE_DIR))
sys.path.append(ROOT_DIR)
sys.path.append(os.path.join(ROOT_DIR, 'ops','pointnet2_ops_lib', 'pointnet2_ops'))
from pointnet2_ops.pointnet2_modules import PointnetFPModule, PointnetSAModule
# from .pointnet2_modules import PointnetSAModule, PointnetSAModuleMSG
from .pointnet2_ssg_cls import PointNet2SSGCls

class PointNet2MSGCls(PointNet2SSGCls):
    """PointNet2 MSG for classification
    """
    def _build_model(self):

        # call the base method and then override SA_modules
        super()._build_model()

        self.SA_modules = nn.ModuleList()

        for i in range(len(self.radii)):
            self.SA_modules.append(
                PointnetSAModuleMSG(
                    npoint=self.npoints[i],
                    radii=self.radii[i],
                    nsamples=self.nsamples[i],
                    mlps=self.mlps[i],
                    use_xyz=self.use_xyz
                )
            )
        self.SA_modules.append(
            PointnetSAModule(
                mlp=self.mlps[-1],
                use_xyz=self.use_xyz
            )
        )


import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import os

def read_image(image):
    return mpimg.imread(image)

def format_image(image):
    return tf.image.resize(image[tf.newaxis, ...], [224, 224]) / 255.0

def get_category(img):
    """Write a Function to Predict the Class Name

    Args:
        img [jpg]: image file

    Returns:
        [str]: Prediction
    """

    path = 'static/model/'
    tflite_model_file = 'converted_model.tflite'

    # Load TFLite model and allocate tensors.
    with open(path + tflite_model_file, 'rb') as fid:
        tflite_model = fid.read()

    # Interpreter interface for TensorFlow Lite Models.
    interpreter = tf.lite.Interpreter(model_content=tflite_model)
    interpreter.allocate_tensors()

    # Gets model input and output details.
    input_index = interpreter.get_input_details()[0]["index"]
    output_index = interpreter.get_output_details()[0]["index"]
    input_img = read_image(img)
    format_img = format_image(input_img)
    # Sets the value of the input tensor
    interpreter.set_tensor(input_index, format_img)
    # Invoke the interpreter.
    interpreter.invoke()

    predictions_array = interpreter.get_tensor(output_index)
    predicted_label = np.argmax(predictions_array)

    class_names = ['rock', 'paper', 'scissors']

    return class_names[predicted_label]

def plot_category(img, current_time):
    """Plot the input image

    Args:
        img [jpg]: image file
    """
    read_img = mpimg.imread(img)
    ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
    file_path = os.path.join(ROOT_DIR + f'/static/images/output_{current_time}.png')
    print(file_path)

    if os.path.exists(file_path):
        os.remove(file_path)

    plt.imsave(file_path, read_img)


import traceback
import sys
from discord.ext import commands
import discord

class ErrorHandler(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

    @commands.Cog.listener()
    async def on_command_error(self, ctx, error):
        if hasattr(ctx.command, 'on_error'):
            return

        ignored = (commands.CommandNotFound)
        error = getattr(error, 'original', error)

        if isinstance(error, ignored):
            print("Command not found: ", error)
            return

        elif isinstance(error, commands.DisabledCommand):
            return await ctx.send(f'{ctx.command} has been disabled.')

        elif isinstance(error, commands.NoPrivateMessage):
            try:
                return await ctx.author.send(f'{ctx.command} can not be used in Private Messages.')
            except:
                pass

        elif isinstance(error, discord.ext.commands.errors.MissingRequiredArgument):
            return await ctx.send(error)

        else:
            print(error)
            return

def setup(bot):
    bot.add_cog(ErrorHandler(bot))


# -*- coding: utf-8 -*-
"""
Created on Mon Apr  1 21:36:27 2019

@author: Rodrigo
"""
import csv
import sqlite3

e = csv.writer(open('output.csv', 'w'))
e.writerow(['cpf','UC'])

conn = sqlite3.connect('enel.db')
cursor = conn.cursor()

# lendo os dados
cursor.execute("""
SELECT * FROM enel;
""")

for linha in cursor.fetchall():
    e.writerow(linha)
    print(linha)

conn.close()


from django.urls import path, re_path
from . import views

app_name = 'adminapp'
urlpatterns = [
    path('', views.login, name='login'),
    path('category/add/', views.add_category, name='add_category'),
    path('article/add/', views.add_post),
    path('article/list/', views.post_list),
    path('category/list/', views.category_list),
    path('event/list/', views.event_list),
    path('event/add/', views.add_event),
    path('page/list/', views.page_list),
    path('menu/list/', views.menu_list),
    path('banner/add/', views.add_banner),
    path('banner/list/', views.banner_list),
    re_path('banner/edit/(?P<pk>[a-zA-Z0-9_-]+)/', views.edit_banner),
    re_path('banner/delete/(?P<pk>[a-zA-Z0-9_-]+)/', views.delete_banner),
    re_path('category/delete/(?P<pk>[a-zA-Z0-9_-]+)/', views.delete_category),
    re_path('article/delete/(?P<pk>[a-zA-Z0-9_-]+)/', views.delete_post),
    re_path('category/edit/(?P<pk>[a-zA-Z0-9_-]+)/', views.edit_category),
    re_path('event/delete/(?P<pk>[a-zA-Z0-9_-]+)/', views.delete_event),
    re_path('article/edit/(?P<pk>[a-zA-Z0-9_-]+)/', views.edit_post),
    re_path('article/change_state/(?P<pk>[a-zA-Z0-9_-]+)/', views.post_state),
    re_path('event/edit/(?P<pk>[a-zA-Z0-9_-]+)/', views.edit_event),
    re_path('event/change_state/(?P<pk>[a-zA-Z0-9_-]+)/', views.event_state),
    path('admin_logout/', views.admin_logout),
    path('menu/add/', views.add_menu),
    re_path('menu/edit/(?P<pk>[a-zA-Z0-9_-]+)/', views.edit_menu),
    re_path('menu/delete/(?P<pk>[a-zA-Z0-9_-]+)/', views.delete_menu),
    re_path('menu/change_state/(?P<pk>[a-zA-Z0-9_-]+)/', views.menu_state),
    re_path('menu/lvl-up/(?P<pk>[a-zA-Z0-9_-]+)/', views.menu_lvl_up),
    re_path('menu/lvl-down/(?P<pk>[a-zA-Z0-9_-]+)/', views.menu_lvl_down),
    re_path('delete_gal_img/(?P<pk>[a-zA-Z0-9_-]+)/(?P<pid>[a-zA-Z0-9_-]+)',
            views.delete_gal_image),
    re_path('delete_page_imgs/(?P<pk>[a-zA-Z0-9_-]+)/(?P<pid>[a-zA-Z0-9_-]+)',
            views.delete_page_images),
    path('page/add/', views.add_page),
    re_path('page/edit/(?P<pk>[a-zA-Z0-9_-]+)/', views.edit_page),
    re_path('page/delete/(?P<pk>[a-zA-Z0-9_-]+)/', views.delete_page),
    path("ajax/photos/upload/", views.upload_photos, name="upload_photos"),
    path("ajax/photos/recent/", views.recent_photos, name="recent_photos"),
    path('change_password/', views.change_password),
    # path("tags/list/", views.tags_list),
    # re_path('tag/delete/(?P<pk>[a-zA-Z0-9_-]+)/', views.delete_tag),
]


import json

from django.core.management import call_command
from django.core.management.base import BaseCommand
from people.models import Person, Address

class Command(BaseCommand):
    help = 'Loads sample data into the database'

    def handle(self, *args, **options):
        # Clear the database
        call_command('flush', '--noinput')

        with open('sample_data.json') as f:
            people_data = json.load(f)

        for person_data in people_data:
            address_data = person_data.pop('address')
            address = Address.objects.create(**address_data)
            Person.objects.create(address=address, **person_data)

        self.stdout.write(self.style.SUCCESS('Successfully loaded sample data'))


# Given an integer n, count the total number of digit 1 appearing in all non-negative integers less than or equal to n.

# Example 1:

# Input: n = 13
# Output: 6
# Example 2:

# Input: n = 0
# Output: 0

# Constraints:

# 0 <= n <= 2 * 109
class Solution:
    def countDigitOne(self, n: int) -> int:
        strn =  str(n)
        lenn = len(strn)
        res = 0
        for i,c in enumerate(strn):
            #print(i,c)
            res += int(c) * (lenn - 1) * 10 ** (lenn - 2)  #比如567 第一位5 lenn为3， 0-99有(lenn-1)*10**(lenn-2)个
                                                           #现在是5，有5个 0-99(0-99 100-199 200-299 300-399 400-499)所以前面乘以int(c)
            if c>'1':
                res += 10**(lenn-1) #如果大于1， 比如还是5的时候， 则100-199 百位上的1，一共100个，就是  10**(lenn-1)
            elif c=='1': #如果等于1比如167,那么就不是所有100-199百位上的，而是100-167 一共68 个
                res += int(strn[i+1:])+1 if i<len(strn)-1 else 1
            lenn -= 1
        return int(res)

# 0 -9 1
# 0 -99 20
# 0 -999 300
# 0 -9999 4000


from src.knowledge_graph import KGEntity, KGProperty
from .kgqa_dataset import KGQADataSet
from .kgqa_data import KGQAData
from typing import List
import logging
import json

class Mintaka(KGQADataSet):

    def load(self, path: str) -> List[KGQAData]:
        datasets: List[KGQAData] = []

        with open(path, encoding='utf-8') as f:
            json_dict = json.load(f)
            for mintaka_data in json_dict:
                question_id = mintaka_data["id"]
                raw_question = mintaka_data["question"]

                answer_data = mintaka_data["answer"]["answer"]
                answers = []
                if answer_data:
                    for answer in answer_data:
                        if type(answer) is dict:
                            answers.append(KGEntity(answer["label"]["en"]))
                        elif type(answer) is bool:
                            # add additional answers yes/no for true/false question
                            if answer == True:
                                answers.append(KGEntity("True"))
                                answers.append(KGEntity("Yes"))
                            elif answer == False:
                                answers.append(KGEntity("False"))
                                answers.append(KGEntity("No"))
                            else:
                                print("Invalid boolean value")
                                continue
                        else:
                            answers.append(KGEntity(str(answer)))
                else:
                    continue
                datasets.append(KGQAData(question_id, raw_question, answers))

        logging.info(f"number of parsed questions: {len(datasets)}")

        return datasets


#coding:utf-8
from weixin import WXAPPAPI

api = WXAPPAPI(appid=APP_ID,
                  app_secret=APP_SECRET)
session_info = api.exchange_code_for_session_key(code=code)

# 获取session_info 后

session_key = session_info.get('session_key')
crypt = WXBizDataCrypt(WXAPP_APPID, session_key)

# encrypted_data 包括敏感数据在内的完整用户信息的加密数据
# iv 加密算法的初始向量
# 这两个参数需要js获取
user_info = crypt.decrypt(encrypted_data, iv)


#!/usr/bin/env python3
"""A denselayer dense block in tensorflow keras"""
import tensorflow.keras as K

def dense_block(X, nb_filters, growth_rate, layers):
    """A dense block, X is the previous layer, nb_filters is the number of
    filters to use, growth rate is the rate to change the number of filters
    by, and layers is how many layers"""
    initializer = K.initializers.he_normal()
    filter_total = 0
    for layer in range(0, layers):
        batch_normalization = K.layers.BatchNormalization()(X)
        activation = K.layers.Activation("relu")(batch_normalization)
        conv2d = K.layers.Conv2D(filters=growth_rate * 4,
                                 kernel_size=(1, 1),
                                 padding="same",
                                 kernel_initializer=initializer)(activation)
        batch_normalization1 = K.layers.BatchNormalization()(conv2d)
        activation1 = K.layers.Activation("relu")(batch_normalization1)
        conv2d1 = K.layers.Conv2D(filters=growth_rate,
                                  kernel_size=(3, 3),
                                  padding="same",
                                  kernel_initializer=initializer)(activation1)
        concatenate = K.layers.concatenate([X, conv2d1])
        X = concatenate
        filter_total += growth_rate
    return X, filter_total + nb_filters


import argparse
import auxil.mydata as mydata
import auxil.mymetrics as mymetrics
import gc
import tensorflow as tf
import keras.backend as K
from keras.callbacks import ModelCheckpoint
from keras.models import load_model
from keras.losses import categorical_crossentropy
from keras.layers import *
from keras.models import Sequential, Model
from keras.optimizers import Adam
from keras import regularizers
from keras.models import Model
from keras.utils import to_categorical as keras_to_categorical
import numpy as np
import sys

class AttentionBlock(Layer):
    def __init__(self, filters):

        super(AttentionBlock, self).__init__()
        self.filters = filters
        #self.init = RandomNormal()
    def call(self, x):
        conv_3d = Conv3D(filters = self.filters, kernel_size=3, strides = 1, padding = 'same')(x)
        conv_3d_shape = conv_3d._keras_shape
        print(conv_3d_shape)
        conv_3d = Reshape((conv_3d_shape[1], conv_3d_shape[2], conv_3d_shape[3]*conv_3d_shape[4]))(conv_3d)

        conv_2d = Conv2D(filters = self.filters, kernel_size=3, strides = 1, padding = 'same')(conv_3d)
        conv_2d_shape = conv_2d._keras_shape
        print(conv_2d_shape)

        conv_2d = Reshape((conv_2d_shape[1],conv_2d_shape[2]*conv_2d_shape[3]))(conv_2d)

        conv_1d = Conv1D(filters = self.filters, kernel_size=3, strides = 1, padding = 'same')(conv_2d)
        conv_1d_shape = conv_1d._keras_shape
        print(conv_1d_shape)

        gap = GlobalAveragePooling1D()(conv_1d)
        fc = Dense(self.filters, use_bias = True)(gap)
        softmax = Activation('softmax')(fc)

        reshape_1d = Reshape((1, self.filters))(softmax)
        deconv_1d = Conv1D(filters = self.filters, kernel_size = 3, strides = 1, padding = 'same')(reshape_1d)
        reshape_2d = Reshape((1,1, self.filters))(deconv_1d)
        deconv_2d = Conv2DTranspose(filters = self.filters, kernel_size=3, strides = 1, padding = 'same')(reshape_2d)
        reshape_3d = Reshape((1,1,1, self.filters))(deconv_2d)
        deconv_3d = Conv3DTranspose(filters = self.filters, kernel_size = 3, strides = 1, padding = 'same')(reshape_3d)
        x = tf.multiply(deconv_3d, x)
        return x

def set_params(args):
    args.batch_size = 64
    args.epochs = 200
    return args

def get_model_compiled(shapeinput, num_class, w_decay=0):
    inputs = Input((shapeinput[0],shapeinput[1],shapeinput[2],1))
    filters = [4,4,4,8]
    x = Conv3D(filters=4,use_bias=False,kernel_size=(3,3,5), padding =       'valid',strides = 1)(inputs)
    x = BatchNormalization()(x)
    x = LeakyReLU()(x)

    for i in range(4):
      x = Conv3D(filters=filters[i],use_bias=False,  kernel_size=(3,3,5),padding = 'valid',strides = 1)(x)
      a1 = AttentionBlock(filters[i])(x)
      #a1 = LeakyReLU()(a1)
      b1 = AttentionBlock(filters[i])(x)
      #b1 = LeakyReLU()(b1)
      x = Add()([a1,b1])
      x = Dropout(0.4)(x)
      x = BatchNormalization()(x)
      x = LeakyReLU()(x)

    x = Dropout(0.85)(x)
    x = Flatten()(x)
    x = Dropout(0.85)(x)
    x = Dense(units=128, use_bias=True)(x)
    x = LeakyReLU()(x)
    x = Dense(units=64, use_bias=True)(x)
    x = LeakyReLU()(x)

    output_layer = Dense(units=num_class, activation='softmax')(x)
    clf = Model(inputs=inputs, outputs=output_layer)
    clf.compile(loss='categorical_crossentropy',
                optimizer=Adam(learning_rate=0.001), metrics=['accuracy'])
    return clf

def main():
    parser = argparse.ArgumentParser(description='Algorithms traditional ML')
    parser.add_argument('--dataset', type=str, required=True,
                        choices=["IP", "UP", "SV", "UH",
                                 "DIP", "DUP", "DIPr", "DUPr"],
                        help='dataset (options: IP, UP, SV, UH, DIP, DUP, DIPr, DUPr)')
    parser.add_argument('--repeat', default=1, type=int, help='Number of runs')
    parser.add_argument('--components', default=None,
                        type=int, help='dimensionality reduction')
    parser.add_argument('--spatialsize', default=9,
                        type=int, help='windows size')
    parser.add_argument('--wdecay', default=0.02, type=float,
                        help='apply penalties on layer parameters')
    parser.add_argument('--preprocess', default="standard",
                        type=str, help='Preprocessing')
    parser.add_argument('--splitmethod', default="sklearn",
                        type=str, help='Method for split datasets')
    parser.add_argument('--random_state', default=42, type=int,
                        help='The seed of the pseudo random number generator to use when shuffling the data')
    parser.add_argument('--tr_percent', default=0.1,
                        type=float, help='samples of train set')
    parser.add_argument('--use_val', action='store_true',
                        help='Use validation set')
    parser.add_argument('--val_percent', default=0.1,
                        type=float, help='samples of val set')
    parser.add_argument(
        '--verbosetrain', action='store_true', help='Verbose train')
    #########################################
    parser.add_argument('--set_parameters', action='store_false',
                        help='Set some optimal parameters')
    ############## CHANGE PARAMS ############
    parser.add_argument('--batch_size', default=64, type=int,
                        help='Number of training examples in one forward/backward pass.')
    parser.add_argument('--epochs', default=100, type=int,
                        help='Number of full training cycle on the training set')
    #########################################

    args = parser.parse_args()
    state = {k: v for k, v in args._get_kwargs()}

    if args.set_parameters:
        args = set_params(args)

    pixels, labels, num_class = \
        mydata.loadData(args.dataset, num_components=args.components,
                        preprocessing=args.preprocess)
    pixels, labels = mydata.createImageCubes(
        pixels, labels, windowSize=args.spatialsize, removeZeroLabels=False)
    stats = np.ones((args.repeat, num_class+3)) * -1000.0  # OA, AA, K, Aclass
    for pos in range(args.repeat):
        rstate = args.random_state+pos if args.random_state != None else None
        if args.dataset in ["UH", "DIP", "DUP", "DIPr", "DUPr"]:
            x_train, x_test, y_train, y_test = \
                mydata.load_split_data_fix(
                    args.dataset, pixels)  # , rand_state=args.random_state+pos)
        else:
            pixels = pixels[labels != 0]
            labels = labels[labels != 0] - 1
            x_train, x_test, y_train, y_test = \
                mydata.split_data(
                    pixels, labels, args.tr_percent, rand_state=rstate)

        if args.use_val:
            x_val, x_test, y_val, y_test = \
                mydata.split_data(
                    x_test, y_test, args.val_percent, rand_state=rstate)

        inputshape = x_train.shape[1:]
        clf = get_model_compiled(inputshape, num_class, w_decay=args.wdecay)
        valdata = (x_val, keras_to_categorical(y_val, num_class)) if args.use_val else (
            x_test, keras_to_categorical(y_test, num_class))
        clf.fit(x_train, keras_to_categorical(y_train, num_class),
                batch_size=args.batch_size,
                epochs=args.epochs,
                verbose=args.verbosetrain,
                validation_data=valdata,
                callbacks=[ModelCheckpoint("/tmp/best_model.h5", monitor='val_accuracy', verbose=0, save_best_only=True)])
        clf.load_weights("/tmp/best_model.h5")
        clf.compile(loss='categorical_crossentropy',
                optimizer=Adam(learning_rate=0.001), metrics=['accuracy'])
        print("PARAMETERS", clf.count_params())
        stats[pos, :] = mymetrics.reports(
            np.argmax(clf.predict(x_test), axis=1), y_test)[2]
    print(args.dataset, list(stats[-1]))

if __name__ == '__main__':
    main()