#!/usr/bin/python3
# -*- coding: utf-8 -*-
# SEE this link for symbols https://latex.wikia.org/wiki/List_of_LaTeX_symbols
# LEARN some python for asigmning expresions as ":=" in https://www.python.org/dev/peps/pep-0572/
from pylatex import Document, Section, Subsection, Command
from pylatex.package import Package
from pylatex.utils import italic, NoEscape
from os import path, chdir, listdir, remove
from subprocess import check_output, STDOUT, CalledProcessError
from re import match
import json
import argparse
import configparser
# Configure Initial Variables
config_file = 'config/nconfig.cfg'
# Configure Enviroment
config_file = path.join(path.abspath(path.dirname(__file__)), config_file)
current_dir_path = path.abspath(path.dirname(__file__))
if (current_dir_path != path.abspath(path.dirname(__file__))):
    chdir(current_dir_path)
assert current_dir_path == path.abspath(path.dirname(__file__))
print("__config file__: {}".format(config_file))
assert path.isfile(config_file)
config = configparser.ConfigParser()
config.read(path.join(path.abspath(path.dirname(__file__)), config_file))
# Start
if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--verbose", "--v", action="store_true", help="Increase output verbosity")
    parser.add_argument("--symbol", "--s", default= "data.json", type=str, help="Symbol path to the symbol '.json' file")
    parser.add_argument("--output", "--o", default= "out.pdf", type=str, help="Path to output file. The extension defines the output.")
    parser.add_argument("--clean", "--c", action="store_true", help="Whether or  not to clean output folder from latex related aux files generating while rendeing the '.tex' file.")
    parser.add_argument("--make_pdf", "--mp", default= True, type=bool, help="Flag wheter to produce '.pdf' output file")
    parser.add_argument("--make_tex", "--mt", default= False, type=bool, help="Flag wheter to produce '.tex' output file")
    args = vars(parser.parse_args())
    if args["verbose"]:
        print("<WORKING DIRECTORY>: {}".format(current_dir_path))
        print("<CONFIG FILE>: {}".format(config_file))
        print("<CONFIG FILE DATA>:")
        for i in ["\t{}: \n\t\t{}".format(section, dict(config.items(section))) for section in dict(config.items()).keys()]:
            print(i)
        print("<SCRIPT ARGUMENTS>: {}".format(args))
# Defined functions
def import_libs_in_tex(tex_file):
    for pkg in config.get("LATEX","pakages").split(','):
        tex_file.write(pkg.strip())
        if(args["verbose"]):
            print("<LATEX PAKAGE LOADED>: {}".format(pkg.strip()))
def latexmk_render(dir_path, file_name, desired_ext, clean = False, latexmk_params = 'CAS_Default'):
    if latexmk_params == 'CAS_Default':
        latexmk_params = [param.strip() for param in config.get("LATEX","latexmk_args").split(',')]
    if desired_ext == '.pdf':
        latexmk_params += ['--dvi-', '--pdf', '--ps-']
    elif desired_ext == '.dvi':
        latexmk_params += ['--dvi', '--pdf-', '--ps-']
    elif desired_ext == '.tex':
        latexmk_params += ['--dvi-', '--pdf-', '--ps-']
    elif desired_ext == '.ps':
        latexmk_params += ['--dvi-', '--pdf-', '--ps']
    else:
        assert False, ("OUTPUT file desired extensión must be either '.pdf', '.dvi', '.tex' or '.ps'")
    latexmk_params += [file_name, '--interaction=nonstopmode']
    command = [str(config.get("LATEX","latexmk_path"))] + latexmk_params
    seen_command = set() # set to filter no redudance in command
    command = [x for x in command if not (x in seen_command or seen_command.add(x))]
    if args["verbose"]:
        print('<LATEXMK COMMAND>: {}'.format(command))
    try:
        # Clean Directory
        output = check_output([str(config.get("LATEX","latexmk_path"))] + ['--C'], stderr=STDOUT)
        # Compile latex
        output = check_output(command, stderr=STDOUT)
        if args["verbose"]:
            print("<LATEXMK STDOUT START>: \n\t{}\n<LATEXMK STDOUT END>".format("\n\t".join(output.decode().split('\n'))))
        if clean: # Clean after folder
            command = [str(config.get("LATEX","latexmk_path"))] + ['--c']
            output = check_output(command, stderr=STDOUT)
            if args["verbose"]:
                print("<LATEXMK STDOUT (clean action) START>: \n\t{}\n<LATEXMK STDOUT (clean action) END>".format("\n\t".join(output.decode().split('\n'))))
    except CalledProcessError as exc:
        print("ERROR from LATEXMK: \n\t{}".format(exc.output))
    except:
        assert False, "ERROR invocando latexmk"
def make_symbol_tex(out = 'out.tex'):
    dir_path =  path.dirname(path.abspath(out))
    file_name, desired_ext = path.splitext(path.basename(out))
    assert desired_ext in [e.strip() for e in config.get("LATEX","latexmk_valid_out").split(',')], "OUTPUT file desired extensión must be either '.pdf', '.dvi', '.tex' or '.ps'"
    tex_file = path.join(dir_path,file_name+'.tex')
    if args["verbose"]:
        print("<MAKE_SYMBOL_TEX> args :\n\tdir_path: {}; \n\tfile_name: {}; \n\tdesired_ext: {}; \n\ttex_file: {}".format(dir_path, file_name, desired_ext, tex_file))
    with open(tex_file, 'w') as tex_file:
        tex_file.write(r"\documentclass[]{article}%"+"\n")
        tex_file.write(r"\usepackage[T1]{fontenc}%"+"\n")
        tex_file.write(r"\usepackage[utf8]{inputenc}%"+"\n")
        tex_file.write(r"\usepackage[]{lmodern}%"+"\n")
        tex_file.write(r"\usepackage[]{textcomp}%"+"\n")
        tex_file.write(r"\usepackage["+config.get("LATEX", "geometry")+r"]{geometry}%"+"\n")
        tex_file.write(r"\usepackage[]{amsmath}%"+"\n")
        tex_file.write(r"\usepackage[]{amsfonts}%"+"\n")
        tex_file.write(r"\usepackage[]{amssymb}%"+"\n")
        tex_file.write(r"\usepackage[]{tabularx}%"+"\n")
        tex_file.write(r"\usepackage[]{multirow}%"+"\n")
        tex_file.write(r"\usepackage[]{graphics}%"+"\n")
        tex_file.write(r"\usepackage[]{graphicx}%"+"\n")
        tex_file.write(r"\usepackage[]{ragged2e}%"+"\n")
        # tex_file.write(r"\newcolumntype{L}{>{\raggedleft\arraybackslash}X}"+"\n")
        # tex_file.write(r"\newcolumntype{U}{>{\raggedright\arraybackslash}X}"+"\n")
        tex_file.write(r"\usepackage{array}%"+"\n")
        tex_file.write(r"\newcolumntype{L}[1]{>{\raggedright\let\newline\\\arraybackslash\hspace{0pt}}m{#1}}%"+"\n")
        tex_file.write(r"\newcolumntype{C}[1]{>{\centering\let\newline\\\arraybackslash\hspace{0pt}}m{#1}}%"+"\n")
        tex_file.write(r"\newcolumntype{R}[1]{>{\raggedleft\let\newline\\\arraybackslash\hspace{0pt}}m{#1}}%"+"\n")
        tex_file.write(r"%"+"\n")
        tex_file.write(r"%"+"\n")
        tex_file.write(r"%"+"\n")
        tex_file.write(r"%"+"\n")
        tex_file.write(r"\begin{document}%"+"\n")
        tex_file.write(r"\normalsize%"+"\n")
        # tex_file.write(r"\begin{table}[]"+"\n")
        # tex_file.write(r"\begin{tabular}{crcll}"+"\n")
        # tex_file.write(r"\multicolumn{5}{c}{top}                                                    \\"+"\n")
        # tex_file.write(r"\multicolumn{1}{r}{a} & b   & \multirow{2}{*}{SYMBOL} & lower & lowerlower \\"+"\n")
        # tex_file.write(r"\multicolumn{1}{r}{c} & RXR &                         & d     & e          \\"+"\n")
        # tex_file.write(r"\multicolumn{5}{c}{bot}                                                   "+"\n")
        # tex_file.write(r"\end{tabular}"+"\n")
        # tex_file.write(r"\end{table}"+"\n")
        # tex_file.write(r"\end{document}")
        # SAecond way of making
        flag = True
        symWidth = 200
        symColsNum = 5
        TOP = {

            'a01':r'\multicolumn',
            'a02':r'{',
            'a03':r'5',
            'a04':r'}',
            'a05':r'{',
            'a06':r'|',
            'a07':r'c',
            'a08':r'|',
            'a09':r'}',
            'a10':r'{',
            'a11':r'TOP',
            'a12':r'}',

            'z01':r'\cline{1-1}',
            'z02':r'\cline{2-2}',
            'z03':r'\cline{4-4}',
            'z04':r'\cline{5-5}'
        }
        UP  = {

            'a01':r'\multicolumn',
            'a02':r'{',
            'a03':r'1',
            'a04':r'}',
            'a05':r'{',
            'a06':r'|',
            'a07':r'C{'+str(int(symWidth/symColsNum))+r'px}',
            'a08':r'|',
            'a09':r'}',
            'a10':r'{',
            'a11':r'a',
            'a12':r'}',

            'b01':r'\multicolumn',
            'b02':r'{',
            'b03':r'1',
            'b04':r'}',
            'b05':r'{',
            'b06':r'|',
            'b07':r'C{'+str(int(symWidth/symColsNum))+r'px}',
            'b08':r'|',
            'b09':r'}',
            'b10':r'{',
            'b11':r'b',
            'b12':r'}',

            'c01':r'\multirow',
            'c02':r'{',
            'c03':r'2',
            'c04':r'}',
            'c05':r'{',
            'c06':r'',
            'c07':r'*',
            'c08':r'',
            'c09':r'}',
            'c10':r'{',
            'c11':r'\fontsize{8}{10}\selectfont $\Omega$',
            'c12':r'}',

            'd01':r'\multicolumn',
            'd02':r'{',
            'd03':r'1',
            'd04':r'}',
            'd05':r'{',
            'd06':r'|',
            'd07':r'C{'+str(int(symWidth/symColsNum))+r'px}',
            'd08':r'|',
            'd09':r'}',
            'd10':r'{',
            'd11':r'f',
            'd12':r'}',

            'e01':r'\multicolumn',
            'e02':r'{',
            'e03':r'1',
            'e04':r'}',
            'e05':r'{',
            'e06':r'|',
            'e07':r'C{'+str(int(symWidth/symColsNum))+r'px}',
            'e08':r'|',
            'e09':r'}',
            'e10':r'{',
            'e11':r'g',
            'e12':r'}',

            'z01':r'\cline{1-1}',
            'z02':r'\cline{2-2}',
            'z03':r'\cline{4-4}',
            'z04':r'\cline{5-5}'
        }
        DOWN = {

            'a01':r'\multicolumn',
            'a02':r'{',
            'a03':r'1',
            'a04':r'}',
            'a05':r'{',
            'a06':r'|',
            'a07':r'C{'+str(int(symWidth/symColsNum))+r'px}',
            'a08':r'|',
            'a09':r'}',
            'a10':r'{',
            'a11':r'c',
            'a12':r'}',

            'b01':r'\multicolumn',
            'b02':r'{',
            'b03':r'1',
            'b04':r'}',
            'b05':r'{',
            'b06':r'|',
            'b07':r'C{'+str(int(symWidth/symColsNum))+r'px}',
            'b08':r'|',
            'b09':r'}',
            'b10':r'{',
            'b11':r'd',
            'b12':r'}',

            'c01':r'',
            'c02':r'',
            'c03':r'',
            'c04':r'',
            'c05':r'',
            'c06':r'',
            'c07':r'',
            'c08':r'',
            'c09':r'',
            'c10':r'',
            'c11':r'',
            'c12':r'',

            'd01':r'\multicolumn',
            'd02':r'{',
            'd03':r'1',
            'd04':r'}',
            'd05':r'{',
            'd06':r'|',
            'd07':r'C{'+str(int(symWidth/symColsNum))+r'px}',
            'd08':r'|',
            'd09':r'}',
            'd10':r'{',
            'd11':r'h',
            'd12':r'}',

            'e01':r'\multicolumn',
            'e02':r'{',
            'e03':r'1',
            'e04':r'}',
            'e05':r'{',
            'e06':r'|',
            'e07':r'C{'+str(int(symWidth/symColsNum))+r'px}',
            'e08':r'|',
            'e09':r'}',
            'e10':r'{',
            'e11':r'i',
            'e12':r'}',

            'z01':r'\cline{1-1}',
            'z02':r'\cline{2-2}',
            'z03':r'\cline{4-4}',
            'z04':r'\cline{5-5}'
        }
        BOTTOM = {

            'a01':r'\multicolumn',
            'a02':r'{',
            'a03':r'5',
            'a04':r'}',
            'a05':r'{',
            'a06':r'|',
            'a07':r'c',
            'a08':r'|',
            'a09':r'}',
            'a10':r'{',
            'a11':r'BOT',
            'a12':r'}',

            'z01':r'',
            'z02':r'',
            'z03':r'',
            'z04':r''
        }
        tex_file.write(r"\begin{table}[!h]"+"\n")
        tex_file.write(r"\begin{minipage}[b]{"+str(int(symWidth))+r"px}%"+"\n")
        tex_file.write(r"\addtolength{\tabcolsep}{-5pt}"+"\n")
        tex_file.write(r"\setlength{\arrayrulewidth}{0.1pt}"+"\n")
        tex_file.write(r"\fontsize{4px}{6px}\selectfont"+"\n")
        tex_file.write(r"\begin{tabular*}{"+str(int(symWidth))+r"px}{C{"+str(int(symWidth/symColsNum))+r"px} C{"+str(int(symWidth/symColsNum))+r"px} C{"+str(int(symWidth/symColsNum))+r"px} C{"+str(int(symWidth/symColsNum))+r"px} C{"+str(int(symWidth/symColsNum))+r"px}}"+"\n")
        tex_file.write("""{a01}{a02}{a03}{a04}{a05}{a06}{a07}{a08}{a09}{a10}{a11}{a12}\\\\{z01}{z02}{z03}{z04}""".format(**TOP)+"\n")
        tex_file.write("""{a01}{a02}{a03}{a04}{a05}{a06}{a07}{a08}{a09}{a10}{a11}{a12}&{b01}{b02}{b03}{b04}{b05}{b06}{b07}{b08}{b09}{b10}{b11}{b12}&{c01}{c02}{c03}{c04}{c05}{c06}{c07}{c08}{c09}{c10}{c11}{c12}&{d01}{d02}{d03}{d04}{d05}{d06}{d07}{d08}{d09}{d10}{d11}{d12}&{e01}{e02}{e03}{e04}{e05}{e06}{e07}{e08}{e09}{e10}{e11}{e12}\\\\{z01}{z02}{z03}{z04}""".format(**UP)+"\n")
        tex_file.write("""{a01}{a02}{a03}{a04}{a05}{a06}{a07}{a08}{a09}{a10}{a11}{a12}&{b01}{b02}{b03}{b04}{b05}{b06}{b07}{b08}{b09}{b10}{b11}{b12}&{c01}{c02}{c03}{c04}{c05}{c06}{c07}{c08}{c09}{c10}{c11}{c12}&{d01}{d02}{d03}{d04}{d05}{d06}{d07}{d08}{d09}{d10}{d11}{d12}&{e01}{e02}{e03}{e04}{e05}{e06}{e07}{e08}{e09}{e10}{e11}{e12}\\\\{z01}{z02}{z03}{z04}""".format(**DOWN)+"\n")
        tex_file.write("""{a01}{a02}{a03}{a04}{a05}{a06}{a07}{a08}{a09}{a10}{a11}{a12}\\\\{z01}{z02}{z03}{z04}""".format(**BOTTOM)+"\n")
        tex_file.write(r"\end{tabular*}"+"\n")
        tex_file.write(r"\end{minipage}"+"\n")
        tex_file.write(r"\end{table}"+"\n")
        tex_file.write(r"\end{document}")
    if config.get("LATEX", "generator").lower().strip() == 'latexmk':
        latexmk_render(dir_path = dir_path, file_name = file_name, desired_ext= desired_ext, clean = args["clean"])
    else: # Not recognized selected latex compiler
        assert False, ("'{}' is not recognized as a valid latex builder. Please configure for {} the variable to be one of the valid generators for latex: {}".format(config.get("LATEX", "generator"), config_file, ['latexmk']))
    # Assert task Ending
    if (not args["verbose"]):
        assert path.isfile(out), ("ERROR: output file not found. run script with --verbose option enable")
    else:
        assert path.isfile(out), ("ERROR: output file not found.")
    print("__output file__: {}".format(path.abspath(out)))
# Ta,latexmk_params = ''sk
if __name__ == '__main__':
    # Document with `\maketitle`  + 'latexmk_params'command activated
    # make_document(name="out", geometry_options={"tmargin": "1pt", "lmargin": "10pt", "paperwidth":"100pt","paperheight":"100pt"})
    make_symbol_tex(out = args["output"])


from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets(".", one_hot=True, reshape=False)

import tensorflow as tf

# parameters
learning_rate = 0.001
training_epochs = 20
batch_size = 128
display_step = 1

n_input = 784
n_classes = 10

n_hidden_layer = 256

weights = {
    'hidden_layer': tf.Variable(tf.random_normal([n_input, n_hidden_layer])),
    'out': tf.Variable(tf.random_normal([n_hidden_layer, n_classes]))
}

biases = {
    'hidden_layer': tf.Variable(tf.random_normal([n_hidden_layer])),
    'out': tf.Variable(tf.random_normal([n_classes]))
}

# tf graph input
x = tf.placeholder(tf.int32, [None, 28, 28, 1])
y = tf.placeholder(tf.int32, [None, n_classes])
x_flat = tf.reshape(tf.cast(x, tf.float32), [-1, n_input])

# hidden layer with RELU activation
layer_1 = tf.add(tf.matmul(x_flat, weights['hidden_layer']), biases["hidden_layer"])
layer_1 = tf.nn.relu(layer_1)

# output layer with linear activation
logits = tf.add(tf.matmul(layer_1, weights["out"]), biases["out"])

# optimizer
# define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cost)

# Evaluate the accuracy of the model
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(logits, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

# session

init = tf.global_variables_initializer()

with tf.Session() as sess:
    sess.run(init)

    for epoch in range(training_epochs):
        total_batch = int(mnist.train.num_examples / batch_size)

        for i in range(total_batch):
            batch_x, batch_y = mnist.train.next_batch(batch_size)
            sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})

    print(sess.run(accuracy, feed_dict={x: mnist.test.images, y: mnist.test.labels}))

# define the input function for training


'''
Question:
You are given a function f(x)=x^2. You are also given K lists. The ith list consists of Ni elements.

You have to pick one element from each list so that the value from the equation below is maximized:
S=(f(x1)+f(x2)+...+f(xk))%M

Xi denotes the element picked from the ith list . Find the maximized value Smax obtained.

% denotes the modulo operator.

Note that you need to take exactly one element from each list, not necessarily the largest element. You add the squares of the chosen elements and perform the modulo operation. The maximum value that you can obtain, will be the answer to the problem.

Input Format
The first line contains  space separated integers K and M.
The next K lines each contains an integer Ni, denoting the number of elements in the ith list, followed by Ni space separated integers denoting the elements in the list.

Output Format
Output a single integer denoting the value Smax.

Sample Input
3 1000
2 5 4
3 7 8 9
5 5 7 8 9 10

Sample Output
206

Explanation
Picking 5 from the 1st list, 9 from the 2nd list and 10 from the 3rd list gives the maximum S value equal to (5^2+9^2+10^2)% 1000= 206.
'''
# Enter your code here. Read input from STDIN. Print output to STDOUT
from itertools import product

def fofx(tup,M):
    lst6 = list()
    res1 = 0
    lst4 = list(tup)
    for i in range(len(lst4)):
        lst6 = lst4[i]
        res1 = res1 + (lst6*lst6)

    return res1%M

def main():
    K, M = map(int,input().split())
    res = 0
    res1 = 0
    lst2 = list()
    lst5 = list()

    for i in range(K):
        lst1 = list(map(int,input().split()))
        lst1.remove(lst1[0])
        lst2.append(lst1)

    lst3 = list(product(*lst2))

    for i in range(len(lst3)):
        res2 = fofx(lst3[i],M)
        lst5.append(res2)

    print(max(lst5))

if __name__=="__main__":
    main()


# 1 den kullanıcının girmiş olduğu sayıya kadar olan tek ve çift sayıların toplamını ayrı ayrı bulan ve sonucu ekranda gösteren Python programini yaziniz.
sayi = input('Sayıyı Girin : ')
tekToplam=0
ciftToplam=0
for i in range(1,int(sayi)):
      if(i%2==0):
            ciftToplam+=i
      else:
            tekToplam+=i
print("Tek Sayıların Toplamı : {0}".format(tekToplam))
print("Çift Sayıların Toplamı : {0}".format(ciftToplam))

# Ekran Ciktisi
# Sayıyı Girin : 5
# Tek Sayıların Toplamı : 4
# Çift Sayıların Toplamı : 6


import pathlib
import shutil

import pytest

import ipsuite as ips

@pytest.mark.parametrize(
    "random, max_rattel_shift, cell_fraction",
    [(True, None, [1.0, 1.0]), (False, 0.1, [0.5, 0.5])],
)
def test_rattle_atoms(proj_path, traj_file, random, max_rattel_shift, cell_fraction):
    traj_file = pathlib.Path(traj_file)
    shutil.copy(traj_file, ".")

    n_conf_per_dist = [5, 5]
    z_dist_list = [1, 2]

    with ips.Project() as project:
        data = ips.AddData(file=traj_file.name)

        scan = ips.bootstrap.SurfaceRasterScan(
            data=data.atoms,
            symbol="O",
            n_conf_per_dist=n_conf_per_dist,
            z_dist_list=z_dist_list,
            random=random,
            max_rattel_shift=max_rattel_shift,
            cell_fraction=cell_fraction,
            seed=0,
        )
    project.run()

    data.load()
    scan.load()
    atoms = scan.atoms

    desired_num_configs = len(z_dist_list) * n_conf_per_dist[0] * n_conf_per_dist[1]
    assert len(atoms) == desired_num_configs

    # check if distances are equally distributed
    dist1 = atoms[0].positions[-1, 1] - atoms[1].positions[-1, 1]
    dist2 = atoms[2].positions[-1, 1] - atoms[3].positions[-1, 1]
    if random is False:
        assert dist1 == dist2
    else:
        assert dist1 != dist2

    # check if bulk hase still the atom positions of the init structure
    pos1 = data.atoms[0].positions[0, 0]
    pos2 = scan.atoms[0].positions[0, 0]
    if max_rattel_shift is None:
        assert pos1 == pos2
    else:
        assert pos1 != pos2

    # check if the additive just is added in the correct cell fraction
    pos_additive = []
    for atom in scan.atoms:
        pos_additive.append(atom.positions[-1, :2])

    max_x = max(pos_additive[0])
    max_y = max(pos_additive[1])

    cell = data.atoms[0].cell

    assert max_x * cell_fraction[0] <= cell[0, 0]
    assert max_y * cell_fraction[1] <= cell[1, 1]


from dateutil import parser
from datetime import datetime

from ..app_logging import app_logging

time_helper_logger = app_logging.getLogger('time_helper')

def validate_time(timestamp_to_validate):
    try:
        datetime_obj = datetime.fromtimestamp(float(timestamp_to_validate))
    except (ValueError, OverflowError, TypeError, OSError) as e:
        time_helper_logger.warning(f'Error when constructing the datetime object from the timestamp {timestamp_to_validate}:\n{e}')
        raise
    return datetime_obj

def format_time(time_str, tz_to_apply):
    # format the time fields, from the format request
    t_parsed = parser.parse(time_str, ignoretz=True)
    t_formatted = t_parsed.astimezone(tz_to_apply).strftime('%Y-%m-%dT%H:%M:%S%z')

    return t_formatted


num = [10, 5, 7, 8, 4, 9, 3, 6, 1, 2]
sorted = [None] * 10

def merge(start, end, mid, num):

    left_idx = start
    right_idx = mid + 1
    index = start

    while left_idx <= mid and right_idx <= end:
        if num[left_idx] <= num[right_idx]:
            sorted[index] = num[left_idx]
            left_idx += 1
        else:
            sorted[index] = num[right_idx]
            right_idx += 1
        index += 1
    if left_idx > mid:
        for i in range(right_idx, end + 1):
            sorted[index] = num[i]
            index += 1
    else:
        for i in range(left_idx, mid + 1):
            sorted[index] = num[i]
            index += 1

    for i in range(start, end + 1):
        num[i] = sorted[i]

def mergesort(start, end, num):
    if start < end:
        mid = (start + end) // 2
        mergesort(start, mid, num)
        mergesort(mid + 1, end, num)
        merge(start, end, mid, num)

mergesort(0, 9, num)
print(num)
print(sorted)


# from core.Optimizers import CholeskyCMAES
# from core.CNN_scorers import TorchScorer
# from core.GAN_utils import upconvGAN

import os
import re
from os.path import join
from glob import glob
from time import time
from easydict import EasyDict
from tqdm import tqdm
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pickle as pkl
import matplotlib as mpl
mpl.rcParams['pdf.fonttype'] = 42
mpl.rcParams['ps.fonttype'] = 42
mpl.rcParams['axes.spines.right'] = False
mpl.rcParams['axes.spines.top'] = False
pd.options.display.max_columns = 10
pd.options.display.max_colwidth = 200
figdir = r"E:\OneDrive - Harvard University\GECCO2022\Figures\TuneGeometry"
#%%
datadir = r"E:\OneDrive - Washington University in St. Louis\HessTune\AlexNet"
data = np.load(join(datadir, "fc8_1_(1_1).npz"))
#%%
for x in data:
    print(x, data[x].shape)
#%%
plt.semilogy(sorted(np.abs(data["eigvals_u"])))
plt.semilogy(sorted(np.abs(data["eigvals"])))
plt.show()
#%% Visualize and plot the eigenvalues
unitfnlist = ["conv1_0_(27_27).npz",
            "conv2_0_(13_13).npz",
            "conv3_0_(6_6).npz",
            "conv4_0_(6_6).npz",
            "conv5_0_(6_6).npz",
            "fc6_0_(1_1).npz",
            "fc7_0_(1_1).npz",
            "fc8_0_(1_1).npz",]

def plot_CNN_spect_comparison(maxnorm=True, colorcyc=plt.rcParams['axes.prop_cycle'].by_key()['color']):
    figh = plt.figure(figsize=[6, 6])
    for k, unitfn in enumerate(unitfnlist):
        match = re.findall("(.*)_(\d*)_\((\d*)_(\d*)\).npz", unitfn, )
        layer, channum, pos = match[0][0], int(match[0][1]), (int(match[0][2]), int(match[0][3]))
        for chi in range(5):
            unitfn = "%s_%d_(%d_%d).npz" % (layer, chi, pos[0], pos[1])
            data = np.load(join(datadir, unitfn))
            eigvals = data["eigvals"].copy()
            normalizer = np.abs(eigvals).max() if maxnorm else np.array(1.0)
            plt.semilogy(sorted(np.abs(eigvals))[::-1] / normalizer,
                         label=layer if chi == 0 else None,
                         lw=2, alpha=0.5, color=colorcyc[k])
    if maxnorm:
        figh.gca().set_ylim(1E-9, 1.5)
    else:
        figh.gca().set_ylim(1E-9, 10)

    plt.ylabel("abs Hessian eigenvalue"+"(norm by max)" if maxnorm else "",fontsize=12)
    plt.xlabel("eigen rank",fontsize=12)
    plt.title("Hessian of Landscape Compared across CNN Hierarchy\nEvaluated at peak of act max",fontsize=14)
    plt.legend()
    plt.savefig(join(figdir, "alexnet_layers_peak_hess_spect_cmp%s%s.png"%("_maxnorm" if maxnorm else "", "")))
    plt.savefig(join(figdir, "alexnet_layers_peak_hess_spect_cmp%s%s.pdf"%("_maxnorm" if maxnorm else "", "")))
    plt.show()
    return figh

cmap = plt.get_cmap("hsv")
colorcyc = [cmap(i/(8)) for i in range(8)]
plot_CNN_spect_comparison(maxnorm=True, colorcyc=colorcyc)
plot_CNN_spect_comparison(maxnorm=False, colorcyc=colorcyc)
#%%
def plot_CNN_spect_comparison_randinit(maxnorm=True, colorcyc=plt.rcParams['axes.prop_cycle'].by_key()['color']):
    figh = plt.figure(figsize=[6, 6])
    for k, unitfn in enumerate(unitfnlist):
        match = re.findall("(.*)_(\d*)_\((\d*)_(\d*)\).npz", unitfn, )
        layer, channum, pos = match[0][0], int(match[0][1]), (int(match[0][2]), int(match[0][3]))
        for chi in range(5):
            unitfn = "%s_%d_(%d_%d).npz" % (layer, chi, pos[0], pos[1])
            data = np.load(join(datadir, unitfn))
            eigvals = data["eigvals_u"].copy()
            normalizer = np.abs(eigvals).max() if maxnorm else np.array(1.0)
            plt.semilogy(sorted(np.abs(eigvals))[::-1] / normalizer,
                         label=layer if chi == 0 else None,
                         lw=2, alpha=0.5, color=colorcyc[k])
    if maxnorm:
        figh.gca().set_ylim(3E-8, 1.5)
    else:
        figh.gca().set_ylim(1E-8, 10)

    plt.ylabel("abs Hessian eigenvalue"+"(norm by max)" if maxnorm else "",fontsize=12)
    plt.xlabel("eigen rank",fontsize=12)
    plt.title("Hessian of Landscape Compared across CNN Hierarchy\nEvaluated at random vector",fontsize=14)
    plt.legend()
    plt.savefig(join(figdir, "alexnet_layers_rnd_hess_spect_cmp%s%s.png"%("_maxnorm" if maxnorm else "", "")))
    plt.savefig(join(figdir, "alexnet_layers_rnd_hess_spect_cmp%s%s.pdf"%("_maxnorm" if maxnorm else "", "")))
    plt.show()
    return figh
#%%
plot_CNN_spect_comparison_randinit(maxnorm=True, colorcyc=colorcyc)
plot_CNN_spect_comparison_randinit(maxnorm=False, colorcyc=colorcyc)


'''
Created on 08.07.2013

@author: vau
'''

import time
import json
import xmlrpc.client
import lemon
import core
import queue

ERROR_NOT_IDENTIFIED  = '0000001'
ERROR_NOT_AUTHORIZED  = '0000002'

TASK_SUCCESSFULLY_ADDED = '11'

RECONNECT_INTERVAL      = 10

class XMLRPC_Client(lemon.BaseAgentLemon):
    '''
    classdocs
    '''

    def __init__(self, _logger, _config, _info):
        self._agentID   = None
        self._taskManager   = None
        self._server_addr   = _config['xmlrpc_server_addr']
        self._server_port   = _config['xmlrpc_server_port']
        self._last_refresh  = 0
        self._reqQueue  = queue.Queue()
        lemon.BaseAgentLemon.__init__(self,_logger,_config, _info)

    def run(self):
        self._taskManager   = core.getCoreInstance().getInstance('TASK_MANAGER')
        self._agentID       = core.getCoreInstance().getItem('agent')['__id']
        self._entityManager = core.getCoreInstance().getInstance('ENTITY_MANAGER')
        self._createConnection(self._server_addr, self._server_port)
        self._setReady()

        while(self._running):
            self._process()
            time.sleep(0.01)

    def _process(self):
        req = self._reqQueue.get()
        method  = req['method']
        args    = req['args']
        callback    = req['callback']
        if callback:
            try:
                result  = method(*args)
                callback(result)
            except ConnectionError as e:
                self._logger.exception(e)
                self._reconnect()
                self._put(method,args,callback)
            except xmlrpc.client.Fault as e:
                self._logger.exception(e)
        req['ready']    = True

    def _createConnection(self, addr, port):
        self._connection    = xmlrpc.client.ServerProxy('http://{0}:{1}'.format(addr, port))

    def _reconnect(self):
        print('Reconnecting...')
        time.sleep(RECONNECT_INTERVAL)
        self._createConnection(self._server_addr, self._server_port)

    def _put(self, method, args=None, callback=None):
        request  = {'method': method, 'args': args, 'callback':callback, 'ready': False}
        self._reqQueue.put(request)
        return request

    def _formRequest(self, method, *args):
        result  = {'return': None}
        def callback(_result):
            result['return']   = _result
        p_task  = self._put(method, args, callback)
        while p_task['ready'] is not True:
            time.sleep(0.01)
        return result['return']

    def _reqREFRESH(self, agentID):
        return self._formRequest(self._connection.refresh, agentID)

    def _reqGET(self, agentID, key, *args, **kwargs):
        return self._formRequest(self._connection.get, agentID, key, *args, **kwargs)

    def _reqPOSTDATA(self, agentID, json_data):
        return self._formRequest(self._connection.postData, agentID, json_data)

    def post(self, data):
        data['time']    = time.time()
        code    = self._reqPOSTDATA(self._agentID,  json.dumps(data))
        if code is TASK_SUCCESSFULLY_ADDED:
            return True
        return False

    def get(self, key=None, *args):
        try:
            if key is None:
                try:
                    result = self._reqREFRESH(self._agentID)
                    if result - self._last_refresh > 0:
                        self._taskManager.new_task('sync')
                        self._last_refresh  = self._entityManager.getRevision()
                except Exception as e:
                    self._logger.exception(e)
            else:
                result  = json.loads(self._reqGET(self._agentID, key, *args))
                return result
        except Exception as e:
            self._logger.exception(e)


import os
from setuptools import setup, find_packages
from os.path import join

# Utility function to read the README file.
# Used for the long_description.  It's nice, because now 1) we have a top level
# README file and 2) it's easier to type in the README file than to put a raw
# string in below ...
def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

setup(
    name = "SoCCo",
    version = read(join('SoCCo','__init__.py')),
    author = "Brian Beckage",
    author_email = "brian.beckage@uvm.edu",
    description = ("Studying the effects of social interactions on climate "
                   "change"),
    license = "BSD",
    keywords = "climate change, social interactions",
    url = "https://github.com/OpenClimate/climate_change_model",
    packages=find_packages(),
    long_description=read('README.md'),
    classifiers=[
        "License :: OSI Approved :: BSD License",
    ],
)


Stack = []
t = 0

def LIFO(a,n):
    global t
    if a == 'remove':
        if t > 0:
            t -=1
            del Stack[t]
        else:
            print('Fifo is already empty')
    elif a == 'add':
        t +=1
        Stack.insert(t,n)

    else:
        print ('invalid')
    return Stack

LIFO('add','toamto')
LIFO('add','potato')
LIFO('add','onion')
LIFO('remove','n')
print(Stack)

Algoritmid2023 Kasper Looga kodutoo 1.py


import os
import json
import sys

sys.path.append(sys.path.append(os.path.join(os.path.dirname(__file__), '..')))

from flask import Flask, request
from datamart_web.src.search_metadata import SearchMetadata
from datamart_web.src.join_datasets import JoinDatasets

class WebApp(Flask):

    def __init__(self):
        super().__init__(__name__, instance_relative_config=True)
        self.search_metadata = SearchMetadata()
        self.join_datasets = JoinDatasets()

    def create_app(self, test_config=None):
        # create and configure the app
        self.config.from_mapping(
            SECRET_KEY='dev',
            DATABASE=os.path.join(self.instance_path, 'flaskr.sqlite'),
        )

        if test_config is None:
            # load the instance config, if it exists, when not testing
            self.config.from_pyfile('config.py', silent=True)
        else:
            # load the test config if passed in
            self.config.from_mapping(test_config)

        # ensure the instance folder exists
        try:
            os.makedirs(self.instance_path)
        except OSError:
            pass

        @self.route('/')
        def hello():
            return 'Datamart Web Service!'

        @self.route('/search/default_search', methods=['POST'])
        def default_search():
            return json.dumps(self.search_metadata.default_search_by_csv(request=request))

        @self.route('/augment/default_join', methods=['POST'])
        def default_join():
            return json.dumps(self.join_datasets.default_join(request=request))

        return self

if __name__ == '__main__':
    WebApp().create_app().run()


from flask import Flask
import sys
import logging
import json

App = Flask(__name__)
@App.route('/palindrome/<word>', methods = ['GET'])
def palindrome(word):
    response = {'message': 'success'}
    logging.info("Word received: [" + str(word) + "]")
    jsonResponse = buildJSON(word)
    logging.info("JSON obtained: {" + jsonResponse + "}")

    return jsonResponse

def buildJSON(word):
    def voltearPal(x):
      return x[::-1]
    palindromo = {}
    palabras = {}
    dct = {}
        # palindromo = str(input("Please type the string: "))
    palindromo = word
        # print(palindromo)

    cant = len (palindromo)
    palabras = voltearPal(palindromo)
    for i in palindromo:
        if i in dct:
            dct[i] +=1
        else:
            dct[i] =1

    if (palindromo == palabras):
        falsePal = True
        x = {
            "name": palindromo,
            "palindrome": falsePal,
            "sorted": dct,
            "length": cant
            }

        y = json.dumps(x)
        # jsonFile = open("data.json", "w")
        # jsonFile.write(y)
        # jsonFile.close()

        print(y)

    else:
        truePal = False
        j = {
            "name": palindromo,
            "palindrome": truePal
            }

        y = json.dumps(j)
        print(y)
    return y

if __name__ == "__main__":
    App.run(host='0.0.0.0', debug=True, port=8080)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os

import numpy as np
from six import PY2
from tensorflow import keras

from google.protobuf import text_format as _text_format
from google.protobuf.message import DecodeError
from tensorflow.core.framework import graph_pb2 as _graph_pb2
from tensorflow.lite.python import convert_saved_model as _convert_saved_model
from tensorflow.lite.python import interpreter as _interpreter
from tensorflow.lite.python import lite as _lite
from tensorflow.lite.python import util as _util
from tensorflow.python.client import session as _session
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework.importer import import_graph_def as _import_graph_def
from tensorflow.python.lib.io import file_io as _file_io
from tensorflow.python.platform import resource_loader as _resource_loader
from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.saved_model import load as _load
from tensorflow.python.saved_model import loader as _loader
from tensorflow.python.saved_model import signature_constants as _signature_constants
from tensorflow.python.saved_model import tag_constants as _tag_constants

_GOLDENS_UPDATE_WARNING = """
  Golden file update requested!
  This test is now going to write new golden files.

  Make sure to package the updates together with your CL.
"""

def get_filepath(filename, base_dir=None):
  """Returns the full path of the filename.

  Args:
    filename: Subdirectory and name of the model file.
    base_dir: Base directory containing model file.

  Returns:
    str.
  """
  if base_dir is None:
    base_dir = "learning/brain/mobile/tflite_compat_models"
  return os.path.join(_resource_loader.get_root_dir_with_all_resources(),
                      base_dir, filename)

def get_golden_filepath(name):
  """Returns the full path to a golden values file.

  Args:
    name: the name of the golden data, usually same as the test name.
  """
  goldens_directory = os.path.join(_resource_loader.get_data_files_path(),
                                   "testdata", "golden")
  return os.path.join(goldens_directory, "%s.npy.golden" % name)

def get_image(size):
  """Returns an image loaded into an np.ndarray with dims [1, size, size, 3].

  Args:
    size: Size of image.

  Returns:
    np.ndarray.
  """
  img_filename = _resource_loader.get_path_to_datafile(
      "testdata/grace_hopper.jpg")
  img = keras.preprocessing.image.load_img(
      img_filename, target_size=(size, size))
  img_array = keras.preprocessing.image.img_to_array(img)
  img_array = np.expand_dims(img_array, axis=0)
  return img_array

def _get_calib_data_func(input_size):
  """Returns a function to generate a representative data set.

  Args:
    input_size: 3D shape of the representative data.
  """
  def representative_data_gen():
    num_calibration = 20
    for _ in range(num_calibration):
      yield [
          np.random.rand(
              1,
              input_size[0],
              input_size[1],
              input_size[2],
          ).astype(np.float32)
      ]

  return representative_data_gen

def _convert(converter, **kwargs):
  """Converts the model.

  Args:
    converter: TFLiteConverter object.
    **kwargs: Additional arguments to be passed into the converter. Supported
      flags are {"target_ops", "post_training_quantize", "quantize_to_float16",
      "post_training_quantize_int8", "post_training_quantize_16x8",
      "model_input_size"}.

  Returns:
    The converted TFLite model in serialized format.

  Raises:
    ValueError: Invalid version number.
  """

  if "target_ops" in kwargs:
    converter.target_spec.supported_ops = kwargs["target_ops"]
  if "post_training_quantize" in kwargs:
    converter.optimizations = [_lite.Optimize.DEFAULT]
  if kwargs.get("quantize_to_float16", False):
    converter.target_spec.supported_types = [dtypes.float16]
  if kwargs.get("post_training_quantize_int8", False):
    input_size = kwargs.get("model_input_size")
    converter.optimizations = [_lite.Optimize.DEFAULT]
    converter.target_spec.supported_ops = [_lite.OpsSet.TFLITE_BUILTINS_INT8]
    converter.representative_dataset = _get_calib_data_func(input_size)
    # Note that the full integer quantization is by the mlir quantizer
    converter.experimental_new_quantizer = True
  if kwargs.get("post_training_quantize_16x8", False):
    input_size = kwargs.get("model_input_size")
    converter.optimizations = [_lite.Optimize.DEFAULT]
    converter.target_spec.supported_ops = \
      [_lite.OpsSet.\
        EXPERIMENTAL_TFLITE_BUILTINS_ACTIVATIONS_INT16_WEIGHTS_INT8]
    converter.representative_dataset = _get_calib_data_func(input_size)
  return converter.convert()

def _check_model_quantized_to_16x8(tflite_model):
  """Checks that the activations are quantized into int16.

  Args:
    tflite_model: Serialized TensorFlow Lite model.

  Raises:
    ValueError: Activations with int16 type are not found.
  """
  interpreter = _get_tflite_interpreter(tflite_model)
  interpreter.allocate_tensors()
  all_tensor_details = interpreter.get_tensor_details()

  found_input = False
  for tensor in all_tensor_details:
    if "_int16" in tensor["name"]:
      found_input = True
      if tensor["dtype"] is not np.int16:
        raise ValueError("Activations should be int16.")

  # Check that we found activations in the correct type: int16
  if not found_input:
    raise ValueError("Could not find int16 activations.")

def _get_tflite_interpreter(tflite_model,
                            input_shapes_resize=None,
                            custom_op_registerers=None):
  """Creates a TFLite interpreter with resized input tensors.

  Args:
    tflite_model: Serialized TensorFlow Lite model.
    input_shapes_resize: A map where the key is the input tensor name and the
      value is the shape of the input tensor. This resize happens after model
      conversion, prior to calling allocate tensors. (default None)
    custom_op_registerers: Op registerers for custom ops.

  Returns:
    lite.Interpreter
  """
  if custom_op_registerers is None:
    custom_op_registerers = []
  interpreter = _interpreter.InterpreterWithCustomOps(
      model_content=tflite_model, custom_op_registerers=custom_op_registerers)
  if input_shapes_resize:
    input_details = interpreter.get_input_details()
    input_details_map = {
        detail["name"]: detail["index"] for detail in input_details
    }
    for name, shape in input_shapes_resize.items():
      idx = input_details_map[name]
      interpreter.resize_tensor_input(idx, shape)
  return interpreter

def _get_input_data_map(tflite_model, input_data, custom_op_registerers=None):
  """Generates a map of input data based on the TFLite model.

  Args:
    tflite_model: Serialized TensorFlow Lite model.
    input_data: List of np.ndarray.
    custom_op_registerers: Op registerers for custom ops.

  Returns:
    {str: [np.ndarray]}.
  """
  interpreter = _get_tflite_interpreter(
      tflite_model, custom_op_registerers=custom_op_registerers)
  interpreter.allocate_tensors()
  input_details = interpreter.get_input_details()
  return {
      input_tensor["name"]: data
      for input_tensor, data in zip(input_details, input_data)
  }

def _generate_random_input_data(tflite_model,
                                seed=None,
                                input_data_range=None,
                                input_shapes_resize=None,
                                custom_op_registerers=None):
  """Generates input data based on the input tensors in the TFLite model.

  Args:
    tflite_model: Serialized TensorFlow Lite model.
    seed: Integer seed for the random generator. (default None)
    input_data_range: A map where the key is the input tensor name and
      the value is a tuple (min_val, max_val) which specifies the value range of
      the corresponding input tensor. For example, '{'input1': (1, 5)}' means to
      generate a random value for tensor `input1` within range [1.0, 5.0)
      (half-inclusive). (default None)
    input_shapes_resize: A map where the key is the input tensor name and the
      value is the shape of the input tensor. This resize happens after model
      conversion, prior to calling allocate tensors. (default None)
    custom_op_registerers: Op registerers for custom ops.

  Returns:
    ([np.ndarray], {str : [np.ndarray]}).
  """
  interpreter = _get_tflite_interpreter(
      tflite_model,
      input_shapes_resize,
      custom_op_registerers=custom_op_registerers)
  interpreter.allocate_tensors()
  input_details = interpreter.get_input_details()

  if seed:
    np.random.seed(seed=seed)

  # Generate random input data. If a tensor's value range is specified, say
  # [a, b), then the generated value will be (b - a) * Unif[0.0, 1.0) + a,
  # otherwise it's Unif[0.0, 1.0).
  input_data = []
  for input_tensor in input_details:
    val = np.random.random_sample(input_tensor["shape"])
    if (input_data_range is not None and
        input_tensor["name"] in input_data_range):
      val = (input_data_range[input_tensor["name"]][1] -
             input_data_range[input_tensor["name"]][0]
            ) * val + input_data_range[input_tensor["name"]][0]
    input_data.append(np.array(val, dtype=input_tensor["dtype"]))

  input_data_map = _get_input_data_map(
      tflite_model, input_data, custom_op_registerers=custom_op_registerers)
  return input_data, input_data_map

def _evaluate_tflite_model(tflite_model,
                           input_data,
                           input_shapes_resize=None,
                           custom_op_registerers=None):
  """Returns evaluation of input data on TFLite model.

  Args:
    tflite_model: Serialized TensorFlow Lite model.
    input_data: List of np.ndarray.
    input_shapes_resize: A map where the key is the input tensor name and the
      value is the shape of the input tensor. This resize happens after model
      conversion, prior to calling allocate tensors. (default None)
    custom_op_registerers: Op registerers for custom ops.

  Returns:
    List of np.ndarray.
  """
  interpreter = _get_tflite_interpreter(
      tflite_model,
      input_shapes_resize,
      custom_op_registerers=custom_op_registerers)
  interpreter.allocate_tensors()

  input_details = interpreter.get_input_details()
  output_details = interpreter.get_output_details()

  for input_tensor, tensor_data in zip(input_details, input_data):
    interpreter.set_tensor(input_tensor["index"], tensor_data)

  interpreter.invoke()
  output_data = [
      interpreter.get_tensor(output_tensor["index"])
      for output_tensor in output_details
  ]
  output_labels = [output_tensor["name"] for output_tensor in output_details]
  return output_data, output_labels

def evaluate_frozen_graph(filename, input_arrays, output_arrays):
  """Returns a function that evaluates the frozen graph on input data.

  Args:
    filename: Full filepath of file containing frozen GraphDef.
    input_arrays: List of input tensors to freeze graph with.
    output_arrays: List of output tensors to freeze graph with.

  Returns:
    Lambda function ([np.ndarray data] : [np.ndarray result]).
  """
  with _file_io.FileIO(filename, "rb") as f:
    file_content = f.read()

  graph_def = _graph_pb2.GraphDef()
  try:
    graph_def.ParseFromString(file_content)
  except (_text_format.ParseError, DecodeError):
    if not isinstance(file_content, str):
      if PY2:
        file_content = file_content.encode("utf-8")
      else:
        file_content = file_content.decode("utf-8")
    _text_format.Merge(file_content, graph_def)

  graph = ops.Graph()
  with graph.as_default():
    _import_graph_def(graph_def, name="")
  inputs = _util.get_tensors_from_tensor_names(graph, input_arrays)
  outputs = _util.get_tensors_from_tensor_names(graph, output_arrays)

  def run_session(input_data):
    with _session.Session(graph=graph) as sess:
      return sess.run(outputs, dict(zip(inputs, input_data)))

  return run_session

def evaluate_saved_model(directory, tag_set, signature_key):
  """Returns a function that evaluates the SavedModel on input data.

  Args:
    directory: SavedModel directory to convert.
    tag_set: Set of tags identifying the MetaGraphDef within the SavedModel to
      analyze. All tags in the tag set must be present.
    signature_key: Key identifying SignatureDef containing inputs and outputs.

  Returns:
    Lambda function ([np.ndarray data] : [np.ndarray result]).
  """
  with _session.Session().as_default() as sess:
    if tag_set is None:
      tag_set = set([_tag_constants.SERVING])
    if signature_key is None:
      signature_key = _signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY

    meta_graph = _loader.load(sess, tag_set, directory)
    signature_def = _convert_saved_model.get_signature_def(
        meta_graph, signature_key)
    inputs, outputs = _convert_saved_model.get_inputs_outputs(signature_def)

    return lambda input_data: sess.run(outputs, dict(zip(inputs, input_data)))

def evaluate_keras_model(filename):
  """Returns a function that evaluates the tf.keras model on input data.

  Args:
    filename: Full filepath of HDF5 file containing the tf.keras model.

  Returns:
    Lambda function ([np.ndarray data] : [np.ndarray result]).
  """
  keras_model = keras.models.load_model(filename)
  return lambda input_data: [keras_model.predict(input_data)]

def compare_models(tflite_model,
                   tf_eval_func,
                   input_shapes_resize=None,
                   input_data=None,
                   input_data_range=None,
                   tolerance=5):
  """Compares TensorFlow and TFLite models.

  Unless the input data is provided, the models are compared with random data.

  Args:
    tflite_model: Serialized TensorFlow Lite model.
    tf_eval_func: Lambda function that takes in input data and outputs the
      results of the TensorFlow model ([np.ndarray data] : [np.ndarray result]).
    input_shapes_resize: A map where the key is the input tensor name and the
      value is the shape of the input tensor. This resize happens after model
      conversion, prior to calling allocate tensors. (default None)
    input_data: np.ndarray to pass into models during inference. (default None)
    input_data_range: A map where the key is the input tensor name and
      the value is a tuple (min_val, max_val) which specifies the value range of
      the corresponding input tensor. For example, '{'input1': (1, 5)}' means to
      generate a random value for tensor `input1` within range [1.0, 5.0)
      (half-inclusive). (default None)
    tolerance: Decimal place to check accuracy to. (default 5).
  """
  if input_data is None:
    input_data, _ = _generate_random_input_data(
        tflite_model=tflite_model,
        input_data_range=input_data_range,
        input_shapes_resize=input_shapes_resize)
  tf_results = tf_eval_func(input_data)
  tflite_results, _ = _evaluate_tflite_model(
      tflite_model, input_data, input_shapes_resize=input_shapes_resize)
  for tf_result, tflite_result in zip(tf_results, tflite_results):
    np.testing.assert_almost_equal(tf_result, tflite_result, tolerance)

def _compare_tf_tflite_results(tf_results,
                               tflite_results,
                               tflite_labels,
                               tolerance=5):
  """Compare the result of TF and TFLite model.

  Args:
    tf_results: results returned by the TF model.
    tflite_results: results returned by the TFLite model.
    tflite_labels: names of the output tensors in the TFlite model.
    tolerance: Decimal place to check accuracy to. (default 5).
  """
  # Convert the output TensorFlow results into an ordered list.
  if isinstance(tf_results, dict):
    if len(tf_results) == 1:
      tf_results = [tf_results[list(tf_results.keys())[0]]]
    else:
      tf_results = [tf_results[tflite_label] for tflite_label in tflite_labels]
  else:
    tf_results = [tf_results]

  for tf_result, tflite_result in zip(tf_results, tflite_results):
    np.testing.assert_almost_equal(tf_result, tflite_result, tolerance)

def compare_models_v2(tflite_model,
                      tf_eval_func,
                      input_data=None,
                      input_data_range=None,
                      tolerance=5):
  """Compares TensorFlow and TFLite models for TensorFlow 2.0.

  Unless the input data is provided, the models are compared with random data.
  Currently only 1 input and 1 output are supported by this function.

  Args:
    tflite_model: Serialized TensorFlow Lite model.
    tf_eval_func: Function to evaluate TensorFlow model. Either a lambda
      function that takes in input data and outputs the results or a TensorFlow
      ConcreteFunction.
    input_data: np.ndarray to pass into models during inference. (default None).
    input_data_range: A map where the key is the input tensor name and
      the value is a tuple (min_val, max_val) which specifies the value range of
      the corresponding input tensor. For example, '{'input1': (1, 5)}' means to
      generate a random value for tensor `input1` within range [1.0, 5.0)
      (half-inclusive). (default None)
    tolerance: Decimal place to check accuracy to. (default 5)
  """
  # Convert the input data into a map.
  if input_data is None:
    input_data, input_data_map = _generate_random_input_data(
        tflite_model=tflite_model, input_data_range=input_data_range)
  else:
    input_data_map = _get_input_data_map(tflite_model, input_data)
  input_data_func_map = {
      input_name: constant_op.constant(input_data)
      for input_name, input_data in input_data_map.items()
  }

  if len(input_data) > 1:
    tf_results = tf_eval_func(**input_data_func_map)
  else:
    tf_results = tf_eval_func(constant_op.constant(input_data[0]))
  tflite_results, tflite_labels = _evaluate_tflite_model(
      tflite_model, input_data)

  _compare_tf_tflite_results(tf_results, tflite_results, tflite_labels,
                             tolerance)

def compare_tflite_keras_models_v2(tflite_model,
                                   keras_model,
                                   input_data=None,
                                   input_data_range=None,
                                   tolerance=5,
                                   custom_op_registerers=None):
  """Similar to compare_models_v2 but accept Keras model.

  Unless the input data is provided, the models are compared with random data.
  Currently only 1 input and 1 output are supported by this function.

  Args:
    tflite_model: Serialized TensorFlow Lite model.
    keras_model: Keras model to evaluate.
    input_data: np.ndarray to pass into models during inference. (default None).
    input_data_range: A map where the key is the input tensor name and the value
      is a tuple (min_val, max_val) which specifies the value range of
      the corresponding input tensor. For example, '{'input1': (1, 5)}' means to
      generate a random value for tensor `input1` within range [1.0, 5.0)
      (half-inclusive). (default None)
    tolerance: Decimal place to check accuracy to. (default 5)
    custom_op_registerers: Op registerers for custom ops.
  """
  # Generate random input data if not provided.
  if input_data is None:
    input_data, _ = _generate_random_input_data(
        tflite_model=tflite_model,
        input_data_range=input_data_range,
        custom_op_registerers=custom_op_registerers)

  if len(input_data) > 1:
    tf_results = keras_model.predict(input_data)
  else:
    tf_results = keras_model.predict(input_data[0])
  tflite_results, tflite_labels = _evaluate_tflite_model(
      tflite_model, input_data, custom_op_registerers=custom_op_registerers)

  _compare_tf_tflite_results(tf_results, tflite_results, tflite_labels,
                             tolerance)

def compare_model_golden(tflite_model,
                         input_data,
                         golden_name,
                         update_golden=False,
                         tolerance=5):
  """Compares the output of a TFLite model against pre-existing golden values.

  Args:
    tflite_model: Serialized TensorFlow Lite model.
    input_data: np.ndarray to pass into models during inference.
    golden_name: Name of the file containing the (expected) golden values.
    update_golden: Whether to update the golden values with the model output
      instead of comparing against them. This should only be done when a change
      in TFLite warrants it.
    tolerance: Decimal place to check accuracy to. (default 5).
  """
  tflite_results, _ = _evaluate_tflite_model(tflite_model, input_data)
  golden_file = get_golden_filepath(golden_name)
  if update_golden:
    logging.warning(_GOLDENS_UPDATE_WARNING)
    logging.warning("Updating golden values in file %s.", golden_file)
    if not os.path.exists(golden_file):
      golden_relative_path = os.path.relpath(
          golden_file, _resource_loader.get_root_dir_with_all_resources())
      logging.warning(
          "Golden file not found. Manually create it first:\ntouch %r",
          golden_relative_path)

    with open(golden_file, "wb") as f:
      np.save(f, tflite_results, allow_pickle=False)
  else:
    golden_data = np.load(golden_file, allow_pickle=False)
    np.testing.assert_almost_equal(golden_data, tflite_results, tolerance)

def test_frozen_graph_quant(filename,
                            input_arrays,
                            output_arrays,
                            input_shapes=None,
                            **kwargs):
  """Sanity check to validate post quantize flag alters the graph.

  This test does not check correctness of the converted model. It converts the
  TensorFlow frozen graph to TFLite with and without the post_training_quantized
  flag. It ensures some tensors have different types between the float and
  quantized models in the case of an all TFLite model or mix-and-match model.
  It ensures tensor types do not change in the case of an all Flex model.

  Args:
    filename: Full filepath of file containing frozen GraphDef.
    input_arrays: List of input tensors to freeze graph with.
    output_arrays: List of output tensors to freeze graph with.
    input_shapes: Dict of strings representing input tensor names to list of
      integers representing input shapes (e.g., {"foo" : [1, 16, 16, 3]}).
      Automatically determined when input shapes is None (e.g., {"foo" : None}).
        (default None)
    **kwargs: Additional arguments to be passed into the converter.

  Raises:
    ValueError: post_training_quantize flag doesn't act as intended.
  """
  # Convert and load the float model.
  converter = _lite.TFLiteConverter.from_frozen_graph(
      filename, input_arrays, output_arrays, input_shapes)
  tflite_model_float = _convert(converter, **kwargs)

  interpreter_float = _get_tflite_interpreter(tflite_model_float)
  interpreter_float.allocate_tensors()
  float_tensors = interpreter_float.get_tensor_details()

  # Convert and load the quantized model.
  converter = _lite.TFLiteConverter.from_frozen_graph(filename, input_arrays,
                                                      output_arrays,
                                                      input_shapes)
  tflite_model_quant = _convert(
      converter, post_training_quantize=True, **kwargs)

  interpreter_quant = _get_tflite_interpreter(tflite_model_quant)
  interpreter_quant.allocate_tensors()
  quant_tensors = interpreter_quant.get_tensor_details()
  quant_tensors_map = {
      tensor_detail["name"]: tensor_detail for tensor_detail in quant_tensors
  }
  quantized_tensors = {
      tensor_detail["name"]: tensor_detail
      for tensor_detail in quant_tensors
      if tensor_detail["quantization_parameters"]
  }

  # Check if weights are of different types in the float and quantized models.
  num_tensors_float = len(float_tensors)
  num_tensors_same_dtypes = sum(
      float_tensor["dtype"] == quant_tensors_map[float_tensor["name"]]["dtype"]
      for float_tensor in float_tensors)
  has_quant_tensor = num_tensors_float != num_tensors_same_dtypes

  # For the "flex" case, post_training_quantize should not alter the graph,
  # unless we are quantizing to float16.
  if ("target_ops" in kwargs and
      not kwargs.get("quantize_to_float16", False) and
      not kwargs.get("post_training_quantize_int8", False) and
      not kwargs.get("post_training_quantize_16x8", False) and
      set(kwargs["target_ops"]) == set([_lite.OpsSet.SELECT_TF_OPS])):
    if has_quant_tensor:
      raise ValueError("--post_training_quantize flag unexpectedly altered the "
                       "full Flex mode graph.")
  elif kwargs.get("post_training_quantize_int8", False):
    # Instead of using tensor names, we use the number of tensors which have
    # quantization parameters to verify the model is quantized.
    if not quantized_tensors:
      raise ValueError("--post_training_quantize flag was unable to quantize "
                       "the graph as expected in TFLite.")
  elif not has_quant_tensor:
    raise ValueError("--post_training_quantize flag was unable to quantize the "
                     "graph as expected in TFLite and mix-and-match mode.")

def test_frozen_graph(filename,
                      input_arrays,
                      output_arrays,
                      input_shapes=None,
                      input_shapes_resize=None,
                      input_data=None,
                      input_data_range=None,
                      **kwargs):
  """Validates the TensorFlow frozen graph converts to a TFLite model.

  Converts the TensorFlow frozen graph to TFLite and checks the accuracy of the
  model on random data.

  Args:
    filename: Full filepath of file containing frozen GraphDef.
    input_arrays: List of input tensors to freeze graph with.
    output_arrays: List of output tensors to freeze graph with.
    input_shapes: Dict of strings representing input tensor names to list of
      integers representing input shapes (e.g., {"foo" : [1, 16, 16, 3]}).
      Automatically determined when input shapes is None (e.g., {"foo" : None}).
        (default None)
    input_shapes_resize: A map where the key is the input tensor name and the
      value is the shape of the input tensor. This resize happens after model
      conversion, prior to calling allocate tensors. (default None)
    input_data: np.ndarray to pass into models during inference. (default None).
    input_data_range: A map where the key is the input tensor name and
      the value is a tuple (min_val, max_val) which specifies the value range of
      the corresponding input tensor. For example, '{'input1': (1, 5)}' means to
      generate a random value for tensor `input1` within range [1.0, 5.0)
      (half-inclusive). (default None)
    **kwargs: Additional arguments to be passed into the converter.
  """
  converter = _lite.TFLiteConverter.from_frozen_graph(
      filename, input_arrays, output_arrays, input_shapes)
  tflite_model = _convert(converter, **kwargs)

  tf_eval_func = evaluate_frozen_graph(filename, input_arrays, output_arrays)
  compare_models(
      tflite_model,
      tf_eval_func,
      input_shapes_resize=input_shapes_resize,
      input_data=input_data,
      input_data_range=input_data_range)

def test_saved_model(directory,
                     input_shapes=None,
                     tag_set=None,
                     signature_key=None,
                     input_data=None,
                     input_data_range=None,
                     **kwargs):
  """Validates the TensorFlow SavedModel converts to a TFLite model.

  Converts the TensorFlow SavedModel to TFLite and checks the accuracy of the
  model on random data.

  Args:
    directory: SavedModel directory to convert.
    input_shapes: Dict of strings representing input tensor names to list of
      integers representing input shapes (e.g., {"foo" : [1, 16, 16, 3]}).
      Automatically determined when input shapes is None (e.g., {"foo" : None}).
        (default None)
    tag_set: Set of tags identifying the MetaGraphDef within the SavedModel to
      analyze. All tags in the tag set must be present.
    signature_key: Key identifying SignatureDef containing inputs and outputs.
    input_data: np.ndarray to pass into models during inference. (default None).
    input_data_range: A map where the key is the input tensor name and
      the value is a tuple (min_val, max_val) which specifies the value range of
      the corresponding input tensor. For example, '{'input1': (1, 5)}' means to
      generate a random value for tensor `input1` within range [1.0, 5.0)
      (half-inclusive). (default None)
    **kwargs: Additional arguments to be passed into the converter.
  """
  converter = _lite.TFLiteConverter.from_saved_model(
      directory,
      input_shapes=input_shapes,
      tag_set=tag_set,
      signature_key=signature_key)
  tflite_model = _convert(converter, **kwargs)

  # 5 decimal places by default
  tolerance = 5
  if kwargs.get("post_training_quantize_16x8", False):
    _check_model_quantized_to_16x8(tflite_model)
    # only 2 decimal places for full quantization
    tolerance = 2

  tf_eval_func = evaluate_saved_model(directory, tag_set, signature_key)
  compare_models(
      tflite_model,
      tf_eval_func,
      input_data=input_data,
      input_data_range=input_data_range,
      tolerance=tolerance)

def test_saved_model_v2(directory,
                        tag_set=None,
                        signature_key=None,
                        input_data=None,
                        input_data_range=None,
                        **kwargs):
  """Validates the TensorFlow SavedModel converts to a TFLite model.

  Converts the TensorFlow SavedModel to TFLite and checks the accuracy of the
  model on random data.

  Args:
    directory: SavedModel directory to convert.
    tag_set: Set of tags identifying the MetaGraphDef within the SavedModel to
      analyze. All tags in the tag set must be present.
    signature_key: Key identifying SignatureDef containing inputs and outputs.
    input_data: np.ndarray to pass into models during inference. (default None).
    input_data_range: A map where the key is the input tensor name and
      the value is a tuple (min_val, max_val) which specifies the value range of
      the corresponding input tensor. For example, '{'input1': (1, 5)}' means to
      generate a random value for tensor `input1` within range [1.0, 5.0)
      (half-inclusive). (default None)
    **kwargs: Additional arguments to be passed into the converter.
  """
  model = _load.load(directory, tags=tag_set)
  if not signature_key:
    signature_key = _signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
  concrete_func = model.signatures[signature_key]

  converter = _lite.TFLiteConverterV2.from_concrete_functions([concrete_func])
  tflite_model = _convert(converter, **kwargs)

  compare_models_v2(
      tflite_model,
      concrete_func,
      input_data=input_data,
      input_data_range=input_data_range)

def _test_conversion_quant_float16(converter,
                                   input_data,
                                   golden_name=None,
                                   update_golden=False,
                                   **kwargs):
  """Validates conversion with float16 quantization.

  Args:
    converter: TFLite converter instance for the model to convert.
    input_data: np.ndarray to pass into models during inference.
    golden_name: Optional golden values to compare the output of the model
      against.
    update_golden: Whether to update the golden values with the model output
      instead of comparing against them.
    **kwargs: Additional arguments to be passed into the converter.
  """
  tflite_model_float = _convert(converter, version=2, **kwargs)

  interpreter_float = _get_tflite_interpreter(tflite_model_float)
  interpreter_float.allocate_tensors()
  float_tensors = interpreter_float.get_tensor_details()

  tflite_model_quant = _convert(
      converter,
      version=2,
      post_training_quantize=True,
      quantize_to_float16=True,
      **kwargs)

  interpreter_quant = _get_tflite_interpreter(tflite_model_quant)
  interpreter_quant.allocate_tensors()
  quant_tensors = interpreter_quant.get_tensor_details()
  quant_tensors_map = {
      tensor_detail["name"]: tensor_detail for tensor_detail in quant_tensors
  }

  # Check if weights are of different types in the float and quantized models.
  num_tensors_float = len(float_tensors)
  num_tensors_same_dtypes = sum(
      float_tensor["dtype"] == quant_tensors_map[float_tensor["name"]]["dtype"]
      for float_tensor in float_tensors)
  has_quant_tensor = num_tensors_float != num_tensors_same_dtypes

  if not has_quant_tensor:
    raise ValueError("--post_training_quantize flag was unable to quantize the "
                     "graph as expected.")

  if golden_name:
    compare_model_golden(tflite_model_quant, input_data, golden_name,
                         update_golden)

def test_saved_model_v2_quant_float16(directory,
                                      input_data,
                                      golden_name=None,
                                      update_golden=False,
                                      **kwargs):
  """Validates conversion of a saved model to TFLite with float16 quantization.

  Args:
    directory: SavedModel directory to convert.
    input_data: np.ndarray to pass into models during inference.
    golden_name: Optional golden values to compare the output of the model
      against.
    update_golden: Whether to update the golden values with the model output
      instead of comparing against them.
    **kwargs: Additional arguments to be passed into the converter.
  """
  converter = _lite.TFLiteConverterV2.from_saved_model(directory)
  _test_conversion_quant_float16(converter, input_data, golden_name,
                                 update_golden, **kwargs)

def test_frozen_graph_quant_float16(filename,
                                    input_arrays,
                                    output_arrays,
                                    input_data,
                                    input_shapes=None,
                                    golden_name=None,
                                    update_golden=False,
                                    **kwargs):
  """Validates conversion of a frozen graph to TFLite with float16 quantization.

  Args:
    filename: Full filepath of file containing frozen GraphDef.
    input_arrays: List of input tensors to freeze graph with.
    output_arrays: List of output tensors to freeze graph with.
    input_data: np.ndarray to pass into models during inference.
    input_shapes: Dict of strings representing input tensor names to list of
      integers representing input shapes (e.g., {"foo" : [1, 16, 16, 3]}).
      Automatically determined when input shapes is None (e.g., {"foo" : None}).
        (default None)
    golden_name: Optional golden values to compare the output of the model
      against.
    update_golden: Whether to update the golden values with the model output
      instead of comparing against them.
    **kwargs: Additional arguments to be passed into the converter.
  """
  converter = _lite.TFLiteConverter.from_frozen_graph(filename, input_arrays,
                                                      output_arrays,
                                                      input_shapes)
  _test_conversion_quant_float16(converter, input_data,
                                 golden_name, update_golden, **kwargs)

def test_keras_model(filename,
                     input_arrays=None,
                     input_shapes=None,
                     input_data=None,
                     input_data_range=None,
                     **kwargs):
  """Validates the tf.keras model converts to a TFLite model.

  Converts the tf.keras model to TFLite and checks the accuracy of the model on
  random data.

  Args:
    filename: Full filepath of HDF5 file containing the tf.keras model.
    input_arrays: List of input tensors to freeze graph with.
    input_shapes: Dict of strings representing input tensor names to list of
      integers representing input shapes (e.g., {"foo" : [1, 16, 16, 3]}).
      Automatically determined when input shapes is None (e.g., {"foo" : None}).
        (default None)
    input_data: np.ndarray to pass into models during inference. (default None).
    input_data_range: A map where the key is the input tensor name and
      the value is a tuple (min_val, max_val) which specifies the value range of
      the corresponding input tensor. For example, '{'input1': (1, 5)}' means to
      generate a random value for tensor `input1` within range [1.0, 5.0)
      (half-inclusive). (default None)
    **kwargs: Additional arguments to be passed into the converter.
  """
  converter = _lite.TFLiteConverter.from_keras_model_file(
      filename, input_arrays=input_arrays, input_shapes=input_shapes)
  tflite_model = _convert(converter, **kwargs)

  tf_eval_func = evaluate_keras_model(filename)
  compare_models(
      tflite_model,
      tf_eval_func,
      input_data=input_data,
      input_data_range=input_data_range)

def test_keras_model_v2(filename,
                        input_shapes=None,
                        input_data=None,
                        input_data_range=None,
                        **kwargs):
  """Validates the tf.keras model converts to a TFLite model.

  Converts the tf.keras model to TFLite and checks the accuracy of the model on
  random data.

  Args:
    filename: Full filepath of HDF5 file containing the tf.keras model.
    input_shapes: List of list of integers representing input shapes in the
      order of the tf.keras model's .input attribute (e.g., [[1, 16, 16, 3]]).
      (default None)
    input_data: np.ndarray to pass into models during inference. (default None).
    input_data_range: A map where the key is the input tensor name and
      the value is a tuple (min_val, max_val) which specifies the value range of
      the corresponding input tensor. For example, '{'input1': (1, 5)}' means to
      generate a random value for tensor `input1` within range [1.0, 5.0)
      (half-inclusive). (default None)
    **kwargs: Additional arguments to be passed into the converter.
  """
  keras_model = keras.models.load_model(filename)
  if input_shapes:
    for tensor, shape in zip(keras_model.inputs, input_shapes):
      tensor.set_shape(shape)

  converter = _lite.TFLiteConverterV2.from_keras_model(keras_model)
  tflite_model = _convert(converter, **kwargs)

  tf_eval_func = evaluate_keras_model(filename)
  compare_models_v2(
      tflite_model,
      tf_eval_func,
      input_data=input_data,
      input_data_range=input_data_range)


import codecs
import struct
from collections import namedtuple
import xml.etree.ElementTree as ET

# lazy implementation, but it's something ¯\_(ツ)_/¯

def hash_label(label, bucket_count):
	val = 0
	for c in label:
		val = ((val * 0x492) + ord(c)) & 0xFFFFFFFF
	return val % bucket_count

def hash_bucket_count(string_count):
	primes = (
		2,3,5,7,11,13,17,19,23,29,31,37,41,43,
		47,53,59,61,67,71,73,79,83,89,97,101
		)
	target = (string_count // 2) + 1
	for prime in primes:
		if prime >= target:
			return prime
	return primes[-1]

def round_up(amount):
	return (amount + 0xF) & ~0xF

StartTag = namedtuple('StartTag', ('a', 'b', 'arg'))
EndTag = namedtuple('EndTag', ('a', 'b'))

# 110, 0 - ellipsis?
# 110, 3 - nickname?
# 110, 5 - phrase?

animEnums = (
	"EmotEnd", "Smiling", "Laughing", "HappyFlower", "Love", "HappyDance", "Angry", "Outraged",
	"Fuming", "Worried", "Sighing", "Thinking", "SadSpiral", "Frantic", "Crying", "Shocked",
	"Aha", "TakenAback", "Surprised", "ColdChill", "Shaking", "Nodding", "IdeaBulb", "QuestionMark",
	"BrokenHeart", "Sleepy", "Blushing", "OhGeez", "Scheming", "Clapping", "Sneezing", "Cheering",
	"Greeting", "SmugFace", "Hot", "Grin", "WrySmile", "FaintSmile", "BigSmile", "Hesitate",
	"Negative", "Oops", "Dance", "AbsentMindedness", "Shaki", "Sleep", "Special", "Silent",
	"Hello", "Confused", "ShyBoy", "Restless", "Serious", "Eh", "Apologize", "Assent",
	"Explain", "NpcShuffle", "Pardon", "NpcHoldout")

# 2e missing ('special')
animMappings = (
	0,0,1,3,4,0x24,0x25,0x26,
	6,7,8,9,0xa,0xb,0xc,0xd,
	0xe,0x18,0x1b,0x31,2,5,0x1d,0x2a,
	0,0x2c,0xf,0x10,0x11,0x12,0x16,0x29,
	0x35,0x1f,0x20,0x36,0x30,0x15,0x17,0x28,
	0x37,0x38,0x13,0x14,0x1e,0x22,0x19,0x2d,
	0,0x2b,0x1a,0x1c,0x27,0x21,0x23,0x32,
	0x33,0x34,0x2f,0x3a,0x39,0x3b)

animIDs = '|'.join([animEnums[x] for x in animMappings])

tagMetadata = {
	(0,0): ('ruby', 'u16 a', 'u16 b', 'str'),
	(0,2): ('size', 'u16 value'),
	(0,3): ('colour', 'u16 id'),
	(0,4): ('pageBreak', ),
	(10,0): ('delay', 'u16 frames'),
	(10,1): ('delay8', ),
	(10,2): ('delay20', ),
	(10,3): ('delay40', ),
	(10,4): ('10_4', ),
	(10,5): ('10_5', ),
	(10,7): ('10_7', 'u32 arg'),
	(10,8): ('10_8', 'u32 arg'),
	(10,9): ('10_9', ),
	(10,10): ('10_10', ),
	(10,11): ('10_11', ),
	(10,14): ('10_14', 'u16 arg'),
	(10,15): ('setNpcTalkChoices', 'u8 chatOption', 'u8 cancelOption'),
	(20,0): ('20_0', 'str arg'),
	(20,1): ('20_1', 'str arg'),
	(20,2): ('20_2', 'u8 arg'),
	(20,3): ('20_3', 'u8 arg'),
	(40,-1): ('anim', 'idB id '+animIDs, 'u8 target actor|player|mainNpc|subNpc', 'align', 'str targetSpec'),
	(45,-1): ('snpcAnim', 'idB id GulCollapsed|GstSurprised|DodWireless|DodRoger|TkkEh|TkkNodding|TkkQuestionMark', 'u8 target actor|player|mainNpc|subNpc', 'align', 'str targetSpec'),
	(50,4): ('capitaliseNextWord', ),
	(50,5): ('gender1', 'str m', 'str f'),
	(50,6): ('gender2', 'str m', 'str f'),
	(50,7): ('gender3', 'str m', 'str f'),
	(50,8): ('50_8', 'u8 x', 'u8 y', 'str a', 'str b', 'str c'),
	(50,9): ('50_9', 'u8 x', 'u8 y', 'str a', 'str b', 'str c'),
	(50,10): ('gender4', 'str m', 'str f'),
	(50,14): ('gender5', 'str m', 'str f'),
	(50,20): ('gender6', 'str m', 'str f'),
	(50,22): ('gender7', 'str m', 'str f'),
	(50,25): ('weatherArea', 'str north', 'str south'),
	(50,27): ('50_27', 'u8 x', 'u8 y', 'str a', 'str b', 'str c'),
	(100,3): ('100_3', 'u8 arg Katsu|PlaneAnnounceOn|PlaneAnnounceOff'),
	(198,0): ('delayByPersonality', 'u16 boy_normal', 'u16 boy_active', 'u16 boy_pride', 'u16 boy_snobby', 'u16 girl_normal', 'u16 girl_active', 'u16 girl_pride', 'u16 girl_big_sis'),
	(199,3): ('199_3', 'u16 a'),
}

def tag_to_xml(tag):
	key = (tag.a, tag.b)
	keyN = (tag.a, -1)
	if key in tagMetadata or keyN in tagMetadata:
		meta = tagMetadata.get(key, tagMetadata.get(keyN))
		elem = ET.Element(meta[0])
		offset = 0
		for argdef in meta[1:]:
			if argdef == 'str':
				elem.text = ''.join(map(chr, tag.arg[offset//2:]))
				break
			if argdef == 'align':
				if (offset & 1) == 1:
					offset += 1
				continue
			bits = argdef.split(' ')
			typ = bits[0]
			name = bits[1]
			if typ == 'str':
				arglen = tag.arg[offset // 2]
				offset += 2
				arg = ''.join(map(chr, tag.arg[offset//2:(offset+arglen)//2]))
				offset += arglen
			elif typ == 'u32':
				arg = str(tag.arg[offset//2] | (tag.arg[(offset//2) + 1] << 16))
				offset += 4
			elif typ == 'u16':
				arg = str(tag.arg[offset//2])
				offset += 2
			elif typ == 'u8':
				if (offset & 1) == 0:
					arg = str(tag.arg[offset // 2] & 0xFF)
				else:
					arg = str(tag.arg[offset // 2] >> 8)
				offset += 1
			elif typ == 'idB':
				arg = str(tag.b)
			if len(bits) > 2:
				enum = bits[2].split('|')
				num = int(arg)
				if num < len(enum):
					arg = enum[num]
			elem.set(name, arg)
		return elem

	# nope
	elem = ET.Element('tag', dict(a=str(tag.a), b=str(tag.b)))
	if tag.arg:
		arg = ' '.join(map(hex, tag.arg))
		elem.set('arg', arg)
	return elem

class Message(namedtuple('Message', ('text', 'attribute'))):
	__slots__ = ()

	@classmethod
	def from_pieces(cls, pieces, attribute):
		chars = []
		for piece in pieces:
			if isinstance(piece, StartTag):
				chars.append(0xE)
				chars.append(piece.a)
				chars.append(piece.b)
				chars.append(len(piece.arg) * 2)
				chars.extend(piece.arg)
			elif isinstance(piece, EndTag):
				chars.append(0xF)
				chars.append(piece.a)
				chars.append(piece.b)
			else:
				chars.extend(map(ord, piece))
		text = struct.pack('<%dH' % len(chars), *chars)
		return cls(text, attribute)

	def parse(self):
		chars = struct.unpack_from('<%dH' % (len(self.text) // 2), self.text)
		last = 0
		pos = 0
		while pos < len(chars):
			c = chars[pos]
			if (c == 0xE) or (c == 0xF):
				if pos > last:
					yield ''.join(map(chr, chars[last:pos]))
				# tag
				a = chars[pos + 1]
				b = chars[pos + 2]
				pos += 3
				if c == 0xE:
					arg_len = chars[pos]
					pos += 1
					assert (arg_len & 1) == 0
					arg = chars[pos:pos+(arg_len>>1)]
					pos += (arg_len >> 1)
					yield StartTag(a, b, arg)
				else:
					yield EndTag(a, b)
				last = pos
			else:
				pos += 1
		if pos > last:
			yield ''.join(map(chr, chars[last:pos]))

	def generate_xml(self, label=None):
		elem = ET.Element('message')
		if label is not None:
			elem.set('label', label)
		if self.attribute:
			elem.set('attrib', self.attribute.hex(' '))
		#elem.set('raw', self.text.hex(' ', -2))
		for bit in self.parse():
			if isinstance(bit, str):
				if len(elem) == 0:
					elem.text = bit
				else:
					elem[-1].tail = bit
			elif isinstance(bit, StartTag):
				elem.append(tag_to_xml(bit))
			elif isinstance(bit, EndTag):
				ET.SubElement(elem, 'endTag', dict(a=str(bit.a), b=str(bit.b)))
		return elem

class MSBT:
	def __init__(self):
		self.data = {}
		self.attributes = []
		self.strings = []

	def load(self, blob):
		assert blob[:8] == b'MsgStdBn'
		endian, version, section_count, file_size = struct.unpack_from('<HxxHII', blob, 8)
		assert version == 0x0301
		offset = 0x20

		for i in range(section_count):
			section_magic, section_size = struct.unpack_from('<4sI', blob, offset)
			offset += 0x10
			section = blob[offset:offset+section_size]
			offset += (section_size + 0xF) & ~0xF

			if section_magic == b'LBL1':
				labels = self._load_lbl1(section)
			elif section_magic == b'ATR1':
				attributes = self._load_atr1(section)
			elif section_magic == b'TXT2':
				strings = self._load_txt2(section)
			else:
				raise 'unknown section %r' % section_magic

		assert(len(labels) == len(strings))
		assert(len(attributes) == len(strings))
		label_list = [None] * len(labels)
		for label, index in labels.items():
			label_list[index] = label

		self.data = {}
		for label, string, attribute in zip(label_list, strings, attributes):
			self.data[label] = Message(string, attribute)

	def _load_lbl1(self, data):
		bucket_count = struct.unpack_from('<I', data, 0)[0]
		results = {}
		for i in range(bucket_count):
			string_count, offset = struct.unpack_from('<II', data, 4 + i * 8)
			for j in range(string_count):
				label_len = data[offset]
				offset += 1
				label = codecs.decode(data[offset:offset+label_len], 'ascii')
				offset += label_len
				index = struct.unpack_from('<I', data, offset)[0]
				offset += 4
				results[label] = index
				assert(hash_label(label, bucket_count) == i)
		return results

	def _load_atr1(self, data):
		count, size = struct.unpack_from('<II', data, 0)
		self.attrib_size = size
		return [data[8+i*size:8+i*size+size] for i in range(count)]

	def _load_txt2(self, data):
		count = struct.unpack_from('<I', data, 0)[0]
		offsets = list(struct.unpack_from('<%dI' % count, data, 4))
		offsets.append(len(data)) # dummy end offset
		return [bytes(data[offsets[i]:offsets[i+1]-2]) for i in range(count)]
		#return [codecs.decode(data[offsets[i]:offsets[i+1]-2], 'utf-16le') for i in range(count)]

	def _get_lbl1_size(self):
		return 4 + (8 * hash_bucket_count(len(self.data))) + (5 * len(self.data)) + sum(map(len, self.data.keys()))
	def _get_atr1_size(self):
		return 8 + (len(self.data) * self.attrib_size)
	def _get_txt2_size(self):
		return 4 + sum([len(msg.text) + 4 + 2 for msg in self.data.values()])
	def _get_file_size(self, block_sizes):
		return 0x20 + sum([0x10 + round_up(size) for size in block_sizes])

	def _pack_section_header(self, buf, offset, magic, size):
		buf[offset:offset+4] = magic
		struct.pack_into('<I', buf, offset + 4, size)
		return offset + 0x10

	def _pack_lbl1(self, buf, base):
		bucket_count = hash_bucket_count(len(self.data))
		buckets = [[] for i in range(bucket_count)]
		for index, label in enumerate(self.data.keys()):
			buckets[hash_label(label, bucket_count)].append((label, index))

		struct.pack_into('<I', buf, base, bucket_count)
		table_offset = 4
		data_offset = 4 + (8 * bucket_count)
		for bucket in buckets:
			struct.pack_into('<II', buf, base + table_offset, len(bucket), data_offset)
			table_offset += 8
			for label, index in bucket:
				buf[base+data_offset] = len(label)
				data_offset += 1
				buf[base+data_offset:base+data_offset+len(label)] = label.encode('ascii')
				data_offset += len(label)
				struct.pack_into('<I', buf, base+data_offset, index)
				data_offset += 4

		return base + data_offset

	def _pack_atr1(self, buf, offset):
		struct.pack_into('<II', buf, offset, len(self.data), self.attrib_size)
		offset += 8
		for msg in self.data.values():
			attrib = msg.attribute
			assert(len(attrib) == self.attrib_size)
			buf[offset:offset+len(attrib)] = attrib
			offset += len(attrib)
		return offset

	def _pack_txt2(self, buf, base):
		struct.pack_into('<I', buf, base, len(self.data))
		table_offset = 4
		data_offset = 4 + (4 * len(self.data))
		for message in self.data.values():
			struct.pack_into('<I', buf, base + table_offset, data_offset)
			table_offset += 4
			string = message.text
			buf[base+data_offset:base+data_offset+len(string)] = string
			data_offset += len(string) + 2
		return base + data_offset

	def pack(self):
		lbl1_size = self._get_lbl1_size()
		atr1_size = self._get_atr1_size()
		txt2_size = self._get_txt2_size()
		file_size = self._get_file_size([lbl1_size, atr1_size, txt2_size])

		buf = bytearray(file_size)
		buf[:8] = b'MsgStdBn'
		struct.pack_into('<HHHII', buf, 8, 0xFEFF, 0, 0x301, 3, file_size)

		offset = 0x20
		offset = self._pack_section_header(buf, offset, b'LBL1', lbl1_size)
		offset = self._pack_lbl1(buf, offset)
		offset = self._round_up_buffer(buf, offset)
		offset = self._pack_section_header(buf, offset, b'ATR1', atr1_size)
		offset = self._pack_atr1(buf, offset)
		offset = self._round_up_buffer(buf, offset)
		offset = self._pack_section_header(buf, offset, b'TXT2', txt2_size)
		offset = self._pack_txt2(buf, offset)
		offset = self._round_up_buffer(buf, offset)
		assert(offset == len(buf))

		return buf

	def _round_up_buffer(self, buf, offset):
		rounded = round_up(offset)
		for i in range(offset, rounded):
			buf[i] = 0xAB
		return rounded

	def generate_xml(self):
		root = ET.Element('msbt', dict(attribsize=str(self.attrib_size)))
		root.text = '\n\t'
		tag = None
		for label, message in self.data.items():
			tag = message.generate_xml(label)
			tag.tail = '\n\t'
			root.append(tag)
		if tag is not None:
			tag.tail = '\n' # no tab on last one
		return root

if __name__ == '__main__':
	import sys, sarc, zstandard

	path = sys.argv[1]
	data = open(path, 'rb').read()
	if path.endswith('.zs'):
		path = path[:-3]
		data = zstandard.ZstdDecompressor().decompress(data)
	if path.endswith('.sarc'):
		arc = sarc.SARC(data)
		if len(sys.argv) == 2:
			for f in sorted(arc.list_files()):
				print(f)
			sys.exit()
		else:
			data = arc.get_file_data(sys.argv[2])

	msbt = MSBT()
	msbt.load(data)
	assert(data == msbt.pack())
	xml = msbt.generate_xml()
	print(ET.tostring(xml, 'unicode'))
	#for label, index in sorted(msbt.labels.items()):
	#	print('%s: %r <%r>' % (label, msbt.strings[index], msbt.attributes[index]))
	#repack = msbt.pack()
	#open('tst.msbt', 'wb').write(repack)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np
import pandas as pd
import pdb
import csv
import math
import matplotlib.pyplot as plt
import time

EPSILON = 1e-4

df_uniprot = pd.read_csv('Assay_UniprotID_subgroup.csv', header = 0, index_col=0)
subgroup_dict = dict(df_uniprot.iloc[:]['Subgroup'])
protID_dict = dict(df_uniprot.iloc[:]['Uniprot_ID'])
protList = list(set(df_uniprot.iloc[:]['Uniprot_ID']))
num_subgroups = len(set(df_uniprot.iloc[:]['Subgroup']))

df_mols = pd.read_csv('mol.csv', header = 0, index_col=False)
mol_mapping = {}
for i in range(len(df_mols)):
  mol_mapping[df_mols.iloc[i][0]] = df_mols.iloc[i][1]

def generate_data(input_csv, binarize=False, head_only=False, head_row_num=300,
  limit_rows=False, limit_row_num=2400, prefix="toxcast_", input_prot=True, output_csv=None):

  df = pd.read_csv(input_csv, header = 0, index_col=2)
  if head_only:
    df=df.head(head_row_num)
  assayList = list(df)[16:]
  molList = []
  for mol in list(df.index):
    if mol != mol:
      continue
    if mol in mol_mapping:
      mol = mol_mapping[mol]
    molList.append(mol)

  row_ind = 0
  col_ind = 0
  INACTIVE_VAL = 1000.0
  min_val = 1000
  max_value = 0
  inactive_counter = 0
  num_pairs = len(molList) * len(protList)
  if input_prot:
    data_matrix = np.empty((num_pairs, num_subgroups))
  else:
    data_matrix = np.empty((len(molList), len(assayList)))
  data_matrix[:] = np.nan
  pair_list = []
  subgroup_list = []
  smiles_set = set()
  pair_set = set()
  subgroup_set = set()
  # The (compound, protein) pair or group id are keys, and row or column indices are values.
  pair_to_row_ind = {}
  group_to_col_ind = {}
  na_counter = 0
  time_start_loop = time.time()
  for j in range(len(df)):
    this_row = dict(df.iloc[j])
    assert df.index[j] == df.index[j]
    smiles = molList[j]
    assert smiles not in smiles_set
    smiles_set.add(smiles)
    if j % 400 == 1:
      print("Now processing dataframe row: ", j + 1)
    for i, assay_name in enumerate(assayList):
      entry_value = this_row[assay_name]
      if entry_value != entry_value:
        na_counter += 1
        continue
      if entry_value == 1000000:
          entry_value = INACTIVE_VAL
          inactive_counter += 1
      elif entry_value > max_value:
        max_value = entry_value

      if entry_value < min_val and entry_value != 0:
        min_val = entry_value

      if input_prot:
        prot_id = protID_dict[assay_name]
        group_id = subgroup_dict[assay_name]
        pair = (smiles, prot_id)

        if pair not in pair_set:
          pair_to_row_ind[pair] = len(pair_list)
          pair_list.append(pair)
          pair_set.add(pair)

        row_ind = pair_to_row_ind[pair]

        if group_id not in subgroup_set:
          group_to_col_ind[group_id] = len(subgroup_list)
          subgroup_list.append(group_id)
          subgroup_set.add(group_id)

        col_ind = group_to_col_ind[group_id]

        assert data_matrix[row_ind, col_ind] != data_matrix[row_ind, col_ind]
        data_matrix[row_ind, col_ind] = entry_value

      else:
        assert data_matrix[j, i] != data_matrix[j, i]
        data_matrix[j, i] = entry_value

  time_end_loop = time.time()
  print("Time used in the loop: ", time_end_loop - time_start_loop)
  print("Minimum value: ", min_val)
  if input_prot:
    data_matrix = data_matrix[:len(pair_list), :len(subgroup_list)]
  # Now we have: na_counter: 4124417, inactive_counter: 954536, goodval_counter (already deleted): 66551.

  data_matrix = 4 - np.log10(data_matrix)
  max_val = 4 - np.log10(min_val)
  inf_indices = np.where(np.isinf(data_matrix))
  #pdb.set_trace()
  num_inf_entries = len(inf_indices[0])
  for i in range(num_inf_entries):
    first_ind = inf_indices[0][i]
    second_ind = inf_indices[1][i]
    assert np.isinf(data_matrix[first_ind, second_ind])
    data_matrix[first_ind, second_ind] = np.ceil(max_val)

  # num_bins = 20
  # fig, ax = plt.subplots()
  # #n, bins, patches = ax.hist(interactions, num_bins, density=1)
  # interaction_flattened = data_matrix.flatten()
  # interaction_flattened = interaction_flattened[~np.isnan(interaction_flattened)]
  # ax.hist(interaction_flattened, num_bins)
  # #ax.hist(interaction_flattened, num_bins, range=(1.01, np.ceil(max_val)))
  # ax.set_xlabel('ToxCast assay values')
  # ax.set_ylabel('Occurrence')
  # ax.set_title('Histogram of ToxCast assay values')
  # #ax.plot(bins)
  # #fig.tight_layout()
  # #plt.show()
  # #pdb.set_trace()

  #counter = 0
  shuffled_pairs = np.random.permutation(range(len(pair_list)))
  start_writing = time.time()
  with open(output_csv, 'w', newline='') as csvfile:
    fieldnames = ['smiles']
    if input_prot:
      subgroup_list = [prefix + str(task_id) for task_id in subgroup_list]
      fieldnames = subgroup_list + fieldnames + ['proteinName', 'protein_dataset']
    else:
      col_names = [prefix + str(subgroup_dict[assay_name]) + '_' + assay_name for assay_name in assayList]
      fieldnames = col_names + fieldnames

    writer = csv.DictWriter(csvfile, fieldnames = fieldnames)
    writer.writeheader()
    if input_prot:
      for i in range(len(pair_list)):
        if limit_rows and i > limit_row_num - 1:
          break
        index = shuffled_pairs[i]
        pair = pair_list[index]
        data_line = ['' if np.isnan(entry) else entry for entry in data_matrix[index]]
        line_values = dict(zip(subgroup_list, data_line))
        out_line = {'smiles': pair[0], 'proteinName': pair[1], 'protein_dataset': 'toxcast'}
        line_values.update(out_line)
        writer.writerow(line_values)

    else:
      shuffled_mol_list = np.random.permutation(range(len(molList)))

      for ind in range(len(molList)):
        if limit_rows and ind > limit_row_num - 1:
          break
        i = shuffled_mol_list[ind]
        output_dict = {'smiles': molList[i]}
        for j, assay_name in enumerate(assayList):
          intxn_value = data_matrix[i, j]
          if intxn_value != intxn_value:
            intxn_value = ''
          col_name = prefix + str(subgroup_dict[assay_name]) + '_' + assay_name
          output_dict[col_name] = intxn_value
        writer.writerow(output_dict)

  end_writing = time.time()
  print("Time spent in writing: ", end_writing - start_writing)

def is_active(interaction_array, direction, inactive_threshold):
  if not direction:
    interaction_array = -interaction_array
    inactive_threshold = -inactive_threshold
  for element in interaction_array:
    if element == element and element > inactive_threshold:
      return True
  return False

def oversample(input_file, output_file, direction=True, inactive_threshold = 1.0+EPSILON):
  df_input = pd.read_csv(input_file, header = 0, index_col=False)
  len_input = len(df_input)
  headers = list(df_input)
  pairs_to_interaction = {}
  inactive_pair_list = []
  active_pair_list = []
  for i in range(len_input):
    this_row = df_input.iloc[i]
    smiles = this_row['smiles']
    protein_name = this_row['proteinName']
    pair = (smiles, protein_name)
    interaction_profile = np.array(this_row[:len(headers)-3])
    active = is_active(interaction_profile, direction, inactive_threshold)
    if active:
      active_pair_list.append(pair)
    else:
      inactive_pair_list.append(pair)
    pairs_to_interaction[pair] = interaction_profile
  assert len(active_pair_list) < len(inactive_pair_list)
  difference = len(inactive_pair_list) - len(active_pair_list)
  oversampled_pairs = np.random.choice(len(active_pair_list), difference)
  increased_active_pairs_ind = np.concatenate((list(range(len(active_pair_list))),
    oversampled_pairs))

  shuffled_inactive = np.random.permutation(range(len(inactive_pair_list)))
  shuffled_active = np.random.permutation(range(len(increased_active_pairs_ind)))
  start_writing = time.time()
  with open(output_file, 'w', newline='') as csvfile:
    fieldnames = headers
    writer = csv.DictWriter(csvfile, fieldnames = fieldnames)
    writer.writeheader()
    inactive_index = 0
    active_index = 0

    while True:
      inactive_remain = len(inactive_pair_list) - inactive_index
      active_remain = len(increased_active_pairs_ind) - active_index
      if inactive_remain <= 0 and active_remain <= 0:
        break
      rand_num = np.random.sample()
      if rand_num <= inactive_remain/(inactive_remain + active_remain):
        shuffled_ind = shuffled_inactive[inactive_index]
        current_pair = inactive_pair_list[shuffled_ind]
        inactive_index += 1
      else:
        shuffled_ind = shuffled_active[active_index]
        current_pair = active_pair_list[increased_active_pairs_ind[shuffled_ind]]
        active_index += 1
      #pdb.set_trace()
      interaction_profile = pairs_to_interaction[current_pair]
      data_line = ['' if np.isnan(entry) else entry for entry in interaction_profile]
      line_values = dict(zip(headers[:-3], data_line))
      out_line = {'smiles': current_pair[0], 'proteinName': current_pair[1],
        'protein_dataset': 'toxcast'}
      line_values.update(out_line)
      writer.writerow(line_values)

  end_writing = time.time()
  print("Time spent in writing: ", end_writing - start_writing)

if __name__ == "__main__":
  generate_data('Assays_smiles_INCHI.csv', input_prot=False, output_csv='restructured_no_prot.csv')
  # oversample('restructured.csv', 'restructured_oversampled.csv')


from PIL import Image
import torchvision.transforms as transforms
import cv2 as cv
import numpy as np

img = Image.open(r'../data/oxbuild_images-v1/all_souls_000001.jpg')
img = img.convert('RGB')
print(img)
transform = transforms.ToTensor()
img = transform(img)
print(img.shape)
img = img.permute((1, 2, 0))[:, :, [2, 1, 0]]
img = img.numpy()
img = img * 255
img = img.astype(np.uint8)
cv.imshow('img', img)

img_cv = cv.imread(r'../data/oxbuild_images-v1/all_souls_000001.jpg')
print(img_cv.shape)

cv.imshow('img_cv', img_cv)

cv.waitKey(0)


input_filename  = "pi1.sh"
output_filename = "pi.sh"
with open(input_filename,'rb') as ifile:
    with open(output_filename, 'wb') as ofile:
        data = ifile.read(1)
        while data:
            if data == '\x80': data = ' '
            if data != '\xe2' and \
               data != '\x80' and \
               data != '\x80' and \
               data != '\xcf':
               print ("%s", data)
               ofile.write(data)
            data = ifile.read(1)