import os
from dragonEyes.align_dlib import AlignDlib
import csv
import glob
import cv2
from PIL import Image, ImageDraw
import pandas as pd
from random import shuffle

align_dlib = AlignDlib(os.path.join(os.path.dirname(__file__), 'transferredSkills/dlib_models/shape_predictor_68_face_landmarks.dat'))

def draw_on_image(image, out_loc, boxes, face_landmarks_list = None):

    #image = utils.loadImage(image_path)

    colors = ['blue', 'red', '#7FFF00', 'yellow', '#BF3EFF', '#121212', '#FF69B4', '#FFA54F']

    # Find all facial features in all the faces in the image
    #face_landmarks_list = utils.facePoints(image)
    #boxes = utils.getFaceBounds(image, 2, 'hog')

    pil_image = Image.fromarray(image)
    d = ImageDraw.Draw(pil_image, 'RGBA')

    if face_landmarks_list:
        for i in range(len(face_landmarks_list)):
            color = colors[i%len(colors)]
            face_landmarks = face_landmarks_list[i]

            # Make the eyebrows into a nightmare
            d.point(face_landmarks['left_eyebrow'], fill=color)
            d.point(face_landmarks['right_eyebrow'], fill=color)

            d.point(face_landmarks['chin'], fill=color)

            # lips
            d.point(face_landmarks['top_lip'], fill=color)
            d.point(face_landmarks['bottom_lip'], fill=color)
            '''
            d.polygon(face_landmarks['top_lip'], fill=(150, 0, 0, 128))
            d.polygon(face_landmarks['bottom_lip'], fill=(150, 0, 0, 128))
            d.line(face_landmarks['top_lip'], fill=(150, 0, 0, 64), width=8)
            d.line(face_landmarks['bottom_lip'], fill=(150, 0, 0, 64), width=8)
            '''
            d.point(face_landmarks['left_eye'] + [face_landmarks['left_eye'][0]], fill=color)
            d.point(face_landmarks['right_eye'] + [face_landmarks['right_eye'][0]], fill=color)

    #print('deferfr', boxes)
    for i in range(len(boxes)):
        #print(boxes[i])
        box = boxes[i][1]
        #print(box[3], box[0]), (box[1], box[2])
        color = colors[i%len(colors)]
        d.rectangle(((box[3], box[0]), (box[1], box[2])), outline = color)
        d.text((box[3], box[2]), text=boxes[i][0], fill = color)
        #d.text((box[3], box[2]), text=str(boxes[i][2]), fill = color)

    pil_image.save(out_loc, "JPEG")

def _buffer_image(filename):
    image = cv2.imread(filename, )
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    return image

def get_image_details(path):
    result = []
    header = []
    isHeader = True
    with open(path) as csvfile:
        reader = csv.reader(csvfile, delimiter=',')
        dict_list = []
        for row in reader:
            if isHeader:
                header = row
                isHeader = False
            else:
                result.append(dict(zip(header, row)))
    return result

def main(img_dir, result):

    #for root, dirs, files in os.walk(img_dir):

    #for file in glob.glob(os.path.join(img_dir,"**/*.csv")):

    file = os.path.join(img_dir, os.path.join(os.path.basename(img_dir).replace('_jpg', '_jpg.csv')))
    df_f = pd.read_csv(file)
    #print(file)
    boxes = []
    #print(file)
    for d in get_image_details(file):
        if len(list(d.keys())) > 0:
            parent_file = os.path.dirname(file).replace('_jpg', '.jpg')
            #print(d)
            bb = [int(s) for s in d['bb'].replace('L','').split(',') if s.isdigit()]
            r = result[d['segment']]

            df_f.loc[df_f['segment']==d['segment'], 'accuracy'] = r.accuracy
            df_f.loc[df_f['segment']==d['segment'], 'name'] = r.person_name

            boxes.append([r.person_name, bb, r.accuracy])

    #print(os.path.dirname(file).replace('_jpg', '.jpg'))
    out_path = os.path.dirname(file).replace('_jpg', '_processed.jpg')
    #print('o',out_path)
    _image = _buffer_image(os.path.dirname(file).replace('_jpg', '.jpg'))
    draw_on_image(_image, out_path, boxes)
    df_f.to_csv(file, index=False)

if __name__ == '__main__':
    import os
    import glob
    import cv2
    from align_dlib import AlignDlib

    align_dlib = AlignDlib(os.path.join(os.path.dirname(__file__), 'transferredSkills/dlib_models/shape_predictor_68_face_landmarks.dat'))

    def _buffer_image(filename):
        image = cv2.imread(filename, )
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        return image

    out_dir = 'tests_output/draw_test'

    if not os.path.exists(out_dir):
        os.makedirs(out_dir)

    image_paths = glob.glob(os.path.join('tests/crop_test', '**/*.jpg'))
    for image in image_paths:
        _image = _buffer_image(image)
        boxes = [ align_dlib._getBoxCoordinates(b) for b in align_dlib.getAllFaceBoundingBoxes(_image)]
        landmarks = align_dlib.facePoints( _image, boxes)

        out_path = os.path.join(out_dir, os.path.basename(image)).replace('.jpg', '') + "_processed.jpg"
        draw_on_image(_image, out_path, boxes, landmarks)


from queue import Queue

import task2
import solver

class HigherLevelSokoban:

    def __init__(self, lower_level_sokoban):
        self.lower_level_sokoban = lower_level_sokoban

    def gen_new_states_with_priority(self, state_with_priority):
        _, state = state_with_priority
        # print('here')
        for new_state, action in self.get_possible_changes(state):
            heuristic = self.get_heuristic(new_state)
            yield (heuristic, new_state), action

    def get_heuristic(self, state):
        _, crates_ordered = state
        return max(
            min(abs(cx - x) + abs(cy - y) for x, y in self.lower_level_sokoban.destinations)
            for cx, cy in crates_ordered
        )

    def get_possible_changes(self, state):
        """Basically a BFS but finds all goals instead of the first one."""
        pos, crates_ordered = state
        crates = set(crates_ordered)

        to_visit = Queue()
        visited = {state}

        to_visit.put((pos, ''))

        while not to_visit.empty():
            sub_root, solution = to_visit.get()
            #print(sub_root, visited)
            if sub_root in self.lower_level_sokoban.walls:
                continue

            if sub_root in crates:
                move = task2.Movement[solution[-1]]
                if move.add_to(sub_root) not in self.lower_level_sokoban.walls | crates:
                    yield (sub_root, tuple(sorted(crates - {sub_root} | {move.add_to(sub_root)}))), solution
                continue

            for movement in task2.Movement:
                child = movement.add_to(sub_root)
                if child in visited:
                    continue
                to_visit.put((child, solution + movement.name))
                visited.add(child)

if __name__ == '__main__':
    with open('zad_input.txt') as inp:
        lower_level_sokoban = task2.Sokoban.from_map(inp)
        game = HigherLevelSokoban(lower_level_sokoban)

    solution = solver.find_solution_with_priority(
        root=(0, lower_level_sokoban.initial_state),
        is_goal=lower_level_sokoban.is_a_win,
        get_successors=game.gen_new_states_with_priority,
    )  # Best-First search in this case

    solution_str = ''.join(solution)
    print(solution_str, len(solution_str))  # for insight

    with open('zad_output.txt', 'w+') as out:
        out.write(solution_str)


from crypt import methods
from flask import Flask , render_template, request,  make_response
from werkzeug.wrappers import Response
import os
from .healpers.CsvHealper import allowed_file, parse_user_file, generate_file

def create_app():
    app = Flask(__name__)
    app.secret_key = os.environ["Secret"]
    return app

app = create_app()

@app.route("/")
def hello_world():
    '''
    index page
    '''
    return render_template("index.html")

@app.route("/upload", methods=['post'])
def handlecsv():
    try:
        if request.method == 'POST':
            # check if the post request has the file part
            if 'file' not in request.files:
                return render_template("index.html", error = 'No file part' )

            file = request.files['file']
            # If the user does not select a file, the browser submits an
            # empty file without a filename.
            if file.filename == '':
                return render_template("index.html", error = 'No selected file' )

            if file and allowed_file(file.filename):

                data = parse_user_file(file)
                if type(data) ==list:

                    # stream the response as the data is generated
                    response = Response(generate_file(data), mimetype='text/csv')

                    # add a filename
                    response.headers.set("Content-Disposition", "attachment", filename=f"{file.filename}-transformed.csv")
                    return response

                else:
                    return render_template("index.html", error = f"error while reading the file : {data}" )

            else:
                return render_template("index.html", error = 'format not supported, please upload csv file' )
    except Exception as e:
        return make_response({'message': f'error {e}'},500)


"""
Graph structure with basic methods built in
"""

from personal_information import (Address, Person, COVID_Status)
from basic_data_structures import (Stack, Queue)
import csv
import random

class Node:
    """
    A single entity in the graph
    """
    pass

class Graph:
    """
    A graph representation of the COVID-19 data
    """

    def iterative_dfs(self, source_id, target_id):
        # source_id -> source objce
        source = self.nodes.get(source_id)
        target = self.nodes.get(target_id)
        stack = Stack()
        path = []

        while not stack.empty():
            # check if visited
            # if not mark as visited
            node = stack.pop()
            if node.visited == True:
                continue

            node.visited = True
            if node == target:
                return path

            path.append(node)

            # node.neighbors --> list of neighbors of node
            for neighbor_id in node.neighbors:
                neigbhor = self.nodes.get(neighbor_id)

        return False

    def shortest_path(self, source_id, target_id):
        """
        loop to iterate over all the nodes -->
            use the id of the source
            use that until you find the target

        how to  use dict : similar idea
            id --> node  <<---
            python dict: dict.value() --> a list of all the right side of the map

        some of bruteforce algo

        """

        source = self.nodes.get(source_id)
        source.path_weight = 0

        target = self.nodes.get(target_id)

        queue = PriorityQueue()
        queue.enqueue(source)

        for node in self.nodes.values():
            queue.enqueue(node)
        while not queue.empty():
            node = queue.dequeue()
            for neigbhor_id in node.neighbors:
                alt_path_weight = node.path_weight + node.weight
                if alt_path_weight <= neighbor.path_weight:
                    neighbor.path_weight = alt_path_weight
                    neighbor.parent = node
            queue.resort()

    def extract_path(self, source, target):
        # base case
        if target == source:
            return [target]

        path = [target]

        parent_path = self.extract_path(source, target.parent)

        # [1,2,3,4] 5 ---> [1,2,3,4,5]
        for node in parent_path:
            path.append(node)

        return path

    def compute_path_probability(self, path):
        """
        [node_id1, node_id2,node_id3,node_id4]
        1. convert from id to node --> look up in the dict
        2. node1.weight * compute_path_probability(path - the first one)
        pop from list: list.pop()
            - this removes the first (last?) items and returns to you

        # product([1,2,3,4])  ==> 1 times product([2,3,4])
                                          ==> 2 * product([3,4])
                                                 ==> 3 * product([4])
                                                         ==> 4 * product([])
                                                                 ==> 1 --> base
        """
        # vvv the weight of the first time

        if len(path) == 0:
            return 1

        node = self.nodes.get(path.pop())
        weight = node.weight
        return weight * self.compute_path_probability(path)


from django.shortcuts import render
from django.http import HttpResponse
from django.template import RequestContext
from .forms import Mp3DetailsForm

from gtts import gTTS

import os
import time

def index(request):

    if request.method == 'POST':
        context = RequestContext(request)
        aform = Mp3DetailsForm(request.POST)

        if aform.is_valid():
            file_name = aform.cleaned_data['file_name'].replace(" ", "")
            text = aform.cleaned_data['text']
            create_sound(file_name, text)
            file_path = '/Users/ivanG/Documents/text_to_speech/Text_to_Speech/' + file_name + ".mp3"

            while not os.path.exists(file_path):
                time.sleep(1)
                if os.path.exists(file_path):
                    break;

            print("file about to be opened")
            fsock = open(file_path, 'r')
            response = HttpResponse(fsock, content_type='audio/mpeg')
            response['Content-Disposition'] = "attachment; filename=%s.mp3" % file_name
            return response

        else:
            aform = Mp3DetailsForm()

        return render(request, 'tts/createmp3.html')

    else:
        form = Mp3DetailsForm()
        context = RequestContext(request)
        return render(request, 'tts/index.html', {'form': form})

def create_sound(file_name, text):
    txt = text
    tts = gTTS(text=txt, lang='en')
    tts.save(file_name + ".mp3")
    return tts


#!/usr/bin/env python3
import re
import csv
import collections

def get_error_message_type(log):
    if re.search(r"ERROR", log):
        return "ERROR"
    if re.search(r"INFO", log):
        return "INFO"

def find_errors():
    error_dict = {}
    with open("syslog.log", "r") as logfile:
        for line in logfile:
            if re.search(r"ERROR", line):
                print(line)
                print("FOUND AN ERROR")
                error_message = str(re.search(r"(ticky).*?\s*\(", line).group(0)).replace(" (", "").replace("ticky: ERROR ", "")
                if error_message not in error_dict:
                    error_dict[error_message] = 0
                error_dict[error_message] += 1
            if re.search(r"INFO", line):
                pass

    return error_dict

def get_per_user_errors():
    per_user_errors = {}
    with open("syslog.log", "r") as logfile:
        for line in logfile:
            username = str(re.search(r"\(.*\)", line).group(0)).replace("(", "").replace(")", "")
            if username not in per_user_errors:
                per_user_errors[username] = {"INFO": 0, "ERROR": 0}
            per_user_errors[username][get_error_message_type(line)] +=1

    return per_user_errors

def write_per_user_dict_to_csv(d):
    with open('user_statistics.csv','w', newline='', encoding='utf-8') as f:
        w = csv.writer(f)
        header = ["Username", "INFO", "ERROR"]
        w.writerow(header)
        od = collections.OrderedDict(sorted(d.items()))
        for k, v in od.items():
            row = [k, v["INFO"], v["ERROR"]]
            w.writerow(row)

def write_error_dict_to_csv(d):
    with open("error_message.csv", "w", newline="", encoding="utf-8") as f:
        w = csv.writer(f)
        header = ["Error", "Count"]
        od = sorted(d.items(), key=lambda x: x[1], reverse=True)
        w.writerow(header)
        for row in od:
            w.writerow(row)

if __name__ == '__main__':
    different_error_messages_dict = find_errors()
    write_error_dict_to_csv(different_error_messages_dict)

    per_user_errors = get_per_user_errors()
    write_per_user_dict_to_csv(per_user_errors)


#!/usr/bin/python3
# 9-model_state_filter_a.py
"""Script that lists all State that contains the letter a
from the database hbtn_0e_6_usa
"""
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from model_state import Base, State
from sys import argv

def firstState():
    """Prints first state
    """
    engine = create_engine('mysql+mysqldb://{}:{}@localhost:3306/{}'.format(
        argv[1],
        argv[2],
        argv[3]),
        pool_pre_ping=True)
    Base.metadata.create_all(engine)

    Session = sessionmaker(bind=engine)
    session = Session()

    states = session.query(State).filter(State.name.contains("a"))
    for state in states:
        print("{}: {}".format(state.id, state.name))
        session.close()

if __name__ == '__main__':
    firstState()


def check_square(matrix, r, c):
    square_list = [matrix[r - 1][c - 1],
                   matrix[r - 1][c],
                   matrix[r - 1][c + 1],
                   matrix[r][c - 1],
                   matrix[r][c + 1],
                   matrix[r + 1][c - 1],
                   matrix[r + 1][c],
                   matrix[r + 1][c + 1]]
    return square_list if matrix[r][c] > max(square_list) else False

def create_edge_neighbors(matrix):
    edge_mat = [[matrix[0][1], matrix[1][0], matrix[1][1]],
                [matrix[0][-2], matrix[1][-2], matrix[1][-1]],
                [matrix[-2][0], matrix[-2][1], matrix[-1][1]],
                [matrix[-2][-2], matrix[-2][-1], matrix[-1][-2]]]
    return edge_mat

def get_edge_number(i, j):
    # upper left edge
    edge_number = 0
    if i == j:
        # lower right edge
        if not i == 0:
            edge_number = 3
    else:
        # upper right edge
        if i == 0:
            edge_number = 1
        # lower left edge
        else:
            edge_number = 2
    return edge_number

def check_middles_frame(mat, i, j):
    if i == 0 or i == len(mat) - 1:
        if i == 0:
            delta = 1
        else:
            delta = -1
        list_neigh = [mat[i][j - 1],
                      mat[i][j + 1],
                      mat[i + delta][j - 1],
                      mat[i + delta][j],
                      mat[i + delta][j + 1]]
    # cols
    else:
        if j == 0:
            delta = 1
        else:
            delta = -1
        list_neigh = [mat[i - 1][j],
                      mat[i + 1][j],
                      mat[i - 1][j + delta],
                      mat[i][j + delta],
                      mat[i + 1][j + delta]]
    return list_neigh

def check_mat(matr):
    counter = 0
    edge_matrix = create_edge_neighbors(matr)
    # iterate on rows
    for i in range(len(matr)):
        # iterate on cols
        for j in range(len(matr[i])):
            # if not first/last row.
            # if not first/last col.
            if (0 < i < len(matr) - 1) and (0 < j < len(matr[i]) - 1):
                checked_list = check_square(matr, i, j)
                if checked_list:
                    counter += 1
                    print("%d. wrapped_matr[%d][%d]=%d > %s" % (counter, i, j, matr[i][j], checked_list))
            # if on edges(checks for diagonal, anti-diagonal).
            elif i == j or (len(matr[i]) - i - 1 == j):
                edge_neig = edge_matrix[get_edge_number(i, j)]
                is_point_big = matr[i][j] > max(edge_neig)
                if is_point_big:
                    counter += 1
                    print("%d. wrapped_matr[%d][%d]=%d > %s" % (counter, i, j, matr[i][j], edge_neig))
            # on middle frame
            else:
                frame_nigh = check_middles_frame(matr, i, j)
                is_big = matr[i][j] > max(frame_nigh)
                if is_big:
                    counter += 1
                    print("%d. wrapped_matr[%d][%d]=%d > %s" % (counter, i, j, matr[i][j], frame_nigh))
    return counter

def main():
    mat = [[2, 3, 4, 5, 6],
           [6, 5, 7, 4, 3],
           [3, 4, 9, 8, 2],
           [5, 4, 8, 7, 6],
           [1, 2, 9, 5, 9]]
    print("count =", check_mat(mat))

main()


#Faça um programa que leia um número inteiro e diga quantas vezes ele foi dividido - utilizar cores

n = int(input('Digite um número: '))
tot = 0
for i in range(1, n + 1):
    if n%i == 0:
        print('\033[33m', end='')
        tot += 1
    else:
        print('\033[31m', end='')
    print('{} '.format(i), end='')
print('\n\033[mO número {} foi divisível {} vezes'.format(n, tot))


#Tool check active subdomain from amass result

from concurrent.futures import ThreadPoolExecutor
import requests
from requests.exceptions import ConnectionError
import argparse

parser = argparse.ArgumentParser(description='Example argument parser')
parser.add_argument('--input_file','-i', help='path to input file')
parser.add_argument('--output-file', '-o', default='output.txt', help='path to output file')

args = parser.parse_args()

domain_list = []
active_domain = []
def read_file(file):
    with open(file, 'r') as f:
        for url in f:
            url = url.replace('\n','')
            domain_list.append(url)

def validate_existence(domain):
        try:
            response = requests.get(domain, timeout=10)
        except ConnectionError:
            print(f'Domain {domain} [is dead]')
        else:
            print(f'Domain {domain} [is active]')
            active_domain.append(domain)

def write_file():
    with open(args.output_file, 'w') as f:
        for url in active_domain:
            f.write(url + '\n')

read_file(args.input_file)
with ThreadPoolExecutor() as executor:
    executor.map(validate_existence, domain_list)
write_file()


__docformat__ = 'restructuredtext'

import sys

from zope import component
from zope.interface import implements
from zope.component.interfaces import ComponentLookupError
from zope.traversing.interfaces import IPathAdapter, ITraversable
from zope.traversing.interfaces import TraversalError
from zope.traversing.adapters import traversePathElement
from zope.security.untrustedpython import rcompile
from zope.security.proxy import ProxyFactory, removeSecurityProxy
from zope.security.untrustedpython.builtins import SafeBuiltins
from zope.i18n import translate

from zope.tales.expressions import PathExpr, StringExpr, NotExpr, DeferExpr
from zope.tales.expressions import SimpleModuleImporter
from zope.tales.pythonexpr import PythonExpr
from zope.tales.tales import ExpressionEngine, Context

from i18n import ZopeMessageFactory as _

class InlineCodeError(Exception):
    pass

class ZopeTraverser(object):

    def __init__(self, proxify=None):
        if proxify is None:
            self.proxify = lambda x: x
        else:
            self.proxify = proxify

    def __call__(self, object, path_items, econtext):
        """Traverses a sequence of names, first trying attributes then items.
        """
        request = getattr(econtext, 'request', None)
        path_items = list(path_items)
        path_items.reverse()

        while path_items:
            name = path_items.pop()

            # special-case dicts for performance reasons
            if getattr(object, '__class__', None) == dict:
                object = object[name]
            else:
                object = traversePathElement(object, name, path_items,
                                             request=request)
            object = self.proxify(object)
        return object

zopeTraverser = ZopeTraverser(ProxyFactory)

class ZopePathExpr(PathExpr):

    def __init__(self, name, expr, engine):
        super(ZopePathExpr, self).__init__(name, expr, engine, zopeTraverser)

trustedZopeTraverser = ZopeTraverser()

class TrustedZopePathExpr(PathExpr):

    def __init__(self, name, expr, engine):
        super(TrustedZopePathExpr, self).__init__(name, expr, engine,
                                                  trustedZopeTraverser)

# Create a version of the restricted built-ins that uses a safe
# version of getattr() that wraps values in security proxies where
# appropriate:

class ZopePythonExpr(PythonExpr):

    def __call__(self, econtext):
        __traceback_info__ = self.text
        vars = self._bind_used_names(econtext, SafeBuiltins)
        return eval(self._code, vars)

    def _compile(self, text, filename):
        return rcompile.compile(text, filename, 'eval')

class ZopeContextBase(Context):
    """Base class for both trusted and untrusted evaluation contexts."""

    def translate(self, msgid, domain=None, mapping=None, default=None):
        return translate(msgid, domain, mapping,
                         context=self.request, default=default)

    evaluateInlineCode = False

    def evaluateCode(self, lang, code):
        if not self.evaluateInlineCode:
            raise InlineCodeError(
                  _('Inline Code Evaluation is deactivated, which means that '
                    'you cannot have inline code snippets in your Page '
                    'Template. Activate Inline Code Evaluation and try again.'))

        # TODO This is only needed when self.evaluateInlineCode is true,
        # so should only be needed for zope.app.pythonpage.
        from zope.app.interpreter.interfaces import IInterpreter
        interpreter = component.queryUtility(IInterpreter, lang)
        if interpreter is None:
            error = _('No interpreter named "${lang_name}" was found.',
                      mapping={'lang_name': lang})
            raise InlineCodeError(error)

        globals = self.vars.copy()
        result = interpreter.evaluateRawCode(code, globals)
        # Add possibly new global variables.
        old_names = self.vars.keys()
        for name, value in globals.items():
            if name not in old_names:
                self.setGlobal(name, value)
        return result

class ZopeContext(ZopeContextBase):
    """Evaluation context for untrusted programs."""

    def evaluateMacro(self, expr):
        """evaluateMacro gets security-proxied macro programs when this
        is run with the zopeTraverser, and in other untrusted
        situations. This will cause evaluation to fail in
        zope.tal.talinterpreter, which knows nothing of security proxies.
        Therefore, this method removes any proxy from the evaluated
        expression.

        >>> output = [('version', 'xxx'), ('mode', 'html'), ('other', 'things')]
        >>> def expression(context):
        ...     return ProxyFactory(output)
        ...
        >>> zc = ZopeContext(ExpressionEngine, {})
        >>> out = zc.evaluateMacro(expression)
        >>> type(out)
        <type 'list'>

        The method does some trivial checking to make sure we are getting
        back a macro like we expect: it must be a sequence of sequences, in
        which the first sequence must start with 'version', and the second
        must start with 'mode'.

        >>> del output[0]
        >>> zc.evaluateMacro(expression) # doctest: +ELLIPSIS
        Traceback (most recent call last):
        ...
        ValueError: ('unexpected result from macro evaluation.', ...)

        >>> del output[:]
        >>> zc.evaluateMacro(expression) # doctest: +ELLIPSIS
        Traceback (most recent call last):
        ...
        ValueError: ('unexpected result from macro evaluation.', ...)

        >>> output = None
        >>> zc.evaluateMacro(expression) # doctest: +ELLIPSIS
        Traceback (most recent call last):
        ...
        ValueError: ('unexpected result from macro evaluation.', ...)
        """
        macro = removeSecurityProxy(Context.evaluateMacro(self, expr))
        # we'll do some basic checks that it is the sort of thing we expect
        problem = False
        try:
            problem = macro[0][0] != 'version' or macro[1][0] != 'mode'
        except (TypeError, IndexError):
            problem = True
        if problem:
            raise ValueError('unexpected result from macro evaluation.', macro)
        return macro

    def setContext(self, name, value):
        # Hook to allow subclasses to do things like adding security proxies
        Context.setContext(self, name, ProxyFactory(value))

class TrustedZopeContext(ZopeContextBase):
    """Evaluation context for trusted programs."""

class AdapterNamespaces(object):
    """Simulate tales function namespaces with adapter lookup.

    When we are asked for a namespace, we return an object that
    actually computes an adapter when called:

    To demonstrate this, we need to register an adapter:

      >>> from zope.component.testing import setUp, tearDown
      >>> setUp()
      >>> from zope.component import provideAdapter
      >>> def adapter1(ob):
      ...     return 1
      >>> adapter1.__component_adapts__ = (None,)
      >>> provideAdapter(adapter1, None, IPathAdapter, 'a1')

    Now, with this adapter in place, we can try out the namespaces:

      >>> ob = object()
      >>> namespaces = AdapterNamespaces()
      >>> namespace = namespaces['a1']
      >>> namespace(ob)
      1
      >>> namespace = namespaces['a2']
      >>> namespace(ob)
      Traceback (most recent call last):
      ...
      KeyError: 'a2'

    Cleanup:

      >>> tearDown()
    """

    def __init__(self):
        self.namespaces = {}

    def __getitem__(self, name):
        namespace = self.namespaces.get(name)
        if namespace is None:
            def namespace(object):
                try:
                    return component.getAdapter(object, IPathAdapter, name)
                except ComponentLookupError:
                    raise KeyError(name)

            self.namespaces[name] = namespace
        return namespace

class ZopeBaseEngine(ExpressionEngine):

    _create_context = ZopeContext

    def __init__(self):
        ExpressionEngine.__init__(self)
        self.namespaces = AdapterNamespaces()

    def getContext(self, __namespace=None, **namespace):
        if __namespace:
            if namespace:
                namespace.update(__namespace)
            else:
                namespace = __namespace

        context = self._create_context(self, namespace)

        # Put request into context so path traversal can find it
        if 'request' in namespace:
            context.request = namespace['request']

        # Put context into context so path traversal can find it
        if 'context' in namespace:
            context.context = namespace['context']

        return context

class ZopeEngine(ZopeBaseEngine):
    """Untrusted expression engine.

    This engine does not allow modules to be imported; only modules
    already available may be accessed::

      >>> modname = 'zope.pagetemplate.tests.trusted'
      >>> engine = _Engine()
      >>> context = engine.getContext(engine.getBaseNames())

      >>> modname in sys.modules
      False
      >>> context.evaluate('modules/' + modname)
      Traceback (most recent call last):
        ...
      KeyError: 'zope.pagetemplate.tests.trusted'

    (The use of ``KeyError`` is an unfortunate implementation detail; I
    think this should be a ``TraversalError``.)

    Modules which have already been imported by trusted code are
    available, wrapped in security proxies::

      >>> m = context.evaluate('modules/sys')
      >>> m.__name__
      'sys'
      >>> m._getframe
      Traceback (most recent call last):
        ...
      ForbiddenAttribute: ('_getframe', <module 'sys' (built-in)>)

    The results of Python expressions evaluated by this engine are
    wrapped in security proxies::

      >>> r = context.evaluate('python: {12: object()}.values')
      >>> type(r)
      <type 'zope.security._proxy._Proxy'>
      >>> r = context.evaluate('python: {12: object()}.values()[0].__class__')
      >>> type(r)
      <type 'zope.security._proxy._Proxy'>

    General path expressions provide objects that are wrapped in
    security proxies as well::

      >>> from zope.component.testing import setUp, tearDown
      >>> from zope.security.checker import NamesChecker, defineChecker

      >>> class Container(dict):
      ...     implements(ITraversable)
      ...     def traverse(self, name, further_path):
      ...         return self[name]

      >>> setUp()
      >>> defineChecker(Container, NamesChecker(['traverse']))
      >>> d = engine.getBaseNames()
      >>> foo = Container()
      >>> foo.__name__ = 'foo'
      >>> d['foo'] = ProxyFactory(foo)
      >>> foo['bar'] = bar = Container()
      >>> bar.__name__ = 'bar'
      >>> bar.__parent__ = foo
      >>> bar['baz'] = baz = Container()
      >>> baz.__name__ = 'baz'
      >>> baz.__parent__ = bar
      >>> context = engine.getContext(d)

      >>> o1 = context.evaluate('foo/bar')
      >>> o1.__name__
      'bar'
      >>> type(o1)
      <type 'zope.security._proxy._Proxy'>

      >>> o2 = context.evaluate('foo/bar/baz')
      >>> o2.__name__
      'baz'
      >>> type(o2)
      <type 'zope.security._proxy._Proxy'>
      >>> o3 = o2.__parent__
      >>> type(o3)
      <type 'zope.security._proxy._Proxy'>
      >>> o1 == o3
      True

      >>> o1 is o2
      False

    Note that this engine special-cases dicts during path traversal:
    it traverses only to their items, but not to their attributes
    (e.g. methods on dicts), because of performance reasons:

      >>> d = engine.getBaseNames()
      >>> d['adict'] = {'items': 123}
      >>> d['anotherdict'] = {}
      >>> context = engine.getContext(d)
      >>> context.evaluate('adict/items')
      123
      >>> context.evaluate('anotherdict/keys')
      Traceback (most recent call last):
        ...
      KeyError: 'keys'

      >>> tearDown()

    """

    def getFunctionNamespace(self, namespacename):
        """ Returns the function namespace """
        return ProxyFactory(
            super(ZopeEngine, self).getFunctionNamespace(namespacename))

class TrustedZopeEngine(ZopeBaseEngine):
    """Trusted expression engine.

    This engine allows modules to be imported::

      >>> modname = 'zope.pagetemplate.tests.trusted'
      >>> engine = _TrustedEngine()
      >>> context = engine.getContext(engine.getBaseNames())

      >>> modname in sys.modules
      False
      >>> m = context.evaluate('modules/' + modname)
      >>> m.__name__ == modname
      True
      >>> modname in sys.modules
      True

    Since this is trusted code, we can look at whatever is in the
    module, not just __name__ or what's declared in a security
    assertion::

      >>> m.x
      42

    Clean up after ourselves::

      >>> del sys.modules[modname]

    """

    _create_context = TrustedZopeContext

class TraversableModuleImporter(SimpleModuleImporter):

    implements(ITraversable)

    def traverse(self, name, further_path):
        try:
            return self[name]
        except KeyError:
            raise TraversalError(self, name)

def _Engine(engine=None):
    if engine is None:
        engine = ZopeEngine()
    engine = _create_base_engine(engine, ZopePathExpr)
    engine.registerType('python', ZopePythonExpr)

    # Using a proxy around sys.modules allows page templates to use
    # modules for which security declarations have been made, but
    # disallows execution of any import-time code for modules, which
    # should not be allowed to happen during rendering.
    engine.registerBaseName('modules', ProxyFactory(sys.modules))

    return engine

def _TrustedEngine(engine=None):
    if engine is None:
        engine = TrustedZopeEngine()
    engine = _create_base_engine(engine, TrustedZopePathExpr)
    engine.registerType('python', PythonExpr)
    engine.registerBaseName('modules', TraversableModuleImporter())
    return engine

def _create_base_engine(engine, pathtype):
    for pt in pathtype._default_type_names:
        engine.registerType(pt, pathtype)
    engine.registerType('string', StringExpr)
    engine.registerType('not', NotExpr)
    engine.registerType('defer', DeferExpr)
    return engine

Engine = _Engine()
TrustedEngine = _TrustedEngine()

class AppPT(object):

    def pt_getEngine(self):
        return Engine

class TrustedAppPT(object):

    def pt_getEngine(self):
        return TrustedEngine


from typing import Optional, List
import numpy as np
import torch
import torch.nn as nn
from transformers import RagSequenceForGeneration, AutoModel, AutoConfig, RagRetriever, BatchEncoding
from transformers.models.dpr.modeling_dpr import DPRQuestionEncoderOutput
from distributed_pytorch_retriever import RagPyTorchDistributedRetriever

class RobertaRetriever(nn.Module):
  def __init__(self, model_name):
    super().__init__()

    config = AutoConfig.from_pretrained(model_name)
    self.encoder = AutoModel.from_pretrained(model_name)
    self.project = nn.Sequential(nn.Linear(config.hidden_size, config.hidden_size),
                                 nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps))

  def encode_seq(self, input_ids, mask):
    cls_rep = self.encoder(input_ids, mask)[0][:, 0, :]
    vector = self.project(cls_rep)
    return vector

  def forward(self,
              input_ids: Optional[torch.Tensor] = None,
              attention_mask: Optional[torch.Tensor] = None,
              token_type_ids: Optional[torch.Tensor] = None,
              inputs_embeds: Optional[torch.Tensor] = None,
              output_attentions=None,
              output_hidden_states=None,
              return_dict=None,):
    pooled = self.encode_seq(input_ids, attention_mask)
    if not return_dict:
      return [pooled]
    return DPRQuestionEncoderOutput(pooler_output=pooled, hidden_states=None, attentions=None)  # TODO: replace none

def load_saved(model, path, exact=True):
  try:
    state_dict = torch.load(path)
  except:
    state_dict = torch.load(path, map_location=torch.device('cpu'))

  def filter(x):
    if x.startswith('rag.question_encoder.'):
      return x[21:]
    if x.startswith('module.'):
      return x[7:]
    return x

  if exact:
    state_dict = {filter(k): v for (k, v) in state_dict.items()}
  else:
    state_dict = {filter(k): v for (k, v) in state_dict.items() if filter(k) in model.state_dict()}
  model.load_state_dict(state_dict, strict=False)  # TODO: embeddings.position_ids missing
  return model

class MyRagRetriever(RagRetriever):
  def __call__(
    self,
    question_input_ids: List[List[int]],
    question_hidden_states: np.ndarray,
    question_strings: List[str]=None,
    prefix=None,
    n_docs=None,
    return_tensors=None,
  ) -> BatchEncoding:
    n_docs = n_docs if n_docs is not None else self.n_docs
    prefix = prefix if prefix is not None else self.config.generator.prefix
    retrieved_doc_embeds, doc_ids, docs = self.retrieve(question_hidden_states, n_docs)

    if question_strings is None:
      input_strings = self.question_encoder_tokenizer.batch_decode(question_input_ids, skip_special_tokens=True)
    else:
      input_strings = [s.replace('[MASK]', '<mask>') for s in question_strings]  # TODO: make it more robust to different models
    context_input_ids, context_attention_mask = self.postprocess_docs(
      docs, input_strings, prefix, n_docs, return_tensors=return_tensors
    )

    return BatchEncoding(
      {
        "context_input_ids": context_input_ids,
        "context_attention_mask": context_attention_mask,
        "retrieved_doc_embeds": retrieved_doc_embeds,
        "doc_ids": doc_ids,
      },
      tensor_type=return_tensors,
    )

class MyRagPyTorchDistributedRetriever(RagPyTorchDistributedRetriever):
  def __call__(
    self,
    question_input_ids: List[List[int]],
    question_hidden_states: np.ndarray,
    question_strings: List[str]=None,
    prefix=None,
    n_docs=None,
    return_tensors=None,
  ) -> BatchEncoding:
    n_docs = n_docs if n_docs is not None else self.n_docs
    prefix = prefix if prefix is not None else self.config.generator.prefix
    retrieved_doc_embeds, doc_ids, docs = self.retrieve(question_hidden_states, n_docs)

    if question_strings is None:
      input_strings = self.question_encoder_tokenizer.batch_decode(question_input_ids, skip_special_tokens=True)
    else:
      input_strings = [s.replace('[MASK]', '<mask>') for s in question_strings]  # TODO: make it more robust to different models
    context_input_ids, context_attention_mask = self.postprocess_docs(
      docs, input_strings, prefix, n_docs, return_tensors=return_tensors
    )

    return BatchEncoding(
      {
        "context_input_ids": context_input_ids,
        "context_attention_mask": context_attention_mask,
        "retrieved_doc_embeds": retrieved_doc_embeds,
        "doc_ids": doc_ids,
      },
      tensor_type=return_tensors,
    )

class MemoryBank(nn.Module):
  def __init__(self, bank_size: int, emb_size: int = None):
    super().__init__()
    self.bank_size = bank_size
    self.emb_size = emb_size
    if emb_size is not None:
      self.register_buffer('queue', torch.randn(bank_size, emb_size))
    self.register_buffer('ptr', torch.zeros(1, dtype=torch.long))
    self.register_buffer('is_full', torch.zeros(1, dtype=bool))

  @property
  def is_initialized(self):
    return hasattr(self, 'queue')

  @torch.no_grad()
  def put(self,
          embeddings: torch.FloatTensor):  # (bs, emb_size)
    bs, es = embeddings.size()
    ptr = int(self.ptr)

    if not self.is_initialized:
      self.emb_size = es
      self.register_buffer('queue', torch.randn(self.bank_size, self.emb_size).to(embeddings.device))

    # update queue
    if ptr + bs > self.bank_size:  # reach the end of the queue
      self.queue[ptr:, :] = embeddings[:self.bank_size - ptr]
      self.queue[:ptr + bs - self.bank_size, :] = embeddings[self.bank_size - ptr:]
      self.is_full = self.is_full | True
    else:
      self.queue[ptr:ptr + bs, :] = embeddings

    # move pointer
    self.ptr[0] = (ptr + bs) % self.bank_size
    return

  @torch.no_grad()
  def get(self):
    if not self.is_initialized:
      raise ValueError('queue not initialized')
    if not self.is_full:
      return self.queue[:self.ptr]  # (<bank_size, emb_size)
    return self.queue

class MyRagSequenceForGeneration(RagSequenceForGeneration):
  def set_additional_retrievers(self, retrievers: List):
    self._retrievers = retrievers

  def retrieve_from_multiple(self, *args, **kwargs):
    if not hasattr(self, '_retrievers'):
      return self.retriever(*args, **kwargs)

    all_retrievers = [self.retriever] + self._retrievers
    question_hidden_states = torch.tensor(args[1])
    n_docs = kwargs['n_docs']

    cii_li = []
    cam_li = []
    rde_li = []
    di_li = []
    ds_li = []

    # collect retrieved results from all retrievers
    for retriever in all_retrievers:
      retriever_outputs = retriever(*args, **kwargs)
      cii, cam, rde, di = (
        retriever_outputs['context_input_ids'],  # (batch_size * n_docs, seq_len)
        retriever_outputs['context_attention_mask'],  # (batch_size * n_docs, seq_len)
        retriever_outputs['retrieved_doc_embeds'],  # (batch_size, n_docs, emb_size)
        retriever_outputs['doc_ids']  # (batch_size, n_docs)
      )
      cii_li.append(cii.view(-1, n_docs, cii.size(-1)))
      cam_li.append(cam.view(-1, n_docs, cam.size(-1)))
      rde_li.append(rde)
      di_li.append(di)
      doc_scores = torch.bmm(question_hidden_states.unsqueeze(1), rde.to(question_hidden_states).transpose(1, 2)).squeeze(1)
      ds_li.append(doc_scores)

    # merge results
    cii = torch.cat(cii_li, 1)
    cam = torch.cat(cam_li, 1)
    rde = torch.cat(rde_li, 1)
    di = torch.cat(di_li, 1)
    doc_scores = torch.cat(ds_li, 1)

    # sort by scores and select
    doc_scores, topk_ind = torch.topk(doc_scores, n_docs, dim=1)
    cii = torch.gather(cii, 1, topk_ind.unsqueeze(-1).repeat(1, 1, cii.size(-1))).view(-1, cii.size(-1))
    cam = torch.gather(cam, 1, topk_ind.unsqueeze(-1).repeat(1, 1, cam.size(-1))).view(-1, cam.size(-1))
    rde = torch.gather(rde, 1, topk_ind.unsqueeze(-1).repeat(1, 1, rde.size(-1)))
    di = torch.gather(di, 1, topk_ind)

    return {'context_input_ids': cii, 'context_attention_mask': cam, 'retrieved_doc_embeds': rde, 'doc_ids': di}

  @torch.no_grad()
  def generate(
    self,
    input_ids: Optional[torch.LongTensor] = None,
    attention_mask: Optional[torch.LongTensor] = None,
    context_input_ids=None,
    context_attention_mask=None,
    doc_scores=None,
    do_deduplication=None,  # defaults to True
    num_return_sequences=None,  # defaults to 1
    num_beams=None,  # defaults to 1
    n_docs=None,
    **model_kwargs
  ):
    """
    Implements RAG sequence "thorough" decoding. Read the :meth:`~transformers.PreTrainedModel.generate``
    documentation for more information on how to set other generate input parameters.
    Args:
        input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
            The sequence used as a prompt for the generation. If :obj:`input_ids` is not passed, then
            :obj:`context_input_ids` has to be provided.
        attention_mask (:obj:`torch.Tensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
            Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``:
            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
            `What are attention masks? <../glossary.html#attention-mask>`__
        context_input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size * config.n_docs, config.max_combined_length)`, `optional`, returned when `output_retrieved=True`):
            Input IDs post-processed from the retrieved documents and the question encoder input_ids by the
            retriever.
        context_attention_mask (:obj:`torch.LongTensor` of shape :obj:`(batch_size * config.n_docs, config.max_combined_length)`, `optional`, returned when `output_retrieved=True`):
            Attention mask post-processed from the retrieved documents and the question encoder :obj:`input_ids` by
            the retriever.
            If the model is not initialized with a ``retriever`` or ``input_ids`` is not given,
            :obj:`context_input_ids` and :obj:`context_attention_mask` have to be provided to the forward pass.
            They are returned by :meth:`~transformers.RagRetriever.__call__`.
        doc_scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.n_docs)`):
            Score between each retrieved document embeddings (see :obj:`retrieved_doc_embeds`) and
            :obj:`question_encoder_last_hidden_state`.
            If the model is not initialized with a ``retriever`` or ``input_ids`` is not given, :obj:`doc_scores`
            has to be provided to the forward pass. :obj:`doc_scores` are returned by
            :meth:`~transformers.RagRetriever.__call__`.
        do_deduplication (:obj:`bool`, `optional`):
            Whether or not to deduplicate the generations from different context documents for a given input. Has
            to be set to :obj:`False` if used while training with distributed backend.
        num_return_sequences(:obj:`int`, `optional`, defaults to 1):
            The number of independently computed returned sequences for each element in the batch. Note that this
            is not the value we pass to the ``generator``'s `:func:`~transformers.PreTrainedModel.generate``
            function, where we set ``num_return_sequences`` to :obj:`num_beams`.
        num_beams (:obj:`int`, `optional`, defaults to 1):
            Number of beams for beam search. 1 means no beam search.
        n_docs (:obj:`int`, `optional`, defaults to :obj:`config.n_docs`)
            Number of documents to retrieve and/or number of documents for which to generate an answer.
        kwargs:
            Additional kwargs will be passed to :meth:`~transformers.PreTrainedModel.generate`.
    Return:
        :obj:`torch.LongTensor` of shape :obj:`(batch_size * num_return_sequences, sequence_length)`: The generated
        sequences. The second dimension (sequence length) is either equal to :obj:`max_length` or shorter if all
        batches finished early due to the :obj:`eos_token_id`.
    """

    n_docs = n_docs if n_docs is not None else self.config.n_docs
    do_deduplication = do_deduplication if do_deduplication is not None else self.config.do_deduplication
    num_doc_return_sequences = (
      num_return_sequences if num_return_sequences is not None else self.config.num_return_sequences
    )
    num_beams = num_beams if num_beams is not None else self.config.num_beams

    assert (
      input_ids is not None or context_input_ids is not None
    ), " At least one of input_ids or context_input_ids must be given"

    if self.retriever is not None and context_input_ids is None:
      question_hidden_states = self.question_encoder(input_ids, attention_mask=attention_mask)[0]
      context_input_ids = self.retriever(
        input_ids,
        question_hidden_states.cpu().detach().to(torch.float32).numpy(),
        prefix=self.generator.config.prefix,
        n_docs=n_docs,
        return_tensors="pt",
      )["context_input_ids"]

      # set to correct device
      context_input_ids = context_input_ids.to(input_ids)

    hypos = []
    logprobs = []
    model_kwargs["num_beams"] = num_beams
    model_kwargs["num_return_sequences"] = num_beams
    model_kwargs["attention_mask"] = None

    batch_size = input_ids.shape[0] if input_ids is not None else context_input_ids.shape[0] // n_docs

    for index in range(batch_size):
      # first, generate beams from documents:
      generator_input_ids = context_input_ids[index * n_docs: (index + 1) * n_docs]  # (n_docs, max_len)

      output_sequences = self.generator.generate(
        generator_input_ids,
        **model_kwargs,
      )  # n_docs * n_beam, tgt_len
      if do_deduplication:
        # do_deduplication, max_output_len
        output_sequences = torch.stack(list({str(k.tolist()): k for k in output_sequences}.values()))

      num_candidates = output_sequences.shape[
        0
      ]  # after deduplication, this number can be less than n_docs*n_beam

      # then, run model forwards to get nll scores:
      if input_ids is not None:
        new_input_ids = input_ids[index: index + 1].repeat(num_candidates, 1)
        outputs = self(new_input_ids, labels=output_sequences, exclude_bos_score=True)
      else:  # input_ids is None, need context_input_ids/mask and doc_scores
        assert (
          context_attention_mask is not None
        ), "Make sure that `context_attention_mask` are passed, if no `input_ids` is set. Alternatively, you can set a retriever using the `set_retriever(...)` function."
        assert (
          doc_scores is not None
        ), "Make sure that `doc_scores` are passed, if no `input_ids` is set. Alternatively, you can set a retriever using the `set_retriever(...)` function."

        individual_input_ids = generator_input_ids.repeat(
          num_candidates, 1
        )  # (num_candidates*n_docs, max_len)

        individual_attention_mask = context_attention_mask[index * n_docs: (index + 1) * n_docs]
        individual_attention_mask = individual_attention_mask.repeat(num_candidates, 1)

        individual_doc_scores = doc_scores[index: (index + 1), :]  # doc_scores.shape = [batch, n_docs]
        individual_doc_scores = individual_doc_scores.repeat(num_candidates, 1)  # [num_candidates, n_docs]

        outputs = self(
          context_input_ids=individual_input_ids,
          context_attention_mask=individual_attention_mask,
          doc_scores=individual_doc_scores,
          labels=output_sequences,
          exclude_bos_score=True,
        )

      lps, top_cand_inds = (-outputs["loss"]).topk(num_doc_return_sequences)

      # add hypothesis
      hypos.append(output_sequences[top_cand_inds])
      logprobs.append(lps)

    logprobs = torch.cat(logprobs, 0)
    return self._cat_and_pad(hypos, pad_token_id=self.config.generator.pad_token_id), logprobs

  @staticmethod
  def load_question_encoder(rag_model, question_encoder_path: str):
    model = RobertaRetriever('roberta-base')
    model = load_saved(model, question_encoder_path, exact=False)
    rag_model.rag.question_encoder = model

  def get_nll(
    self, seq_logits, doc_scores, target, reduce_loss=False, epsilon=0.0, exclude_bos_score=False, n_docs=None
  ):
    # shift tokens left
    target = torch.cat(
      [target[:, 1:], target.new(target.shape[0], 1).fill_(self.config.generator.pad_token_id)], 1
    )

    n_docs = n_docs if n_docs is not None else self.config.n_docs

    # bos_token_id is None for T5
    bos_token_id = self.config.bos_token_id or self.config.generator.bos_token_id
    use_bos = bos_token_id is not None and target[:, 0].eq(bos_token_id).all()

    def _mask_pads(ll, smooth_obj):
      pad_mask = target.eq(self.config.generator.pad_token_id)
      if pad_mask.any():
        ll.masked_fill_(pad_mask, 0.0)
        smooth_obj.masked_fill_(pad_mask, 0.0)
      return ll.squeeze(-1), smooth_obj.squeeze(-1)

    # seq_logits dim = (batch*n_docs, tgt_len , #vocabs)
    seq_logprobs = torch.nn.functional.log_softmax(seq_logits, dim=-1).view(
      seq_logits.shape[0] // n_docs, n_docs, -1, seq_logits.size(-1)
    )  # batch_size x n_docs x tgt_len x #vocab_size

    # (bs, n_docs) or (bs, bs * n_docs) or (bs, bs * n_docs + memory_bank_size)
    doc_logprobs = torch.nn.functional.log_softmax(doc_scores, dim=1)

    bs = doc_logprobs.size(0)
    if doc_logprobs.size(1) == n_docs:
      pass
    elif doc_logprobs.size(1) >= bs * n_docs:  # contain doc log probs over the whole batch (plus memory bank)
      # (bs, bs, n_docs)
      doc_logprobs = doc_logprobs[:, :bs * n_docs].view(bs, bs, n_docs)  # remove memory bank
      # (bs, n_docs)
      doc_logprobs = torch.masked_select(
        doc_logprobs,
        torch.eye(bs).unsqueeze(-1).bool().to(doc_logprobs.device)).view(bs, n_docs)
    else:
      raise Exception(f'the size of document log prob {doc_logprobs.size()} is unexpected')

    doc_logprobs = doc_logprobs.unsqueeze(-1).unsqueeze(-1)

    # RAG-sequence marginalization
    first_token_scores = seq_logprobs[:, :, :1, :]
    second_token_scores = seq_logprobs[:, :, 1:2, :]
    remainder = seq_logprobs[:, :, 2:, :]
    rag_logprobs = torch.cat([first_token_scores, second_token_scores + doc_logprobs, remainder], dim=2)

    # calculate loss
    target = target.unsqueeze(1).unsqueeze(-1).repeat(1, n_docs, 1, 1)
    assert target.dim() == rag_logprobs.dim()

    ll = rag_logprobs.gather(dim=-1, index=target)
    smooth_obj = rag_logprobs.sum(dim=-1, keepdim=True)  # total sum of all (normalised) logits

    ll, smooth_obj = _mask_pads(ll, smooth_obj)

    # sum over tokens, exclude bos while scoring
    ll = ll[:, :, 1:].sum(2) if exclude_bos_score and use_bos else ll.sum(2)
    smooth_obj = smooth_obj.sum(2)
    ll = ll.logsumexp(1)  # logsumexp over docs
    smooth_obj = smooth_obj.logsumexp(1)

    nll_loss = -ll
    smooth_loss = -smooth_obj

    if reduce_loss:
      nll_loss = nll_loss.sum()
      smooth_loss = smooth_loss.sum()

    eps_i = epsilon / rag_logprobs.size(-1)
    loss = (1.0 - epsilon) * nll_loss + eps_i * smooth_loss
    return loss


import keras
from keras.datasets import mnist
from keras.models import load_model

def Getdata():
    (x_train, y_train), (x_test, y_test) = mnist.load_data()
    x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
    x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)

    x_train = x_train / 255
    x_test = x_test / 255

    y_train = keras.utils.to_categorical(y_train, 10)
    y_test = keras.utils.to_categorical(y_test, 10)

    data = x_test[0].reshape(-1)
    ans = y_test[0]
    for i in range(28):
        for j in range(28):
            print("%.6f , " % data[i*28+j],end='')
        print('')
    print(ans)
Getdata()


import logging
from enum import Enum
from typing import Optional

import pandas as pd
from pandas import DataFrame

import config
from asseeibot import runtime_variables
from asseeibot.models.cache import Cache
from asseeibot.models.fuzzy_match import FuzzyMatch, MatchBasedOn
from asseeibot.models.ontology_dataframe import OntologyDataframeColumn
from asseeibot.models.wikimedia.wikidata.entity import EntityId

logger = logging.getLogger(__name__)

# This code is adapted from https://github.com/dpriskorn/WikidataMLSuggester and LexUtils

# lookups where inspired by
# https://stackoverflow.com/questions/24761133/pandas-check-if-row-exists-with-certain-values

class CacheDataframeColumn(Enum):
    QID = "qid"
    CROSSREF_SUBJECT = "crossref_subject"
    MATCH_BASED_ON = "match_based_on"
    SPLIT_SUBJECT = "split_subject"
    ORIGINAL_SUBJECT = "original_subject"

class MatchCache(Cache):
    """This models our cache of matches"""
    crossref_subject_found: bool = None
    dataframe: DataFrame = None
    qid_dropped: bool = None
    qid_found: bool = None
    pickle: str = config.cache_pickle_filename

    class Config:
        # Because of DataFrame
        arbitrary_types_allowed = True

    def __check_variables__(self):
        logger.debug("Checking variables")
        if self.match.qid is None:
            raise ValueError("match.qid was None")
        if self.match.original_subject is None:
            raise ValueError("match.original_subject was None")
        if self.match.crossref_subject is None:
            raise ValueError("match.crossref_subject was None")
        if self.match.split_subject is None:
            raise ValueError("match.split_subject was None")
        if self.match.match_based_on is None:
            raise ValueError("match.match_based_on was None")

    def __append_new_match_to_the_dataframe__(self):
        self.__check_variables__()
        logger.debug("Adding to cache")
        data = {
            CacheDataframeColumn.QID.value: self.match.qid.value,
            CacheDataframeColumn.CROSSREF_SUBJECT.value: self.match.crossref_subject,
            CacheDataframeColumn.MATCH_BASED_ON.value: self.match.match_based_on.value,
            CacheDataframeColumn.ORIGINAL_SUBJECT.value: self.match.original_subject,
            CacheDataframeColumn.SPLIT_SUBJECT.value: self.match.split_subject,
        }
        if self.dataframe is None:
            self.dataframe = pd.DataFrame(data=[data])
        else:
            self.dataframe = self.dataframe.append(pd.DataFrame(data=[data]))

    def __check_crossref_subject__(self):
        if self.match.crossref_subject is None:
            raise ValueError("crossref_subject was None")
        if self.match.crossref_subject == "":
            raise ValueError("crossref_subject was empty string")

    def __check_qid__(self):
        if self.match.qid is None:
            raise ValueError("qid was None")

    def __check_if_drop_was_successful__(self):
        if config.loglevel == logging.DEBUG:
            logging.debug("Checking if the qid is still in the cache")
            match = (self.dataframe[CacheDataframeColumn.QID.value] == self.match.qid.value).any()
            logger.debug(f"match:{match}")
            print(self.dataframe.info())
            logger.debug(f"Saving pickle without {self.match.qid.value}")

    def __drop_qid_from_dataframe__(self):
        self.__read_dataframe_from_disk__()
        # This tests whether any row matches
        match = (self.dataframe[CacheDataframeColumn.QID.value] == self.match.qid.value).any()
        logger.debug(f"match:{match}")
        if match:
            logger.debug("Deleting the item from the cache now")
            self.dataframe = self.dataframe[self.dataframe[CacheDataframeColumn.QID.value] != self.match.qid.value]
            self.qid_dropped = True
        else:
            self.qid_dropped = False

    def __extract_match__(self):
        """Here we find the row that matches and extract the
        result column and extract the value using any()
        """
        logger.info("Extracting match from cache")
        found_match = self.dataframe.loc[
            self.dataframe[CacheDataframeColumn.CROSSREF_SUBJECT.value] == self.match.crossref_subject].any()
        logger.debug(f"result:{found_match}")
        if found_match is not None:
            logger.info("Already matched QID found in the cache")
            row: DataFrame = self.dataframe[
                self.dataframe[CacheDataframeColumn.CROSSREF_SUBJECT.value] == self.match.crossref_subject
            ]
            if isinstance(row, DataFrame):
                # print(row)
                # print(row[CacheDataframeColumn.QID.value].values[0])
                # exit()
                qid: EntityId = EntityId(row[CacheDataframeColumn.QID.value][0])
                original_subject: str = row[CacheDataframeColumn.ORIGINAL_SUBJECT.value][0]
                crossref_subject: str = row[CacheDataframeColumn.ORIGINAL_SUBJECT.value][0]
                match_based_on = MatchBasedOn(row[CacheDataframeColumn.MATCH_BASED_ON.value][0])
                split_subject: bool = bool(row[CacheDataframeColumn.SPLIT_SUBJECT.value][0])
                ontology_dataframe = runtime_variables.ontology_dataframe
                label = ontology_dataframe.loc[
                    ontology_dataframe[OntologyDataframeColumn.ITEM.value] == qid.url(),
                    OntologyDataframeColumn.LABEL.value].head(1).values[0]
                description = ontology_dataframe.loc[
                    ontology_dataframe[OntologyDataframeColumn.ITEM.value] == qid.url(),
                    OntologyDataframeColumn.DESCRIPTION.value].head(1).values[0]
                alias = ontology_dataframe.loc[
                    ontology_dataframe[OntologyDataframeColumn.ITEM.value] == qid.url(),
                    OntologyDataframeColumn.ALIAS.value].head(1).values[0]
                if label is None:
                    raise ValueError("label was None")
                self.match = FuzzyMatch(
                    qid=qid,
                    original_subject=original_subject,
                    match_based_on=match_based_on,
                    split_subject=split_subject,
                    crossref_subject=crossref_subject,
                    alias=alias,
                    label=label,
                    description=description,
                )
                # print(self.match)
                # exit()
            else:
                logger.error("Did not get pandas dataframe, got")
                print(row)
                exit()

    def __lookup_crossref_subject__(self):
        if len(self.dataframe) > 0:
            match = (self.dataframe[CacheDataframeColumn.CROSSREF_SUBJECT.value] == self.match.crossref_subject).any()
            logger.debug(f"match:{match}")
            self.crossref_subject_found = match
        else:
            self.crossref_subject_found = False

    def __lookup_qid__(self):
        match = (self.dataframe[CacheDataframeColumn.QID.value] == self.match.qid.value).any()
        logger.debug(f"match:{match}")
        self.qid_found = match

    def read(self) -> Optional[FuzzyMatch]:
        """Returns None or result from the cache"""
        self.__check_crossref_subject__()
        self.__verify_that_the_cache_file_exists_and_read__()
        self.__lookup_crossref_subject__()
        if self.crossref_subject_found:
            self.__extract_match__()
            return self.match

    def add(self) -> bool:
        """Add a match to the cache
        It returns True if it was added and False if either QID
        or the crossref subject was found in the cache.
        :return: bool"""
        self.__check_crossref_subject__()
        self.__check_qid__()
        self.__verify_that_the_cache_file_exists_and_read__()
        self.__lookup_crossref_subject__()
        if not self.crossref_subject_found and not self.qid_found:
            self.__append_new_match_to_the_dataframe__()
            self.__save_dataframe_to_disk__()
            return True
        else:
            return False

    def delete(self) -> bool:
        """Delete from the cache.
        Returns True if success and False if not found"""
        self.__check_qid__()
        logger.debug("Deleting from the cache")
        self.__verify_that_the_cache_file_exists_and_read__()
        self.__drop_qid_from_dataframe__()
        if self.qid_dropped:
            self.__save_dataframe_to_disk__()
            return True
        else:
            return False


import numpy as np
import cv2
import nibabel as nib
import os
import yaml
from pytorch_fid import fid_score

#from pytorch_lightning import metrics

import argparse

parser = argparse.ArgumentParser(description='Metrics Script')
parser.add_argument('--bodymask_path',  type=str)
parser.add_argument('--real_slices_path',  type=str)
parser.add_argument('--fake_slices_path',  type=str)
parser.add_argument('--results_path',  type=str, default='./metrics_result.yaml')
parser.add_argument('--FID', action='store_true')
parser.add_argument('--debug_images', action='store_true')

args = parser.parse_args()

def build_volume(slices, patient):
    # build 3D volume w*h*d
    names_slices = slices.keys()
    filtered_slice_keys =[s for s in names_slices if (s[:4] == patient)]

    filtered_slice_keys.sort()
    volume_dimensions = (slices[filtered_slice_keys[0]].shape[0], slices[filtered_slice_keys[0]].shape[0], len(filtered_slice_keys))

    volume = np.zeros(volume_dimensions)

    for i, sl in enumerate(filtered_slice_keys):
        if len(slices[sl].shape) == 2:
            volume[:,:,i] = slices[sl]
        else:
            volume[:,:,i] = slices[sl][:,:,0]

    return volume

def read_slices(path):
    # assume folder is for one patient
    filelist = os.listdir(path)
    imglist = {}
    for f in filelist:
        if f[-4:] == '.png':
            imglist[f] = cv2.imread(os.path.join(path, f))

        elif f[-4:] == '.nii':
            A_img_nifti = nib.load(os.path.join(path, f))
            imglist[f] = A_img_nifti.get_fdata(caching = "unchanged").astype(np.int16)

            #print(imglist[f][0,0])
        else:
            print(f + ' does not fit specified input')
    return imglist

def rescale_slices(slices):
    for key in slices:
        slices[key] = 4095*slices[key] -1024

    return slices

def read_mask(path):
    # assume folder is for one patient
    filelist = os.listdir(path)
    imglist = {}

    for f in filelist:
        A_img_nifti = nib.load(os.path.join(path, f))
        imglist[f] = A_img_nifti.get_fdata(caching = "unchanged")
    return imglist

def mask_volume(volume, mask):
    return np.multiply(volume, mask)

if __name__ == "__main__":
    bodymask_path = args.bodymask_path
    real_slices_path = args.real_slices_path
    fake_slices_path = args.fake_slices_path
    results_path = args.results_path
    print('reading real slices from: '+ real_slices_path)
    real_slices = read_slices(real_slices_path)
    print('reading fake slices from: '+ fake_slices_path)
    fake_slices = read_slices(fake_slices_path)
    print('rescaling fake slices')

    mask_slices = read_slices(bodymask_path)
    results = {}
    results['real_path'] = real_slices_path
    results['fake_path'] = fake_slices_path
    results['masks_path'] = bodymask_path

    # patient list with registered data, this needs to be changed if more
    # or other registered training data is available. The all the files with the
    # filename beginning with one of the list elements are considered.

    pat = ['PAT1', 'PAT3']

    diff_all_pat = []
    for p in pat:
        # make volumes
        print(p)
        real_vol = build_volume(real_slices, p)
        fake_vol = build_volume(fake_slices, p)

        mask = build_volume(mask_slices, p)

        # mask volumes
        fake_vol = mask_volume(fake_vol, mask)
        real_vol = mask_volume(real_vol, mask)

        diff = real_vol - fake_vol
        diff_masked = diff[mask>0]
        diff_all_pat.append(diff_masked)
        # calculate MAE
        mae = (np.abs(diff_masked)).mean()
        sd_ae = np.std(np.abs(diff_masked))

        print(p + ' MAE: '+ str(mae) + ', SD: '+ str(sd_ae))

        # calculate MSE
        mse = ((diff_masked)**2).mean()
        sd_se = np.std((diff_masked)**2)
        print(p + ' MSE: '+ str(mse) + ', SD: '+ str(sd_se))

        # calculate ME
        me = diff_masked.mean()
        sd_e = np.std(diff_masked)
        print(p + ' ME: '+ str(me)+ ', SD: '+ str(sd_e))

        # calculate MRSE
        mrse = np.sqrt(mse)
        print(p + ' MRSE: '+ str(mrse))

        # add to dict
        results[p] = {
            'mse': float(mse), 'sd_mse': float(sd_se),
            'mae': float(mae), 'sd_mae': float(sd_ae),
            'me': float(me), 'sd_me': float(sd_e)}

    # calculations for all patients
    p = 'PAT1+PAT3'
    # calculate MAE
    diff_all_pat = np.concatenate(diff_all_pat)
    # calculate MAE
    mae = (np.abs(diff_all_pat)).mean()
    sd_ae = np.std(np.abs(diff_all_pat))

    print(p + ' MAE: '+ str(mae) + ', SD: '+ str(sd_ae))

    # calculate MSE
    mse = ((diff_all_pat)**2).mean()
    sd_se = np.std((diff_all_pat)**2)
    print(p + ' MSE: '+ str(mse) + ', SD: '+ str(sd_se))

    # calculate ME
    me = diff_all_pat.mean()
    sd_e = np.std(diff_all_pat)
    print(p + ' ME: '+ str(me)+ ', SD: '+ str(sd_e))
    results[p] = {
        'mse': float(mse), 'sd_mse': float(sd_se),
        'mae': float(mae), 'sd_mae': float(sd_ae),
        'me': float(me), 'sd_me': float(sd_e)}

    print('FID Calculations: ' + str(args.FID))
    if args.FID:
        print('Calculating FID score, this may take a while...')
        fid_paths =  [real_slices_path,fake_slices_path]
        fid_value = fid_score.calculate_fid_given_paths(fid_paths,
                                                     batch_size=50,
                                                     device=None,
                                                     dims=2048)
        results['FID'] = float(fid_value)
    with open(results_path, 'w') as file:
        documents = yaml.dump(results, file)
    print(results)

    if args.debug_images:
        def on_change(i):
            img = np.hstack(((real_vol[:,:,i]+1024)/4095, (fake_vol[:,:,i]+1024)/4095, mask[:,:,i]))
            cv2.imshow('real-fake-mask', img)
        img = np.hstack(((real_vol[:,:,0]+1024)/4095, (fake_vol[:,:,0]+1024)/4095, mask[:,:,0]))
        cv2.imshow('real-fake-mask', img)
        cv2.createTrackbar('slider', 'real-fake-mask', 0, 299, on_change)
        while True:
            k = cv2.waitKey(0)
            if k==27:    # Esc key to stop
                cv2.destroyAllWindows()
                break


import gc
import numpy as np
import pandas as pd
import tensorflow as tf
import os
import cv2
import matplotlib.pyplot as plt
import matplotlib as mpl

mpl.style.use("seaborn-v0_8")
from tqdm import tqdm

# pathlib to my loccal file

files = os.listdir("/home/michael/Desktop/archive/dataset/")
files
# initialisation of array
image_array = []  # it's a list later i will convert it to array
label_array = []
path = "/home/michael/Desktop/archive/dataset/"
# loop through each sub-folder in train data
for i in range(len(files)):
    # files in sub-folder
    file_sub = os.listdir(path + files[i])

    for k in tqdm(range(len(file_sub))):
        try:
            img = cv2.imread(path + files[i] + "/" + file_sub[k])
            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
            img = cv2.resize(img, (96, 96))
            image_array.append(img)
            label_array.append(i)
        except:
            pass

# collecting garbae
gc.collect()
# importing process utils
import psutil

print(psutil.virtual_memory())
# image arrays
image_array = np.array(image_array) / 255.0
label_array = np.array(label_array)
# importing sckit data
from sklearn.model_selection import train_test_split

X_train, X_test, Y_train, Y_test = train_test_split(image_array, label_array, test_size=0.15)
from keras import layers, callbacks, utils, applications, optimizers
from keras.models import Sequential, Model, load_model

# checking file length
len(files)
model = Sequential()
# I will use MobileNetV2 as an pretrained model
pretrained_model = tf.keras.applications.EfficientNetB0(input_shape=(96, 96, 3), include_top=False,
                                                        weights="imagenet")
model.add(pretrained_model)
model.add(layers.GlobalAveragePooling2D())
# add dropout to increase accuracy by not overfitting
model.add(layers.Dropout(0.3))
# add dense layer as final output
model.add(layers.Dense(1))
model.summary()
# model seuen

model.compile(optimizer="adam", loss="mean_squared_error", metrics=["mae"])
# creating a chechpoint to save model at best accuarcy

ckp_path = "trained_model/model"
model_checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath=ckp_path,
                                                      monitor="val_mae",
                                                      mode="auto",
                                                      save_best_only=True,
                                                      save_weights_only=True)

# create a lr reducer which decrease learning rate when accuarcy does not increase
reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(factor=0.9, monitor="val_mae",
                                                 mode="auto", cooldown=0,
                                                 patience=5, verbose=1, min_lr=1e-6)
# patience : wait till 5 epoch
# verbose : show accuracy every 1 epoch
# min_lr=minimum learning rate

# model train_test_split
EPOCHS = 300
BATCH_SIZE = 64

history = model.fit(X_train,
                    Y_train,
                    validation_data=(X_test, Y_test),
                    batch_size=BATCH_SIZE,
                    epochs=EPOCHS,
                    callbacks=[model_checkpoint, reduce_lr]
                    )

model.load_weights(ckp_path)

# converting
converter = tf.lite.TFLiteConverter.from_keras_model(model)
tflite_model = converter.convert()

# Save the model.
with open('model.tflite', 'wb') as f:
    f.write(tflite_model)

prediction_val = model.predict(X_test, batch_size=BATCH_SIZE)

# p@rediction
prediction_val[:20]

Y_test[:20]

def print_hi(name):
    # Use a breakpoint in the code line below to debug your script.
    print(f'Hi, {name}')  # Press Ctrl+F8 to toggle the breakpoint.

# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    print_hi('hey am coding using python')

# See PyCharm help at https://www.jetbrains.com/help/pycharm/


import shutil
import webbrowser
import numpy as np
from math import ceil
from time import time
from random import randint, shuffle, seed
from tec.ic.ia.pc2.model.file_utils import *

class Gen:
    """
    'Escructura' para manejar cada gen
    :var array: numpy array con los contenidos del tablero
    :var score: puntaje fitness del gen
    """

    def __init__(self, array, score=0):
        self.gen_array = array
        self.score = score

    def set_score(self, score): self.score = score

    def get_score(self): return self.score

    def get_array(self): return self.gen_array

__console_output__ = True
__print_frecuency__ = 100
__print_per_generation__ = 3
__weigths__ = {}

def set_weights(mat_shape):
    # ['pc'] = picked carrots weight      (zanahorias pisadas)
    # ['s'] = steps weight                (pisadas)
    # ['af'] = arrows found weight        (flechas pisadas)
    # ['anf'] = arrows not found weight   (flechas sin usar)
    # ['apc'] = arrows pointing carrots   (flechas que apuntan directo a Z)
    # ['180°t'] = 180 turn                (giro en 180 grados)
    # ['bfc'] = best first carrot         (una flecha inicial idónea)

    _scalar = mat_shape[0] * mat_shape[1]

    global __weigths__
    __weigths__ = {
        'pc': round_up(_scalar, 1000),
        's': -1,
        'af': -5,
        'anf': -15,
        'apc': 10,
        '180°t': -(round_up(_scalar, 100)),
        'bfc': round_up(_scalar, 500)
    }

def round_up(number, base=10):
    return int(base * ceil(float(number)/base))

def gen_in_list(gen, gen_list):
    """
    Revisa si el genotipo de un determinado gen es igual a alguno de los
    genes en una lista
    :param gen: objeto Gen a buscar
    :param gen_list: lista con objetos Gen
    :return: True si encuentra un genotipo igual, False de lo contrario
    """
    return any((gen.get_array() == g.get_array()).all() for g in gen_list)

def direction_to_arrow():
    """
    Define la flecha correspondiente para el parámetro inicial del programa
    :return: diccionario con las correspondencias
    """
    return {'arriba': 'A', 'abajo': 'V', 'derecha': '>', 'izquierda': '<'}

def a_idx_to_m_idx(a_idx, mat_shape):
    """
    Convierte un índice de array al correspondiente índide de matriz según
    las dimensiones de la matriz
    :param a_idx: índice del arreglo
    :param mat_shape: dimensiones de la matriz (f, c)
    :return: par ordenado, file x columna para el índice de una matriz
    """
    return int(a_idx / mat_shape[1]), a_idx % mat_shape[1]

def initialization(initial_board, individuals, direction, fitness=True):
    """
    Convierte a Gen y genera la población inicial
    :param initial_board: matriz de numpy del estado inicial
    :param individuals: cantidad de individuos por generacion
    :param direction: dirección inicial del conejo
    :param fitness: define si se calcula el fitness para la 1ra generación o no
    :return: lista con los Genes de la generación inicial, shape de initial
    board
    """
    set_weights(initial_board.shape)
    first_generation = list()
    content = initial_board.getA1().tolist()
    if fitness:
        score = eval_fitness(Gen(np.array(content, object)), direction,
                             initial_board.shape)
    else:
        score = None

    for _ in range(0, individuals):
        first_generation.append(Gen(np.array(content, object), score))

    return first_generation, initial_board.shape

def mutate(gen, mutation_chance):
    """
    Dada una probabilidad de mutación, muta o no al gen recibido,
    la mutación puede ser la adición de una flecha, eliminación de flecha,
    o el cambio de dirección de una flecha
    :param gen: objeto Gen
    :param mutation_chance: probabilidad de mutación de 0 a 100
    :return: el gen luego de la mutación o sin mutar, y la posición del
    array donde muto, o en su defecto -1 en caso de no haber mutado
    """
    temp = gen.get_array().copy()
    arrow_symbols = ['<', '>', 'A', 'V']

    mutates = True if randint(0, 99) < mutation_chance else False

    if mutates:

        cell_idx = randint(0, len(temp) - 1)
        cell_content = temp[cell_idx]

        if cell_content is ' ':
            temp[cell_idx] = arrow_symbols[randint(0, 3)]
        elif cell_content in arrow_symbols:
            arrow_symbols += [' ']
            arrow_symbols.remove(cell_content)
            temp[cell_idx] = arrow_symbols[randint(0, 3)]

    return Gen(temp), mutates

def analyze_carrots(board, carrot_positions, rabbit_row, rabbit_col):
    """
    Para cada zanahoria, cuenta la cantidad de direcciones desde las que la
    apunta una flecha, para aprovechar el ciclo de recorrido de zanahorias,
    se identifica también la zanahoria más cercana
    :param board: tablero escenario
    :param carrot_positions: tupla de arrays con filas y columnas de las
    zanahorias
    :param rabbit_col: columna inicial del conejo
    :param rabbit_row: fila inicial del conejo
    :return: cantidad de flechas apuntando zanahorias, y la posición de la
    zanahoria más cercana
    """
    arrow_count = 0  # acumulador de flechas
    nearest_so_far = carrot_positions[0][0], carrot_positions[1][0]

    def arrow_count_in_subpath(subpath, arrow):
        if len(subpath) == 0:
            return 0

        temp = np.array(subpath, object)
        return 1 if np.count_nonzero(temp == arrow) > 0 else 0

    for row, col in zip(carrot_positions[0], carrot_positions[1]):

        current_distance = abs(row - rabbit_row) + abs(col - rabbit_col)
        best_distance = abs(nearest_so_far[0] - rabbit_row) + abs(
            nearest_so_far[1] - rabbit_col)

        if current_distance < best_distance:
            nearest_so_far = row, col

        arrow_count += arrow_count_in_subpath(
            board[:, col][:row].flatten().tolist()[0][::-1], 'V'
        )
        arrow_count += arrow_count_in_subpath(
            board[:, col][row + 1:].flatten().tolist(), 'A'
        )
        arrow_count += arrow_count_in_subpath(
            board[row, :col].flatten().tolist()[::-1], '>'
        )
        arrow_count += arrow_count_in_subpath(
            board[row, col + 1:].flatten().tolist(), '<'
        )

    return arrow_count, nearest_so_far

def walk_rabbit_path(board, direction, row, col, carrot_count, mat_shape,
                     nearest_carrot):
    """
    Simula el recorrido del conejo en el tablero para obtener datos usados
    por el fitness
    :param board: numpy matrix con el tablero disponible
    :param direction: dirección inicial del conejo
    :param row: fila de la posición inicial del conejo
    :param col: columna de la posición inicial del conejo
    :param carrot_count: cantidad de zanahorias existentes
    :param mat_shape: tupla con dimensiones de la matriz (filas, columnas)
    :param nearest_carrot: tupla con la fila y columna de la zanahoria más
    cercana
    :return: contadores usados por el fitness
    """
    def move_up():
        nonlocal row
        row -= 1
        return False if row < 0 else True

    def move_down():
        nonlocal row
        row += 1
        return False if row == mat_shape[0] else True

    def move_left():
        nonlocal col
        col -= 1
        return False if col < 0 else True

    def move_right():
        nonlocal col
        col += 1
        return False if col == mat_shape[1] else True

    def valid_move():
        if direction == 'A':
            return move_up()
        elif direction == 'V':
            return move_down()
        elif direction == '>':
            return move_right()
        else:
            return move_left()

    picked_carrots = 0
    steps = 0
    arrows_found = 0
    _180_degree_turns = 0

    arrow_symbols = ['<', '>', 'A', 'V']
    opposite_direction = {'A': 'V', 'V': 'A', '>': '<', '<': '>'}
    last_cell = 'C'

    is_first_carrot = True
    first_carrot_is_optimal = 0
    # Recorrer el camino del conejo para determinar las zanahorias, pasos y
    # las flechas que encuentra
    while True:
        if picked_carrots == carrot_count:
            rabbit_fall = False
            break

        if valid_move():
            steps += 1
            cell_content = board[row][col]
            if cell_content is 'Z':
                picked_carrots += 1
                board[row][col] = ' '
                if is_first_carrot:
                    is_first_carrot = False
                    if (row, col) == nearest_carrot:
                        first_carrot_is_optimal = 1

            elif cell_content in arrow_symbols:
                # Revisar los giros en 180 grados
                if cell_content is opposite_direction[direction]:
                    if last_cell is not 'Z':
                        _180_degree_turns += 1
                arrows_found += 1
                direction = cell_content
                board[row][col] = ' '

        else:
            rabbit_fall = True
            break

    return picked_carrots, steps, arrows_found, _180_degree_turns, \
        rabbit_fall, first_carrot_is_optimal

def eval_fitness(gen, direction, mat_shape):
    """
    calcula la aptitud para un gen
    :param gen: objeto tipo Gen
    :param direction: dirección inicial del conejo
    :param mat_shape: dimensiones de la matriz
    :return: Gen con el atributo score alterado
    """
    temp = (gen.get_array().copy()).reshape(mat_shape)

    carrot_positions = np.nonzero(temp == 'Z')

    init_position = np.where(temp == 'C')
    row = init_position[0][0]
    col = init_position[1][0]

    arrows_pointing, nearest_carrot_pos = analyze_carrots(temp.copy(),
                                                          carrot_positions,
                                                          row, col)

    carrot_count = len(carrot_positions[0])
    arrow_count = np.count_nonzero(temp != ' ') - carrot_count - 1

    results = walk_rabbit_path(board=temp, direction=direction,
                               row=row, col=col, mat_shape=mat_shape,
                               carrot_count=carrot_count,
                               nearest_carrot=nearest_carrot_pos)

    picked_carrots = results[0]
    steps = results[1]
    arrows_found = results[2]
    _180_degree_turns = results[3]
    rabbit_fall = results[4]
    first_carrot_is_optimal = results[5]

    pcw = picked_carrots * __weigths__['pc']
    sw = steps * __weigths__['s']
    if rabbit_fall:
        sw = -sw
    afw = arrows_found * __weigths__['af']
    anfw = (arrow_count - arrows_found) * __weigths__['anf']
    apcw = arrows_pointing * __weigths__['apc']
    _180dtw = _180_degree_turns * __weigths__['180°t']
    bfcw = first_carrot_is_optimal * __weigths__['bfc']

    score = pcw + sw + afw + anfw + apcw + _180dtw + bfcw

    gen.set_score(score)

    return score

def cross(parent1, parent2, cross_type):
    """
    Realiza el tipo de cruce entre dos genes padres
    :param parent1: objeto Gen
    :param parent2: objeto Gen
    :param cross_type: 1 -> cruce en 1 punto, 2 -> cruce en dos puntos
    :return: los dos np array genes hijos resultantes
    """
    child1 = parent1.get_array().copy()
    child2 = parent2.get_array().copy()

    idx1 = randint(1, len(child1) - 1)
    if cross_type == 1:
        child1[:idx1], child2[:idx1] = child2[:idx1].tolist(), child1[
                                                               :idx1].tolist()
    else:
        idx2 = randint(1, len(child1) - 1)
        idx1, idx2 = min(idx1, idx2), max(idx1, idx2)
        child1[idx1:idx2], child2[idx1:idx2] = \
            child2[idx1:idx2].tolist(), child1[idx1:idx2].tolist()

    child1 = Gen(child1)
    child2 = Gen(child2)

    return child1, child2

def get_children(parent1, parent2, cross_type, direction, mat_shape,
                 mutation_chance):
    """
    Dados un par de padres, se crean los hijos y sus mutaciones en caso de
    haberlas
    :param parent1: objeto tipo Gen
    :param parent2: objeto tipo Gen
    :param cross_type: tipo de cruce, 1 o 2
    :param direction: dirección inicial del conejo para el fitness
    :param mat_shape: dimensiones del tablero para el fitness
    :param mutation_chance: porcentaje de mutación entre 0 y 100
    :return:
    """
    children = list()

    child1, child2 = cross(parent1, parent2, cross_type)

    eval_fitness(child1, direction, mat_shape)
    eval_fitness(child2, direction, mat_shape)

    mutation1, mutated = mutate(child1, mutation_chance)
    if mutated:
        eval_fitness(mutation1, direction, mat_shape)
        children.append(mutation1)

    mutation2, mutated = mutate(child2, mutation_chance)
    if mutated:
        eval_fitness(mutation2, direction, mat_shape)
        children.append(mutation2)

    children.append(child1)
    children.append(child2)

    return children

def generate(parents, direction, mat_shape,
             mutation_chance, cross_type):
    """
    Realiza el cruce de los genes según dos tipos de selección diferente
    :param parents: lista de Genes para cruzar
    :param direction: dirección inicial del conejo para evaluar fitness
    :param mat_shape: dimensiones de la matriz para evaluar fitness
    :param mutation_chance: probabilidad de 0 a 100 de mutación
    :param cross_type: 1 -> corte en un punto, 2 -> corte en dos puntos
    :return: una lista con todos los Genes (hijos, mutos, padres) resultantes
    """
    resulting_generation = list()
    resulting_generation += parents

    index_list = list(range(0, len(parents)))
    shuffle(index_list)

    for gen1_idx, gen2_idx in zip(*[iter(index_list)] * 2):
        children = get_children(parents[gen1_idx], parents[gen2_idx],
                                cross_type, direction, mat_shape,
                                mutation_chance)

        resulting_generation += [child for child in children if not
                                 gen_in_list(child, resulting_generation)]

    return resulting_generation

def replacement(generation, individuals):
    """
    Escoge los genes más aptos para conformar la siguiente generación
    :param generation: lista con todos los padres, hijos y mutos
    :param individuals: cantidad de individuos por generación
    :return: lista con la generación ya recortada
    """
    # Se ordena de mayor a menor
    generation.sort(key=lambda gen: gen.get_score(), reverse=True)
    return generation[:individuals]

def create_folder(folder):
    if os.path.exists(folder):
        shutil.rmtree(folder)
    if not os.path.exists(folder):
        os.makedirs(folder)

def get_folder(direction):
    folder = get_default_folder()
    folder = os.path.join(folder, "genetic_algoritms")
    folder = os.path.join(folder, direction)
    create_folder(folder)
    return folder

def get_generation_folder(base_folder, generation_number):
    folder = base_folder[:]
    folder = os.path.join(folder, generation_number)
    create_folder(folder)
    return folder

def save_single_gen(generation_folder, single_gen, gen_shape, gen_number):
    filename = str(gen_number).zfill(5) + ".txt"
    filename = os.path.join(generation_folder, filename)
    save_file(filename, single_gen.get_array().reshape(gen_shape))

def save_generation(base_folder, generation, generation_number, gen_shape):

    generation_number = str(generation_number).zfill(5)
    generation_folder = get_generation_folder(base_folder, generation_number)

    for i in range(len(generation)):
        save_single_gen(generation_folder, generation[i], gen_shape, i)

def run_carrot_finder(initial_direction, individuals, max_generations,
                      mutation_chance, initial_board, cross_type=1,
                      custom_seed=-1, return_generation_number=False):
    """
    Función principal de ejecución del algoritmo genético
    :param initial_direction: dirección en la que comienza a moverse el conejo
    :param individuals: cantidad de invididuos por generación
    :param max_generations: número máximo de generaciones por corrida
    :param mutation_chance: número de 0 a 100 para la probabilidad de mutación
    :param initial_board: numpy matrix con el tablero inicial
    :param cross_type: 1 -> corte en un punto 2 -> corte en dos puntos
    :param custom_seed: semilla de random para reproducibilidad de resultados
    :param return_generation_number: bandera para retornar el número de la
    generación donde se encontró el mejor gen
    :return: lista con la última generación ordenada por fitness de Genes
    """

    base_folder = get_folder(initial_direction)

    if custom_seed != -1:
        seed(custom_seed)

    initial_direction = direction_to_arrow()[initial_direction]

    optimal_origin_generation = -1
    current_best_score = 0

    # Definición de la población inicial
    generation, dimensions = initialization(initial_board=initial_board,
                                            individuals=individuals,
                                            direction=initial_direction)

    if __console_output__:
        print('Pesos calculados: ', __weigths__)

    for generation_number in range(1, max_generations + 1):

        # Se obtiene la generación completa sin ordenar
        generation = generate(parents=generation,
                              direction=initial_direction,
                              mutation_chance=mutation_chance,
                              mat_shape=dimensions,
                              cross_type=cross_type)

        # Se seleccionan los mejores
        generation = replacement(generation, individuals)
        save_generation(base_folder, generation, generation_number, dimensions)

        if generation[0].get_score() > current_best_score:
            optimal_origin_generation = generation_number
            current_best_score = generation[0].get_score()

        print_this_generation = generation_number % __print_frecuency__ == 0

        if __console_output__ and print_this_generation:
            print('\nGENERACIÓN: {:05}'.format(generation_number))
            for i in range(0, len(generation[:__print_per_generation__])):
                print(
                    '\tINDIVIDUO: {:05}'.format(i),
                    ' -> APTITUD: ' + str(generation[i].get_score())
                )

    if __console_output__:
        print('\nMÁS APTO ENCONTRADO: ')
        print(generation[0].get_array().reshape(initial_board.shape))
        print('\nAPTITUD DEL MEJOR ENCONTRADO:', generation[0].get_score())
        print('\nGENERACIÓN DEL MEJOR ENCONTRADO:',
              '{:05}'.format(optimal_origin_generation))

    # Abrir carpeta al finalizar
    webbrowser.open(base_folder)

    if return_generation_number:
        return generation, optimal_origin_generation

    return generation


import os

from kivy.app import App
from kivy.core.window import Window
from kivy.metrics import dp
from kivy.uix.button import Button
from kivy.uix.gridlayout import GridLayout
from kivy.uix.image import Image
from kivy.uix.screenmanager import ScreenManager, Screen
from kivy.uix.scrollview import ScrollView

class SquareImage(Image):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.allow_stretch = True
        self.keep_ratio = False
        self.size_hint_y = None
        self.height = dp(self.width)

class MyNewGallery(Screen):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        my_scroll = ScrollView(size_hint=(1, None), size=(Window.width, Window.height))
        self.my_gallery_grid = GridLayout(cols=3, spacing=dp(5), padding=dp(5),
                                          size_hint=(1, None), height=1000)
        self.add_images_to_gallery()
        my_scroll.add_widget(self.my_gallery_grid)

        self.add_widget(my_scroll)

    def add_images_to_gallery(self, *args):
        path = "/home/petchorine/Desktop/monPyhon/mes_projets_python/Double_view/Images"
        for filename in os.listdir(path):
            if os.path.isfile(os.path.join(path, filename)):
                img = SquareImage(source=f"/home/petchorine/Desktop/monPyhon/mes_projets_python/Double_view/Images/{filename}")
                self.my_gallery_grid.add_widget(img)

    def refresh_gallery(self, *args):
        self.my_gallery_grid.clear_widgets()
        self.add_images_to_gallery()

class MenuB(Screen):
    pass

class MyScreenManager(ScreenManager):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        self.screen1 = MyNewGallery(name="screen1")
        screen2 = MenuB(name="screen2")

        btn_to_screen2 = Button(text="to screen 2",
                                size_hint = (None, None),
                                size = (100,100),
                                pos_hint = {"right": 1})
        btn_to_screen2.bind(on_press=self.to_b)
        self.screen1.add_widget(btn_to_screen2)

        btn_to_gallery = Button(text="to gallery",
                                size_hint=(None, None),
                                size=(100, 100),
                                pos_hint={"left": 1})
        btn_to_gallery.bind(on_press=self.to_gallery)
        screen2.add_widget(btn_to_gallery)

        self.add_widget(self.screen1)
        self.add_widget(screen2)

    def to_b(self, *args):
        self.transition.direction = "left"
        self.current = "screen2"

    def to_gallery(self, *args):
        self.transition.direction = "right"
        self.current = "screen1"

        self.screen1.refresh_gallery()

class RefreshGalleryApp(App):
    def build(self):
        Window.size = (350,700)
        return MyScreenManager()

RefreshGalleryApp().run()