from __future__ import absolute_import
from __future__ import print_function

import os
from collections import namedtuple, MutableMapping
from copy import deepcopy, copy
from itertools import ifilter
import logging

import pyros_utils
import rospy
import rosservice, rostopic, rosparam

import re
import ast
import socket
import threading
import Queue

import time

# create logger
_logger = logging.getLogger(__name__)
# and let it propagate to parent logger, or other handler
# the user of pyros should configure handlers

from pyros_interfaces_common.transient_if_pool import DiffTuple
from .baseinterface import BaseInterface
from .param_if_pool import RosParamIfPool
from .service_if_pool import RosServiceIfPool
from .subscriber_if_pool import RosSubscriberIfPool
from .publisher_if_pool import RosPublisherIfPool

from .connection_cache_utils import connection_cache_proxy_create, connection_cache_marshall, connection_cache_merge_marshalled

try:
    import rocon_python_comms
except ImportError:
    rocon_python_comms = None

# To make sure things dont get messed up between threads
CacheTuple = (namedtuple("CacheTuple", "complete added removed"))

class RosInterface(BaseInterface):

    """
    RosInterface.
    """
    def __init__(self, node_name, publishers=None, subscribers=None, services=None, params=None, enable_cache=False, argv=None):
        # This runs in a child process (managed by PyrosROS) and as a normal ros node)

        # First thing to do : find the rosmaster...
        # master has to be running here or we just wait for ever
        # TODO : improve...
        m, _ = pyros_utils.get_master(spawn=False)
        while not m.is_online():
            _logger.warning("ROSMASTER not found !!!...")
            time.sleep(0.5)

        # Second thing to do : initialize the ros node and disable signals to avoid overriding callers behavior
        rospy.init_node(node_name, argv=argv, disable_signals=True)
        rospy.loginfo('RosInterface {name} node started with args : {argv}'.format(name=node_name, argv=argv))
        # note on dynamic update (config reload, etc.) this is reinitialized
        # rospy.init_node supports being reinitialized with the exact same arguments

        self.enable_cache = enable_cache
        if self.enable_cache and rocon_python_comms is None:
            rospy.logerr("Connection Cache enabled for RosInterface, but rocon_python_comms not found. Disabling.")
            self.enable_cache = False

        # only one queue to avoid sync issues
        self.cb_ss = Queue.Queue()

        # we add params from ROS environment, if we get something there (bwcompat behavior)
        services = services or []
        publishers = publishers or []
        subscribers = subscribers or []
        params = params or []
        services += list(set(ast.literal_eval(rospy.get_param('~services', "[]"))))

        # bwcompat
        publishers += list(set(ast.literal_eval(rospy.get_param('~topics', "[]"))))
        subscribers += list(set(ast.literal_eval(rospy.get_param('~topics', "[]"))))
        # new
        publishers += list(set(ast.literal_eval(rospy.get_param('~publishers', "[]"))))
        subscribers += list(set(ast.literal_eval(rospy.get_param('~subscribers', "[]"))))

        params += list(set(ast.literal_eval(rospy.get_param('~params', "[]"))))
        enable_cache = rospy.get_param('~enable_cache', enable_cache)

        if enable_cache is not None:
            self.enable_cache = enable_cache
        # Note : None means no change ( different from [] )
        rospy.loginfo("""[{name}] ROS Interface initialized with:
        -    services : {services}
        -    publishers : {publishers}
        -    subscribers : {subscribers}
        -    params : {params}
        -    enable_cache : {enable_cache}
        """.format(
            name=__name__,
            publishers="\n" + "- ".rjust(10) + "\n\t- ".join(publishers) if publishers else [],
            subscribers="\n" + "- ".rjust(10) + "\n\t- ".join(subscribers) if subscribers else [],
            services="\n" + "- ".rjust(10) + "\n\t- ".join(services) if services else [],
            params="\n" + "- ".rjust(10) + "\n\t- ".join(params) if params else [],
            enable_cache=enable_cache)
        )

        # This base constructor assumes the system to interface with is already available ( can do a get_svc_available() )
        params_pool = RosParamIfPool(params)
        services_pool = RosServiceIfPool(services)
        subscribers_pool = RosSubscriberIfPool(subscribers)
        publishers_pool = RosPublisherIfPool(publishers)

        super(RosInterface, self).__init__(publishers_pool, subscribers_pool, services_pool, params_pool)

        # connecting to the master via proxy object
        self._master = rospy.get_master()

        #: If enabled, connection cache proxy will be setup in update() to allow dynamic update via config.
        # TODO : double check : maybe useless now since we completely reinit the interface for dynamic update...
        self.connection_cache = None

        # Setup our debug log
        # We need this debug log since rospy.logdebug does NOT store debug messages in the log.
        # But it should Ref : http://wiki.ros.org/rospy/Overview/Logging#Reading_log_messages
        # TODO : find why.
        ros_home = os.environ.get('ROS_HOME', os.path.join(os.path.expanduser("~"), '.ros'))
        self._debug_logger = logging.getLogger('pyros.ros_interface')
        if not os.path.exists(os.path.join(ros_home, 'logdebug')):
            os.makedirs(os.path.join(ros_home, 'logdebug'))
        logfilename = os.path.join(ros_home, 'logdebug', rospy.get_name()[1:].replace(os.path.sep, "-") + '_pyros_rosinterface.log')
        file_handler = logging.handlers.RotatingFileHandler(
            logfilename,
            maxBytes=100 * 131072,
            backupCount=10
        )
        self._debug_logger.propagate = False  # to avoid propagating to root logger
        self._debug_logger.setLevel(logging.DEBUG)
        self._debug_logger.addHandler(file_handler)

    def retrieve_system_state(self):
        """
        This will retrieve the system state from ROS master if needed, and apply changes to local variable to keep
        a local representation of the connections available up to date.
        """
        try:
            # we call the master only if we dont get system_state from connection cache
            if self.enable_cache and self.connection_cache is not None:
                publishers, subscribers, services = self.connection_cache.getSystemState()
                topic_types = self.connection_cache.getTopicTypes()
                try:
                    service_types = self.connection_cache.getServiceTypes()
                    # handling fallback here since master doesnt have the API
                except rocon_python_comms.UnknownSystemState as exc:
                    service_types = []
            else:
                publishers, subscribers, services = self._master.getSystemState()[2]
                topic_types = self._master.getTopicTypes()[2]
                service_types = []  # master misses this API to be consistent

            # Getting this doesnt depend on cache for now
            params = set(rospy.get_param_names())

            return publishers, subscribers, services, params, topic_types, service_types

        except socket.error:
            rospy.logerr("[{name}] couldn't get system state from the master ".format(name=__name__))

    # for use with line_profiler or memory_profiler
    # Not working yet... need to solve multiprocess profiling issues...
    #@profile
    def update(self):

        backedup_complete_cb_ss = None

        #update will retrieve system state here
        publishers = []
        subscribers = []
        services = []
        params = []
        topic_types = []
        service_types = []

        # and populate these to represent the changes done on the interface
        params_if_dt = DiffTuple([], [])
        services_if_dt = DiffTuple([], [])
        subscribers_if_dt = DiffTuple([], [])
        publishers_if_dt = DiffTuple([], [])

        # Destroying connection cache proxy if needed
        if self.connection_cache is not None and not self.enable_cache:
            # removing existing connection cache proxy to force a reinit of everything
            # to make sure we dont get a messed up system state with wrong list/diff from
            # dynamically switching cache on and off.
            self.connection_cache = None

        # TODO Instead of one or the other, we should have "two layer behaviors" with different frequencies
        # Fast loop checking only diff
        # Slow loop checking full state
        # It will allow recovering from any mistakes because of wrong diffs (update speed/race conditions/etc.)
        if self.enable_cache:
            if self.connection_cache is None:  # Building Connection Cache Proxy if needed
                self.connection_cache = connection_cache_proxy_create(self._proxy_cb)
                if self.connection_cache:
                    self.enable_cache = True
                else:
                    self.enable_cache = False

        # TMP until it s implemented in the connection cache
        # Because the cache doesnt currently do it
        params = set(rospy.get_param_names())
        # determining params diff despite lack of API
        params_dt = DiffTuple(
            added=[p for p in params if p not in self.params_available],
            removed=[p for p in self.params_available if p not in params]
        )

        # If we have the connection_cache and a callback setup we process the diff (and maybe param changes)
        if self.connection_cache and (params_dt.added or params_dt.removed or backedup_complete_cb_ss is not None or self.cb_ss.qsize() > 0):
            try:
                if backedup_complete_cb_ss is not None:
                    # using the message backed up after latest diff process
                    cb_ss = backedup_complete_cb_ss
                    backedup_complete_cb_ss = None  #CAREFUL : untested...
                else:
                    cb_ss = self.cb_ss.get_nowait()
                # print("CC MSG !")  # if we didn't except on empty queue, so we got a message

            except Queue.Empty:

                return self.update_nodelta(params_dt)

            else:
                # if there was no change,
                # it means it s the first and we need to initialize with the full list
                if cb_ss.added is None and cb_ss.removed is None:
                    #print("CC COMPLETE !")
                    publishers = cb_ss.complete.get('publishers', [])
                    subscribers = cb_ss.complete.get('subscribers', [])
                    services = cb_ss.complete.get('services', [])
                    # NOT YET...
                    #params = cb_ss.complete.get('params', params)
                    topic_types = cb_ss.complete.get('topic_types', [])
                    service_types = cb_ss.complete.get('service_types', [])

                    # To convert to ROS master API format :
                    publishers = [[p, [n[0] for n in nset]] for p, nset in publishers.iteritems()]
                    subscribers = [[s, [n[0] for n in nset]] for s, nset in subscribers.iteritems()]
                    services = [[s, [n[0] for n in nset]] for s, nset in services.iteritems()]

                    #print("CC COMPLETEDDDD !")

                    # we go back to normal flow
                    #print("UPDATE FULLSTATE ON CACHE LIST")
                    return self.update_fullstate(publishers, subscribers, services, params, topic_types, service_types)

                else:  # we have a delta, we can use it directly and skip the rest

                    #print("CC DELTA !")
                    next_cb_ss = cb_ss
                    while self.cb_ss.qsize() > 0 and not (next_cb_ss.added is None and next_cb_ss is None):
                        # merging multiple diff messages as fast as possible, until the next complete status
                        next_cb_ss = self.cb_ss.get_nowait()
                        if not (next_cb_ss.added is None and next_cb_ss is None):
                            cb_ss = CacheTuple(
                                complete=None,  # this is ignored here
                                # CAREFUL one of added or remove can still be None here...
                                added=connection_cache_merge_marshalled(next_cb_ss.added or {}, cb_ss.added),
                                removed=connection_cache_merge_marshalled(next_cb_ss.removed or {}, cb_ss.removed)
                            )
                        else:  # we need to pass next_cb_ss to handle complete list again...
                            backedup_complete_cb_ss = next_cb_ss

                    added_publishers = cb_ss.added.get('publishers', [])
                    added_subscribers = cb_ss.added.get('subscribers', [])
                    added_services = cb_ss.added.get('services', [])
                    # NOT YET
                    #added_params = cb_ss.added.get('params', params_dt.added)
                    added_params = params_dt.added
                    added_topic_types = cb_ss.added.get('topic_types', [])
                    added_service_types = cb_ss.added.get('service_types', [])

                    removed_publishers = cb_ss.removed.get('publishers', [])
                    removed_subscribers = cb_ss.removed.get('subscribers', [])
                    removed_services = cb_ss.removed.get('services', [])
                    # NOT YET
                    #removed_params = cb_ss.removed.get('params', params_dt.removed)
                    removed_params = params_dt.removed
                    removed_topic_types = cb_ss.removed.get('topic_types', [])
                    removed_service_types = cb_ss.removed.get('service_types', [])

                    #print("UPDATE STATEDELTA ON CACHE DIFF")
                    return self.update_statedelta(
                        added_publishers, added_subscribers, added_services, added_params, added_topic_types, added_service_types,
                        removed_publishers, removed_subscribers, removed_services, removed_params, removed_topic_types, removed_service_types
                    )
        elif not self.connection_cache:  # make sure we are not using connection cache (otherwise state representations might be out of sync !!)
            #print("GETTING STATE FROM MASTER")
            publishers, subscribers, services, params, topic_types, service_types = self.retrieve_system_state()  # This will call the master

            #print("UPDATE FULLSTATE")
            # NOTE : we want to be certain here that we do not mix full state representation from master with representation from cache (out of sync !!!)
            return self.update_fullstate(publishers, subscribers, services, params, topic_types, service_types)
        else:
            # This happens when the update is triggered, we are supposed to use the cache, but we got no message from the connection cache proxy.
            # WE DO NOT WANT TO GET FULLSTATE FROM CACHE, since it is out of date (ex: removed publishers are not removed until next message...)
            # print("GETTING STATE FROM CACHE")
            # publishers, subscribers, services, params, topic_types, service_types = self.retrieve_system_state()  # This will call the cache
            #
            # print("UPDATE FULLSTATE")
            # return self.update_fullstate(publishers, subscribers, services, params, topic_types, service_types)
            # ==> We need to wait for next message...
            return self.update_nodelta(params_dt)

    def update_fullstate(self, publishers, subscribers, services, params, topic_types, service_types):
        # NORMAL full update
        self._debug_logger.debug("""SYSTEM STATE :
                    - publishers : {publishers}
                    - subscribers : {subscribers}
                    - services : {services}
                    - params : {params}
                    - topic_types : {topic_types}
                    - service_types : {service_types}
                """.format(**locals()))

        # TODO : unify with the reset behavior in case of cache...

        # Needs to be done first, since topic algorithm depends on it
        # print("PARAMS : {params}".format(**locals()))
        params_if_dt = self.params_if_pool.update(params=params)
        # print("PARAM IF DT : {params_if_dt}".format(**locals()))

        # print("SERVICES : {services}".format(**locals()))
        services_if_dt = self.services_if_pool.update(services, service_types)
        # print("SERVICE IF DT : {services_if_dt}".format(**locals()))

        # print("SUBSCRIBERS : {subscribers}".format(**locals()))
        subscribers_if_dt = self.subscribers_if_pool.update(subscribers, topic_types)
        # print("SUBSCRIBER IF DT : {subscribers_if_dt}".format(**locals()))

        # print("PUBLISHERS : {publishers}".format(**locals()))
        publishers_if_dt = self.publishers_if_pool.update(publishers, topic_types)
        # print("PUBLISHER IF DT : {publishers_if_dt}".format(**locals()))

        dt = DiffTuple(
            added=params_if_dt.added + services_if_dt.added + subscribers_if_dt.added + publishers_if_dt.added,
            removed=params_if_dt.removed + services_if_dt.removed + subscribers_if_dt.removed + publishers_if_dt.removed
        )

        self._debug_logger.debug("""
                    ROS INTERFACE ADDED : {dt.added}
                    ROS INTERFACE REMOVED : {dt.removed}
                """.format(**locals()))

        return dt

    def update_nodelta(self, params_dt):
        """
        Running updates of nothing (workaround until params are handled by cache)
        :param params_dt:
        :return:
        """
        # we re NOT done here, we might still need to update params
        params_if_dt = self.params_if_pool.update_delta(params_dt=params_dt)

        topic_types_dt = DiffTuple(
            added=[],
            removed=[]  # shouldnt matter
        )

        services_if_dt = DiffTuple([], [])
        # TMP : NOT dropping topics early (just be patient and wait for the cache callback to come...)
        # topics_if_dt = self.topics_pool.update_delta(topics_dt, topic_types_dt)
        subscribers_if_dt = DiffTuple([], [])
        publishers_if_dt = DiffTuple([], [])

        # and here we need to return to not do the normal full update
        dt = DiffTuple(
            added=params_if_dt.added + services_if_dt.added + subscribers_if_dt.added + publishers_if_dt.added,
            removed=params_if_dt.removed + services_if_dt.removed + subscribers_if_dt.removed + publishers_if_dt.removed
        )

        self._debug_logger.debug("""
                            ROS INTERFACE DIFF ADDED : {dt.added}
                            ROS INTERFACE DIFF REMOVED : {dt.removed}
                        """.format(**locals()))

        return dt

    def update_statedelta(
            self,
            added_publishers, added_subscribers, added_services, added_params, added_topic_types, added_service_types,
            removed_publishers, removed_subscribers, removed_services, removed_params, removed_topic_types, removed_service_types
    ):
        params_dt = DiffTuple(
            added=added_params,
            removed=removed_params
        )

        # Needs to be done first, since topic algorithm depends on it
        params_if_dt = self.params_if_pool.update_delta(params_dt=params_dt)

        # here we need to get only the nodes' names to match ROs master API format
        services_dt = DiffTuple(
            added=[[k, [n[0] for n in nset]] for k, nset in added_services.iteritems()],
            removed=[[k, [n[0] for n in nset]] for k, nset in removed_services.iteritems()]
        )

        service_types_dt = DiffTuple(
            added=added_service_types,
            removed=removed_service_types
        )

        services_if_dt = self.services_if_pool.update_delta(services_dt, service_types_dt)

        # here we need to get only the nodes' names to match ROs master API format
        # CAREFUL about unicity here, a Pub|Sub can have multiple endpoint within the same node...
        # We do not care about that information. If we have to care, then we should pass also the uri...
        publishers_dt = DiffTuple(
            added=[[k, [n[0] for n in set(nset)]] for k, nset in added_publishers.iteritems()],
            removed=[[k, [n[0] for n in set(nset)]] for k, nset in removed_publishers.iteritems()]
        )
        subscribers_dt = DiffTuple(
            added=[[k, [n[0] for n in set(nset)]] for k, nset in added_subscribers.iteritems()],
            removed=[[k, [n[0] for n in set(nset)]] for k, nset in removed_subscribers.iteritems()]
        )

        # NOW DONE IN update_delta
        # # CAREFUL, topic interface by itself also makes the topic detected on system
        # # Check if there are any pyros interface with it and ignore them
        # topics_to_drop = TopicBack.get_interface_only_topics()
        # # TODO : simplify : same as early_topics_to_drop ?
        #
        # # filtering the topic dict
        # for td, tnode in added_topics.iteritems():
        #     tnode = [n for n in tnode if n not in topics_to_drop.get(td, [])]
        #
        # added_topics_list = [[td, added_topics[td]] for td in added_topics if added_topics[td]]  # filtering out topics with no endpoints
        #
        # # we also need to simulate topic removal here (only names), to trigger a cleanup of interface if it s last one
        # removed_topics_list = [[td, removed_topics[td]] for td in removed_topics] + early_topics_to_drop

        topic_types_dt = DiffTuple(
            added=added_topic_types,
            removed=removed_topic_types
        )

        subscribers_if_dt = self.subscribers_if_pool.update_delta(subscribers_dt, topic_types_dt)
        publishers_if_dt = self.publishers_if_pool.update_delta(publishers_dt, topic_types_dt)

        if publishers_if_dt.added or publishers_if_dt.removed:
            self._debug_logger.debug(
                rospy.get_name() + " Pyros.ros : Publishers Delta {publishers_if_dt}".format(
                    **locals()))
        if subscribers_if_dt.added or subscribers_if_dt.removed:
            self._debug_logger.debug(
                rospy.get_name() + " Pyros.ros : Subscribers Delta {subscribers_if_dt}".format(
                    **locals()))
        if services_if_dt.added or services_if_dt.removed:
            self._debug_logger.debug(
                rospy.get_name() + " Pyros.ros : Services Delta {services_if_dt}".format(
                    **locals()))

        # TODO : put that in debug log and show based on python logger configuration
        # print("Pyros ROS interface UPDATE")
        # print("Params ADDED : {0}".format([p for p in params_dt.added]))
        # print("Params GONE : {0}".format([p for p in params_dt.removed]))
        # print("Topics ADDED : {0}".format([t[0] for t in topics_dt.added] + early_topics_dt.added))
        # print("Topics GONE : {0}".format([t[0] for t in topics_dt.removed] + early_topics_dt.removed))
        # print("Srvs ADDED: {0}".format([s[0] for s in services_dt.added]))
        # print("Srvs GONE: {0}".format([s[0] for s in services_dt.removed]))

        # update_on_diff wants only names
        # dt = super(RosInterface, self).update_on_diff(
        #         DiffTuple([s[0] for s in services_dt.added], [s[0] for s in services_dt.removed]),
        #         DiffTuple([t[0] for t in topics_dt.added] + early_topics_dt.added, [t[0] for t in topics_dt.removed] + early_topics_dt.removed),
        #         # Careful params_dt has a different content than service and topics, due to different ROS API
        #         # TODO : make this as uniform as possible
        #         DiffTuple([p for p in params_dt.added], [p for p in params_dt.removed])
        # )

        # and here we need to return to not do the normal full update
        dt = DiffTuple(
            added=params_if_dt.added + services_if_dt.added + subscribers_if_dt.added + publishers_if_dt.added,
            removed=params_if_dt.removed + services_if_dt.removed + subscribers_if_dt.removed + publishers_if_dt.removed
        )

        self._debug_logger.debug("""
                                        ROS INTERFACE DIFF ADDED : {dt.added}
                                        ROS INTERFACE DIFF REMOVED : {dt.removed}
                                    """.format(**locals()))

        return dt

    def _proxy_cb(self, system_state, added_system_state, lost_system_state):
        self.cb_ss.put(CacheTuple(
            complete=connection_cache_marshall(system_state),
            added=connection_cache_marshall(added_system_state) if added_system_state is not None else None,
            removed=connection_cache_marshall(lost_system_state) if lost_system_state is not None else None
        ))


import matplotlib
import matplotlib.pyplot as plt
import dataset
import numpy as np
import tensorflow as tf
import os
import time
tf.compat.v1.reset_default_graph()
np.random.seed(42)
tf.compat.v1.set_random_seed(42)

###################################################################################
###################Uncomment this part to run capsulenetworks for artist classification
output_classes     = {0: 'Albrecht_Durer',
                     1: 'Boris_Kustodiev',
                     2: 'Camille_Pissarro',
                     3: 'Childe_Hassam',
                     4: 'Claude_Monet',
                     5: 'Edgar_Degas',
                     6: 'Eugene_Boudin',
                     7: 'Gustave_Dore',
                     8: 'Ilya_Repin',
                     9: 'Ivan_Aivazovsky',
                     10: 'Ivan_Shishkin',
                     11: 'John_Singer_Sargent',
                     12: 'Marc_Chagall',
                     13: 'Martiros_Saryan',
                     14: 'Nicholas_Roerich',
                     15: 'Pablo_Picasso',
                     16: 'Paul_Cezanne',
                     17: 'Pierre_Auguste_Renoir',
                     18: 'Pyotr_Konchalovsky',
                     19: 'Raphael_Kirchner',
                     20: 'Rembrandt',
                     21: 'Salvador_Dali',
                     22: 'Vincent_van_Gogh'}
train_path='./processed_Data/Artist/dataartisttrain_old'
val_path='./processed_Data/Artist/dataartistval_old'

###################################################################################
###################Uncomment this part to run capsulenetworks for genre classification
#output_classes     =  {0: 'abstract_painting',
#                        1: 'cityscape',
#                        2: 'genre_painting',
#                        3: 'illustration',
#                        4: 'landscape',
#                        5: 'nude_painting',
#                        6: 'portrait',
#                        7: 'religious_painting',
#                        8: 'sketch_and_study',
#                        9: 'still_life'}
# train_path='./processed_Data/Genre/datagenretrain_old'
# val_path='./processed_Data/Genre/datagenreval_old'
###################################################################################
###################Uncomment this part to run capsulenetworks for style classification
# output_classes = {0: 'Abstract_Expressionism',
#                   1: 'Action_painting',
#                   2: 'Analytical_Cubism',
#                   3: 'Art_Nouveau',
#                   4: 'Baroque',
#                   5: 'Color_Field_Painting',
#                   6: 'Contemporary_Realism',
#                   7: 'Cubism',
#                   8: 'Early_Renaissance',
#                   9: 'Expressionism',
#                   10: 'Fauvism',
#                   11: 'High_Renaissance',
#                   12: 'Impressionism',
#                   13: 'Mannerism_Late_Renaissance',
#                   14: 'Minimalism',
#                   15: 'Naive_Art_Primitivism',
#                   16: 'New_Realism',
#                   17: 'Northern_Renaissance',
#                   18: 'Pointillism',
#                   19: 'Pop_Art',
#                   20: 'Post_Impressionism',
#                   21: 'Realism',
#                   22: 'Rococo',
#                   23: 'Romanticism',
#                   24: 'Symbolism',
#                   25: 'Synthetic_Cubism',
#                   26: 'Ukiyo_e'}

# train_path='./processed_Data/Style/datastyletrain_old'
# val_path='./processed_Data/Style/datastyleval_old'
#######################################################################
#Parameterssss
batch_size = 128
num_epochs = 80
iterations = 1
#20% of the data will automatically be used for validation
test_size = 0.2
img_size = 28

start = time.time()
print('Starteddddd', start)

if not os.path.exists(train_path):
    print("No such directory")
    raise Exception
classes = os.listdir(train_path)
num_classes = len(classes)

# We shall load all the training and validation images and labels into memory using openCV and use that during training
data = dataset.read_train_sets(train_path, val_path, img_size, classes, test_size)

# Display the stats
print("Complete reading input data. Will Now print a snippet of it")
print("Number of files in Training-set:\t\t{}".format(len(data.train.labels)))
print("Number of files in Validation-set:\t{}".format(len(data.valid.labels)))
print("Number of files in Test-set:\t{}".format(len(data.test.labels)))

print('Doneeeee', time.time()-start)
X_train = data.train.images
y_train = data.train.labels

X_test = data.test.images
y_test = data.test.labels

X_valid = data.valid.images
y_valid = data.valid.labels

print('#training dataaaaa')
print(np.shape(X_train),np.shape(y_train))
print(type(y_train[1]), y_train[:10])

print('#testing dataaaaa')
print(np.shape(X_test),np.shape(y_test))
print(type(y_test[1]), y_test[:10])

print('#Validation dataaaaa')
print(np.shape(X_valid),np.shape(y_valid))
print(type(y_valid[1]), y_valid[:10])

n_samples = 5

print('And Corresponding Labelsssss')
print(y_train[:n_samples])

X = tf.compat.v1.placeholder(shape=[None, 28, 28, 1], dtype=tf.float32, name="X")

#primary capsuless

caps1_n_maps = 32
caps1_n_caps = caps1_n_maps * 6 * 6  # 1152 primary capsules
caps1_n_dims = 8

conv1_params = {
    "filters": 256,
    "kernel_size": 9,
    "strides": 1,
    "padding": "valid",
    "activation": tf.nn.relu,
}

conv2_params = {
    "filters": caps1_n_maps * caps1_n_dims, # 256 convolutional filters
    "kernel_size": 9,
    "strides": 2,
    "padding": "valid",
    "activation": tf.nn.relu
}

conv1 = tf.layers.conv2d(X, name="conv1", **conv1_params)
conv2 = tf.layers.conv2d(conv1, name="conv2", **conv2_params)

caps1_raw = tf.reshape(conv2, [-1, caps1_n_caps, caps1_n_dims],
                       name="caps1_raw")

def squash(s, axis=-1, epsilon=1e-7, name=None):
    with tf.name_scope(name, default_name="squash"):
        squared_norm = tf.reduce_sum(tf.square(s), axis=axis,
                                     keepdims=True)
        safe_norm = tf.sqrt(squared_norm + epsilon)
        squash_factor = squared_norm / (1. + squared_norm)
        unit_vector = s / safe_norm
        return squash_factor * unit_vector

caps1_output = squash(caps1_raw, name="caps1_output")

#Digital Capsules Now

caps2_n_caps = 10
caps2_n_dims = 16

init_sigma = 0.1

W_init = tf.random.normal(
    shape=(1, caps1_n_caps, caps2_n_caps, caps2_n_dims, caps1_n_dims),
    stddev=init_sigma, dtype=tf.float32, name="W_init")
W = tf.Variable(W_init, name="W")

batch_size = tf.shape(X)[0]
W_tiled = tf.tile(W, [batch_size, 1, 1, 1, 1], name="W_tiled")

caps1_output_expanded = tf.expand_dims(caps1_output, -1,
                                       name="caps1_output_expanded")
caps1_output_tile = tf.expand_dims(caps1_output_expanded, 2,
                                   name="caps1_output_tile")
caps1_output_tiled = tf.tile(caps1_output_tile, [1, 1, caps2_n_caps, 1, 1],
                             name="caps1_output_tiled")

caps2_predicted = tf.matmul(W_tiled, caps1_output_tiled,
                            name="caps2_predicted")

#Routing by agreement
#round 1
raw_weights = tf.zeros([batch_size, caps1_n_caps, caps2_n_caps, 1, 1],
                       dtype=np.float32, name="raw_weights")
routing_weights = tf.nn.softmax(raw_weights, axis=2, name="routing_weights")

weighted_predictions = tf.multiply(routing_weights, caps2_predicted,
                                   name="weighted_predictions")
weighted_sum = tf.reduce_sum(weighted_predictions, axis=1, keepdims=True,
                             name="weighted_sum")

caps2_output_round_1 = squash(weighted_sum, axis=-2,
                              name="caps2_output_round_1")

#round2

caps2_output_round_1_tiled = tf.tile(
    caps2_output_round_1, [1, caps1_n_caps, 1, 1, 1],
    name="caps2_output_round_1_tiled")

agreement = tf.matmul(caps2_predicted, caps2_output_round_1_tiled,
                      transpose_a=True, name="agreement")

raw_weights_round_2 = tf.add(raw_weights, agreement,
                             name="raw_weights_round_2")

routing_weights_round_2 = tf.nn.softmax(raw_weights_round_2,
                                        axis=2,
                                        name="routing_weights_round_2")
weighted_predictions_round_2 = tf.multiply(routing_weights_round_2,
                                           caps2_predicted,
                                           name="weighted_predictions_round_2")
weighted_sum_round_2 = tf.reduce_sum(weighted_predictions_round_2,
                                     axis=1, keepdims=True,
                                     name="weighted_sum_round_2")
caps2_output_round_2 = squash(weighted_sum_round_2,
                              axis=-2,
                              name="caps2_output_round_2")

caps2_output = caps2_output_round_2

#Estimating class probabilities

def safe_norm(s, axis=-1, epsilon=1e-7, keep_dims=False, name=None):
    with tf.name_scope(name, default_name="safe_norm"):
        squared_norm = tf.reduce_sum(tf.square(s), axis=axis,
                                     keepdims=keep_dims)
        return tf.sqrt(squared_norm + epsilon)

y_proba = safe_norm(caps2_output, axis=-2, name="y_proba")

y_proba_argmax = tf.argmax(y_proba, axis=2, name="y_proba")

y_pred = tf.squeeze(y_proba_argmax, axis=[1,2], name="y_pred")

#Labels

y = tf.compat.v1.placeholder(shape=[None], dtype=tf.int64, name="y")

#computing margin loss

m_plus = 0.9
m_minus = 0.1
lambda_ = 0.5

T = tf.one_hot(y, depth=caps2_n_caps, name="T")

caps2_output_norm = safe_norm(caps2_output, axis=-2, keep_dims=True,
                              name="caps2_output_norm")

present_error_raw = tf.square(tf.maximum(0., m_plus - caps2_output_norm),
                              name="present_error_raw")
present_error = tf.reshape(present_error_raw, shape=(-1, 10),
                           name="present_error")

absent_error_raw = tf.square(tf.maximum(0., caps2_output_norm - m_minus),
                             name="absent_error_raw")
absent_error = tf.reshape(absent_error_raw, shape=(-1, 10),
                          name="absent_error")

L = tf.add(T * present_error, lambda_ * (1.0 - T) * absent_error,
           name="L")

margin_loss = tf.reduce_mean(tf.reduce_sum(L, axis=1), name="margin_loss")

#Reconstruction

mask_with_labels = tf.placeholder_with_default(False, shape=(),
                                               name="mask_with_labels")

reconstruction_targets = tf.cond(mask_with_labels, # condition
                                 lambda: y,        # if True
                                 lambda: y_pred,   # if False
                                 name="reconstruction_targets")

reconstruction_mask = tf.one_hot(reconstruction_targets,
                                 depth=caps2_n_caps,
                                 name="reconstruction_mask")

reconstruction_mask_reshaped = tf.reshape(
    reconstruction_mask, [-1, 1, caps2_n_caps, 1, 1],
    name="reconstruction_mask_reshaped")

caps2_output_masked = tf.multiply(
    caps2_output, reconstruction_mask_reshaped,
    name="caps2_output_masked")

decoder_input = tf.reshape(caps2_output_masked,
                           [-1, caps2_n_caps * caps2_n_dims],
                           name="decoder_input")

#Decoder

n_hidden1 = 512
n_hidden2 = 1024
n_output = 28 * 28

with tf.name_scope("decoder"):
    hidden1 = tf.layers.dense(decoder_input, n_hidden1,
                              activation=tf.nn.relu,
                              name="hidden1")
    hidden2 = tf.layers.dense(hidden1, n_hidden2,
                              activation=tf.nn.relu,
                              name="hidden2")
    decoder_output = tf.layers.dense(hidden2, n_output,
                                     activation=tf.nn.sigmoid,
                                     name="decoder_output")

X_flat = tf.reshape(X, [-1, n_output], name="X_flat")
squared_difference = tf.square(X_flat - decoder_output,
                               name="squared_difference")
reconstruction_loss = tf.reduce_mean(squared_difference,
                                    name="reconstruction_loss")

alpha = 0.05

loss = tf.add(margin_loss, alpha * reconstruction_loss, name="loss")

#Computing accuracies and initializing paramters
correct = tf.equal(y, y_pred, name="correct")
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name="accuracy")

optimizer = tf.train.AdamOptimizer()
training_op = optimizer.minimize(loss, name="training_op")

init = tf.compat.v1.global_variables_initializer()
saver = tf.compat.v1.train.Saver()

train_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train))
valid_dataset =  tf.data.Dataset.from_tensor_slices((X_valid, y_valid))
test_dataset = tf.data.Dataset.from_tensor_slices((X_test, y_test))

#Traininggggggggggggggggg

def next_batch(num, data, labels):
    '''
    Return a total of `num` random samples and labels.
    '''
    idx = np.arange(0 , len(data))
    np.random.shuffle(idx)
    idx = idx[:num]
    data_shuffle = [data[ i] for i in idx]
    labels_shuffle = [labels[ i] for i in idx]

    return np.asarray(data_shuffle), np.asarray(labels_shuffle)

n_epochs = 80
batch_size = 100
restore_checkpoint = True

n_iterations_per_epoch =  np.shape(X_train)[0] // batch_size
n_iterations_validation = np.shape(X_valid)[0] // batch_size
best_loss_val = np.infty
checkpoint_path = "./my_capsule_network"

def plot_train_val_test(train_acc, val_acc, train_loss, val_loss):
   fig, (ax1, ax2) = plt.subplots(1, 2)

   # Accuracy
   ax1.set_title('Model accuracy')
   ax1.plot(train_acc)
   ax1.plot(val_acc)
   ax1.set_xlabel('epoch')
   ax1.set_ylabel('accuracy')
   ax1.legend(['train', 'validation'], loc='upper left')

   # Loss
   ax2.set_title('Model loss')
   ax2.plot(train_loss)
   ax2.plot(val_loss)
   ax2.set_xlabel('epoch')
   ax2.set_ylabel('loss')
   ax2.legend(['train', 'validation',], loc='upper left')

   fig.set_size_inches(20, 5)
   plt.show()

train_acc = []
val_acc = []
train_loss = []
val_loss = []

with tf.Session() as sess:
    if restore_checkpoint and tf.train.checkpoint_exists(checkpoint_path):
        saver.restore(sess, checkpoint_path)
    else:
        init.run()

    for epoch in range(n_epochs):
        loss_trains = []
        acc_trains = []
        for iteration in range(1, n_iterations_per_epoch + 1):
            X_batch, y_batch = next_batch(batch_size, X_train, y_train)
            #print(train.shape)
            #X_batch, y_batch = train
            # Run the training operation and measure the loss:
            loss_train, acc_train = sess.run(
                [loss, accuracy],
                feed_dict={X: X_batch.reshape([-1, 28, 28, 1]),
                           y: y_batch,
                           mask_with_labels: True})

            print("\rIteration: {}/{} ({:.1f}%)  Loss: {:.5f}".format(
                      iteration, n_iterations_per_epoch,
                      iteration * 100 / n_iterations_per_epoch,
                      loss_train),
                  end="")
            loss_trains.append(loss_train)
            acc_trains.append(acc_train)
        loss_train = np.mean(loss_trains)
        acc_train = np.mean(acc_trains)
        # At the end of each epoch,
        # measure the validation loss and accuracy:
        loss_vals = []
        acc_vals = []
        for iteration in range(1, n_iterations_validation + 1):
            X_batch, y_batch = next_batch(batch_size, X_valid, y_valid)
            #X_batch, y_batch = valid
            loss_val, acc_val = sess.run(
                    [loss, accuracy],
                    feed_dict={X: X_batch.reshape([-1, 28, 28, 1]),
                               y: y_batch})
            loss_vals.append(loss_val)
            acc_vals.append(acc_val)
            print("\rEvaluating the model: {}/{} ({:.1f}%)".format(
                      iteration, n_iterations_validation,
                      iteration * 100 / n_iterations_validation),
                  end=" " * 10)
        loss_val = np.mean(loss_vals)
        acc_val = np.mean(acc_vals)

        print("\rEpoch: {}  Val accuracy: {:.4f}%  Loss: {:.6f}{}".format(
            epoch + 1, acc_val * 100, loss_val,
            " (improved)" if loss_val < best_loss_val else ""))
        train_acc.append(acc_train)
        val_acc.append(acc_val)
        train_loss.append(loss_train)
        val_loss.append(loss_val)
        # And save the model if it improved:
        if loss_val < best_loss_val:
            save_path = saver.save(sess, checkpoint_path)
            best_loss_val = loss_val

n_iterations_test = np.shape(X_test)[0] // batch_size

with tf.Session() as sess:
    saver.restore(sess, checkpoint_path)
    loss_tests = []
    acc_tests = []
    for iteration in range(1, n_iterations_test + 1):
        X_batch, y_batch = next_batch(batch_size, X_test, y_test)
        #[X_batch, y_batch] = test
        loss_test, acc_test = sess.run(
                [loss, accuracy],
                feed_dict={X: X_batch.reshape([-1, 28, 28, 1]),
                           y: y_batch})
        loss_tests.append(loss_test)
        acc_tests.append(acc_test)
        print("\rEvaluating the model: {}/{} ({:.1f}%)".format(
                  iteration, n_iterations_test,
                  iteration * 100 / n_iterations_test),
              end=" " * 10)
    loss_test = np.mean(loss_tests)
    acc_test = np.mean(acc_tests)
    print("\rFinal test accuracy: {:.4f}%  Loss: {:.6f}".format(
        acc_test * 100, loss_test))

#predictions = cnn.predict_classes(X_test, verbose=0)
#plot_confusion_matrix(confusion_matrix(y_test, predictions), list(output_classes.keys()))

#print(classification_report(y_test, predictions))

plot_train_val_test(train_acc, val_acc, train_loss, val_loss)


from flask import Flask, render_template
from flask import jsonify, request
from flask import Blueprint, abort
from flask_mysqldb import MySQL
from flask_cors import CORS
from ..Config import Config as cfg
from ..run import db

kriteria = Blueprint('kriteria',__name__)

# READ
@kriteria.route('/kriteria', methods=['GET'])
def get_all_kriteria():
    cur = db.connection.cursor()
    # cur.execute("SELECT id_nilai_buku, nama_buku, kode, nilai_buku "
    #             "FROM tbl_nilai_buku as nb "
    #             "INNER JOIN tbl_buku as b "
    #             "ON nb.id_buku = b.id_buku "
    #             "INNER JOIN tbl_kriteria as k  "
    #             "ON nb.id_kriteria = k.id_kriteria ")
    cur.execute("SELECT * FROM tbl_kriteria ")

    rv = cur.fetchall()
    return jsonify(rv)

# CREATE
@kriteria.route('/kriteria', methods=['POST'])
def add_kriteria():
    cur = db.connection.cursor()

    kode = request.get_json()['kode']
    nama_kriteria = request.get_json()['nama_kriteria']
    bobot = request.get_json()['bobot']
    tipe_kriteria = request.get_json()['tipe_kriteria']

    cur.execute(
        "INSERT INTO tbl_kriteria (id_kriteria,kode,nama_kriteria,bobot,tipe_kriteria) "
        " SELECT MAX(id_kriteria)+1, "
        "'" + str(kode) + "', "
        "'" + str(nama_kriteria) + "', "
        "'" + str(bobot) + "', "
        "'" + str(tipe_kriteria) + "' " +
        "FROM tbl_kriteria")

    db.connection.commit()
    result = \
        {

            'kode': kode,
            'nama_kriteria' : nama_kriteria,
            'bobot' : bobot,
            'tipe_kriteria' : tipe_kriteria
        }

    return jsonify({"result":result})

# Update
@kriteria.route('/kriteria/<id_kriteria>', methods=['PUT'])
def update_kriteria(id_kriteria):
    cur = db.connection.cursor()

    kode = request.get_json()['kode']
    nama_kriteria = request.get_json()['nama_kriteria']
    bobot = request.get_json()['bobot']
    tipe_kriteria = request.get_json()['tipe_kriteria']

    cur.execute("UPDATE tbl_kriteria SET kode = '"+ str(kode) + "',"
                                          "nama_kriteria ='" + str(nama_kriteria) + "',"
                                          "bobot         ='" + str(bobot) + "',"
                                          "tipe_kriteria ='" + str(tipe_kriteria) + "'"
                                          + "WHERE id_kriteria=" + id_kriteria)

    db.connection.commit()

    result = \
        {
            'id_kriteria': id_kriteria,
            'kode': kode,
            'nama_kriteria' : nama_kriteria,
            'bobot' : bobot,
            'tipe_kriteria' : tipe_kriteria,
        }

    return jsonify({"result":result})

# DELETE
@kriteria.route('/kriteria/<id_kriteria>', methods=['DELETE'])
def delete_kriteria(id_kriteria):
    cur = db.connection.cursor()
    response = cur.execute("DELETE FROM tbl_kriteria where id_kriteria=" + id_kriteria)
    db.connection.commit()

    if response> 0:
        result = {'message' : 'record deleted'}
    else:
        result = {'message' : 'no record found'}
    return jsonify({"result": result})


import gettext
import tempfile
from collections import defaultdict

import polib

from carcade.utils import get_template_source

def get_translations(po_file_path):
    """Creates :class:`gettext.GNUTranslations` from PO file `po_file_path`."""
    po_file = polib.pofile(po_file_path)
    with tempfile.NamedTemporaryFile() as mo_file:
        po_file.save_as_mofile(mo_file.name)
        return gettext.GNUTranslations(mo_file)

def extract_translations(jinja2_env, target_pot_file):
    """Produces a `target_pot_file` which contains a list of all
    the translatable strings extracted from the templates.
    """
    po = polib.POFile()
    po.metadata = {'Content-Type': 'text/plain; charset=utf-8'}

    messages = defaultdict(list)
    for template in jinja2_env.list_templates():
        template_source = get_template_source(jinja2_env, template)
        for (lineno, _, message) in jinja2_env.extract_translations(template_source):
            message = unicode(message)
            occurence = (template, lineno)
            messages[message].append(occurence)

    for message, occurrences in messages.iteritems():
        entry = polib.POEntry(msgid=message, msgstr=message, occurrences=occurrences)
        po.append(entry)

    po.save(target_pot_file)


import random

def yaziTura():
    yaziTuraListesi = []
    for para in range(100000):
        yaziTuraListesi.append(random.randrange(1,3))
    return yaziTuraListesi

def altiDefaPespese():
    listem = yaziTura()

    altiliGrup = []
    n = 0
    altiDefaPesPeseTura = 0

    while len(listem) >= 6:
        sayi = []
        for counter in listem[0:6]:
            sayi.append(counter)
        if sayi[0] == 2 and sayi[1] == 2 and sayi[2] == 2 and sayi[3] == 2 and sayi[4] == 2 and sayi[5] == 2:
            altiDefaPesPeseTura = altiDefaPesPeseTura + 1
            print(altiDefaPesPeseTura,": ",sayi)

        altiliGrup.append(sayi)
        listem.pop(0)
    print("Peşpeşe 6 defa tura gelme miktari :", altiDefaPesPeseTura)
    #print(altiliGrup)

print(yaziTura())
altiDefaPespese()


import tkinter as tk

class Aplicacion:
    def __init__(self):
        self.ventana1=tk.Tk()
        self.ventana1.title("Ventana con botones")
        self.seleccion=tk.IntVar()
        self.seleccion.set(1)
        self.radio1=tk.Radiobutton(self.ventana1, text="Rojo", variable=self.seleccion, value=1, command=self.cambio_color)
        self.radio1.grid(column=0, row=0)
        self.radio2=tk.Radiobutton(self.ventana1, text="Verde", variable=self.seleccion, value=2, command=self.cambio_color)
        self.radio2.grid(column=1, row=0)
        self.radio3=tk.Radiobutton(self.ventana1, text="Azul", variable=self.seleccion, value=3, command=self.cambio_color)
        self.radio3.grid(column=2, row=0)
        self.ventana1.mainloop()

    def cambio_color(self):
        if self.seleccion.get()==1:
            self.ventana1.configure(bg="red")
        elif self.seleccion.get()==2:
            self.ventana1.configure(bg="green")
        else:
            self.ventana1.configure(bg="blue")

aplicacion1=Aplicacion()


import joblib
from pickle import load

import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_absolute_error,r2_score,accuracy_score

from preprocessing import for_evaluate
from preprocessing import slope_acc
from preprocessing import to_dataFrame
from preprocessing import smooth_candlestick

import requests
import json

from collections import namedtuple

def moving_average(x, w):
    return np.convolve(x, np.ones(w), 'valid') / w

def slope(predict):
    predict_pd = pd.DataFrame(predict)

    def calc_slope(x):
        slope = np.polyfit(range(len(x)), x, 1)[0]
        return slope
    pre_slope = predict_pd.rolling(48).apply(calc_slope,raw=False).values

    return pre_slope

def evaluate_historical(data,c_pair):
    pip = 0
    if c_pair == "EURUSD" or c_pair == "GBPUSD":
            pip = 10000
    elif c_pair == "USDJPY":
            pip = 100
    # ----------------------------------------
    # ! Create Features
    features,targets = for_evaluate(data,scale=c_pair)
    # ----------------------------------------
    # print(targets)
    # ----------------------------------------
    # ! Normalization
    # sc_X = joblib.load('model/scaler/x_scaler.bin')

    # print(sc_X)
    sc_X = StandardScaler()
    sc_y = StandardScaler()

    f_train = pd.read_csv('save_data\{}_features.csv'.format(c_pair))
    f_train.set_index('date', inplace=True, drop=True)

    labels = f_train[['open_24', 'close_24']].copy(deep=False)
    labels = labels*pip

    f_train = f_train.drop(['open_24', 'close_24'], axis=1)

    sc_X.fit_transform(f_train.values)
    sc_y.fit_transform(labels.values)
    # sc_y = joblib.load('model/scaler/y_scaler.bin')
    x = sc_X.transform(features.values)
    sc_y.transform(targets.values)

    # ----------------------------------------

    # ----------------------------------------
    # ! Run model & Denormalization
    # model = joblib.load('model/{}_SVR_1.joblib'.format(c_pair))
    model = joblib.load('model/2020_{}.joblib'.format(c_pair))
    yhat = model.predict(x)
    yhat = sc_y.inverse_transform(yhat)
    # y_pre = yhat[-24:,:].copy()/pip
    # yhat = yhat[:-24,:].copy()/pip
    # yhat = yhat
    # ----------------------------------------
    # print(yhat)
    # ----------------------------------------
    # ! Evaluate

    date = data['date'].values.tolist()
    # date = data['date'].iloc[-240:].str.split(n=1,expand=True).values[:,0].tolist()

    time_stamp = data['t'].values.tolist()
    MAE = int(mean_absolute_error(targets.values, yhat[:-24,:]))
    SMA_predict = (np.around(moving_average(yhat[:,1],24),4)/pip)
    SMA_true = (np.around(moving_average(targets['close'].values,24),4)/pip)
    trend_acc = slope_acc(SMA_predict[:-24],SMA_true,48)
    slope_values = slope(yhat[:,1])[-24:].tolist()
    slope_values = [item for sublist in slope_values for item in sublist]

    yhat_smooth = smooth_candlestick(yhat)
    values = np.around((yhat_smooth/pip),decimals=5).tolist()
    # pre_values = np.around((y_pre),decimals=5).tolist()

    score = {
        'Date' : date,
        # 'Date_24hr' : date_24,
        'Time_stamp' : time_stamp,
        # 'Time_stamp_24hr' : time_stamp_24,
        'MAE' : MAE,
        'R2_SCORE' : int(r2_score(targets.values, yhat[:-24,:])*100),
        'SMA_predict' : SMA_predict.tolist(),
        'SMA_true' : SMA_true.tolist(),
        'Slope_acc' : int(trend_acc),
        'Predict_ohlc' : values,
        'Slope_values' : slope_values
        # 'Predict_24hr' : pre_values
    }
    # ----------------------------------------

    # score = json.dumps(score)
    # score = json.loads(score)

    return score

# data = pd.read_csv(r'save_data\test_EUR.csv')
# result = evaluate_historical(data,'EURUSD')
# print((result["R2_SCORE"]))
# print(type(result['Date']))
# from collections import namedtuple
# d_named = namedtuple("Evaluate", result.keys())(*result.values())
# print(type(d_named))


'''
Create input features YAML for Ludwig configuration.
'''

import yaml
import argparse
import pandas as pd

if __name__ == "__main__":
    # Argument parsing.
    parser = argparse.ArgumentParser()
    parser.add_argument("input_file", help="Input file")
    parser.add_argument("output_file", help="Output file")
    args = parser.parse_args()

    # Read features from input.
    feature_names = pd.read_csv(args.input_file, delimiter="\t", header=0, nrows=0).columns.tolist()

    # Define inputs.
    inputs = [{"name": c, "type": "numerical"} for c in feature_names[1:-1]]
    config = {
        "input_features": inputs
    }

    # Write config.
    with open(args.output_file, "w") as output_file:
        yaml.dump(config, output_file)


# 739. Daily Temperatures
# Medium
#
# 4298
#
# 130
#
# Add to List
#
# Share
# Given a list of daily temperatures temperatures, return a list such that, for each day in the input, tells you how many days you would have to wait until a warmer temperature. If there is no future day for which this is possible, put 0 instead.
#
# For example, given the list of temperatures temperatures = [73, 74, 75, 71, 69, 72, 76, 73], your output should be [1, 1, 4, 2, 1, 1, 0, 0].
#
# Note: The length of temperatures will be in the range [1, 30000]. Each temperature will be an integer in the range [30, 100].
from typing import List

class Solution:
    def dailyTemperatures(self, temperatures: List[int]) -> List[int]:
        ans = [0] * len(temperatures)
        stack = []

        for i, t in enumerate(temperatures):
            while stack and temperatures[stack[-1]] < t:
                curr = stack.pop()
                ans[curr] = i - curr
            stack.append(i)

        return ans


# 预处理Node Extraction数据，生成字粒度标注序列
# 可配置生成: 有类别(E/V/T/VT) / 无类别(仅BIO) 的标签序列

import json
import sys
sys.path.append('../')
from transformers import DebertaV2Tokenizer, RobertaTokenizer
from tqdm import tqdm

from annotation.preprocess import parse_sparql
from QG.util import qg_tag2id

# tokenizer标志空格的标志
TOKENIZER_START_SIGN = 'Ġ'  # '▁' for deberta

def check_mention_identity(token_l, token_r, tokens, raw_mention, dat):
    # 比较tokenize后的token序列与原始char mention
    assert(token_l <= token_r)
    cur_mention = ''.join([ct.replace(TOKENIZER_START_SIGN, ' ') for ct in tokens[token_l:token_r+1]]).strip().replace(')?', ')').replace(').', ')')
    if cur_mention[-1] == ',':
        cur_mention = cur_mention[:-1]
    if cur_mention[-1] == '?':
        cur_mention = cur_mention[:-1]
    if cur_mention != raw_mention:
        print(token_l, '\t', token_r, '\t', dat['_id'], '\n', cur_mention, '\n', raw_mention, sep='', end='\n\n')

def has_token_span_overlap(token_l, token_r, gold_nodes):
    for cur_l, cur_r in gold_nodes.keys():
        if (token_l, token_r) == (cur_l, cur_r):
            return (token_l, token_r)
        elif token_l <= cur_r and token_r >= cur_l:
            return (cur_l, cur_r)
    return -1

def form_NEQG_gold_data(src, dst_ne, dst_qg, tokenizer, add_tag_type=False):
    # 根据src文件中的char级别l/r_pos信息, 运行tokenize转换成token级别位置
    # 再根据这些位置生成NE标签序列, 支持重叠标签的生成(VT, VE...)
    # TODO: QG的 <pointer, type> 序列实际上也可以一块生成

    with open(src) as fin:
        annotated_data = json.load(fin)
    type_relations = set()      # 记录除#type以外仍被认为是type (mention重叠) 的relation
    ne_output_data = []        # 记录token和标签序列, 即NE训练数据
    qg_output_data = []        # 记录node <type, ptr> 序列, 即QG训练数据

    for dat in (annotated_data):
        tokens = tokenizer.tokenize(dat['question'])
        # 先通过find计算token在question中的位置对应关系, 然后反过来计算char到token的映射
        char2token_pos, token_start_pos = dict(), []
        pos = 0
        for ix, token in enumerate(tokens):
            token_mention = token.replace(TOKENIZER_START_SIGN, '')
            if len(token_mention) == 0:     # 空token直接跳过
                continue
            l_pos = dat['question'].find(token_mention, pos)
            r_pos = l_pos + len(token_mention) - 1
            assert(l_pos != -1)
            assert(dat['question'][l_pos:r_pos+1].strip() == dat['question'][l_pos:r_pos+1])
            pos = r_pos + 1

            # 从l_pos到r_pos都映射到ix号token
            for cur_pos in range(l_pos, r_pos+1):
                assert((cur_pos) not in char2token_pos.keys())
                char2token_pos[cur_pos] = ix
            # 为sanity check, 也保存token开始的位置
            token_start_pos.append(l_pos)

        # 计算char级别位置到token位置的映射
        node_set = dat['node_set']
        # 在标签中会出现的node列表, 处理overlap, 包含node type
        gold_nodes = dict()
        # 处理NE数据时不重复计算同一个node
        already_annotated = set()
        # QG的gold数据, 包含node_sequence, 每个node的start/end token-pos
        # 注意需要单独添加target节点, 其中ASK类target规定为 <None, Variable>
        qg_node_sequence = []

        for triple in dat['triples']:
            head, tail = triple[0], triple[2]
            # 确定head和tail的类别, 考虑type类型
            head_type = 'variable' if '?' in head else 'entity'
            tail_type = 'variable' if '?' in tail else ('type' if ('type' in triple[1].lower()) else 'entity')
            if head_type != 'variable' and tail_type == 'type':
                tail_type = 'entity'
            # 对于NASA的几条数据特判一下
            if tail == 'http://dbpedia.org/resource/NASA':
                tail_type = 'entity'
            head_l, head_r = node_set[head]['l_pos'], node_set[head]['r_pos']
            tail_l, tail_r = node_set[tail]['l_pos'], node_set[tail]['r_pos']

            # 这里假设所有char级别l/r都不是空格 (因为之前算char2token_pos没考虑未被token包括的char pos)
            if head_l == -1:
                assert(head_r == -1 and node_set[head]['mention'] == 'None')
            if tail_l == -1:
                assert(tail_r == -1 and node_set[tail]['mention'] == 'None')

            # sanity checks
            if head_l not in token_start_pos and head_l != -1:
                print(1)
                print(head_l, tokens, dat['question'], head, node_set[head]['mention'])
            if tail_l not in token_start_pos and tail_l != -1:
                print(2)
                print(tail_l, tokens, dat['question'], tail, node_set[tail]['mention'])
            #if head_r in token_start_pos:
            #    print(3)
            #    print(head_r, tokens, dat['question'], head, node_set[head]['mention'])
            #if tail_r in token_start_pos:
            #    print(4)
            #    print(tail_r, tokens, dat['question'], tail, node_set[tail]['mention'])

            # 构建标注区间-head
            if head_l != -1 and (head not in already_annotated):
                already_annotated.add(head)
                head_token_l, head_token_r = char2token_pos[head_l], char2token_pos[head_r]
                # 检查tokens和原始char级别mention是否一致
                #check_mention_identity(head_token_l, head_token_r, tokens, node_set[head]['mention'], dat)
                res = has_token_span_overlap(head_token_l, head_token_r, gold_nodes)
                if res == -1:
                    gold_nodes[(head_token_l, head_token_r)] = {head_type}
                elif res == (head_token_l, head_token_r):
                    print('Overlap head: ', dat['_id'], '\t', dat['question'], '\n', triple[0], dat['triples'], sep='', end='\n\n')
                    gold_nodes[(head_token_l, head_token_r)].add(head_type)
                else:
                    # 少量case出现部分重叠, 由于我们的序列标注模型无法处理重叠序列, 这里仅保留更长的区间
                    print('Partial overlap head: ', dat['_id'], '\t', dat['question'], '\n', triple[0], '\n', tokens, '\n', (head_token_l, head_token_r), '\t', res, sep='', end='\n\n')
                    if head_token_r-head_token_l > res[1]-res[0]:
                        print('Replace the old span')
                        del gold_nodes[res]
                        gold_nodes[(head_token_l, head_token_r)] = {head_type}

            # 在 QG sequence 中的 head node
            if head_l == -1:
                head_token_l, head_token_r = -1, -1
            else:
                head_token_l, head_token_r = char2token_pos[head_l], char2token_pos[head_r]
            qg_head = {
                "tag": head_type,   # head-type没啥异议, 就entity和variable两类, 直接分即可
                "start": head_token_l,
                "end": head_token_r,
                "sparql_tok": head,
            }

            # tail
            if tail_l != -1 and (tail not in already_annotated):
                already_annotated.add(tail)
                tail_token_l, tail_token_r = char2token_pos[tail_l], char2token_pos[tail_r]
                #check_mention_identity(tail_token_l, tail_token_r, tokens, node_set[tail]['mention'], dat)
                res = has_token_span_overlap(tail_token_l, tail_token_r, gold_nodes)
                if res == -1:
                    gold_nodes[(tail_token_l, tail_token_r)] = {tail_type}
                elif res == (tail_token_l, tail_token_r):
                    if tail_type != 'type':
                        type_relations.add(triple[1])
                        #print('Non-type overlap tail: ', dat['_id'], '\t', dat['question'], '\n', triple[2], sep='', end='\n\n')
                    if gold_nodes[(tail_token_l, tail_token_r)] != {'variable'}:
                        print('Not variable-type: ', dat['_id'], '\t', gold_nodes[(tail_token_l, tail_token_r)], '\t', dat['question'], '\n', triple[2], sep='', end='\n\n')
                    # NOTE: 这里直接强制当做variable-type了!
                    gold_nodes[(tail_token_l, tail_token_r)].add('type')
                    # NOTE: 这里不管mention overlap的是entity还是variable, 都转成了type
                    # 强转variable是为了消除自环便于训练, 这一做法值得商榷, 但是影响面只有~3条, 估计差别不大
                    tail_type = 'type'
                else:
                    print('Partial overlap tail: ', dat['_id'], '\t', dat['question'], '\n', triple[2], '\n', tokens, '\n', (tail_token_l, tail_token_r), '\t', res, sep='', end='\n\n')
                    if tail_token_r-tail_token_l > res[1]-res[0]:
                        print('Replace the old span\n')
                        del gold_nodes[res]
                        gold_nodes[(tail_token_l, tail_token_r)] = {tail_type}

            # 在 QG sequence 中的 tail node
            if tail_l == -1:
                tail_token_l, tail_token_r = -1, -1
            else:
                tail_token_l, tail_token_r = char2token_pos[tail_l], char2token_pos[tail_r]
            qg_tail = {
                "tag": tail_type,   # tail-type 在上面处理了, 部分强转成type
                "start": tail_token_l,
                "end": tail_token_r,
                "sparql_tok": tail,
            }

            # 将当前triple (node pair) 加入 qg_sequence
            qg_node_sequence.append([qg_head, qg_tail])

        # 到这里, gold_nodes计算完成, 根据它生成NE标签
        cur_tags = ['O'] * len(tokens)
        for (node_l, node_r), node_type in gold_nodes.items():
            if node_type == {'variable', 'type'}:
                tag_type = 'VT'
            elif node_type == {'type'}:
                tag_type = 'CT'
            elif node_type == {'entity'}:
                tag_type = 'E'
            elif node_type == {'variable'}:
                tag_type = 'V'
            else:
                assert(False), node_type
            # 输出 BIO 标签序列
            cur_tags[node_l] = 'B_' + tag_type if add_tag_type else 'B'
            for token_ix in range(node_l+1, node_r+1):
                cur_tags[token_ix] = 'I_' + tag_type if add_tag_type else 'I'

        cur_output_data = {
            '_id': dat['_id'],
            'tokens': tokens,
            'tag_seq': cur_tags
        }
        ne_output_data.append(cur_output_data)

        # QG规范化: 将triple按 mention pointer (start token ix) 顺序排序, 先head后tail
        qg_node_sequence.sort(key=lambda x:10000*x[0]['start']+x[1]['start'])
        # triple的head和tail也按照 <start, tag_type_id> 从小到大, 因为QG是不考虑head-tail顺序的
        qg_node_sequence = [sorted(tri, key=lambda x:10000*x['start'] + qg_tag2id[x['tag']]) for tri in qg_node_sequence]

        # 扁平化, 直接弄成node sequence即可
        qg_node_sequence = sum(qg_node_sequence, [])

        # 加入 qg_sequence 的 target
        raw_spq = dat['raw_sparql']
        res = parse_sparql(raw_spq)
        if res['type'] != 'ask':
            target_node = res['target']
            assert('?' in target_node)
            target_l, target_r = node_set[target_node]['l_pos'], node_set[target_node]['r_pos']
            if target_l == -1:
                assert(target_r == -1)
                target_token_l, target_token_r = -1, -1
            else:
                target_token_l, target_token_r = char2token_pos[target_l], char2token_pos[target_r]
            qg_target = {
                "tag": 'variable',
                "start": target_token_l,
                "end": target_token_r,
                "sparql_tok": target_node,
            }
        else:
            qg_target = {
                "tag": 'variable',
                "start": -1,
                "end": -1,
                "sparql_tok": '?ASK_target',
            }
        qg_node_sequence.insert(0, qg_target)

        cur_output_data = {
            '_id': dat['_id'],
            'tokens': tokens,
            'node_seq': qg_node_sequence
        }
        qg_output_data.append(cur_output_data)

    print('Type-like relations:\n', type_relations, '\n', sep='')
    with open(dst_ne, 'w') as fout:
        json.dump(ne_output_data, fout, indent=4)
    with open(dst_qg, 'w') as fout:
        json.dump(qg_output_data, fout, indent=4)

def main():
    train_src = '../annotation/post_processed/Train_full_data_annotated.json'
    train_dst_ne = '../data/NEQG_gold/Train_NE_gold.json'
    train_dst_qg = '../data/NEQG_gold/Train_QG_gold.json'
    test_src = '../annotation/post_processed/Test_full_data_annotated.json'
    test_dst_ne = '../data/NEQG_gold/Test_NE_gold.json'
    test_dst_qg = '../data/NEQG_gold/Test_QG_gold.json'
    add_tag_type = False

    global TOKENIZER_START_SIGN
    #tokenizer_dir = '../data/pretrained/deberta_xlarge_tokenizer/'
    #tokenizer_class = DebertaV2Tokenizer
    #TOKENIZER_START_SIGN = '▁'
    tokenizer_dir = '../data/pretrained/roberta_large_tokenizer/'
    tokenizer_class = RobertaTokenizer
    TOKENIZER_START_SIGN = 'Ġ'
    tokenizer = tokenizer_class.from_pretrained(tokenizer_dir)

    form_NEQG_gold_data(train_src, train_dst_ne, train_dst_qg, tokenizer, add_tag_type=add_tag_type)
    print('====================================================')
    form_NEQG_gold_data(test_src, test_dst_ne, test_dst_qg, tokenizer, add_tag_type=add_tag_type)

if __name__ == '__main__':
    main()


import tensorflow as tf
from defs import *
import numpy as np

#------ State encoder decoder -----#
class state_encoder_decoder():
    def __init__(self, name):
        self.name = name
        self.en_name = name+"_encode_"
        self.dec_name = name+"_decode_"
        self.opt = tf.train.AdamOptimizer(0.0001)

    def encode(self, state, reuse):
        with tf.variable_scope(self.en_name, reuse=reuse):
            conv1 = tf.layers.conv2d(state, 32, 5, activation=tf.nn.relu, name=self.en_name+"conv1")
            conv1 = tf.layers.max_pooling2d(conv1, 2, 2, name=self.en_name+"conv1_pool")
            conv2 = tf.layers.conv2d(conv1, 64, 5, activation=tf.nn.relu, name=self.en_name+"conv2")
            conv2 = tf.layers.max_pooling2d(conv2, 2, 2, name=self.en_name+"conv2_pool")
            conv3 = tf.layers.conv2d(conv2, 128, 3, activation=tf.nn.relu, name=self.en_name+"conv3")
            conv3 = tf.layers.max_pooling2d(conv3, 2, 2, name=self.en_name+"conv3_pool")
            h_conv3_flat = tf.contrib.layers.flatten(conv3)
            fc1 = tf.layers.dense(h_conv3_flat, 1024, activation=tf.nn.tanh, name=self.en_name+"fc1")
            z = tf.layers.dense(fc1, LATENT_DIM, activation=tf.nn.tanh, name=self.en_name+"fc2")
        return z

    def decode(self, state_latent, reuse):
        with tf.variable_scope(self.dec_name, reuse=reuse):
            fc1 = tf.layers.dense(state_latent, IMAGE_PIXELS * 4, name=self.dec_name+"fc")
            D_fc1 = tf.reshape(fc1, [-1, IMAGE_HEIGHT * 2, IMAGE_WIDTH * 2, IMAGE_CHANNEL])
            D_fc1 = tf.contrib.layers.batch_norm(D_fc1, epsilon=1e-5)
            D_fc1 = tf.nn.tanh(D_fc1)
            dconv1 = tf.layers.conv2d(D_fc1, 64, 5, activation=tf.nn.tanh, name=self.dec_name+"conv1")
            dconv1 = tf.contrib.layers.batch_norm(dconv1, epsilon=1e-5)
            dconv1 = tf.image.resize_images(dconv1, [IMAGE_HEIGHT * 2, IMAGE_WIDTH * 2])
            dconv2 = tf.layers.conv2d(dconv1, 32, 3, activation=tf.nn.tanh, name=self.dec_name+"conv2")
            dconv2 = tf.contrib.layers.batch_norm(dconv2, epsilon=1e-5)
            dconv2 = tf.image.resize_images(dconv2, [IMAGE_HEIGHT * 1, IMAGE_WIDTH * 1])
            image = tf.layers.conv2d(dconv2, 3, 1, activation=tf.nn.sigmoid, name=self.dec_name+"conv3")
        return image

    def get_loss(self, source, target):
        batch_flatten = tf.reshape(target, [BATCH_SIZE, -1])
        batch_reconstruct_flatten = tf.reshape(source, [BATCH_SIZE, -1])
        loss1 = batch_flatten * tf.log(1e-10 + batch_reconstruct_flatten)
        loss2 = (1 - batch_flatten) * tf.log(1e-10 + 1 - batch_reconstruct_flatten)
        loss = loss1 + loss2
        reconstruction_loss = -tf.reduce_sum(loss)
        loss = tf.reduce_mean(reconstruction_loss)
        return loss

    def train_step(self, loss):
        tvars = tf.trainable_variables()
        phi_vars = [var for var in tvars if self.name in var.name]
        with tf.variable_scope(tf.get_variable_scope()) as scope:
            train_ = self.opt.minimize(loss, var_list=phi_vars)
        return train_

    def get_vars(self):
        tvars = tf.trainable_variables()
        vars = [var for var in tvars if self.name in var.name]
        return vars

#------ Policy -----#
class policy():
    def __init__(self, name):
        self.name = name
        self.opt = tf.train.AdamOptimizer(learning_rate=0.001, epsilon=1e-4)

    def sample(self, state_latent, reuse=False):
        n = self.name
        with tf.variable_scope(self.name, reuse=reuse):
            ex_fc1 = tf.layers.dense(state_latent, 32, activation=tf.nn.relu, name=n+"Ex_fc1")
            ex_fc2 = tf.layers.dense(ex_fc1, 20, activation=tf.nn.tanh, name=n+"Ex_fc2")
            ex_out = tf.layers.dense(ex_fc2, 3, activation=tf.nn.sigmoid, name=n+"Ex_out")

            in_fc1 = tf.layers.dense(state_latent, 32, activation=tf.nn.relu, name=n+"In_fc1")
            in_fc2 = tf.layers.dense(in_fc1, 20, activation=tf.nn.tanh, name=n+"In_fc2")
            in_out = tf.layers.dense(in_fc2, 3, activation=tf.nn.sigmoid, name=n+"In_out")
            in_out = tf.multiply(-1.0, in_out)
        return ex_out, in_out

    def get_vars(self):
        tvars = tf.trainable_variables()
        vars = [var for var in tvars if self.name in var.name]
        return vars

#------ Tensorboard ----#
def tensorboard_summary(data):
    st = data["source_image"]
    st_recon = data["reconstructed_image"]
    #sr_feature_recon = data["sr_feature"]
    with tf.name_scope("summary/images"):
        source_image = (st * 127.5) + 1
        recon_image = (st_recon * 127.5) + 1
        #sr_feature_recon = sr_feature_recon * 255.0
        image_to_tb = tf.concat([source_image, recon_image], axis=1)
        #image_to_tb = tf.concat([image_to_tb, sr_feature_recon], axis=1)
        tf.summary.image('src', image_to_tb, 5)

    with tf.name_scope("summary/losses"):
        tf.summary.scalar("StRecon_loss", data["reconstruction_loss"])
        #tf.summary.scalar("SR_loss", data["sr_loss"])


# Photo.py
import io
import os
from enum import Enum, unique
import hashlib
from typing import List
from datetime import datetime
import base64

from mongoengine import (
    Document,
    IntField,
    FloatField,
    StringField,
    ListField,
    ReferenceField,
    BooleanField,
    ImageField,
    DateTimeField,
    DictField,
    EmailField,
    EnumField,
    URLField,
)
import numpy as np
from tqdm import tqdm
from mongoengine import signals
from PIL import Image, ImageDraw, ImageFont
from pkg_resources import require
from PIL.ExifTags import GPSTAGS, TAGS
from PIL.TiffImagePlugin import IFDRational
from retinaface import RetinaFace
from deepface.basemodels import ArcFace
from deepface.commons import distance as dst
from hachoir.parser import createParser
from hachoir.metadata import extractMetadata

from . import logger, db
from .config import Config
from .Face import Face
from .utils.db import handler
from .Address import Address, get_decimal_coordinates
from .Face import Face
from .Person import Person

def read_metadata(pth: str, write_md: bool = False) -> dict:
    im = Image.open(pth)
    exif_data = im._getexif()
    _, fn = os.path.split(pth)
    bn, _ = os.path.splitext(fn)

    if write_md:
        os.makedirs("md", exist_ok=True)
        f_md = open("md/metadata_%s.txt" % fn, "src_w")
        f_md.write("%s\n" % pth)
        f_md.write("%s\n" % exif_data)
        im.close()

    if exif_data is None:
        exif_data = {}

    lkeys = list(exif_data.keys())
    for key in lkeys:
        name = TAGS.get(key, key)
        exif_data[name] = exif_data.pop(key)

    if "GPSInfo" in exif_data:
        lkeys = list(exif_data["GPSInfo"].keys())
    elif "34853" in exif_data:
        lkeys = list(exif_data["34853"].keys())
        exif_data["GPSInfo"] = {}
    else:
        lkeys = []
        exif_data["GPSInfo"] = {}

    for key in lkeys:
        name = GPSTAGS.get(key, key)
        exif_data["GPSInfo"][name] = exif_data["GPSInfo"].pop(key)

    orig_exif = stringify_keys(exif_data)

    if not "Orientation" in exif_data.keys():
        exif_data["Orientation"] = 1

    if "DateTimeOriginal" in exif_data.keys():
        date_time_str = exif_data["DateTimeOriginal"]
        # date_time_str='2020:04:02 12:58:12'
        exif_data["InferredDateTime"] = False
        dt = datetime.strptime(date_time_str, "%Y:%m:%d %H:%M:%S")
    else:
        # bn="Photo 20-03-13 17-53-52 0558"
        exif_data["InferredDateTime"] = True
        dt = datetime.strptime(bn, "Photo %y-%m-%d %H-%M-%S %f")

    exif_data["DateTimeOriginal"] = dt

    lat, lon = get_decimal_coordinates(exif_data["GPSInfo"])
    exif_data["GPSInfo"]["Latitude"] = lat
    exif_data["GPSInfo"]["Longitude"] = lon

    if write_md:
        f_md.write("%s\n" % exif_data)
        f_md.close()

    return orig_exif, exif_data

def crop_face(img: Image, height: int, force_square=True) -> Image:
    src_w, src_h = img.size
    sze_h = height
    sze_w = (sze_h * src_w) // src_h

    if force_square:
        tgt_h = height
        tgt_w = height
    else:
        tgt_h = height
        tgt_w = tgt_h * src_w / src_h

    rect_face = img.resize((sze_w, sze_h))

    # Resizing the face
    # and adding black pixels if the extracted face is not square
    img = Image.new(mode="RGB", size=(tgt_w, tgt_h), color=(0, 0, 0))
    if sze_w > sze_h:
        img.paste(rect_face, (0, tgt_w // 2 - sze_h // 2))
    else:
        img.paste(rect_face, (tgt_h // 2 - sze_w // 2, 0))

    return img

def stringify_keys(md: dict) -> dict:
    ret = {}
    for k in md.keys():
        sk = str(k)
        if isinstance(md[k], dict):
            sv = stringify_keys(md[k])
        else:
            sv = md[k]
            if isinstance(sv, IFDRational):
                sv = float(sv)
            elif hasattr(sv, "__iter__") and not isinstance(sv, str):
                svl = []
                for svk in sv:
                    if isinstance(svk, IFDRational):
                        if svk.denominator == 0:
                            svk = np.nan
                        else:
                            svk = float(svk)
                    svl.append(svk)
                sv = svl

        ret[sk] = sv

    return ret

face_detector = RetinaFace.build_model()
face_recog = ArcFace.loadModel()
face_recog.load_weights(
    os.path.expandvars("$HOME/.deepface/weights/arcface_weights.h5")
)

def detect_face(photo: Image) -> List[dict]:
    orig_arr = np.array(photo)

    detec = RetinaFace.detect_faces(orig_arr, model=face_detector)
    if not hasattr(detec, "keys"):
        detec = {}

    faces = []
    for kd in detec.keys():
        # Coordinates of the face the picture
        left, upper, right, lower = detec[kd]["facial_area"]

        # Crop the picture to extract the face
        rect_face = photo.crop((left, upper, right, lower))

        # Resizing the face (112,112)
        img = crop_face(rect_face, height=112, force_square=True)

        # Hash of the face
        buf = io.BytesIO()
        img.save(buf, format="JPEG")
        buf = buf.getvalue()
        h = hashlib.sha224(buf).hexdigest()

        # Embedding of the face
        img = np.array(img, dtype=np.float32)
        img_pixels = np.expand_dims(img, axis=0)
        img_pixels /= 255  # normalize input in [0, 1]
        embedding = face_recog.predict(img_pixels)[0]

        face_db = dict(
            hash=h,
            embedding=embedding,
            left=left,
            right=right,
            lower=lower,
            upper=upper,
            manually_tagged=False,
            detection_score=detec[kd]["score"],
            # landmarks=detec[kd]["landmarks"],
        )
        faces.append(face_db)

    logger.debug("%i faces detected" % len(faces))

    return faces

@handler(signals.pre_delete)
def photo_suppressed(sender, document):
    logger.debug("Suppress photo %s (%s)" % (document.id, document.original_path))
    for face in document.faces:
        logger.debug("   Suppress face %s" % face.id)
        face.delete()

    if not document.place_taken is None:
        document.place_taken.photos.remove(document)

    if not document.album is None:
        document.album.photos.remove(document)

@photo_suppressed.apply
class Photo(db.Document):
    photo = ImageField()
    original_path = StringField(required=True)
    hash = StringField(unique=True, required=True)
    place_taken = ReferenceField("Address")
    miniature = ImageField(thumbnail_size=(200, 200))
    date_taken = DateTimeField(required=True)
    faces = ListField(ReferenceField(Face))
    album = ReferenceField("Album")
    inferred_date = BooleanField()
    original_exif = DictField()

    meta = {"ordering": ["date_taken"]}

    @classmethod
    def showPhoto(cls, show_faces=False, **kwargs):
        photo = cls.objects(**kwargs).first()
        image = photo.photo.read()
        img = Image.open(io.BytesIO(image))
        font = ImageFont.truetype("Arial Unicode.ttf", size=15)

        if show_faces:
            # Drawing a red rectangle on the photo to locate the person
            img_with_red_box_draw = ImageDraw.Draw(img)
            for face in photo.faces:
                if face.manually_tagged:
                    color = "green"
                else:
                    color = "red"

                img_with_red_box_draw.rectangle(
                    [(face.left, face.upper), (face.right, face.lower)],
                    outline=color,
                    width=3,
                )
                if not face.person is None:
                    rec = face.person.complete_name
                    nom = rec["Nom complet"]
                else:
                    nom = "Unknown"
                img_with_red_box_draw.text(
                    (face.left, face.lower), nom, font=font, fill=color
                )

        img.show()

    def show(self, show_faces=True):
        image = self.photo.read()
        img = Image.open(io.BytesIO(image))

        if show_faces:
            img_with_red_box_draw = ImageDraw.Draw(img)

            for face in self.faces:
                if face.manually_tagged:
                    color = "green"
                else:
                    color = "red"
                img_with_red_box_draw.rectangle(
                    [(face.left, face.upper), (face.right, face.lower)],
                    outline=color,
                    width=3,
                )

        img.show()

    @classmethod
    def importFile(cls, pth: str, recognize: bool = False) -> "Photo":
        face_dst_dir = "faces"
        os.makedirs(face_dst_dir, exist_ok=True)

        logger.info(72 * "=")
        logger.info(pth)

        # ======================
        # Lecture des metadata
        # ======================
        orig_exif, metadata = read_metadata(pth)
        gps_info = metadata.get("GPSInfo", {})
        lat = gps_info.get("Latitude", None)
        lon = gps_info.get("Longitude", None)
        alt = gps_info.get("GPSAltitude", 0)
        if not lon is None and not lat is None:
            place_taken = Address.fromLatLonAlt(lat, lon, alt)
        else:
            logger.warning("No location data")
            place_taken = None
        inferred_dt = metadata["InferredDateTime"]
        date_taken = metadata.get("DateTimeOriginal", None)

        # ======================
        # Photo creation
        # ======================
        img = Image.open(pth)
        buf = io.BytesIO()
        img.save(buf, format="JPEG")
        src_h = hashlib.sha224(buf.getvalue()).hexdigest()

        if cls.objects(hash=src_h).count() > 0:
            dup = cls.objects(hash=src_h).first()
            logger.warning("Duplicate photo (skipping):")
            logger.warning("%s" % pth)
            logger.warning("%s" % dup.original_path)
            return

        if date_taken is None:
            photo = cls(
                hash=src_h,
                original_exif=orig_exif,
                inferred_date=inferred_dt,
                original_path=pth,
            )
            logger.warning("No date and time data")
        else:
            photo = cls(
                hash=src_h,
                original_exif=orig_exif,
                inferred_date=inferred_dt,
                original_path=pth,
                date_taken=date_taken,
            )

        if not place_taken is None:
            photo.save()
            photo.place_taken = place_taken
            place_taken.photos.append(photo)
            place_taken.save()

        # ======================
        # Orientation correction
        # ======================
        # angle – In degrees counter clockwise
        if metadata["Orientation"] == 8:
            angle = 90
        elif metadata["Orientation"] == 3:
            angle = 180
        elif metadata["Orientation"] == 6:
            angle = -90
        else:
            angle = 0

        if angle != 0:
            logger.debug("Rotated a=%i deg, %s" % (angle, photo.id))
            img = img.rotate(angle=angle)

        # ======================
        # Miniature creation
        # ======================
        src_w, src_h = img.size
        if src_w < src_h:
            mini = img.resize(
                size=(200, 200),
                resample=Image.HAMMING,
                box=(0, (src_h - src_w) // 2, src_w, src_w),
            )
        else:
            mini = img.resize(
                size=(200, 200),
                resample=Image.HAMMING,
                box=((src_w - src_h) // 2, 0, src_h, src_h),
            )

        # ======================
        # Enregistrement photo et miniature
        # ======================
        with io.BytesIO() as buf:
            mini.save(buf, format="JPEG")
            photo.miniature.replace(buf, filename=pth)

        with io.BytesIO() as buf:
            img.save(buf, format="JPEG")
            photo.photo.replace(buf, filename=pth)

        photo.save()

        # ======================
        # Faces detection
        # ======================
        lfaces = detect_face(img)
        for face_info in lfaces:
            q = Face.objects(hash=face_info["hash"])
            if q.count() > 0:
                face = q.first()
                logger.debug("Found Face already existing")
            else:
                face = Face(photo=photo, **face_info)
                face.save()
            if recognize:
                matching, corr = face.recognize_face()
                if not matching is None:
                    face.person = matching
                    face.recognition_score = corr
                    face.save()
            photo.faces.append(face)
        photo.save()

        return photo

    def getB64Miniature(self) -> bytes:
        image = self.miniature.read()
        b64 = base64.b64encode(image)
        return b64

    @property
    def date_elements(self):
        photo_year = self.date_taken.year
        photo_month = self.date_taken.month
        photo_month_name = datetime.strftime(self.date_taken, "%B")
        photo_day = self.date_taken.day
        photo_id = self.id
        return photo_year, photo_month, photo_month_name, photo_day, photo_id


import py_trees

class GlobalBlackboard:

    _instance = None

    @staticmethod
    def get_instance():

        if GlobalBlackboard._instance is None:
            GlobalBlackboard._instance = py_trees.blackboard.Client(name="Global")

        return GlobalBlackboard._instance

def get_port_content(port_value):

    if port_value.startswith("{") and port_value.endswith("}"):
        bb_key = port_value.strip("{}")
        blackboard = GlobalBlackboard.get_instance()

        # Return the value of the blackboard entry if it exists, otherwise None
        return getattr(blackboard, bb_key, "")
    else:
        return port_value

def set_port_content(port_value, value):

    if not (port_value.startswith("{") and port_value.endswith("}")):
        raise ValueError(f"'{port_value}' is a read-only port. Only ports connected to the blackboard are writable")

    # Extract the key from the port_value
    key = port_value.strip("{}")

    blackboard = GlobalBlackboard.get_instance()

    # Lazy creation: if key doesn't exist, register it
    if not hasattr(blackboard, key):
        blackboard.register_key(key=key, access=py_trees.common.Access.WRITE, required=True)

    # Set the value for the key in the blackboard
    setattr(blackboard, key, value)


'''
Code taken from rochakgupta repository
Github Reference : https://github.com/rochakgupta/aco-tsp.git
Reference : http://www.theprojectspot.com/tutorial-post/ant-colony-optimization-for-hackers/10
'''
import random
import math
import numpy as np
from aco_tsp import *
from kmeans import *

depot = []
# capacity = 100

def load_data(filename):
    global depot
    cities = []
    isdepot = 1
    with open(filename, 'r') as f:
        data = f.readlines()
        for line in data:
            # arr = line.strip('\n').split(', ')
            # x = int(arr[0]); y = int(arr[1])
            arr = line.strip('\n').split(' ')
            x = int(arr[1]); y = int(arr[2])
            if isdepot:         # first pair is depot by default
                depot = np.array([x,y])
                isdepot = 0
            else:
                cities.append([x,y])

    print("loading data complete")
    return cities

def get_distance_from_depot(elem):
    return math.sqrt((elem[0] - depot[0])**2 + (elem[1] - depot[1])**2)

_nodes = load_data("benchmarks/eil101.txt")

# print(_nodes)
num_salesmen = 5
nod = np.array(_nodes)
km = K_Means(num_salesmen)
km.fit(nod)

km.add_depot(depot)
# print(km.classes)

_colony_size = 5
_steps = 200
capacity = 120
time_dist_factor = 10
total_dist_sum = 0
total_jobs_sum = 0
capacity_wastage = 0
for c_id in km.classes:
    cities_base = list(km.classes[c_id])
    min_cost = 999999999999999999999
    cities_base.sort(key=get_distance_from_depot)
    # print(cities_base)
    prev_aco = None
    prev_cost = None
    rem_cap = 0
    for i in range(2, len(cities_base)):
        cities = cities_base[:i]
        aco = SolveTSPUsingACO(mode='MaxMin', colony_size=_colony_size, steps=_steps, nodes=cities)
        aco.run()
        cost = aco.get_best_total_distance() + (i-1)*time_dist_factor   # here we assume each job takes same amount of time, future work - variable time per job
        if cost > capacity:
            break
        prev_aco = aco
    total_dist_sum += prev_aco.get_best_total_distance()
    total_jobs_sum += prev_aco.get_num_nodes() - 1  # don't count depot
    capacity_wastage += capacity - (prev_aco.get_best_total_distance() + (prev_aco.get_num_nodes() - 1)*time_dist_factor)
    prev_aco.plot()

print("TOTAL DISTANCE", total_dist_sum)
print("TOTAL JOBS COMPLETED", total_jobs_sum)
print("CAPACITY WASTAGE", str(capacity_wastage))
print("CAPACITY WASTAGE PERCENT = %.2f" % (capacity_wastage/(capacity*num_salesmen)*100))

colors = 5*["r", "c", "k", "g", "b", "y", "m"]

for classification in km.classes:
    color = colors[classification]
    for features in km.classes[classification]:
        plt.scatter(features[0], features[1], color = color,s = 30)

plt.show()

# --------------------------------------------------------------------------------------------------------

# _nodes = [(random.uniform(-400, 400), random.uniform(-400, 400)) for _ in range(0, 15)]
# acs = SolveTSPUsingACO(mode='ACS', colony_size=_colony_size, steps=_steps, nodes=_nodes)
# acs.run()
# acs.plot()
# elitist = SolveTSPUsingACO(mode='Elitist', colony_size=_colony_size, steps=_steps, nodes=_nodes)
# elitist.run()
# elitist.plot()

# max_min = SolveTSPUsingACO(mode='MaxMin', colony_size=_colony_size, steps=_steps, nodes=_nodes)
# max_min.run()
# max_min.plot()

# Plotting
# colors = 10*["r", "c", "k", "g", "b"]

# for centroid in km.centroids:
#     plt.scatter(km.centroids[centroid][0], km.centroids[centroid][1], s = 130, marker = "x")

# for classification in km.classes:
#     color = colors[classification]
#     for features in km.classes[classification]:
#         plt.scatter(features[0], features[1], color = color,s = 30)

# plt.show()