from abc import ABCMeta, abstractmethod

import cPickle as pkl
import logging
import matplotlib.pyplot as plt
import numpy as np
import os
import sys
from time import time

from sklearn.mixture import GaussianMixture

import autolab_core.utils as utils
from autolab_core import Point
from perception import BinaryImage, ColorImage, DepthImage, RgbdImage, SegmentationImage, CameraIntrinsics

from gqcnn import Grasp2D, SuctionPoint2D, ImageGraspSamplerFactory, GQCNN, InputDataMode, GraspQualityFunctionFactory, GQCnnQualityFunction, NoValidGraspsException
from gqcnn import Visualizer as vis

FIGSIZE = 16
SEED = 5234709

class RgbdImageState(object):
    """ State to encapsulate RGB-D images.

    Attributes
    ----------
    rgbd_im : :obj:`perception.RgbdImage`
        an RGB-D image to plan grasps on
    camera_intr : :obj:`perception.CameraIntrinsics`
        intrinsics of the RGB-D camera
    segmask : :obj:`perception.BinaryImage`
        segmentation mask for the image
    obj_segmask : :obj:`perception.SegmentationImage`
        segmentation mask for the different objects in the image
    full_observed : :obj:`object`
        representation of the fully observed state
    """
    def __init__(self, rgbd_im, camera_intr,
                 segmask=None,
                 obj_segmask=None,
                 fully_observed=None):
        self.rgbd_im = rgbd_im
        self.camera_intr = camera_intr
        self.segmask = segmask
        self.obj_segmask = obj_segmask
        self.fully_observed = fully_observed
        logging.debug(self.camera_intr)

    def save(self, save_dir):
        if not os.path.exists(save_dir):
            os.mkdir(save_dir)
        color_image_filename = os.path.join(save_dir, 'color.png')
        depth_image_filename = os.path.join(save_dir, 'depth.npy')
        camera_intr_filename = os.path.join(save_dir, 'camera.intr')
        segmask_filename = os.path.join(save_dir, 'segmask.npy')
        obj_segmask_filename = os.path.join(save_dir, 'obj_segmask.npy')
        state_filename = os.path.join(save_dir, 'state.pkl')
        self.rgbd_im.color.save(color_image_filename)
        self.rgbd_im.depth.save(depth_image_filename)
        self.camera_intr.save(camera_intr_filename)
        if self.segmask is not None:
            self.segmask.save(segmask_filename)
        if self.obj_segmask is not None:
            self.obj_segmask.save(obj_segmask_filename)
        if self.fully_observed is not None:
            pkl.dump(self.fully_observed, open(state_filename, 'wb'))

    @staticmethod
    def load(save_dir):
        if not os.path.exists(save_dir):
            raise ValueError('Directory %s does not exist!' %(save_dir))
        color_image_filename = os.path.join(save_dir, 'color.png')
        depth_image_filename = os.path.join(save_dir, 'depth.npy')
        camera_intr_filename = os.path.join(save_dir, 'camera.intr')
        segmask_filename = os.path.join(save_dir, 'segmask.npy')
        obj_segmask_filename = os.path.join(save_dir, 'obj_segmask.npy')
        state_filename = os.path.join(save_dir, 'state.pkl')
        camera_intr = CameraIntrinsics.load(camera_intr_filename)
        color = ColorImage.open(color_image_filename, frame=camera_intr.frame)
        depth = DepthImage.open(depth_image_filename, frame=camera_intr.frame)
        segmask = None
        if os.path.exists(segmask_filename):
            segmask = BinaryImage.open(segmask_filename, frame=camera_intr.frame)
        obj_segmask = None
        if os.path.exists(obj_segmask_filename):
            obj_segmask = SegmentationImage.open(obj_segmask_filename, frame=camera_intr.frame)
        fully_observed = None
        if os.path.exists(state_filename):
            fully_observed = pkl.load(open(state_filename, 'rb'))
        return RgbdImageState(RgbdImage.from_color_and_depth(color, depth),
                              camera_intr,
                              segmask=segmask,
                              obj_segmask=obj_segmask,
                              fully_observed=fully_observed)

class ParallelJawGrasp(object):
    """ Action to encapsulate parallel jaw grasps.
    """
    def __init__(self, grasp, q_value, image):
        self.grasp = grasp
        self.q_value = q_value
        self.image = image

    def save(self, save_dir):
        if not os.path.exists(save_dir):
            os.mkdir(save_dir)
        grasp_filename = os.path.join(save_dir, 'grasp.pkl')
        q_value_filename = os.path.join(save_dir, 'pred_robustness.pkl')
        image_filename = os.path.join(save_dir, 'tf_image.npy')
        pkl.dump(self.grasp, open(grasp_filename, 'wb'))
        pkl.dump(self.q_value, open(q_value_filename, 'wb'))
        self.image.save(image_filename)

    @staticmethod
    def load(save_dir):
        if not os.path.exists(save_dir):
            raise ValueError('Directory %s does not exist!' %(save_dir))
        grasp_filename = os.path.join(save_dir, 'grasp.pkl')
        q_value_filename = os.path.join(save_dir, 'pred_robustness.pkl')
        image_filename = os.path.join(save_dir, 'tf_image.npy')
        grasp = pkl.load(open(grasp_filename, 'rb'))
        q_value = pkl.load(open(q_value_filename, 'rb'))
        image = DepthImage.open(image_filename)
        return ParallelJawGrasp(grasp, q_value, image)

class GraspAction(object):
    """ Action to encapsulate parallel jaw grasps.
    """
    def __init__(self, grasp, q_value, image):
        self.grasp = grasp
        self.q_value = q_value
        self.image = image

    def save(self, save_dir):
        if not os.path.exists(save_dir):
            os.mkdir(save_dir)
        grasp_filename = os.path.join(save_dir, 'grasp.pkl')
        q_value_filename = os.path.join(save_dir, 'pred_robustness.pkl')
        image_filename = os.path.join(save_dir, 'tf_image.npy')
        pkl.dump(self.grasp, open(grasp_filename, 'wb'))
        pkl.dump(self.q_value, open(q_value_filename, 'wb'))
        self.image.save(image_filename)

class Policy(object):
    """ Abstract policy class. """
    __metaclass__ = ABCMeta

    def __call__(self, state):
        return self.action(state)

    @abstractmethod
    def action(self, state):
        """ Returns an action for a given state.
        """
        pass

class GraspingPolicy(Policy):
    """ Policy for robust grasping with Grasp Quality Convolutional Neural Networks (GQ-CNN).
    Attributes
    ----------
    config : dict
        dictionary of parameters for the policy

    Notes
    -----
    Required configuration parameters are specified in Other Parameters

    Other Parameters
    ----------------
    sampling : dict
        dictionary of parameters for grasp sampling, see gqcnn/image_grasp_sampler.py
    gqcnn_model : str
        string path to a trained GQ-CNN model see gqcnn/neural_networks.py
    """
    def __init__(self, config):
        # store parameters
        self._config = config
        self._gripper_width = np.inf
        if 'gripper_width' in config.keys():
            self._gripper_width = config['gripper_width']

        # init grasp sampler
        self._sampling_config = config['sampling']
        self._sampling_config['gripper_width'] = self._gripper_width
        sampler_type = self._sampling_config['type']
        self._grasp_sampler = ImageGraspSamplerFactory.sampler(sampler_type,
                                                               self._sampling_config)

        # init grasp quality function
        self._metric_config = config['metric']
        metric_type = self._metric_config['type']
        self._grasp_quality_fn = GraspQualityFunctionFactory.quality_function(metric_type,
                                                                              self._metric_config)

    @property
    def config(self):
        """ Returns the policy parameters. """
        return self._config

    @property
    def grasp_sampler(self):
        """ Returns the grasp sampler. """
        return self._grasp_sampler

    @property
    def grasp_quality_fn(self):
        """ Returns the grasp sampler. """
        return self._grasp_quality_fn

    @property
    def gqcnn(self):
        """ Returns the GQ-CNN. """
        return self._gqcnn

    @abstractmethod
    def action(self, state):
        """ Returns an action for a given state.
        """
        pass

class UniformRandomGraspingPolicy(GraspingPolicy):
    """ Returns a grasp uniformly at random. """
    def __init__(self, config):
        GraspingPolicy.__init__(self, config)
        self._num_grasp_samples = 1

    def action(self, state):
        """ Plans the grasp with the highest probability of success on
        the given RGB-D image.

        Attributes
        ----------
        state : :obj:`RgbdImageState`
            image to plan grasps on

        Returns
        -------
        :obj:`GraspAction`
            grasp to execute
        """
        # check valid input
        if not isinstance(state, RgbdImageState):
            raise ValueError('Must provide an RGB-D image state.')

        # parse state
        rgbd_im = state.rgbd_im
        camera_intr = state.camera_intr
        segmask = state.segmask

        # sample grasps
        grasps = self._grasp_sampler.sample(rgbd_im, camera_intr,
                                            self._num_grasp_samples,
                                            segmask=segmask,
                                            visualize=self.config['vis']['grasp_sampling'],
                                            seed=None)
        num_grasps = len(grasps)
        if num_grasps == 0:
            logging.warning('No valid grasps could be found')
            raise NoValidGraspsException()

        # set grasp
        grasp = grasps[0]

        # form tensors
        return GraspAction(grasp, 0.0, state.rgbd_im.depth)

class RobustGraspingPolicy(GraspingPolicy):
    """ Samples a set of grasp candidates in image space,
    ranks the grasps by the predicted probability of success from a GQ-CNN,
    and returns the grasp with the highest probability of success.

    Notes
    -----
    Required configuration parameters are specified in Other Parameters

    Other Parameters
    ----------------
    num_grasp_samples : int
        number of grasps to sample
    gripper_width : float, optional
        width of the gripper in meters
    logging_dir : str, optional
        directory in which to save the sampled grasps and input images
    """
    def __init__(self, config, filters=None):
        GraspingPolicy.__init__(self, config)
        self._parse_config()
        self._filters = filters

    def _parse_config(self):
        """ Parses the parameters of the policy. """
        self._num_grasp_samples = self.config['sampling']['num_grasp_samples']
        self._max_grasps_filter = 1
        if 'max_grasps_filter' in self.config.keys():
            self._max_grasps_filter = self.config['max_grasps_filter']
        self._gripper_width = np.inf
        if 'gripper_width' in self.config.keys():
            self._gripper_width = self.config['gripper_width']
        self._logging_dir = None
        if 'logging_dir' in self.config.keys():
            self._logging_dir = self.config['logging_dir']

    def select(self, grasps, q_value):
        """ Selects the grasp with the highest probability of success.
        Can override for alternate policies (e.g. epsilon greedy).
        """
        # sort grasps
        num_grasps = len(grasps)
        grasps_and_predictions = zip(np.arange(num_grasps), q_value)
        grasps_and_predictions.sort(key = lambda x : x[1], reverse=True)

        # return top grasps
        if self._filters is None:
            return grasps_and_predictions[0][0]

        # filter grasps
        logging.info('Filtering grasps')
        i = 0
        while i < self._max_grasps_filter and i < len(grasps_and_predictions):
            index = grasps_and_predictions[i][0]
            grasp = grasps[index]
            valid = True
            for filter_name, is_valid in self._filters.iteritems():
                valid = is_valid(grasp)
                logging.debug('Grasp {} filter {} valid: {}'.format(i, filter_name, valid))
                if not valid:
                    valid = False
                    break
            if valid:
                return index
            i += 1
        raise NoValidGraspsException('No grasps satisfied filters')

    def action(self, state):
        """ Plans the grasp with the highest probability of success on
        the given RGB-D image.

        Attributes
        ----------
        state : :obj:`RgbdImageState`
            image to plan grasps on

        Returns
        -------
        :obj:`GraspAction`
            grasp to execute
        """
        # check valid input
        if not isinstance(state, RgbdImageState):
            raise ValueError('Must provide an RGB-D image state.')

        # parse state
        rgbd_im = state.rgbd_im
        camera_intr = state.camera_intr
        segmask = state.segmask

        # sample grasps
        grasps = self._grasp_sampler.sample(rgbd_im, camera_intr,
                                            self._num_grasp_samples,
                                            segmask=segmask,
                                            visualize=self.config['vis']['grasp_sampling'],
                                            seed=None)
        num_grasps = len(grasps)
        if num_grasps == 0:
            logging.warning('No valid grasps could be found')
            raise NoValidGraspsException()

        # save if specified
        if self._logging_dir is not None:
            if not os.path.exists(self._logging_dir):
                raise ValueError('Logging directory %s does not exist' %(self._logging_dir))

            # create output dir
            test_case_id = utils.gen_experiment_id()
            test_case_dir = os.path.join(self._logging_dir, 'test_case_%s' %(test_case_id))

            # re-sample if the test case dir exists
            while os.path.exists(test_case_dir):
                test_case_id = utils.gen_experiment_id()
                test_case_dir = os.path.join(self._logging_dir, 'test_case_%s' %(test_case_id))

            # create the directory and save
            os.mkdir(test_case_dir)
            candidate_actions_filename = os.path.join(test_case_dir, 'actions.pkl')
            pkl.dump(grasps, open(candidate_actions_filename, 'wb'))
            image_state_filename = os.path.join(test_case_dir, 'state.pkl')
            pkl.dump(state, open(image_state_filename, 'wb'))

        # compute grasp quality
        compute_start = time()
        q_values = self._grasp_quality_fn(state, grasps, params=self._config)
        logging.debug('Grasp evaluation took %.3f sec' %(time()-compute_start))

        if self.config['vis']['grasp_candidates']:
            # display each grasp on the original image, colored by predicted success
            norm_q_values = (q_values - np.min(q_values)) / (np.max(q_values) - np.min(q_values))
            vis.figure(size=(FIGSIZE,FIGSIZE))
            vis.imshow(rgbd_im.depth,
                       vmin=self.config['vis']['vmin'],
                       vmax=self.config['vis']['vmax'])
            for grasp, q in zip(grasps, norm_q_values):
                vis.grasp(grasp, scale=1.0,
                          grasp_center_size=10,
                          grasp_center_thickness=2.5,
                          jaw_width=2.5,
                          show_center=False,
                          show_axis=True,
                          color=plt.cm.RdYlGn(q))
            vis.title('Sampled grasps')
            vis.show()

        # select grasp
        index = self.select(grasps, q_values)
        grasp = grasps[index]
        q_value = q_values[index]
        if self.config['vis']['grasp_plan']:
            vis.figure()
            vis.imshow(rgbd_im.depth)#,
                      # vmin=self.config['vis']['vmin'],
                      # vmax=self.config['vis']['vmax'])
            vis.grasp(grasp, scale=2.0, show_axis=True)
            vis.title('Best Grasp: d=%.3f, q=%.3f' %(grasp.depth, q_value))
            vis.show()

        return GraspAction(grasp, q_value, state.rgbd_im.depth)

class CrossEntropyRobustGraspingPolicy(GraspingPolicy):
    """ Optimizes a set of grasp candidates in image space using the
    cross entropy method:
    (1) sample an initial set of candidates
    (2) sort the candidates
    (3) fit a GMM to the top P%
    (4) re-sample grasps from the distribution
    (5) repeat steps 2-4 for K iters
    (6) return the best candidate from the final sample set

    Parameters
    ----------
    filters : :obj:`dict` mapping names to functions
        list of functions to apply to filter invalid grasps

    Notes
    -----
    Required configuration parameters are specified in Other Parameters

    Other Parameters
    ----------------
    num_seed_samples : int
        number of candidate to sample in the initial set
    num_gmm_samples : int
        number of candidates to sample on each resampling from the GMMs
    num_iters : int
        number of sample-and-refit iterations of CEM
    gmm_refit_p : float
        top p-% of grasps used for refitting
    gmm_component_frac : float
        percentage of the elite set size used to determine number of GMM components
    gmm_reg_covar : float
        regularization parameters for GMM covariance matrix, enforces diversity of fitted distributions
    deterministic : bool, optional
        whether to set the random seed to enforce deterministic behavior
    gripper_width : float, optional
        width of the gripper in meters
    """
    def __init__(self, config, filters=None):
        GraspingPolicy.__init__(self, config)
        self._parse_config()
        self._filters = filters

    def _parse_config(self):
        """ Parses the parameters of the policy. """
        # cross entropy method parameters
        self._num_seed_samples = self.config['num_seed_samples']
        self._num_gmm_samples = self.config['num_gmm_samples']
        self._num_iters = self.config['num_iters']
        self._gmm_refit_p = self.config['gmm_refit_p']
        self._gmm_component_frac = self.config['gmm_component_frac']
        self._gmm_reg_covar = self.config['gmm_reg_covar']

        self._max_grasps_filter = 1
        if 'max_grasps_filter' in self.config.keys():
            self._max_grasps_filter = self.config['max_grasps_filter']

        self._max_resamples_per_iteration = 100
        if 'max_resamples_per_iteration' in self.config.keys():
            self._max_resamples_per_iteration = self.config['max_resamples_per_iteration']

        self._max_approach_angle = np.inf
        if 'max_approach_angle' in self.config.keys():
            self._max_approach_angle = np.deg2rad(self.config['max_approach_angle'])

        # gripper parameters
        self._seed = None
        if self.config['deterministic']:
            self._seed = SEED
        self._gripper_width = np.inf
        if 'gripper_width' in self.config.keys():
            self._gripper_width = self.config['gripper_width']

        # optional, logging dir
        self._logging_dir = None
        if 'logging_dir' in self.config.keys():
            self._logging_dir = self.config['logging_dir']
            if not os.path.exists(self._logging_dir):
                os.mkdir(self._logging_dir)

    def select(self, grasps, q_values):
        """ Selects the grasp with the highest probability of success.
        Can override for alternate policies (e.g. epsilon greedy).
        """
        # sort
        logging.info('Sorting grasps')
        num_grasps = len(grasps)
        if num_grasps == 0:
            raise NoValidGraspsException('Zero grasps')
        grasps_and_predictions = zip(np.arange(num_grasps), q_values)
        grasps_and_predictions.sort(key = lambda x : x[1], reverse=True)

        # return top grasps
        if self._filters is None:
            return grasps_and_predictions[0][0]

        # filter grasps
        logging.info('Filtering grasps')
        i = 0
        while i < self._max_grasps_filter and i < len(grasps_and_predictions):
            index = grasps_and_predictions[i][0]
            grasp = grasps[index]
            valid = True
            for filter_name, is_valid in self._filters.iteritems():
                valid = is_valid(grasp)
                logging.info('Grasp {} filter {} valid: {}'.format(i, filter_name, valid))
                if not valid:
                    valid = False
                    break
            if valid:
                return index
            i += 1
        raise NoValidGraspsException('No grasps satisfied filters')

    def action(self, state):
        """ Plans the grasp with the highest probability of success on
        the given RGB-D image.

        Attributes
        ----------
        state : :obj:`RgbdImageState`
            image to plan grasps on

        Returns
        -------
        :obj:`GraspAction`
            grasp to execute
        """
        # check valid input
        if not isinstance(state, RgbdImageState):
            raise ValueError('Must provide an RGB-D image state.')

        if self._logging_dir is not None:
            policy_id = utils.gen_experiment_id()
            policy_dir = os.path.join(self._logging_dir, 'policy_output_%s' %(policy_id))
            while os.path.exists(policy_dir):
                policy_id = utils.gen_experiment_id()
                policy_dir = os.path.join(self._logging_dir, 'policy_output_%s' %(policy_id))
            os.mkdir(policy_dir)
            state_dir = os.path.join(policy_dir, 'state')
            state.save(state_dir)

        # parse state
        seed_set_start = time()
        rgbd_im = state.rgbd_im
        depth_im = rgbd_im.depth
        camera_intr = state.camera_intr
        segmask = state.segmask
        point_cloud_im = camera_intr.deproject_to_image(depth_im)
        normal_cloud_im = point_cloud_im.normal_cloud_im()

        # sample grasps
        grasps = self._grasp_sampler.sample(rgbd_im, camera_intr,
                                            self._num_seed_samples,
                                            segmask=segmask,
                                            visualize=self.config['vis']['grasp_sampling'],
                                            seed=self._seed)
        num_grasps = len(grasps)
        if num_grasps == 0:
            logging.warning('No valid grasps could be found')
            raise NoValidGraspsException()

        grasp_type = 'parallel_jaw'
        if isinstance(grasps[0], SuctionPoint2D):
            grasp_type = 'suction'

        logging.info('Sampled %d grasps' %(len(grasps)))
        logging.info('Computing the seed set took %.3f sec' %(time() - seed_set_start))

        # iteratively refit and sample
        for j in range(self._num_iters):
            logging.info('CEM iter %d' %(j))

            # predict grasps
            predict_start = time()
            q_values = self._grasp_quality_fn(state, grasps, params=self._config)
            logging.info('Prediction took %.3f sec' %(time()-predict_start))

            # sort grasps
            resample_start = time()
            q_values_and_indices = zip(q_values, np.arange(num_grasps))
            q_values_and_indices.sort(key = lambda x : x[0], reverse=True)

            if self.config['vis']['grasp_candidates']:
                # display each grasp on the original image, colored by predicted success
                norm_q_values = q_values #(q_values - np.min(q_values)) / (np.max(q_values) - np.min(q_values))
                vis.figure(size=(FIGSIZE,FIGSIZE))
                vis.imshow(rgbd_im.depth)#,
                           #vmin=self.config['vis']['vmin'],
                           #vmax=self.config['vis']['vmax'])
                for grasp, q in zip(grasps, norm_q_values):
                    vis.grasp(grasp, scale=2.0,
                              jaw_width=2.0,
                              show_center=False,
                              show_axis=True,
                              color=plt.cm.RdYlGn(q))
                vis.title('Sampled grasps iter %d' %(j))
                filename = None
                if self._logging_dir is not None:
                    filename = os.path.join(self._logging_dir, 'cem_iter_%d.png' %(j))
                vis.show(filename)

            # fit elite set
            elite_start = time()
            num_refit = max(int(np.ceil(self._gmm_refit_p * num_grasps)), 1)
            elite_q_values = [i[0] for i in q_values_and_indices[:num_refit]]
            elite_grasp_indices = [i[1] for i in q_values_and_indices[:num_refit]]
            elite_grasps = [grasps[i] for i in elite_grasp_indices]
            elite_grasp_arr = np.array([g.feature_vec for g in elite_grasps])

            if self.config['vis']['elite_grasps']:
                # display each grasp on the original image, colored by predicted success
                norm_q_values = (elite_q_values - np.min(elite_q_values)) / (np.max(elite_q_values) - np.min(elite_q_values))
                vis.figure(size=(FIGSIZE,FIGSIZE))
                vis.imshow(rgbd_im.depth)#,
                           #vmin=self.config['vis']['vmin'],
                           #vmax=self.config['vis']['vmax'])
                for grasp, q in zip(elite_grasps, norm_q_values):
                    vis.grasp(grasp, scale=1.5, show_center=False, show_axis=True,
                              color=plt.cm.RdYlGn(q))
                vis.title('Elite grasps iter %d' %(j))
                filename = None
                if self._logging_dir is not None:
                    filename = os.path.join(self._logging_dir, 'elite_set_iter_%d.png' %(j))
                vis.show(filename)

            # normalize elite set
            elite_grasp_mean = np.mean(elite_grasp_arr, axis=0)
            elite_grasp_std = np.std(elite_grasp_arr, axis=0)
            elite_grasp_std[elite_grasp_std == 0] = 1.0
            elite_grasp_arr = (elite_grasp_arr - elite_grasp_mean) / elite_grasp_std
            logging.info('Elite set computation took %.3f sec' %(time()-elite_start))

            # fit a GMM to the top samples
            num_components = max(int(np.ceil(self._gmm_component_frac * num_refit)), 1)
            uniform_weights = (1.0 / num_components) * np.ones(num_components)
            gmm = GaussianMixture(n_components=num_components,
                                  weights_init=uniform_weights,
                                  reg_covar=self._gmm_reg_covar)
            train_start = time()
            gmm.fit(elite_grasp_arr)
            logging.info('GMM fitting with %d components took %.3f sec' %(num_components, time()-train_start))

            # sample the next grasps
            grasps = []
            loop_start = time()
            num_tries = 0
            while len(grasps) < self._num_gmm_samples and num_tries < self._max_resamples_per_iteration:
                # sample from GMM
                sample_start = time()
                grasp_vecs, _ = gmm.sample(n_samples=self._num_gmm_samples)
                grasp_vecs = elite_grasp_std * grasp_vecs + elite_grasp_mean
                logging.info('GMM sampling took %.3f sec' %(time()-sample_start))

                # convert features to grasps and store if in segmask
                for grasp_vec in grasp_vecs:
                    feature_start = time()
                    if grasp_type == 'parallel_jaw':
                        # form grasp object
                        grasp = Grasp2D.from_feature_vec(grasp_vec,
                                                         width=self._gripper_width,
                                                         camera_intr=camera_intr)
                    elif grasp_type == 'suction':
                        # read depth and approach axis
                        u = int(min(max(grasp_vec[1], 0), depth_im.height-1))
                        v = int(min(max(grasp_vec[0], 0), depth_im.width-1))
                        grasp_depth = depth_im[u, v]

                        # approach_axis
                        grasp_axis = -normal_cloud_im[u, v]

                        # form grasp object
                        grasp = SuctionPoint2D.from_feature_vec(grasp_vec,
                                                                camera_intr=camera_intr,
                                                                depth=grasp_depth,
                                                                axis=grasp_axis)
                    logging.debug('Feature vec took %.5f sec' %(time()-feature_start))

                    bounds_start = time()
                    # check in bounds
                    if state.segmask is None or \
                        (grasp.center.y >= 0 and grasp.center.y < state.segmask.height and \
                         grasp.center.x >= 0 and grasp.center.x < state.segmask.width and \
                         np.any(state.segmask[int(grasp.center.y), int(grasp.center.x)] != 0) and \
                         grasp.approach_angle < self._max_approach_angle):

                        # check validity according to filters
                        grasps.append(grasp)
                    logging.debug('Bounds took %.5f sec' %(time()-bounds_start))
                    num_tries += 1

            # check num grasps
            num_grasps = len(grasps)
            if num_grasps == 0:
                logging.warning('No valid grasps could be found')
                raise NoValidGraspsException()
            logging.info('Resample loop took %.3f sec' %(time()-loop_start))
            logging.info('Resampling took %.3f sec' %(time()-resample_start))

        # predict final set of grasps
        predict_start = time()
        q_values = self._grasp_quality_fn(state, grasps, params=self._config)
        logging.info('Final prediction took %.3f sec' %(time()-predict_start))

        if self.config['vis']['grasp_candidates']:
            # display each grasp on the original image, colored by predicted success
            norm_q_values = q_values #(q_values - np.min(q_values)) / (np.max(q_values) - np.min(q_values))
            vis.figure(size=(FIGSIZE,FIGSIZE))
            vis.imshow(rgbd_im.depth)#,
                       #vmin=self.config['vis']['vmin'],
                       #vmax=self.config['vis']['vmax'])
            for grasp, q in zip(grasps, norm_q_values):
                vis.grasp(grasp, scale=2.0,
                          jaw_width=2.0,
                          show_center=False,
                          show_axis=True,
                          color=plt.cm.RdYlGn(q))
            vis.title('Final sampled grasps')
            filename = None
            if self._logging_dir is not None:
                filename = os.path.join(self._logging_dir, 'final_grasps.png')
            vis.show(filename)

        # select grasp
        index = self.select(grasps, q_values)
        grasp = grasps[index]
        q_value = q_values[index]
        if self.config['vis']['grasp_plan']:
            vis.figure()
            vis.imshow(rgbd_im.depth)#,
                       #vmin=self.config['vis']['vmin'],
                       #vmax=self.config['vis']['vmax'])
            vis.grasp(grasp, scale=5.0, show_center=False, show_axis=True, jaw_width=1.0, grasp_axis_width=0.2)
            vis.title('Best Grasp: d=%.3f, q=%.3f' %(grasp.depth,
                                                     q_value))
            filename = None
            if self._logging_dir is not None:
                filename = os.path.join(self._logging_dir, 'planned_grasp.png')
            vis.show(filename)

        # form return image
        image = state.rgbd_im.depth
        if isinstance(self._grasp_quality_fn, GQCnnQualityFunction):
            image_arr, _ = self._grasp_quality_fn.grasps_to_tensors([grasp], state)
            image = DepthImage(image_arr[0,...],
                               frame=state.rgbd_im.frame)

        # return action
        action = GraspAction(grasp, q_value, image)

        if self._logging_dir is not None:
            action_dir = os.path.join(policy_dir, 'action')
            action.save(action_dir)

        return action

class QFunctionRobustGraspingPolicy(CrossEntropyRobustGraspingPolicy):
    """ Optimizes a set of antipodal grasp candidates in image space using the
    cross entropy method with a GQ-CNN that estimates the Q-function
    for use in Q-learning.

    Notes
    -----
    Required configuration parameters are specified in Other Parameters

    Other Parameters
    ----------------
    reinit_pc1 : bool
        whether or not to reinitialize the pc1 layer of the GQ-CNN
    reinit_fc3: bool
        whether or not to reinitialize the fc3 layer of the GQ-CNN
    reinit_fc4: bool
        whether or not to reinitialize the fc4 layer of the GQ-CNN
    reinit_fc5: bool
        whether or not to reinitialize the fc5 layer of the GQ-CNN
    num_seed_samples : int
        number of candidate to sample in the initial set
    num_gmm_samples : int
        number of candidates to sample on each resampling from the GMMs
    num_iters : int
        number of sample-and-refit iterations of CEM
    gmm_refit_p : float
        top p-% of grasps used for refitting
    gmm_component_frac : float
        percentage of the elite set size used to determine number of GMM components
    gmm_reg_covar : float
        regularization parameters for GMM covariance matrix, enforces diversity of fitted distributions
    deterministic : bool, optional
        whether to set the random seed to enforce deterministic behavior
    gripper_width : float, optional
        width of the gripper in meters
    """
    def __init__(self, config):
        CrossEntropyRobustGraspingPolicy.__init__(self, config)
        QFunctionRobustGraspingPolicy._parse_config(self)
        self._setup_gqcnn()

    def _parse_config(self):
        """ Parses the parameters of the policy. """
        self._reinit_pc1 = self.config['reinit_pc1']
        self._reinit_fc3 = self.config['reinit_fc3']
        self._reinit_fc4 = self.config['reinit_fc4']
        self._reinit_fc5 = self.config['reinit_fc5']

    def _setup_gqcnn(self):
        """ Sets up the GQ-CNN. """
        # close existing session (from superclass initializer)
        self.gqcnn.close_session()

        # check valid output size
        if self.gqcnn.fc5_out_size != 1 and not self._reinit_fc5:
            raise ValueError('Q function must return scalar values')

        # reinitialize layers
        if self._reinit_fc5:
            self.gqcnn.fc5_out_size = 1

        # TODO: implement reinitialization of pc0
        self.gqcnn.reinitialize_layers(self._reinit_fc3,
                                       self._reinit_fc4,
                                       self._reinit_fc5)
        self.gqcnn.initialize_network(add_softmax=False)

class EpsilonGreedyQFunctionRobustGraspingPolicy(QFunctionRobustGraspingPolicy):
    """ Optimizes a set of antipodal grasp candidates in image space
    using the cross entropy method with a GQ-CNN that estimates the
    Q-function for use in Q-learning, and chooses a random antipodal
    grasp with probability epsilon.

    Notes
    -----
    Required configuration parameters are specified in Other Parameters

    Other Parameters
    ----------------
    epsilon : float
    """
    def __init__(self, config):
        QFunctionRobustGraspingPolicy.__init__(self, config)
        self._parse_config()

    def _parse_config(self):
        """ Parses the parameters of the policy. """
        self._epsilon = self.config['epsilon']

    @property
    def epsilon(self):
        return self._epsilon

    @epsilon.setter
    def epsilon(self, val):
        self._epsilon = val

    def greedy_action(self, state):
        """ Plans the grasp with the highest probability of success on
        the given RGB-D image.

        Attributes
        ----------
        state : :obj:`RgbdImageState`
            image to plan grasps on

        Returns
        -------
        :obj:`GraspAction`
            grasp to execute
        """
        return CrossEntropyRobustGraspingPolicy.action(self, state)

    def action(self, state):
        """ Plans the grasp with the highest probability of success on
        the given RGB-D image.

        Attributes
        ----------
        state : :obj:`RgbdImageState`
            image to plan grasps on

        Returns
        -------
        :obj:`GraspAction`
            grasp to execute
        """
        # take the greedy action with prob 1 - epsilon
        if np.random.rand() > self.epsilon:
            logging.debug('Taking greedy action')
            return CrossEntropyRobustGraspingPolicy.action(self, state)

        # otherwise take a random action
        logging.debug('Taking random action')

        # check valid input
        if not isinstance(state, RgbdImageState):
            raise ValueError('Must provide an RGB-D image state.')

        # parse state
        rgbd_im = state.rgbd_im
        camera_intr = state.camera_intr
        segmask = state.segmask

        # sample random antipodal grasps
        grasps = self._grasp_sampler.sample(rgbd_im, camera_intr,
                                            self._num_seed_samples,
                                            segmask=segmask,
                                            visualize=self.config['vis']['grasp_sampling'],
                                            seed=self._seed)

        num_grasps = len(grasps)
        if num_grasps == 0:
            logging.warning('No valid grasps could be found')
            raise NoValidGraspsException()

        # choose a grasp uniformly at random
        grasp_ind = np.random.choice(num_grasps, size=1)[0]
        grasp = grasps[grasp_ind]
        depth = grasp.depth

        # create transformed image
        image_tensor, pose_tensor = self.grasps_to_tensors([grasp], state)
        image = DepthImage(image_tensor[0,...])

        # predict prob success
        output_arr = self.gqcnn.predict(image_tensor, pose_tensor)
        q_value = output_arr[0,-1]

        # visualize planned grasp
        if self.config['vis']['grasp_plan']:
            scale_factor = float(self.gqcnn.im_width) / float(self._crop_width)
            scaled_camera_intr = camera_intr.resize(scale_factor)
            vis_grasp = Grasp2D(Point(image.center), 0.0, depth,
                                width=self._gripper_width,
                                camera_intr=scaled_camera_intr)
            vis.figure()
            vis.imshow(image)
            vis.grasp(vis_grasp, scale=1.5, show_center=False, show_axis=True)
            vis.title('Best Grasp: d=%.3f, q=%.3f' %(depth, q_value))
            vis.show()

        # return action
        return GraspAction(grasp, q_value, image)


from app.api import bp
from flasgger import swag_from
from flask import jsonify, request
from marshmallow import ValidationError
from app.schemas import ZoneSchema, ZoneEntrySchema
from app.api.zones.service import zone_list, zone_history, deactivate
from app.api.errors import bad_request
from app.swagger.zones_specs import get_spec, post_spec, put_spec, get_history_spec, post_deactive_spec
from app import db
from app.models import Zone
from app.api.users.service import token_auth
from app.utils.helpers import build_page

@bp.route('/zones', methods=['GET'])
@swag_from(get_spec)
def get_zones():
    zones = zone_list()
    zone_schema = ZoneSchema(many=True)
    return jsonify(zone_schema.dump(zones))

@bp.route('/zones', methods=['POST'])
@swag_from(post_spec)
@token_auth.login_required
def create_zone():
    data = request.get_json()
    zone_schema = ZoneSchema()

    try:
        zone = zone_schema.load(
            data, session=db.session)
        print(zone)
        db.session.add(zone)
        db.session.commit()

    except ValidationError as err:
        print("error")
        return bad_request(err.messages)

    return jsonify(zone_schema.dump(zone))

@bp.route('/zones/<id>', methods=['PUT'])
@swag_from(put_spec)
@token_auth.login_required
def edit_zone(id):
    data = request.get_json()
    zone_schema = ZoneSchema()
    zone = Zone.query.get(id)
    if(zone):
        data['id'] = zone.id
    else:
        bad_request(f'Zone with id={id} does not exist')

    try:
        zone = zone_schema.load(
            data, session=db.session)
        db.session.add(zone)
        db.session.commit()

    except ValidationError as err:
        print("error")
        return bad_request(err.messages)

    return jsonify(zone_schema.dump(zone))

@bp.route('/zones/<id>', methods=['GET'])
@swag_from(get_history_spec)
def get_history(id):
    zone = Zone.query.get(id)
    if(not zone):
        return bad_request(f'Zone with id={id} does not exist')

    page = request.args.get('page', 1, type=int)
    per_page = request.args.get('per_page', 10, type=int)
    order_by = request.args.get('order_by', "")
    order = request.args.get('order', "asc")

    history = zone_history(id, order_by, order).paginate(page, per_page, False)
    zone_entry_schema = ZoneEntrySchema(many=True)

    return jsonify(build_page(zone_entry_schema, history, order_by, order))

@bp.route('/zones/deactivate/<id>', methods=['POST'])
@swag_from(post_deactive_spec)
@token_auth.login_required
def deactivate_zone(id):
    zone = deactivate(id)
    if(not zone):
        return bad_request(f'Zone with id={id} does not exist')
    zone_schema = ZoneSchema()
    return jsonify(zone_schema.dump(zone))


import torch
import argparse
from timeit import default_timer as time
from PIL import Image
import yaml
import numpy as np
from denoising_pipeline.model_estimator.model import Model
from denoising_pipeline.architectures.configure_model import \
    build_model_from_config

def parse_args():
    parser = argparse.ArgumentParser(description='Video denoising train script')
    parser.add_argument('--config', required=False, type=str,
                          default='../configuration/example_train_config.yml',
                          help='Path to configuration yml file.'
                        )
    parser.add_argument('--model-weights', required=True, type=str,
                        help='Path to model weights')
    parser.add_argument('--input-image', required=True, type=str,
                        help='Path to image file')
    parser.add_argument('--output-image', required=True, type=str,
                        help='Path to result image file')
    parser.add_argument('--batch-size', required=False, type=int, default=16,
                        help='Inference model batch size')
    parser.add_argument('--no-cuda', action='store_true', default=False,
                        help='disables CUDA training')
    return parser.parse_args()

def main():
    args = parse_args()

    use_cuda = not args.no_cuda and torch.cuda.is_available()
    device = 'cuda' if use_cuda else 'cpu'

    with open(args.config, 'r') as f:
        config = yaml.safe_load(f)

    window_size = config['model']['window_size']

    series_size = 1 \
        if config['model']['architecture'] != 'sequential' else \
        config['model'][config['model']['architecture']]['series_size']

    denoising_model = build_model_from_config(config)

    model = Model(
        denoising_model,
        device
    )

    print(
        'Count of model trainable parameters: {}'.format(
            model.get_parameters_count()
        )
    )

    model.load(args.model_weights)

    print('Model out shape: {}'.format(
        model.model.inference(*tuple([
            torch.FloatTensor(
                np.zeros((1, 3, window_size, window_size))
            ).to(device)
        ] * series_size)).shape
    ))

    inp_image = np.array(Image.open(args.input_image).convert('RGB'))
    start_time = time()
    out_image = model.predict(
        image=inp_image,
        window_size=window_size,
        batch_size=args.batch_size,
        verbose=True
    )
    finish_time = time()
    Image.fromarray(out_image).save(args.output_image)

    print(
        'Inference time: {:.2f} sec'.format(
            finish_time - start_time
        )
    )

if __name__ == '__main__':
    main()


import asyncio
import logging
import socket

CHUNK_SIZE_HTTP = 1024
QUEUE_SOCKET = 8
TIMEOUT_CLIENT = 5

async def get_data(client):
    loop = asyncio.get_event_loop()
    data = b''
    while True:
        chunk = await loop.sock_recv(client, CHUNK_SIZE_HTTP)
        data += chunk
        if len(chunk) < CHUNK_SIZE_HTTP:
            break
    return data

def get_socket(host, port):
    server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    server_sock.bind((host, port))
    server_sock.listen(QUEUE_SOCKET)
    server_sock.setblocking(False)
    logging.info('Start on {}'.format(server_sock.getsockname()))
    return server_sock

async def handle_client(client, core_engine, loop):
    data = await get_data(client)

    data_generator = await core_engine(data)

    for chunk in data_generator:
        await loop.sock_sendall(client, chunk)

    client.close()

async def run(loop, server_sock, core_engine):
    while True:
        client, _ = await loop.sock_accept(server_sock)
        client.settimeout(TIMEOUT_CLIENT)
        loop.create_task(handle_client(client, core_engine, loop))


import pyodbc
from email.message import EmailMessage
import ssl
import smtplib

connection_string = ""
table_name = ''

def fetch_entries(conn):
    cursor = conn.cursor()
    cursor.execute("SELECT * FROM AllMenuData WHERE Name LIKE '%chicken%' OR Name LIKE '%kiwi%'")
    rows = cursor.fetchall()
    conn.commit()
    conn.close()
    return rows

def send_text():
    email_sender = ""
    email_password = ""
    email_receiver = ''
    subject = "DiningCourtMessenger"

    em = EmailMessage()
    em['From'] = email_sender
    em['To'] = email_receiver
    em['subject'] = subject

    context = ssl.create_default_context()

    with smtplib.SMTP_SSL('smtp.gmail.com', 465, context=context) as smtp:
        smtp.login(email_sender, email_password)

        for entry in entries:
            date = entry[0]
            dining_court = entry[1]
            meal = entry[2]
            food = entry[3]
            body = f"""
                    {food} is being served at {dining_court} dining court for {meal} today, {date}!
                """
            em.set_content(body)
            smtp.sendmail(email_sender, email_receiver, em.as_string())

def main():
    conn = pyodbc.connect(connection_string)
    entries = fetch_entries(conn)

    if len(entries) == 0:
        print('No matches found.')
        return

    print(entries)

    send_text()

    print('finished!')

if __name__ == "__main__":
    main()


import sys
import math
import time
import random
import numpy as np
import scipy.stats as stats
from itertools import product
sys.path.append('../')
from common import call_prolog
from common import parmap

num_trials = 10
num_tasks = 50
trials = list(range(1,num_trials+1))
mrules = ['u-2-s','u-2-e','u-2-d','u-2-dm']
tasks = list(range(0,num_tasks))
jobs = list(product(mrules,tasks,trials))
max_time = 600

def get_train_file(task, trial):
    return f'data/train/{task}-{trial}.pl'

def get_test_file(task, trial):
    return f'data/test/{task}-{trial}.pl'

def get_res_file(ms, task, trial):
    return f'results/{ms}/{task}-{trial}.txt'

def get_program_file(ms, task, trial):
    return f'programs/{ms}/{task}-{trial}.pl'

def load_probs(nowrites=True):
    dic = {}
    with open('probs.txt', 'r') as f:
        for line in f:
            if line.startswith('# -*- coding: utf-8 -*-'):
                continue
            xs=line.strip().split('=>')
            if len(xs)==1 and len(xs[0])>0:
                try:
                    problem=int(xs[0][1:])
                    if prob_num > 273:
                        return dic
                except:
                    continue
            elif len(xs)>1:
                if problem not in dic:
                    dic[problem] = []
                dic[problem].append(xs)
    return dic

def gen_data_(dic, trial):
    for problem, examples in dic.items():
        if problem not in tasks:
            continue
        # if len(examples) != 10:
            # continue
        random.shuffle(examples)
        train = examples[:5]
        test = examples[5:]
        filename = get_train_file(problem, trial)
        with open(filename, 'w') as f:
            for x in train:
                a = list(x[0].strip())
                b = list(x[1].strip())
                atom = f'pos({a},{b}).'
                f.write(atom + '\n')
        filename = get_test_file(problem, trial)
        with open(filename, 'w') as f:
            for x in test:
                a = list(x[0].strip())
                b = list(x[1].strip())
                atom = f'pos({a},{b}).'
                f.write(atom + '\n')

def gen_data():
    dic = load_probs()
    for trial in trials:
        gen_data_(dic, trial)

def learn_(args):
    (m, task, k) = args
    data_file = get_train_file(task, k)
    data = f'data/{k}.pl'
    mf = f'../mrules/{m}.pl'
    program_file = get_program_file(m, task, k)
    with open(program_file, 'w') as f:
        a = time.time()
        load_files = ['experiment', data_file, mf]
        prog = call_prolog('learn', load_files, max_time)
        b = time.time()
        if prog == '':
            prog = 'f(_,_):-false.'
        f.write(prog + '\n')
        f.write(f'%time,{b-a}')

def learn():
    list(parmap(learn_,jobs))

def test_(args):
    (m, task, k) = args
    data_file = get_test_file(task, k )
    program_file = get_program_file(m, task, k)
    results_file = get_res_file(m, task, k)
    with open(results_file, 'w') as f:
        f.write(call_prolog('do_test', ['experiment', data_file, program_file]))

def test():
    list(parmap(test_,jobs))

def get_times(ms, task, trial):
    filename = get_program_file(ms, task, trial)
    with open(filename) as f:
        for line in f:
            xs = line.strip().split(',')
            if xs[0]!='%time':
              continue
            return float(xs[1])
    return max_time

def get_accuracies(metarules, task, trial):
    results_file = get_res_file(metarules, task, trial)
    accs = []
    with open(results_file, 'r') as f:
        for line in f:
            line = line.strip()
            if line == '1':
                accs.append(1)
            if line == '0':
                accs.append(0)
    return accs

def acc_results():
    all_accs = {ms:[] for ms in mrules}
    for ms in mrules:
        for trial in trials:
            accs = []
            for task in tasks:
                k_accs = get_accuracies(ms, task, trial)
                accs.append(np.mean(k_accs))
            mu_acc = int(np.mean(accs)*100)
            sem_acc = int(stats.sem(accs)*100)
            all_accs[ms].append(mu_acc)
    out = ''
    for m in mrules:
        mu_acc = int(np.mean(all_accs[m]))
        sem_acc = int(stats.sem(all_accs[m]))
        out += f' & {mu_acc} $\pm$ {sem_acc}'
    print(out)

def time_results():
    all_times = {ms:[] for ms in mrules}
    for ms in mrules:
        for trial in trials:
            times = []
            for task in tasks:
                k_times = get_times(ms, task, trial)
                times.append(np.mean(k_times))
            mu = int(np.mean(times))
            sem = int(stats.sem(times))
            all_times[ms].append(mu)

    out = ''
    # out+='mean'
    for m in mrules:
        mu = int(np.mean(all_times[m]))
        sem = int(stats.sem(all_times[m]))
        out += f' & {mu} $\pm$ {sem}'
    print(out)

def results():
    print('accuracies')
    acc_results()
    print('times')
    time_results()

arg = sys.argv[1]

if arg == 'gen':
    gen_data()
if arg == 'learn':
    learn()
if arg == 'test':
    test()
if arg == 'results':
    results()


# -*- coding: utf-8 -*-

#    This file is part of buscarTitulosFacebook.
#
#    buscarTitulosFacebook is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 3 of the License, or
#    (at your option) any later version.
#
#    buscarTitulosFacebook is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with buscarTitulosFacebook; if not, write to the Free Software
#    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

from pyfbutils.DataSetCSV import DataSetCSV
from pyfbutils.PostFacebook import PostFacebook

def agregarInfoPostFacebook(datasetCSV):
    posts = datasetCSV.dataset

    for i in range(datasetCSV.inicio, datasetCSV.fin):
        try:
            print(i)
            url = posts[i][1]
            postFacebook = PostFacebook(url)
            datosPost = postFacebook.getInfoPostFacebook()
            posts[i].append(datosPost[0])
            posts[i].append(datosPost[1])
            posts[i].append(datosPost[2])
            posts[i].append(datosPost[3])
        except Exception as ex:
            print(ex)
            columnas = len(posts[i]) + 1
            for j in range(columnas, datasetCSV.cantidadColumnas):
                posts[i].append("Error exception")

# programaPrincipal
nombreArchivoEntrada = 'post_input.csv'
nombreArchivoSalida = 'post_output.csv'
columnas = ['post_id', 'post_link', 'titulo_facebook', 'subtitulo_facebook', 'menciones_facebook', 'hashtags_facebook']

inicio = 0
fin = None

datasetCSV = DataSetCSV(nombreArchivoEntrada, nombreArchivoSalida, columnas, inicio, fin)
agregarInfoPostFacebook(datasetCSV)
datasetCSV.guardar()


from data_utils import *
from collections import Counter
from tqdm import tqdm
import utils
from sklearn import svm
from sklearn.linear_model import SGDClassifier
from torch.nn import functional as F
from torch import nn
from torch import optim
import train_utils

class KNNModel(object):
    def __init__(self, h=2, k=5):
        self.train_x = None
        self.train_y = None
        self.h = h
        self.k = k

    @utils.logging_time_wrapper
    def train(self, train_x, train_y):
        self.train_x = train_x
        self.train_y = train_y if isinstance(train_y, np.ndarray) else np.array(train_y)

    @utils.logging_time_wrapper
    def predict(self, X):
        dis = self.distance(self.train_x, X)
        neighbor_k = np.argsort(dis, axis=1)[:, :self.k]
        neighbor_labels = self.train_y[neighbor_k]
        labels = [Counter(neighbor_labels[i, :]).most_common(1)[0][0] for i in range(neighbor_labels.shape[0])]
        return np.array(labels)

    def distance(self, X1, X2):
        assert X1.shape[1] == X2.shape[1]
        dis = [np.sum(np.abs(X1 - X2[i]) ** self.h, axis=1) ** (1 / self.h) for i in tqdm(range(X2.shape[0]), desc='Distance :')]
        dis = np.array(dis)
        return dis

class TwoLayerNN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(TwoLayerNN, self).__init__()
        self.hidden_layer = nn.Linear(input_size, hidden_size)
        self.output_layer = nn.Linear(hidden_size, output_size)

    def forward(self, x):
        x = self.hidden_layer(x)
        x = F.relu(x)
        output = self.output_layer(x)
        output = F.log_softmax(output)
        return output

class SVMModel(object):
    def __init__(self):
        self.model = svm.LinearSVC()

    @utils.logging_time_wrapper
    def train(self, train_x, train_y):
        self.model.fit(train_x, train_y)

    @utils.logging_time_wrapper
    def predict(self, test_x):
        return self.model.predict(test_x)

class LRModel(object):
    def __init__(self):
        self.model = SGDClassifier()

    @utils.logging_time_wrapper
    def train(self, train_x, train_y):
        self.model.fit(train_x, train_y)

    @utils.logging_time_wrapper
    def predict(self, test_x):
        return self.model.predict(test_x)

if __name__ == '__main__':
    train, test, train_loader, test_loader, classes = cifar10_datasets(train_sample_num=5000, test_sample_num=1000, flat_pix=False)
    model = TwoLayerNN(3072, 256, 10)
    criterion = nn.CrossEntropyLoss()
    optimizer = optim.Adam(params=model.parameters(), lr=0.001)
    train_utils.train(train_loader, model, criterion, optimizer, epoch_size=1)
    # report = utils.evaluate_report(test.targets, labels, target_names=classes)
    # print(report)


from indico.core.db import db
from indico.util.string import format_repr

class EventSeries(db.Model):
    """A series of events."""

    __tablename__ = 'series'
    __table_args__ = {'schema': 'events'}

    #: The ID of the series
    id = db.Column(
        db.Integer,
        primary_key=True
    )
    #: Whether to show the sequence number of an event in its title
    #: on category display pages and on the main event page.
    show_sequence_in_title = db.Column(
        db.Boolean,
        nullable=False,
        default=True
    )
    #: Whether to show links to the other events in the same series
    #: on the main event page.
    show_links = db.Column(
        db.Boolean,
        nullable=False,
        default=True
    )
    #: A pattern to auto-generate event titles when cloning events from
    #: the series.
    event_title_pattern = db.Column(
        db.String,
        nullable=False,
        default=''
    )

    # relationship backrefs:
    # - events (Event.series)

    def __repr__(self):
        return format_repr(self, 'id')


with open('input.txt', 'r') as f:
    data = f.read()

def parse_line(line):
    l, w, h = line.split('x')
    return int(l), int(w), int(h)

def smallest_side_area(l, w, h):
    a1 = l * w
    a2 = l * h
    a3 = w * h
    return min(a1, a2, a3)

def total_surface_area(l, w, h):
    return 2 * ((l * w) + (l * h) + (w * h))

def shortest_perimiter(l, w, h):
    p1 = 2 * (l + w)
    p2 = 2 * (l + h)
    p3 = 2 * (w + h)
    return min(p1, p2, p3)

def volume(l, w, h):
    return l * w * h

total_paper_needed = 0
total_ribbon_needed = 0
for line in data.splitlines():
    l, w, h = parse_line(line)
    paper_needed = total_surface_area(l, w, h) + smallest_side_area(l, w, h)
    ribbon_needed = shortest_perimiter(l, w, h) + volume(l, w, h)
    total_paper_needed += paper_needed
    total_ribbon_needed += ribbon_needed

print("Part 1:", total_paper_needed)
print("Part 2:", total_ribbon_needed)


import os
import pathlib
import shutil
from os.path import realpath, dirname

from src.create_similarity_features.lexical_similarity import tokenize_and_filter_out_stop_words
from src.create_similarity_features.referential_similarity import get_named_entities_of_sentence, get_text_synonyms
from src.create_similarity_features.string_similarity import match_sequences, levenshtein_sim
from src.get_data import DATA_PATH
import pandas as pd

from src.utils import get_queries, get_targets

filepath = realpath(__file__)
dir_of_file = dirname(filepath)
parent_dir_of_file = dirname(dir_of_file)
parent_parent_dir_of_file = dirname(parent_dir_of_file)
parents_parent_parent_dir_of_file = dirname(parent_parent_dir_of_file)
repo_path = parents_parent_parent_dir_of_file

def group_dataset(dataset_name):
    gold_path = DATA_PATH + dataset_name + "/gold.tsv"
    query_path = DATA_PATH + dataset_name + "/queries.tsv"
    target_path = DATA_PATH + dataset_name + "/corpus"

    gold_df = pd.read_csv(gold_path, sep='\t', dtype=str, names=["query_id", "0", "target_id", "score"])
    query_df = pd.read_csv(query_path, sep='\t', dtype=str, names=["query_id", "query_text"])
    target_df_orig = pd.read_csv(target_path, sep='\t', dtype=str, names=["target_id", "target_text_1"])#, "target_text_2"])
    #print(target_df_orig)
    target_df = pd.DataFrame(columns=['target_id', 'target_text'])
    target_df["target_id"] = target_df_orig["target_id"]
    target_df["target_text"] = target_df_orig["target_text_1"]#target_df_orig.target_text_1.str.cat(target_df_orig.target_text_2, sep=" ")
    gold_df = gold_df.drop_duplicates()
    gold_text_df = gold_df[["query_id", "target_id"]].merge(query_df, on="query_id", how="left").merge(target_df, on="target_id", how="left")
    queries = gold_text_df["query_text"].tolist()
    targets = list(map(str, gold_text_df["target_text"].tolist()))
    target_ids = gold_text_df["target_id"].tolist()

    #gold_text_df.loc[gold_text_df["target_text"]]

    query_tokens = [set(tokenize_and_filter_out_stop_words(query)) for query in queries]
    target_tokens = [set(tokenize_and_filter_out_stop_words(target)) for target in targets]
    lexical_overlaps = [len(query_tokens[i].intersection(target_tokens[i])) for i in range(len(gold_text_df))]
    gold_text_df["lexical_similarity"] = lexical_overlaps

    gold_text_df = gold_text_df.sort_values("lexical_similarity")
    scores = [-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 100]
    for idx, score in enumerate(scores[:-1]):
        group_df = gold_text_df[(gold_text_df["lexical_similarity"] <= scores[idx + 1]) & (gold_text_df["lexical_similarity"] > score)].copy()
        group_df["0"] = str(0)
        group_df["score"] = str(1)
        group_gold_df = group_df[["query_id", "0", "target_id", "score"]]
        group_query_df = group_df[["query_id", "query_text"]]
        group_query_df = group_query_df.drop_duplicates()
        lexical_output_path = DATA_PATH + "lexical/" + dataset_name + "/" + str(score)
        pathlib.Path(lexical_output_path).mkdir(parents=True, exist_ok=True)
        lexical_gold_path = lexical_output_path + "/gold.tsv"
        lexical_query_path = lexical_output_path + "/queries.tsv"
        group_gold_df.to_csv(lexical_gold_path, sep='\t', header=False, index=False)
        group_query_df.to_csv(lexical_query_path, sep='\t', header=False, index=False)

def analyse_subsets(dataset_name):
    similarity_features = ["ne"]#["levenshtein", "sequence_matching", "synonym", "lexical"]
    features_dict = {}
    for feature in similarity_features:
        output_path = DATA_PATH + feature + "/" + dataset_name
        n_queries = []
        for file in os.listdir(output_path):
            gold_path = output_path + "/" + file + "/gold.tsv"
            gold_df = pd.read_csv(gold_path, sep='\t', dtype=str, names=["query_id", "0", "target_id", "score"])
            n_queries.append(len(gold_df))
        features_dict[feature] = n_queries
    return features_dict

def analyse_subsets_lexical(dataset_name):
    features_dict = {}
    output_path = DATA_PATH + "lexical/" + dataset_name
    for file in os.listdir(output_path):
        gold_path = output_path + "/" + file + "/gold.tsv"
        gold_df = pd.read_csv(gold_path, sep='\t', dtype=str, names=["query_id", "0", "target_id", "score"])
        features_dict[file] = len(gold_df)
    return features_dict

def print_subset_examples_lexical():

    dataset_names = ["clef_2020_checkthat_2_english",
                     "clef_2021_checkthat_2a_english",
                     "clef_2022_checkthat_2a_english",
                     "clef_2021_checkthat_2b_english",
                     "clef_2022_checkthat_2b_english"]

    input_claims_highest_lexical = {}
    verified_claims_highest_lexical = {}

    input_claims_lowest_lexical = {}
    verified_claims_lowest_lexical = {}

    for dataset_name in dataset_names:

        query_path = DATA_PATH + dataset_name + "/queries.tsv"
        target_path = DATA_PATH + dataset_name + "/corpus"

        lexical_output_path = DATA_PATH + "lexical/" + dataset_name
        highest_lexical_output_path = lexical_output_path + "/8"
        lowest_lexical_output_path = lexical_output_path + "/-1"

        highest_lexical_gold_df = pd.read_csv(highest_lexical_output_path + "/gold.tsv", sep='\t', dtype=str, names=["query_id", "0", "target_id", "score"])
        highest_lexical_gold_sample = highest_lexical_gold_df.sample(1, random_state=2).values[0]
        highest_lex_query_id = highest_lexical_gold_sample[0]
        highest_lex_target_id = highest_lexical_gold_sample[2]
        highest_lex_query_text = get_queries(query_path)[highest_lex_query_id]
        highest_lex_target_text = get_targets(target_path)[highest_lex_target_id]
        input_claims_highest_lexical[dataset_name] = highest_lex_query_text
        verified_claims_highest_lexical[dataset_name] = highest_lex_target_text

        lowest_lexical_gold_df = pd.read_csv(lowest_lexical_output_path + "/gold.tsv", sep='\t', dtype=str, names=["query_id", "0", "target_id", "score"])
        lowest_lexical_gold_sample = lowest_lexical_gold_df.sample(1, random_state=2).values[0]
        lowest_lex_query_id = lowest_lexical_gold_sample[0]
        lowest_lex_target_id = lowest_lexical_gold_sample[2]
        lowest_lex_query_text = get_queries(query_path)[lowest_lex_query_id]
        lowest_lex_target_text = get_targets(target_path)[lowest_lex_target_id]
        input_claims_lowest_lexical[dataset_name] = lowest_lex_query_text
        verified_claims_lowest_lexical[dataset_name] = lowest_lex_target_text

    print("\\begin{figure}[hbtp!]")
    print("\\begin{quote}")

    print("\\textbf{Input Claim ct 2020 2a:} \\textit{"+input_claims_highest_lexical["clef_2020_checkthat_2_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2020 2a:} \\textit{"+verified_claims_highest_lexical["clef_2020_checkthat_2_english"]+"}\\\\")
    print("\hline")

    print("\\textbf{Input Claim ct 2021 2a:} \\textit{"+input_claims_highest_lexical["clef_2021_checkthat_2a_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2021 2a:} \\textit{"+verified_claims_highest_lexical["clef_2021_checkthat_2a_english"]+"}\\\\")
    print("\hline")

    print("\\textbf{Input Claim ct 2022 2a:} \\textit{"+input_claims_highest_lexical["clef_2022_checkthat_2a_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2022 2a:} \\textit{"+verified_claims_highest_lexical["clef_2022_checkthat_2a_english"]+"}\\\\")
    print("\hline")

    print("\\textbf{Input Claim ct 2021 2b:} \\textit{"+input_claims_highest_lexical["clef_2021_checkthat_2b_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2021 2b:} \\textit{"+verified_claims_highest_lexical["clef_2021_checkthat_2b_english"]+"}\\\\")
    print("\hline")

    print("\\textbf{Input Claim ct 2022 2b:} \\textit{"+input_claims_highest_lexical["clef_2022_checkthat_2b_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2022 2b:} \\textit{"+verified_claims_highest_lexical["clef_2022_checkthat_2b_english"]+"}\\\\")
    print("\hline")

    print("\end{quote}")
    print("\caption{Example of true pairs with high lexical overlap.}")
    print("\end{figure}")

    print()

    print("\\begin{figure}[hbtp!]")
    print("\\begin{quote}")

    print("\\textbf{Input Claim ct 2020 2a:} \\textit{"+input_claims_lowest_lexical["clef_2020_checkthat_2_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2020 2a:} \\textit{"+verified_claims_lowest_lexical["clef_2020_checkthat_2_english"]+"}\\\\")
    print("\hline")

    print("\\textbf{Input Claim ct 2021 2a:} \\textit{"+input_claims_lowest_lexical["clef_2021_checkthat_2a_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2021 2a:} \\textit{"+verified_claims_lowest_lexical["clef_2021_checkthat_2a_english"]+"}\\\\")
    print("\hline")

    print("\\textbf{Input Claim ct 2022 2a:} \\textit{"+input_claims_lowest_lexical["clef_2022_checkthat_2a_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2022 2a:} \\textit{"+verified_claims_lowest_lexical["clef_2022_checkthat_2a_english"]+"}\\\\")
    print("\hline")

    print("\\textbf{Input Claim ct 2021 2b:} \\textit{"+input_claims_lowest_lexical["clef_2021_checkthat_2b_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2021 2b:} \\textit{"+verified_claims_lowest_lexical["clef_2021_checkthat_2b_english"]+"}\\\\")
    print("\hline")

    print("\\textbf{Input Claim ct 2022 2b:} \\textit{"+input_claims_lowest_lexical["clef_2022_checkthat_2b_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2022 2b:} \\textit{"+verified_claims_lowest_lexical["clef_2022_checkthat_2b_english"]+"}\\\\")
    print("\hline")

    print("\end{quote}")
    print("\caption{Example of true pairs with no lexical overlap.}")
    print("\end{figure}")

def print_lowest_lexical_subset():

    dataset_names = ["clef_2020_checkthat_2_english",
                     "clef_2021_checkthat_2a_english",
                     "clef_2022_checkthat_2a_english",
                     "clef_2021_checkthat_2b_english",
                     "clef_2022_checkthat_2b_english"]

    input_claims_lowest_lexical = {}
    verified_claims_lowest_lexical = {}

    for dataset_name in dataset_names:
        query_path = DATA_PATH + dataset_name + "/queries.tsv"
        target_path = DATA_PATH + dataset_name + "/corpus"

        lexical_output_path = DATA_PATH + "lexical/" + dataset_name
        lowest_lexical_output_path = lexical_output_path + "/-1"

        lowest_lexical_gold_df = pd.read_csv(lowest_lexical_output_path + "/gold.tsv", sep='\t', dtype=str,
                                             names=["query_id", "0", "target_id", "score"])
        lowest_lex_query_ids = lowest_lexical_gold_df["query_id"].values
        lowest_lex_target_ids = lowest_lexical_gold_df["target_id"].values
        queries = get_queries(query_path)
        targets = get_targets(target_path)
        lowest_lex_query_texts = [queries[query_id] for query_id in lowest_lex_query_ids]
        lowest_lex_target_texts = [targets[target_id] for target_id in lowest_lex_target_ids]
        input_claims_lowest_lexical[dataset_name] = lowest_lex_query_texts
        verified_claims_lowest_lexical[dataset_name] = lowest_lex_target_texts
        for idx, query in enumerate(lowest_lex_query_texts):
            print(query)
            print(lowest_lex_target_texts[idx])
            print("---------------")

#print_lowest_lexical_subset()

#print_subset_examples_lexical()

# dataset_names = ["clef_2020_checkthat_2_english",
#                  "clef_2021_checkthat_2a_english",
#                  "clef_2022_checkthat_2a_english",
#                  "clef_2021_checkthat_2b_english",
#                  "clef_2022_checkthat_2b_english"]

dataset_names = ["climate-fever", "fever", "scifact"]

# shutil.rmtree(DATA_PATH + "levenshtein")
# shutil.rmtree(DATA_PATH + "sequence_matching")
# shutil.rmtree(DATA_PATH + "synonym")
# shutil.rmtree(DATA_PATH + "lexical")
# shutil.rmtree(DATA_PATH + "ne")
for dataset_name in dataset_names:
    group_dataset(dataset_name)
#
# for dataset_name in dataset_names:
#     print(analyse_subsets(dataset_name))


# Напишите программу, которая по заданному номеру четверти, показывает диапазон возможных
# координат точек в этой четверти (x и y).

def InputNumbers(inputText):
    check_input = False
    while not check_input:
        try:
            number = int(input(f'{inputText}'))
            check_input = True
        except ValueError:
            print('Введите корректные данные')
    return number

def CheckCoordinatesPoints(number):
    if number < 1 or number > 4:
        print('Введите число, соответствующее четверти системы координат')
    elif number == 1:
        print('x > 0, y > 0')
    elif number == 2:
        print('x < 0, y > 0')
    elif number == 3:
        print('x < 0, y < 0')
    elif number == 4:
        print('x > 0, y < 0')

number = InputNumbers('Введите номер четверти системы координат: ')
CheckCoordinatesPoints(number)


def gcd(n1, n2) :

    small = int(min(n1, n2))

    for i in range(small, 0, -1) :

        if n1 % i == 0 and n2 % i == 0:
            return i

def lcm(n1, n2) :

    result = n1 * n2 / gcd(n1, n2)
    return result

ls = [int(x) for x in input("Enter list\n").split()]

found = ls[0]

for i in range(1, len(ls)) :

    found = lcm(found, ls[i])

print(found)


import kfp
from kfp import dsl

CONTAINER_REGISTRY="<Todo_Insert_container_registry_here>"
CONTAINER_TAG="latest"
MINIO_SERVER='minio.cloudtraining-mavencode.com:9000'
MINIO_ACCESS_KEY='some_access_key'
MINIO_SECRET_KEY='some_secret_key'

def datadownload_op(minio_server, data_dir):
    return dsl.ContainerOp(
        name='Data Download',
        image=f'{CONTAINER_REGISTRY}/datadownloader:{CONTAINER_TAG}',
        arguments=[
            '--minio_server', minio_server,
            '--data_dir', data_dir
        ],
        command=["python3", "datadownloader.py"]
    )

def data_preprocess_op(data_dir, clean_data_dir):
    return dsl.ContainerOp(
        name='Data Preprocessing',
        image=f'{CONTAINER_REGISTRY}/preprocess_dataset:{CONTAINER_TAG}',
        arguments=[
            '--data_dir', data_dir,
            '--clean_data_dir', clean_data_dir
        ],
        command=["python3", "preprocess_dataset.py"]
    )

def pca_analysis_op(clean_data_dir, pca_dir):
    return dsl.ContainerOp(
        name='PCA Analysis',
        image=f'{CONTAINER_REGISTRY}/pca_analysis:{CONTAINER_TAG}',
        arguments=[
            '--clean_data_dir', clean_data_dir,
            '--pca_dir', pca_dir
        ],
        command=["python3", "pca_analysis.py"]
    )

def train_op(clean_data_dir, model_dir):
    return dsl.ContainerOp(
        name='Model Training',
        image=f'{CONTAINER_REGISTRY}/train_model:{CONTAINER_TAG}',
        arguments=[
            '--clean_data_dir', clean_data_dir,
            '--model_dir', model_dir
        ],
        command=["python3", "train.py"]
    )

def model_test_op(clean_data_dir, model_dir, metrics_path):
    return dsl.ContainerOp(
        name='Model Testing',
        image=f'{CONTAINER_REGISTRY}/test_model:{CONTAINER_TAG}',
        arguments=[
            '--clean_data_dir', clean_data_dir,
            '--model_dir', model_dir,
            '--metrics_path', metrics_path
        ],
        command=["python3", "test_model.py"]
    )

def model_export_op(model_dir, metrics_path, export_bucket, model_name, model_version, minio_server,
                    minio_access_key, minio_secret_key):
    return dsl.ContainerOp(
        name='Model Export',
        image=f'{CONTAINER_REGISTRY}/export_models:{CONTAINER_TAG}',
        arguments=[
            '--model_dir', model_dir,
            '--metrics_path', metrics_path,
            '--export_bucket', export_bucket,
            '--model_name', model_name,
            '--model_version', model_version,
            '--minio_server', minio_server,
            '--minio_access_key', minio_access_key,
            '--minio_secret_key', minio_secret_key
        ],
        command=["python3", "export_models.py"]
    )

def print_results_op(pca_dir, metrics_path):
    return dsl.ContainerOp(
        name='Result Printing',
        image=f'{CONTAINER_REGISTRY}/print_results:{CONTAINER_TAG}',
        arguments=[
            '--pca_dir', pca_dir,
            '--metrics_path', metrics_path
        ],
        command=["python3", "print_results.py"]
    )

def op_transformer(op):
    op.add_pod_annotation(name="sidecar.istio.io/inject", value="false")
    return op

@dsl.pipeline(
   name='End-to-End Industrial IoT Pipeline',
   description='A sample pipeline to demonstrate multi-step model training, evaluation and export'
)
def monitoring_pipeline(
    model_dir: str = "/train/model",
    data_dir: str = "/train/data",
    clean_data_dir: str = "/train/data",
    metrics_path: str ="/train/metrics",
    pca_dir: str ="/train/metrics",
    export_bucket: str = "monitoring",
    model_name: str = "monitoring",
    model_version: int = 1,
    minio_server: str=MINIO_SERVER,
    minio_access_key: str=MINIO_ACCESS_KEY,
    minio_secret_key: str=MINIO_SECRET_KEY
):

    pvc_op = dsl.VolumeOp(name='Persistent Storage',
            resource_name='data-volume',
            size='2Gi',
            modes=dsl.VOLUME_MODE_RWO)

    datadownload_task = datadownload_op(MINIO_SERVER, data_dir).add_pvolumes({"/train": pvc_op.volume})

    data_preprocess_task = data_preprocess_op(data_dir=data_dir, clean_data_dir=clean_data_dir).add_pvolumes({"/train": pvc_op.volume}).after(datadownload_task)

    pca_analysis_task = pca_analysis_op(clean_data_dir=clean_data_dir, pca_dir=pca_dir).add_pvolumes({"/train": pvc_op.volume}).after(data_preprocess_task)

    train_task = train_op(clean_data_dir=clean_data_dir, model_dir=model_dir).add_pvolumes({"/train": pvc_op.volume}).after(data_preprocess_task)

    test_task = model_test_op(clean_data_dir=clean_data_dir, model_dir=model_dir, metrics_path=metrics_path).add_pvolumes({"/train": pvc_op.volume}).after(train_task)

    model_export_task = model_export_op(
                            model_dir=model_dir,
                            metrics_path=metrics_path,
                            export_bucket=export_bucket,
                            model_name=model_name,
                            model_version=model_version,
                            minio_server=minio_server,
                            minio_access_key=minio_access_key,
                            minio_secret_key=minio_secret_key).add_pvolumes({"/train": pvc_op.volume}).after(test_task)

    results_task = print_results_op(pca_dir=pca_dir, metrics_path=metrics_path).add_pvolumes({"/train": pvc_op.volume}).after(test_task, pca_analysis_task)

    dsl.get_pipeline_conf().add_op_transformer(op_transformer)

if __name__ == '__main__':
    kfp.compiler.Compiler().compile(monitoring_pipeline, 'monitoring_pipeline.yaml')
    # client = kfp.Client(host='pipelines-api.kubeflow.svc.cluster.local:8888')


class Solution:
    def isInterleave(self, s1: str, s2: str, s3: str) -> bool:
        x, y, z = len(s1), len(s2), len(s3)
        if x + y != z:
            return False

        dp = [[False for _ in range(y + 1)] for _ in range(x + 1)]
        dp[0][0] = True
        for i in range(x + 1):
            for j in range(y + 1):
                if i - 1 >= 0 and s1[i - 1] == s3[i + j - 1]:
                    dp[i][j] |= dp[i - 1][j]
                if j - 1 >= 0 and s2[j - 1] == s3[i + j - 1]:
                    dp[i][j] |= dp[i][j - 1]
        return dp[x][y]


import pandas as pd
import numpy as np
from analysis import blocks
from numpy import nan
from sklearn.multioutput import MultiOutputRegressor
from sklearn.metrics import mean_squared_error
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import SGDRegressor
from sklearn import svm
from sklearn import preprocessing
import pickle
from math import floor

MODELS_PATH = '/Users/apple/Desktop/mag/dane/DANE_PO_MODERNIZACJI_VRM/modele/regresja/'

COL_NAMES = ['in', 'control', 'out']

def load_data(block_name):
    df = pd.read_csv(blocks.LOAD_PATH + block_name + '.csv')
    print('data loaded')
    return df

def load_block_vars():
    df = pd.read_excel(blocks.BLOCK_VARS_PATH_XLSX, sheetname=None)
    print('blocks vars loaded')
    return df

def linear_regresion(x, y):
    r = LinearRegression()
    r.fit(np.transpose(np.matrix(x)), np.transpose(np.matrix(y)))
    return r

def logistic_regresion(x, y):
    r = LogisticRegression()
    lab_enc = preprocessing.LabelEncoder()
    y_enc = lab_enc.fit_transform(y)
    # matrix = np.asmatrix(y_enc)
    # transpose = np.transpose(matrix)
    r.fit(np.transpose(np.matrix(x)), y_enc)
    return r

def single_output_regression(x, y):
    print('Single output regression')
    models = []
    for output in y:
        print('new model')
        r = linear_regresion(x, output)
        models.append(r)
    return models

def single_output_logistic_regression(x, y):
    print('Single output log regression')
    models = []
    for output in y:
        print('new model')
        r = logistic_regresion(x, output)
        models.append(r)
    return models

def multi_output_regression(x, y, regressor = svm.SVR()):
    print('multi output regression')
    r = MultiOutputRegressor(regressor)

    r.fit(np.transpose(np.matrix(x)), np.transpose(np.matrix(y)))
    return r

def evaluate(model, x, y):
    y_pred = model.predict(np.transpose(np.matrix(x)))
    r2 = model.score(np.transpose(np.matrix(x)), np.transpose(np.matrix(y)))
    mse = mean_squared_error(y, y_pred)
    return r2, mse

def save_model(model, file_name):
    pickle.dump(model, open(file_name, 'wb'))

def main():
    blocks_vars = load_block_vars()
    for block_name in blocks.names:
        df = load_data(block_name)
        vars = blocks_vars[block_name][COL_NAMES]
        vars_in = [x for x in vars['in'] if not x is nan and not '#' in x]
        vars_out = [x for x in vars['out'] if not x is nan and not '#' in x]
        input_data = [df[x].tolist() for x in vars_in]
        output_data = [df[x].tolist() for x in vars_out]
        last_train_idx = floor(len(input_data[0]) * 0.875)
        x_train = [x[:last_train_idx] for x in input_data]
        y_train = [x[:last_train_idx] for x in output_data]
        x_test = [x[last_train_idx:len(input_data[0])-1] for x in input_data]
        y_test = [x[last_train_idx:len(output_data[0])-1] for x in output_data]

        r_list = single_output_regression(x_train, y_train)
        for i, model in enumerate(r_list):
            r2, mse = evaluate(model, x_test, y_test[i])
            print('r^2: {0}\n mse: {1}'.format(r2, mse))
            save_model(model, MODELS_PATH  + block_name + '/' + 'single_linear_{0}'.format(i) + '.sav')

        r_list = single_output_logistic_regression(x_train, y_train)
        for i, model in enumerate(r_list):
            r2, mse = evaluate(model, x_test, y_test[i])
            print('r^2: {0}\n mse: {1}'.format(r2, mse))
            save_model(model, MODELS_PATH + block_name + '/' + 'single_logistic_{0}'.format(i) + '.sav')

        r = multi_output_regression(x_train, y_train)
        r2, mse = evaluate(r, x_test, y_test)
        print('r^2: {0}\n mse: {1}'.format(r2, mse))
        save_model(model, MODELS_PATH + block_name + '/' + 'multi_SVR' + '.sav')

        r = multi_output_regression(x_train, y_train, SGDRegressor())
        r2, mse = evaluate(r, x_test, y_test)
        print('r^2: {0}\n mse: {1}'.format(r2, mse))
        save_model(model, MODELS_PATH + block_name + '/' + 'multi_SGD' + '.sav')

if __name__ == '__main__':
    main()


import sys
from cx_Freeze import setup, Executable

# Dependencies are automatically detected, but it might need to be fine-tuned.
build_exe_options = {
    "excludes": ['PyQt5'],
    'packages': ['imagej', 'PySimpleGUI', 'caiman', 'ipyparallel', 'skimage', 'seaborn'],
    'include_files': ['config.ini', 'cache'],
    'silent_level': ['1']
}

msi_data = {
    "Icon": [
        ("IconId", "icon.ico"),
    ],
}

bdist_msi_options = {}

# base="Win32GUI" should be used only for Windows GUI app
base = "Win32GUI" if sys.platform == "win32" else None

executables = [
    Executable(
        'main_GUI.py',
        base=base
    )
]

setup(
    name="pipeline",
    version="0.1",
    description="Test on build for incomplete GUI application",
    options={
        "build_exe": build_exe_options,
        "build_msi": bdist_msi_options,
    },
    executables=executables,
)


from rest_framework import serializers
from rest_framework.exceptions import ValidationError

from swp.models import Monitor, Publication, Thinktank
from .validators import css_selector

class ThinktankField(serializers.ModelSerializer):

    class Meta:
        model = Thinktank
        fields = ['id', 'name']
        read_only_fields = ['id']

class MonitorField(serializers.ModelSerializer):
    class Meta:
        model = Monitor
        fields = ['id', 'name']
        read_only_fields = ['id']

class PublicationField(serializers.ModelSerializer):
    class Meta:
        model = Publication
        fields = ['id', 'url', 'title']
        read_only_fields = ['id']

class CSSSelectorField(serializers.CharField):
    default_validators = [css_selector]

class RecipientsField(serializers.ListField):
    child = serializers.EmailField()

    def run_child_validation(self, data):
        result = []
        errors = []

        for idx, item in enumerate(data):
            try:
                result.append(self.child.run_validation(item))
            except ValidationError as e:
                errors.extend(e.detail)

        if not errors:
            return result

        raise ValidationError(errors)


# Author: Jochen Gast <jochen.gast@visinf.tu-darmstadt.de>

import logging
import os
import shutil

import torch
from torch import nn

from utils import json
from utils import strings
from utils import system

def load_state_dict_into_module(state_dict, module, strict=True):
    own_state = module.state_dict()
    for name, param in state_dict.items():
        if name in own_state:
            if isinstance(param, nn.Parameter):
                # backwards compatibility for serialized parameters
                param = param.data
            try:
                own_state[name].resize_as_(param)
                own_state[name].copy_(param)
            except Exception:
                raise RuntimeError('While copying the parameter named {}, '
                                   'whose dimensions in the model are {} and '
                                   'whose dimensions in the checkpoint are {}.'
                                   .format(name, own_state[name].size(), param.size()))
        elif strict:
            logging.info('Unexpected key "{}" in state_dict'.format(name))
            raise KeyError('')
    if strict:
        missing = set(own_state.keys()) - set(state_dict.keys())
        if len(missing) > 0:
            logging.info('Missing keys in state_dict: ')
            logging.value("{}".format(missing))
            raise KeyError('')

def restore_module_from_state_dict(module,
                                   checkpoint_state_dict,
                                   include_params='*',
                                   exclude_params=(),
                                   translations=(),
                                   fuzzy_translation_keys=()):
    include_params = list(include_params)
    exclude_params = list(exclude_params)
    fuzzy_translation_keys = list(fuzzy_translation_keys)
    translations = dict(translations)

    # This list is only for book keeping
    actual_translations = []

    # Fuzzy matching is just a lazy way to specify translations:
    # A given match will get searcher for and added to the manual translations,
    # if not already existing
    for checkpoint_key in checkpoint_state_dict.keys():
        for module_key in module.state_dict().keys():
            for fuzzy_key in fuzzy_translation_keys:
                if fuzzy_key in checkpoint_key and fuzzy_key in module_key:
                    new_translation_key = checkpoint_key.replace(fuzzy_key, '')
                    new_translation_value = module_key.replace(fuzzy_key, '')
                    if new_translation_key not in translations.keys():
                        translations[new_translation_key] = new_translation_value

    def _translate(key_in):
        key_out = key_in
        # first check fuzz matching
        for k, v in translations.items():
            key_out = key_out.replace(k, v)
        if key_out != key_in:
            actual_translations.append((key_in, key_out))
        return key_out

    # -----------------------------------------------------------------------------------------
    # translate keys in checkpoint_state_dict according to translations
    # -----------------------------------------------------------------------------------------
    checkpoint_state_dict = {_translate(key): value for key, value in checkpoint_state_dict.items()}

    # -----------------------------------------------------------------------------------------
    # Filter afterwards
    # -----------------------------------------------------------------------------------------
    restore_keys = strings.filter_list_of_strings(
        checkpoint_state_dict.keys(),
        include=include_params,
        exclude=exclude_params)
    checkpoint_state_dict = {key: value for key, value in checkpoint_state_dict.items() if key in restore_keys}

    # if parameter lists are given, don't be strict with loading from checkpoints
    default_parameters_given = \
        len(include_params) == 1 and include_params[0] == '*' and len(exclude_params) == 0

    strict = True
    if not default_parameters_given:
        strict = False

    load_state_dict_into_module(checkpoint_state_dict, module, strict=strict)

    return restore_keys, actual_translations

def restore_module_from_filename(module,
                                 filename,
                                 key='state_dict',
                                 include_params='*',
                                 exclude_params=(),
                                 translations=(),
                                 fuzzy_translation_keys=()):
    include_params = list(include_params)
    exclude_params = list(exclude_params)
    fuzzy_translation_keys = list(fuzzy_translation_keys)
    translations = dict(translations)

    # ------------------------------------------------------------------------------
    # Make sure file exists
    # ------------------------------------------------------------------------------
    if not os.path.isfile(filename):
        logging.info("Could not find checkpoint file '%s'!" % filename)
        quit()

    # ------------------------------------------------------------------------------
    # Load checkpoint from file including the state_dict
    # ------------------------------------------------------------------------------
    cpu_device = torch.device('cpu')
    checkpoint_dict = torch.load(filename, map_location=cpu_device)
    checkpoint_state_dict = checkpoint_dict[key]

    try:
        restore_keys, actual_translations = restore_module_from_state_dict(
            module,
            checkpoint_state_dict,
            include_params=include_params,
            exclude_params=exclude_params,
            translations=translations,
            fuzzy_translation_keys=fuzzy_translation_keys)
    except KeyError:
        with logging.block('Checkpoint keys:'):
            logging.value(checkpoint_state_dict.keys())
        with logging.block('Module keys:'):
            logging.value(module.state_dict().keys())
        logging.info("Could not load checkpoint because of key errors. Checkpoint translations gone wrong?")
        quit()

    return checkpoint_dict, restore_keys, actual_translations

# --------------------------------------------------------------------------
# Checkpoint loader/saver.
# --------------------------------------------------------------------------
class CheckpointSaver:
    def __init__(self,
                 prefix="checkpoint",
                 latest_postfix="_latest_",
                 best_postfix="_best_",
                 model_key="state_dict",
                 extension=".ckpt"):

        self.prefix = prefix
        self.model_key = model_key
        self.latest_postfix = latest_postfix
        self.best_postfix = best_postfix
        self.extension = extension

    def restore(self,
                filename,
                model_and_loss,
                include_params='*',
                exclude_params=(),
                translations={},
                fuzzy_translation_keys=()):

        # -----------------------------------------------------------------------------------------
        # Make sure file exists
        # -----------------------------------------------------------------------------------------
        if not os.path.isfile(filename):
            logging.info("Could not find checkpoint file '%s'!" % filename)
            quit()

        checkpoint_dict, restore_keys, actual_translations = \
            restore_module_from_filename(model_and_loss,
                                         filename,
                                         key=self.model_key,
                                         include_params=include_params,
                                         exclude_params=exclude_params,
                                         translations=translations,
                                         fuzzy_translation_keys=fuzzy_translation_keys)

        if len(actual_translations) > 0:
            logging.info("  Translations:")
            for pair in actual_translations:
                logging.info("    %s  =>  %s" % pair)

        logging.info("  Restore keys:")
        for key in restore_keys:
            logging.info("    %s" % key)

        # -----------------------------------------------------------------------------------------
        # Get checkpoint statistics without the state dict
        # -----------------------------------------------------------------------------------------
        checkpoint_stats = {
            key: value for key, value in checkpoint_dict.items() if key != self.model_key
        }

        return checkpoint_stats, filename

        # # -----------------------------------------------------------------------------------------
        # # Load checkpoint from file including the state_dict
        # # -----------------------------------------------------------------------------------------
        # cpu_device = torch.device('cpu')
        # checkpoint_with_state = torch.load(filename, map_location=cpu_device)

        # # -----------------------------------------------------------------------------------------
        # # Load filtered state dictionary
        # # -----------------------------------------------------------------------------------------
        # state_dict = checkpoint_with_state[self.model_key]
        # restore_keys = strings.filter_list_of_strings(
        #     state_dict.keys(),
        #     include=include_params,
        #     exclude=exclude_params)

        # state_dict = {key: value for key, value in state_dict.items() if key in restore_keys}

        # # if parameter lists are given, don't be strict with loading from checkpoints
        # strict = True
        # if include_params != "*" or len(exclude_params) != 0:
        #     strict = False

        # checkpoints.load_state_dict_into_module(state_dict, model_and_loss, strict=strict)
        # logging.info("  Restore keys:")
        # for key in restore_keys:
        #     logging.info("    %s" % key)

        # # -----------------------------------------------------------------------------------------
        # Get checkpoint statistics without the state dict
        # -----------------------------------------------------------------------------------------
        # checkpoint_stats = {
        #     key: value for key, value in checkpoint_with_state.items() if key != self._model_key
        # }
        # return checkpoint_stats, filename

    def restore_latest(self,
                       directory,
                       model_and_loss,
                       include_params='*',
                       exclude_params=(),
                       translations={},
                       fuzzy_translation_keys=()):

        latest_checkpoint_filename = os.path.join(
            directory, self.prefix + self.latest_postfix + self.extension)

        return self.restore(
            latest_checkpoint_filename,
            model_and_loss=model_and_loss,
            include_params=include_params,
            exclude_params=exclude_params,
            translations=translations,
            fuzzy_translation_keys=fuzzy_translation_keys)

    def restore_best(self,
                     directory,
                     model_and_loss,
                     include_params='*',
                     exclude_params=(),
                     translations={},
                     fuzzy_translation_keys=()):

        best_checkpoint_filename = os.path.join(
            directory, self.prefix + self.best_postfix + self.extension)

        return self.restore(
            best_checkpoint_filename,
            model_and_loss=model_and_loss,
            include_params=include_params,
            exclude_params=exclude_params,
            translations=translations,
            fuzzy_translation_keys=fuzzy_translation_keys)

    def save_latest(self, directory, model_and_loss, stats_dict,
                    store_as_best=False, store_prefixes='total_loss'):

        # -----------------------------------------------------------------------------------------
        # Mutable default args..
        # -----------------------------------------------------------------------------------------
        store_as_best = list(store_as_best)

        # -----------------------------------------------------------------------------------------
        # Make sure directory exists
        # -----------------------------------------------------------------------------------------
        system.ensure_dir(directory)

        # -----------------------------------------------------------------------------------------
        # Save
        # -----------------------------------------------------------------------------------------
        save_dict = dict(stats_dict)
        save_dict[self.model_key] = model_and_loss.state_dict()
        latest_checkpoint_filename = os.path.join(
            directory, self.prefix + self.latest_postfix + self.extension)
        latest_statistics_filename = os.path.join(
            directory, self.prefix + self.latest_postfix + '.json')

        # convert to cpu tensors before writing the checkpoint files
        cpu_device = torch.device('cpu')

        def map_dict_to_cpu(x):
            if torch.is_tensor(x):
                return x.to(cpu_device)
            elif isinstance(x, dict):
                return dict((key, map_dict_to_cpu(value)) for key, value in x.items())
            else:
                return x

        save_dict = map_dict_to_cpu(save_dict)
        torch.save(save_dict, latest_checkpoint_filename)
        json.write_dictionary_to_file(stats_dict, filename=latest_statistics_filename)

        # -----------------------------------------------------------------------------------------
        # Possibly store as best
        # -----------------------------------------------------------------------------------------
        for store, prefix in zip(store_as_best, store_prefixes):
            if store:
                best_checkpoint_filename = os.path.join(
                    directory, self.prefix + self.best_postfix + prefix + self.extension)

                best_statistics_filename = os.path.join(
                    directory, self.prefix + self.best_postfix + prefix + ".json")

                shortfile = best_checkpoint_filename.rsplit("/", 1)[1]
                shortpath = os.path.dirname(best_checkpoint_filename).rsplit("/", 1)[1]
                shortname = os.path.join(shortpath, shortfile)
                logging.info("Save ckpt to ../%s" % shortname)
                shutil.copyfile(latest_checkpoint_filename, best_checkpoint_filename)
                shutil.copyfile(latest_statistics_filename, best_statistics_filename)