"""
 This module contains the Distribution class which defines a standard
 interface for distributions It also provides several implemented
 distributions, which inherit from Distribution Any user-specified
 distributions should inherit from Distribution
"""
import numpy as np
from .utils import overrides, package_path
import os
from scipy import stats
import pickle

class Distribution(object):
    """
    Interface/abstract class for distributions.
    Any user-specified distributions should be defined by inheriting from this class and
     overriding the appropriate methods.
    """

    def __init__(self, ndims=2, nbatch=100):
        """ Creates a Distribution object

        :param ndims: the dimension of the state space for this distribution
        :param nbatch: the number of sampling particles to run simultaneously
        :returns: a Distribution object
        :rtype: Distribution

        """

        # distribution dimensions
        self.ndims = ndims
        # number of sampling particles to use
        self.nbatch = nbatch

        # TensorflowDistributions require some special treatment
        # this attribute is to be used instead of isinstance, as that would require
        # tensorflow to be imported globally
        if not hasattr(self, 'backend'):
            self.backend = 'numpy'

        # true iff being sampled with a jump process
        self.mjhmc = None

        # number of times energy op has been called
        self.E_count = 0

        # number of times gradient op has been called
        self.dEdX_count = 0

        # only set to true when I have a bias initialization and am being burned in
        # to generate and cache a fair initialization for continuous samplers
        self.generation_instance = False

        # so some distributions may modify the default
        if not hasattr(self, 'max_n_particles'):
            self.max_n_particles = None

        # set the state fairly. calls out to a cache
        self.init_X()

    def E(self, X):
        self.E_count += X.shape[1]
        return self.E_val(X)

    def E_val(self, X):
        """
        Subclasses should implement this with the correct energy function
        """
        raise NotImplementedError()

    def dEdX(self, X):
        self.dEdX_count += X.shape[1]
        return self.dEdX_val(X)

    def dEdX_val(self, X):
        """
        Subclasses should implement this with the correct energy gradient function
        """
        raise NotImplementedError()

    def __hash__(self):
        """ Subclasses should implement this as the hash of the tuple of all parameters
        that effect the distribution, including ndims. This is very important!!
        nbatch should not be part of the hash!! Including it will break everything

        As an example, see how this is implemented in Gaussian

        :returns: a hash of the relevant parameters of self
        :rtype: int
        """
        raise NotImplementedError()

    def init_X(self):
        """
        Sets self.Xinit to a good initial value
        """
        # TODO: make production ready by adding global flag to disable
        #  research options like this
        self.cached_init_X()

    def cached_init_X(self):
        """ Sets self.Xinit to cached (serialized) initial states for continuous-time samplers, generated by burn in
        *For use with continuous-time samplers only*

        :returns: None
        :rtype: none
        """
        distr_name = type(self).__name__
        distr_hash = hash(self)
        file_name = '{}_{}.pickle'.format(distr_name, distr_hash)
        file_prefix = '{}/initializations'.format(package_path())
        if file_name in os.listdir(file_prefix):
            with open('{}/{}'.format(file_prefix, file_name), 'rb') as cache_file:
                mjhmc_endpt, _, _, control_endpt  = pickle.load(cache_file)
                if self.mjhmc:
                    self.Xinit = mjhmc_endpt[:, :self.nbatch]
                else:
                    self.Xinit = control_endpt[:, :self.nbatch]
        else:
            from mjhmc.misc.gen_mj_init import MAX_N_PARTICLES, cache_initialization
            # modify this object so it can be used by gen_mj_init
            old_nbatch = self.nbatch
            self.nbatch = self.max_n_particles or MAX_N_PARTICLES
            self.generation_instance = True

            # must rebuild now that nbatch is changed back
            if self.backend == 'tensorflow':
                self.build_graph()

            # start with biased initializations
            # changes self.nbatch
            try:
                self.gen_init_X()
            except NotImplementedError:
                # completely arbitrary choice
                self.Xinit = np.random.randn(self.ndims, self.nbatch)

            #generate and cache fair initialization
            cache_initialization(self)
            # reconstruct this object using fair initialization
            self.nbatch = old_nbatch
            self.generation_instance = False
            # must rebuild now that nbatch is changed back
            if self.backend == 'tensorflow':
                 self.build_graph()
            self.cached_init_X()

    def gen_init_X(self):
        """ Sets self.Xinit to generated initial states for the sampling particles
        *For use with discrete-time samplers only*

        :returns: None
        :rtype: None
        """
        raise NotImplementedError()

    def reset(self):
        """
        resets the object. returns self for convenience
        """
        self.E_count = 0
        self.dEdX_count = 0
        if not self.generation_instance:
            self.init_X()
        return self

    def __call__(self, X):
        """
        Convenience method for NUTS compatibility
        returns -E, -dEdX
        """
        rshp_X = X.reshape(len(X), 1)
        E = float(self.E(rshp_X))
        dEdX = self.dEdX(rshp_X).T[0]
        return -E, -dEdX

    def load_cache(self):
        """ Loads and returns the cached fair initializations and
         estimated variances associated with this
         distribution. Throws an error if the cache does not exist

        :returns: the loaded cache: (fair_initialization, emc_var_estimate, true_var_estimate)
        :rtype: (np.ndarray, float, float)
        """
        distr_name = type(self).__name__
        distr_hash = hash(self)
        file_name = '{}_{}.pickle'.format(distr_name, distr_hash)
        file_prefix = '{}/initializations'.format(package_path())
        with open('{}/{}'.format(file_prefix, file_name)) as cache_file:
            return pickle.load(cache_file)

class LambdaDistribution(Distribution):
    """ An `anonymous' distribution object for quick
    experimentation. Due to the initialization time that is required
    at first run it, one shouldn't use this object in the
    long-term. Rather create your own distribution class that inherits
    from Distribution.

    You should give your LambdaDistribution objects a name. Use a
    descriptive name, and use the same for functionally equivalent
    LambdaDistributions - the hash of the name is used to label the
    initialization information which is generated at first run time of
    a new distribution. This requirement is a side effect of the
    unfortunate fact that there is no computable hash function which
    assigns functionally identical programs to the same number.
    """

    #pylint: disable=too-many-arguments
    def __init__(self, energy_func=None, energy_grad_func=None, init=None, name=None):
        """ Creates an anonymous distribution object.

        :param ndims: the dimension of the state space for this distribution
        :param nbatch: the number of sampling particles to run simultaneously
        :param energy_func: function specifying the energy
        :param energy_grad_func: function specifying gradient of the energy
        :param name: name of this distribution. use the same name for
          functionally identical distributions
        :param init: fair initialization for this distribution. array of shape (ndims, nbatch)
        :returns: an anonymous distribution object
        :rtype: LambdaDistribution

        """
        self.energy_func = energy_func
        self.energy_grad_func = energy_grad_func
        self.init = init
        # TODO: raise warning if name is not passed
        self.name = name or str(np.random())
        super(LambdaDistribution, self).__init__(ndims=init.shape[0], nbatch=init.shape[1])

    @overrides(Distribution)
    def E_val(self, X):
        return np.sum(X*np.dot(self.J,X), axis=0).reshape((1,-1))/2.

    @overrides(Distribution)
    def dEdX_val(self, X):
        return np.dot(self.J,X)/2. + np.dot(self.J.T,X)/2.

    @overrides(Distribution)
    def gen_init_X(self):
        self.Xinit = self.init

    @overrides(Distribution)
    def __hash__(self):
        return hash((self.ndims, self.nbatch, self.name))

class Gaussian(Distribution):
    def __init__(self, ndims=2, nbatch=100, log_conditioning=6):
        """
        Energy function, gradient, and hyperparameters for the "ill
        conditioned Gaussian" example from the LAHMC paper.
        """
        self.conditioning = 10**np.linspace(-log_conditioning, 0, ndims)
        self.J = np.diag(self.conditioning)
        self.description = '%dD Anisotropic Gaussian, %g self.conditioning'%(ndims, 10**log_conditioning)
        super(Gaussian, self).__init__(ndims, nbatch)

    @overrides(Distribution)
    def E_val(self, X):
        return np.sum(X*np.dot(self.J,X), axis=0).reshape((1,-1))/2.

    @overrides(Distribution)
    def dEdX_val(self, X):
        return np.dot(self.J,X)/2. + np.dot(self.J.T,X)/2.

    @overrides(Distribution)
    def gen_init_X(self):
        self.Xinit = (1./np.sqrt(self.conditioning).reshape((-1,1))) * np.random.randn(self.ndims,self.nbatch)

    @overrides(Distribution)
    def __hash__(self):
        return hash((self.ndims, hash(tuple(self.conditioning))))

class RoughWell(Distribution):
    def __init__(self, ndims=2, nbatch=100, scale1=100, scale2=4):
        """
        Energy function, gradient, and hyperparameters for the "rough well"
        example from the LAHMC paper.
        """
        self.scale1 = scale1
        self.scale2 = scale2
        self.description = '{} Rough Well'.format(ndims)
        super(RoughWell, self).__init__(ndims, nbatch)

    @overrides(Distribution)
    def E_val(self, X):
        cosX = np.cos(X*2*np.pi/self.scale2)
        E = np.sum((X**2) / (2*self.scale1**2) + cosX, axis=0).reshape((1,-1))
        return E

    @overrides(Distribution)
    def dEdX_val(self, X):
        sinX = np.sin(X*2*np.pi/self.scale2)
        dEdX = X/self.scale1**2 + -sinX*2*np.pi/self.scale2
        return dEdX

    @overrides(Distribution)
    def gen_init_X(self):
        self.Xinit = self.scale1 * np.random.randn(self.ndims, self.nbatch)

    @overrides(Distribution)
    def __hash__(self):
        return hash((self.ndims, self.scale1, self.scale2))

class MultimodalGaussian(Distribution):
    def __init__(self, ndims=2, nbatch=100, separation=3):
        self.sep_vec = np.array([separation] * nbatch +
                                [0] * (ndims - 1) * nbatch).reshape(ndims, nbatch)
        # separated along first axis
        self.sep_vec[0] += separation
        super(MultimodalGaussian, self).__init__(ndims, nbatch)

    @overrides(Distribution)
    def E_val(self, X):
        trim_sep_vec = self.sep_vec[:, :X.shape[1]]
        return -np.log(np.exp(-np.sum((X + trim_sep_vec)**2, axis=0)) +
                       np.exp(-np.sum((X - trim_sep_vec)**2, axis=0)))

    @overrides(Distribution)
    def dEdX_val(self, X):
        # allows for partial batch size
        trim_sep_vec = self.sep_vec[:, :X.shape[1]]
        common_exp = np.exp(np.sum(4 * trim_sep_vec * X, axis=0))
        # floating point hax
        return ((2 * ((X - trim_sep_vec) * common_exp + trim_sep_vec + X)) /
                (common_exp + 1))

    @overrides(Distribution)
    def init_X(self):
        # okay, this is pointless... sep vecs cancel
        self.Xinit = ((np.random.randn(self.ndims, self.nbatch) + self.sep_vec) +
                (np.random.randn(self.ndims, self.nbatch) - self.sep_vec))

    @overrides(Distribution)
    def __hash__(self):
        return hash((self.ndims, self.separation))

class TestGaussian(Distribution):

    def __init__(self, ndims=2, nbatch=100, sigma=1.):
        """Simple default unit variance gaussian for testing samplers
        """
        self.sigma = sigma
        super(TestGaussian, self).__init__(ndims, nbatch)

    @overrides(Distribution)
    def E_val(self, X):
        return np.sum(X**2, axis=0).reshape((1, -1)) / (2. * self.sigma ** 2)

    @overrides(Distribution)
    def dEdX_val(self, X):
        return X/self.sigma**2

    @overrides(Distribution)
    def gen_init_X(self):
        self.Xinit = np.random.randn(self.ndims, self.nbatch)

    @overrides(Distribution)
    def __hash__(self):
        return hash((self.ndims, self.sigma))

#pylint: disable=too-many-instance-attributes
class ProductOfT(Distribution):
    """ Provides the product of T experts distribution
    """

    #pylint: disable=too-many-arguments
    def __init__(self, ndims=36, nbasis=36, nbatch=100, lognu=None, W=None, b=None):
        """ Product of T experts, assumes a fixed W that is sparse and alpha that is
        """
        # awkward hack to import theano in poe only
        try:
            import theano.tensor as T
            import theano
            self.theano = theano
            self.T = T
        except:
            raise ImportError("Theano could not be imported")

        if ndims != nbasis:
            raise NotImplementedError("Initializer only works for ndims == nbasis")
        self.ndims = ndims
        self.nbasis = nbasis
        self.nbatch = nbatch
        if W is None:
            W = np.eye(ndims, nbasis)
        self.weights = self.theano.shared(np.array(W, dtype='float32'), 'W')
        if lognu is None:
            pre_nu = np.random.rand(nbasis,) * 2 + 2.1
        else:
            pre_nu = np.exp(lognu)
        self.nu = self.theano.shared(np.array(pre_nu, dtype='float32'), 'nu')
        if b is None:
            b = np.zeros((nbasis,))
        self.bias = self.theano.shared(np.array(b, dtype='float32'), 'b')

        state = T.matrix()
        energy = self.E_def(state)
        gradient = T.grad(T.sum(energy), state)

        #@overrides(Distribution)
        self.E_val = self.theano.function([state], energy, allow_input_downcast=True)
        #@overrides(Distribution)
        self.dEdX_val = self.theano.function([state], gradient, allow_input_downcast=True)

        super(ProductOfT,self).__init__(ndims,nbatch)
        self.backend = 'theano'

    def E_def(self,X):
        """
        energy for a POE with student's-t expert in terms of:
                samples [# dimensions]x[# samples] X
                receptive fields [# dimensions]x[# experts] W
                biases [# experts] b
                degrees of freedom [# experts] nu
        """
        rshp_b = self.bias.reshape((1,-1))
        rshp_nu = self.nu.reshape((1, -1))
        alpha = (rshp_nu + 1.)/2.
        energy_per_expert = alpha * self.T.log(1 + ((self.T.dot(X.T, self.weights) + rshp_b) / rshp_nu) ** 2)
        energy = self.T.sum(energy_per_expert, axis=1).reshape((1, -1))
        return energy

    @overrides(Distribution)
    def gen_init_X(self):
        #hack to remap samples from a generic product of experts to
        #the model we are actually going to generate samples from
        Zinit = np.zeros((self.ndims, self.nbatch))
        for ii in xrange(self.ndims):
            Zinit[ii] = stats.t.rvs(self.nu.get_value()[ii], size=self.nbatch)

        Yinit = Zinit - self.bias.get_value().reshape((-1, 1))
        self.Xinit = np.dot(np.linalg.inv(self.weights.get_value()), Yinit)

    @overrides(Distribution)
    def __hash__(self):
        return hash((self.ndims,
                     self.nbasis,
                     hash(tuple(self.nu.get_value())),
                     hash(tuple(self.weights.get_value().ravel())),
                     hash(tuple(self.bias.get_value().ravel()))))


# -*- coding: utf-8 -*-

# ***************************************************
# * File        : main.py
# * Author      : Zhefeng Wang
# * Email       : wangzhefengr@163.com
# * Date        : 2023-04-11
# * Version     : 0.1.041123
# * Description : description
# * Link        : link
# * Requirement : 相关模块版本需求(例如: numpy >= 2.1.0)
# ***************************************************

# python libraries
import os
import sys
from typing import List, Optional, Callable, Iterable

import torch
from torch import nn
from gluonts.torch.model.predictor import PyTorchPredictor
from gluonts.torch.distributions import StudentTOutput
from gluonts.model.forecast_generator import DistributionForecastGenerator
import pytorch_lightning as pl

# global variable
LOGGING_LABEL = __file__.split('/')[-1][:-3]

def mean_abs_scaling(context, min_scale = 1e-5):
    return context.abs().mean().clamp(min_scale, None).unsqueeze(1)

class FeedForwardNetwork(nn.Module):

    def __init__(self, prediction_length: int, context_length: int,
                 hidden_dimensions: List[int], batch_norm: bool = False,
                 distr_output: Callable = StudentTOutput(),
                 scaling: Callable = mean_abs_scaling) -> None:
        super(FeedForwardNetwork, self).__init__()
        # ------------------------------
        # Parameters
        # ------------------------------
        # check params
        assert prediction_length > 0
        assert context_length > 0
        assert len(hidden_dimensions) > 0
        # params init
        self.prediction_length = prediction_length  # 预测长度
        self.context_length = context_length  # 10
        self.hidden_dimensions = hidden_dimensions  # 隐藏层维度 []
        self.distr_output = distr_output  # 分布输出
        self.batch_norm = batch_norm  # 是否进行 BatchNormalization
        # ------------------------------
        # Layers
        # ------------------------------
        # layer1: 数据转换
        self.scaling = scaling
        # layer2:
        modules = []
        dimensions = [context_length] + hidden_dimensions[:-1] # dimensions=[0, 1, 2, ..., n]
        for in_size, out_size in zip(dimensions[:-1], dimensions[1:]):
            # layer2.1
            modules += [
                self.__make_linear(in_size, out_size),
                nn.ReLU()
            ]
            # layer2.2
            if batch_norm:
                modules.append(nn.BatchNorm1d(out_size))
        # layer3:
        modules.append(
            self.__make_linear(dimensions[-1], prediction_length * hidden_dimensions[-1])
        )
        # layer4: output
        self.nn = nn.Sequential(*modules)
        self.args_proj = self.distr_output.get_args_proj(hidden_dimensions[-1])

    @staticmethod
    def __make_linear(dim_in, dim_out):
        linear = nn.Linear(dim_in, dim_out)
        torch.nn.init.uniform_(linear.weight, -0.07, 0.07)
        torch.nn.init.zeros_(linear.bias)
        return linear

    def forward(self, context):
        # data scaling
        scale = self.scaling(context)
        scaled_context = context / scale
        # output
        nn_out = self.nn(scaled_context)
        nn_out_reshaped = nn_out.reshape(-1, self.prediction_length, self.hidden_dimensions[-1])
        # student t distribution outout
        distr_args = self.args_proj(nn_out_reshaped)
        return distr_args, torch.zeros_like(scale), scale

    def get_predictor(self, input_transform, batch_size = 32, device = None):
        return PyTorchPredictor(
            prediction_length = self.prediction_length,
            input_names = ["past_target"],
            prediction_net = self,
            batch_size = batch_size,
            input_transform = input_transform,
            forecast_generator = DistributionForecastGenerator(self.distr_output),
            device = device,
        )

class LightningFeedForwardNetwork(FeedForwardNetwork, pl.LightningModule):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def training_step(self, batch, batch_idx):
        # TODO
        context = batch["past_target"]
        target = batch["future_target"]
        assert context.shape[-1] == self.context_length
        assert target.shape[-1] == self.prediction_length

        distr_args, loc, scale = self(context)
        distr = self.distr_output.distribution(distr_args, loc, scale)
        # loss function
        loss = -distr.log_prob(target)
        return loss.mean()

    def configure_optimizers(self):
        optimizer = torch.optim.Adam(self.parameters(), lr = 1e-3)
        return optimizer

# 测试代码 main 函数
def main():
    from gluonts.dataset.repository.datasets import get_dataset
    dataset = get_dataset("electricity")

if __name__ == "__main__":
    main()


'''
select_subhalos

library of functions for selecting subhalos (for purpose of cw reconstruction) and padding
set of positions in periodic cube (to counteract disperse's problems with boundary conds.)

Chris J Duckworth cduckastro@gmail.com
'''

import numpy as np
import groupcat as gc

def return_stel_tracers(basepath, snapnum, blen, min_mass=10**8.5):
	'''
	returns stellar tracer positions for all objects with a defined minimum mass at a
	defined snapshot. basepath should direct to /output directory of chosen simulation
	and blen should be the corresponding box side length.
	'''
	# Subhalo fields
	sfields = ['SubhaloMassType','SubhaloPos']
	subhalos = gc.loadSubhalos(basepath, snapnum, fields=sfields)

	# selecting based on total mass within the subhalo.
	return subhalos['SubhaloPos'][()][subhalos['SubhaloMassType'][()][:,4]*10**10 > min_mass]

def box_extend(blen, pos, frac=0.1):
	'''
	Given a periodic cube of equal side length blen, this script extends (or pads) each
	side by an additional fraction set by frac. This is simply repeating all
	positions (defined by pos) on each side.

	This is required to ensure disperse is doing its job at the edges of sims.
	'''

	# wrapping x, y and then z directions in order. wrapping is performed on updated set
	# of positions at each step to ensure there is enough padding at the corners.

	# creating copy of positions to update.
	pos_extend = pos

	# looping over dimensions in order.
	for ind in np.arange(3):
		# slicing each side.
		low_slice = pos_extend[pos_extend[:, ind] < blen * frac]
		low_slice[:, ind] += blen

		high_slice = pos_extend[pos_extend[:, ind] > blen * (1 - frac)]
		high_slice[:, ind] -= blen

		# adding new points.
		pos_extend = np.append(pos_extend, low_slice, axis=0)
		pos_extend = np.append(pos_extend, high_slice, axis=0)

	return pos_extend


import os

import numpy as np
import pytest
from numpy.testing import assert_array_equal

import tiledb

def test_schema_evolution(tmp_path):
    ctx = tiledb.default_ctx()
    se = tiledb.ArraySchemaEvolution(ctx)

    uri = str(tmp_path)

    attrs = [
        tiledb.Attr(name="a1", dtype=np.float64),
        tiledb.Attr(name="a2", dtype=np.int32),
    ]
    dims = [tiledb.Dim(domain=(0, 3), dtype=np.uint64)]
    domain = tiledb.Domain(*dims)
    schema = tiledb.ArraySchema(domain=domain, attrs=attrs, sparse=False)
    tiledb.Array.create(uri, schema)

    data1 = {
        "a1": np.arange(5, 9),
        "a2": np.random.randint(0, 1e7, size=4).astype(np.int32),
    }

    with tiledb.open(uri, "w") as A:
        A[:] = data1

    with tiledb.open(uri) as A:
        res = A[:]
        assert_array_equal(res["a1"], data1["a1"])
        assert_array_equal(res["a2"], data1["a2"])
        assert "a3" not in res.keys()

    newattr = tiledb.Attr("a3", dtype=np.int8)
    se.add_attribute(newattr)

    with pytest.raises(tiledb.TileDBError) as excinfo:
        se.add_attribute(newattr)
    assert "Input attribute name is already there" in str(excinfo.value)
    assert "tiledb/schema_evolution.cc" in str(excinfo.value)

    se.array_evolve(uri)

    data2 = {
        "a1": np.arange(5, 9),
        "a2": np.random.randint(0, 1e7, size=4).astype(np.int32),
        "a3": np.random.randint(0, 255, size=4).astype(np.int8),
    }

    with tiledb.open(uri, "w") as A:
        A[:] = data2

    def test_it():
        with tiledb.open(uri) as A:
            res = A[:]
            assert_array_equal(res["a1"], data2["a1"])
            assert_array_equal(res["a2"], data2["a2"])
            assert_array_equal(res["a3"], data2["a3"])

    test_it()
    tiledb.consolidate(uri)
    test_it()

    se = tiledb.ArraySchemaEvolution(ctx)
    se.drop_attribute("a1")
    se.array_evolve(uri)

    data3 = {
        "a2": np.random.randint(0, 1e7, size=4).astype(np.int32),
        "a3": np.random.randint(0, 255, size=4).astype(np.int8),
    }

    def test_it2():
        with tiledb.open(uri) as A:
            res = A[:]
            assert "a1" not in res.keys()
            assert_array_equal(res["a2"], data3["a2"])
            assert_array_equal(res["a3"], data3["a3"])

    with tiledb.open(uri, "w") as A:
        A[:] = data3

    test_it2()
    tiledb.consolidate(uri)
    test_it2()

def test_schema_evolution_timestamp(tmp_path):
    ctx = tiledb.default_ctx()
    se = tiledb.ArraySchemaEvolution(ctx)
    vfs = tiledb.VFS()

    uri = str(tmp_path)
    schema_uri = os.path.join(uri, "__schema")

    attrs = [tiledb.Attr(name="a1", dtype=np.float64)]
    domain = tiledb.Domain(tiledb.Dim(domain=(0, 3), dtype=np.uint64))
    schema = tiledb.ArraySchema(domain=domain, attrs=attrs, sparse=False)
    tiledb.Array.create(uri, schema)

    def get_schema_timestamps(schema_uri):
        schema_files = filter(lambda x: "__enumerations" not in x, vfs.ls(schema_uri))
        return [int(os.path.basename(file).split("_")[2]) for file in schema_files]

    assert 123456789 not in get_schema_timestamps(schema_uri)

    newattr = tiledb.Attr("a2", dtype=np.int8)
    se.timestamp(123456789)
    se.add_attribute(newattr)
    se.array_evolve(uri)

    assert 123456789 in get_schema_timestamps(schema_uri)

def test_schema_evolution_with_enmr(tmp_path):
    ctx = tiledb.default_ctx()
    se = tiledb.ArraySchemaEvolution(ctx)

    uri = str(tmp_path)

    attrs = [
        tiledb.Attr(name="a1", dtype=np.float64),
        tiledb.Attr(name="a2", dtype=np.int32),
    ]
    dims = [tiledb.Dim(domain=(0, 3), dtype=np.uint64)]
    domain = tiledb.Domain(*dims)
    schema = tiledb.ArraySchema(domain=domain, attrs=attrs, sparse=False)
    tiledb.Array.create(uri, schema)

    data1 = {
        "a1": np.arange(5, 9),
        "a2": np.random.randint(0, 1e7, size=4).astype(np.int32),
    }

    with tiledb.open(uri, "w") as A:
        A[:] = data1

    with tiledb.open(uri) as A:
        assert not A.schema.has_attr("a3")

    newattr = tiledb.Attr("a3", dtype=np.int8, enum_label="e3")
    se.add_attribute(newattr)

    with pytest.raises(tiledb.TileDBError) as excinfo:
        se.array_evolve(uri)
    assert " Attribute refers to an unknown enumeration" in str(excinfo.value)

    se.add_enumeration(tiledb.Enumeration("e3", True, np.arange(0, 8)))
    se.array_evolve(uri)

    se = tiledb.ArraySchemaEvolution(ctx)

    with tiledb.open(uri) as A:
        assert A.schema.has_attr("a3")
        assert A.attr("a3").enum_label == "e3"

    se.drop_enumeration("e3")

    with pytest.raises(tiledb.TileDBError) as excinfo:
        se.array_evolve(uri)
    assert "Unable to drop enumeration" in str(excinfo.value)

    se.drop_attribute("a3")
    se.array_evolve(uri)

    with tiledb.open(uri) as A:
        assert not A.schema.has_attr("a3")

@pytest.mark.parametrize(
    "type,data",
    (
        ("int", [0]),
        ("bool", [True, False]),
        ("str", ["abc", "defghi", "jk"]),
        ("bytes", [b"abc", b"defghi", b"jk"]),
    ),
)
def test_schema_evolution_extend_enmr(tmp_path, type, data):
    uri = str(tmp_path)
    enmr = tiledb.Enumeration("e", True, dtype=type)
    attrs = [tiledb.Attr(name="a", dtype=int, enum_label="e")]
    domain = tiledb.Domain(tiledb.Dim(domain=(0, 3), dtype=np.uint64))
    schema = tiledb.ArraySchema(domain=domain, attrs=attrs, enums=[enmr])
    tiledb.Array.create(uri, schema)

    with tiledb.open(uri) as A:
        assert A.schema.has_attr("a")
        assert A.attr("a").enum_label == "e"
        assert A.enum("e") == enmr

    se = tiledb.ArraySchemaEvolution()
    updated_enmr = enmr.extend(data)
    se.extend_enumeration(updated_enmr)
    se.array_evolve(uri)

    with tiledb.open(uri) as A:
        assert A.schema.has_attr("a")
        assert A.attr("a").enum_label == "e"
        assert A.enum("e") == updated_enmr

def test_schema_evolution_extend_check_bad_type():
    enmr = tiledb.Enumeration("e", True, dtype=str)
    with pytest.raises(tiledb.TileDBError):
        enmr.extend([1, 2, 3])
    with pytest.raises(tiledb.TileDBError):
        enmr.extend([True, False])
    enmr.extend(["a", "b"])

    enmr = tiledb.Enumeration("e", True, dtype=int)
    with pytest.raises(tiledb.TileDBError):
        enmr.extend(["a", "b"])
    with pytest.raises(tiledb.TileDBError):
        enmr.extend([True, False])
    enmr.extend([1, 2, 3])

    enmr = tiledb.Enumeration("e", True, dtype=bool)
    with pytest.raises(tiledb.TileDBError):
        enmr.extend(["a", "b"])
    with pytest.raises(tiledb.TileDBError):
        enmr.extend([1, 2, 3])
    enmr.extend([True, False])


#!/usr/bin/env python
# -*- coding: utf-8 -*-

""" Scripts to plot Figure 2, co-occurrence and spearman rho for top 1000 videos in terms of views and watch times.

Usage: python plot_intersection_spearman_top1000.py
Time: ~2M
"""

from __future__ import division, print_function
import os
from scipy import stats
import operator
import matplotlib.pyplot as plt

def plot_intersection(ax, view_rank_dict, watch_rank_dict, color, linestyle, label):
    sorted_view_rank = sorted(view_rank_dict.items(), key=operator.itemgetter(1), reverse=True)[:1000]
    sorted_watch_rank = sorted(watch_rank_dict.items(), key=operator.itemgetter(1), reverse=True)[:1000]

    x_axis = []
    y_axis = []
    # iterate from 50 to 1000, with gap 10
    for i in range(50, 1001, 10):
        view_set = set([item[0] for item in sorted_view_rank[:i]])
        watch_set = set([item[0] for item in sorted_watch_rank[:i]])
        x_axis.append(i)
        y_axis.append(len(view_set.intersection(watch_set))/i)

    ax.plot(x_axis, y_axis, color=color, linestyle=linestyle, label=label)

def plot_spearman(ax, view_rank_dict, watch_rank_dict, color, linestyle, label):
    sorted_view_rank = sorted(view_rank_dict.items(), key=operator.itemgetter(1), reverse=True)[:1000]
    sorted_watch_rank = sorted(watch_rank_dict.items(), key=operator.itemgetter(1), reverse=True)[:1000]

    x_axis = []
    y_axis = []
    # iterate from 50 to 1000, with gap 10
    for i in range(50, 1001, 10):
        view_set = set([item[0] for item in sorted_view_rank[:i]])
        watch_set = set([item[0] for item in sorted_watch_rank[:i]])
        unit_list = list(view_set.union(watch_set))
        view_rank = [view_rank_dict[x] for x in unit_list]
        watch_rank = [watch_rank_dict[x] for x in unit_list]

        x_axis.append(i)
        y_axis.append(stats.spearmanr(view_rank, watch_rank)[0])

    ax.plot(x_axis, y_axis, color=color, linestyle=linestyle, label=label)

if __name__ == '__main__':
    # == == == == == == == == Part 1: Set up experiment parameters == == == == == == == == #
    # setting parameters
    view_rank_dict = {}
    watch_rank_dict = {}
    music_view_rank_dict = {}
    music_watch_rank_dict = {}
    news_view_rank_dict = {}
    news_watch_rank_dict = {}

    # == == == == == == == == Part 2: Load dataset == == == == == == == == #
    input_doc = '../../production_data/new_tweeted_dataset_norm/'
    for subdir, _, files in os.walk(input_doc):
        for f in files:
            with open(os.path.join(subdir, f), 'r') as fin:
                fin.readline()
                for line in fin:
                    vid, dump = line.rstrip().split('\t', 1)
                    view30 = float(dump.split('\t')[7])
                    watch30 = float(dump.split('\t')[8])

                    view_rank_dict[vid] = view30
                    watch_rank_dict[vid] = watch30

                    if f.startswith('10'):
                        music_view_rank_dict[vid] = view30
                        music_watch_rank_dict[vid] = watch30

                    if f.startswith('25'):
                        news_view_rank_dict[vid] = view30
                        news_watch_rank_dict[vid] = watch30
            print('>>> Loading data: {0} done!'.format(os.path.join(subdir, f)))

    fig = plt.figure()
    ax1 = fig.add_subplot(1, 1, 1)
    plot_intersection(ax1, view_rank_dict, watch_rank_dict, color='r', linestyle='-', label='ALL')
    plot_intersection(ax1, music_view_rank_dict, music_watch_rank_dict, color='k', linestyle='--', label='Music')
    plot_intersection(ax1, news_view_rank_dict, news_watch_rank_dict, color='k', linestyle=':', label='News')
    ax1.set_ylim([0, 1])
    ax1.set_xlabel('top $n$ videos', fontsize=18)
    ax1.set_ylabel('Intersection size', fontsize=18)
    ax1.set_yticks([0.0, 0.2, 0.4, 0.6, 0.8, 1.0])
    ax1.tick_params(axis='both', which='major', labelsize=16)
    ax1.legend(loc='lower right', handlelength=1, frameon=False, fontsize=20)
    ax1.spines['right'].set_visible(False)
    ax1.spines['top'].set_visible(False)
    ax1.set_title('(a) co-occurrence rate', fontsize=18)
    plt.tight_layout()
    plt.show()

    fig = plt.figure()
    ax2 = fig.add_subplot(1, 1, 1)
    plot_spearman(ax2, view_rank_dict, watch_rank_dict, color='r', linestyle='-', label='ALL')
    plot_spearman(ax2, music_view_rank_dict, music_watch_rank_dict, color='k', linestyle='--', label='Music')
    plot_spearman(ax2, news_view_rank_dict, news_watch_rank_dict, color='k', linestyle=':', label='News')
    ax2.set_ylim([-1, 1])
    ax2.set_xlabel('top $n$ videos', fontsize=18)
    ax2.set_ylabel("Spearman's $\\rho$", fontsize=18)
    ax2.set_yticks([-1.0, -0.5, 0.0, 0.5, 1.0])
    ax2.tick_params(axis='both', which='major', labelsize=16)
    ax2.legend(loc='lower right', handlelength=1, frameon=False, fontsize=20)
    ax2.spines['right'].set_visible(False)
    ax2.spines['top'].set_visible(False)
    ax2.set_title("(b) Spearman's $\\rho$", fontsize=18)
    plt.tight_layout()
    plt.show()


from spectractor import parameters
from spectractor.config import set_logger

import matplotlib.pyplot as plt
import pandas as pd
import os
import numpy as np

class LogBook:
    """Class to load_image and analyse observation logbook csv files."""

    def __init__(self, logbook="./tests/data/ctiofulllogbook_jun2017_v5.csv"):
        """Load and initialise the logbook

        Parameters
        ----------
        logbook: str
            Path to the logbook. Must be a CSV file.

        Examples
        ----------

        >>> logbook = LogBook('./tests/data/ctiofulllogbook_jun2017_v5.csv')
        >>> assert logbook.df is not None
        >>> print(logbook.logbook)
        ./tests/data/ctiofulllogbook_jun2017_v5.csv
        >>> print(logbook.df['disperser'][:2])
        0    Ron400
        1    Ron400
        Name: disperser, dtype: object
        >>> logbook = LogBook('./log.csv')

        """
        self.my_logger = set_logger(self.__class__.__name__)
        self.logbook = logbook
        if not os.path.isfile(logbook):
            self.my_logger.error('CSV logbook file {} not found.'.format(logbook))
            return
        # self.csvfile = open(self.logbook, 'rU', encoding='latin-1')
        # self.reader = csv.DictReader(self.csvfile, delimiter=';', dialect=csv.excel_tab)
        self.df = pd.read_csv(self.logbook, sep=";", decimal=",", encoding='latin-1', header='infer')
        self.df['date'] = pd.to_datetime(self.df.date)

    def search_for_image(self, filename):
        """
        Look for an image file name in the logbook and load_image properties:

        - Obj-posXpix and Obj-posYpix: the [x0,y0] guessed pixel position in the image
        - Dx and Dy: the x and y windows in pixel to search for the target; set XWINDOW and YWINDOW variables in parameters.py
        - object: the name of the target

        Parameters
        ----------
        filename: str
            the fits image file name (not the path, only the file name.)

        Returns
        -------
        disperser_label: str
            the name of the disperser
        target: str
            the name of the target
        xpos: int
            the x position of the target (in pixel)
        ypos: int
            the y position of the target (in pixel)

        Examples
        --------

        >>> logbook = LogBook('./tests/data/ctiofulllogbook_jun2017_v5.csv')
        >>> disperser_label, target, xpos, ypos = logbook.search_for_image("unknown_file.fits")
        >>> print(disperser_label, target, xpos, ypos)
        None None None None
        >>> disperser_label, target, xpos, ypos = logbook.search_for_image("reduc_20170605_028.fits")
        >>> print(disperser_label, target, xpos, ypos)
        HoloPhAg PNG321.0+3.9 814 585
        >>> disperser_label, target, xpos, ypos = logbook.search_for_image("reduc_20170608_119.fits")
        >>> print(disperser_label, target, xpos, ypos)
        None HD205905 None None
        >>> disperser_label, target, xpos, ypos = logbook.search_for_image("reduc_20170630_001.fits")
        >>> print(disperser_label, target, xpos, ypos)
        None bias None None

        """
        disperser_label = None
        target = None
        xpos = None
        ypos = None
        skip = False
        try:
            row = self.df.loc[(self.df['file'] == filename)].iloc[0]
            target = row['object']
            if row['object'] == 'bias' or row['object'] == 'flat' or row['object'] == 'zero':
                self.my_logger.error(
                    'Fits file %s in logbook %s has flag %s. Skip file.' % (filename, self.logbook, target))
                skip = True
            if row['skip'] == 'skip':
                self.my_logger.error('Fits file %s in logbook has flag "skip". Skip file.' % filename)
                skip = True
            if np.isnan(row['Obj-posXpix']):
                self.my_logger.error(
                    'Fits file %s in logbook %s has no target x position. Skip file.' % (filename, self.logbook))
                skip = True
            if np.isnan(row['Obj-posYpix']):
                self.my_logger.error(
                    'Fits file %s in logbook %s has no target y position. Skip file.' % (filename, self.logbook))
                skip = True
            if not np.isnan(row['Dx']):
                parameters.XWINDOW = int(row['Dx'])
                parameters.XWINDOW_ROT = int(row['Dx'])
            if not np.isnan(row['Dy']):
                parameters.YWINDOW = int(row['Dy'])
                parameters.YWINDOW_ROT = int(row['Dy'])
            if not skip:
                xpos = int(row['Obj-posXpix'])
                ypos = int(row['Obj-posYpix'])
                disperser_label = row['disperser']
        except IndexError:
            if target is None and skip is False:
                self.my_logger.error('Fits file %s not found in logbook %s.' % (filename, self.logbook))
        return disperser_label, target, xpos, ypos

    def plot_columns_vs_date(self, column_names):
        """Plot of the column property with respect to the dates.

        Parameters
        ----------
        column_names: list, str
            List of column names to plot versus time from the log book.

        Examples
        --------
        >>> logbook = LogBook('./tests/data/ctiofulllogbook_jun2017_v5.csv')
        >>> logbook.plot_columns_vs_date(['seeing'])
        >>> logbook.plot_columns_vs_date(['P', 'T'])
        """
        if isinstance(column_names, str):
            column_names = [column_names]
        self.df.plot(x='date', y=column_names)
        if parameters.DISPLAY:
            plt.show()
        if parameters.PdfPages:
            parameters.PdfPages.savefig()

if __name__ == "__main__":
    import doctest

    doctest.testmod()


from maltego_trx.transform import DiscoverableTransform
from db import db
from utils import row_dict_to_conversation_email

class DomainToEnronUsers(DiscoverableTransform):
    """
    Given a  maltego.Domain Entity, return a list of Users for this Domain.
    """

    @classmethod
    def create_entities(cls, request, response):
        domain = request.Value
        res = db.get_ppl_by_domain(domain, limit=request.Slider)
        for d in res:
            ent = response.addEntity('maltego.EmailAddress', d['email'])


import torch
import torch.nn as nn
from conf import device

class Encoder(nn.Module):
    def __init__(self, vocab_size, hidden_size=256):
        super(Encoder, self).__init__()
        self.hidden_size = hidden_size
        self.embedding = nn.Embedding(vocab_size, hidden_size)
        self.gru = nn.GRU(hidden_size, hidden_size)

    def forward(self, input, hidden):
        embedd = self.embedding(input).view(1, 1, -1)
        output, hidden = self.gru(embedd, hidden)
        return output, hidden

    def init_hidden(self):
        return torch.zeros(1, 1, self.hidden_size, dtype=torch.float32, device=device)


import csv
from datetime import datetime

from django.conf import settings
from django.core.management import BaseCommand

from exo_accounts.models import EmailAddress
from exo_certification.tasks import HubspotCertificationDealSyncTask

from ...models import ExOCertification, CertificationRequest

class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument(
            '-f', '--file', nargs='+', type=str, help='CSV file')

    def handle(self, *args, **kwargs):
        path = kwargs.get('file')[0]
        certification = ExOCertification.objects.get(level='L2A')
        with open(path) as csvfile:
            spamreader = csv.reader(csvfile, delimiter=',')
            for row in spamreader:
                try:
                    str_email = row[1]
                    email = EmailAddress.objects.get(email=str_email)
                    user = email.user
                    data = {
                        'price': 0,
                        'created': datetime.strptime(row[2], '%Y-%m-%d'),
                        'status': settings.EXO_CERTIFICATION_REQUEST_STATUS_CH_DRAFT,
                        'user': user,
                        'requester_email': str_email,
                        'requester_name': user.full_name,
                    }

                    certification_request, created = CertificationRequest.objects.get_or_create(
                        user=user,
                        certification=certification,
                        defaults=data,
                    )
                    if created:
                        HubspotCertificationDealSyncTask().s(pk=certification_request.pk).apply()
                        certification_request.refresh_from_db()
                        self.stdout.write(
                            self.style.SUCCESS('email [{}]: Successfully created {}'.format(
                                str_email, certification_request)
                            )
                        )
                        certification_request.status = settings.EXO_CERTIFICATION_REQUEST_STATUS_CH_APPROVED
                        certification_request.save(update_fields=['status'])
                    else:
                        self.stdout.write(
                            self.style.WARNING('email [{}]: Already exists CertRequest {}'.format(
                                str_email, certification_request)
                            )
                        )
                except Exception as exc:
                    self.stdout.write(
                        self.style.ERROR('email [{}]: Errored {}'.format(str_email, exc)))


from typing import Optional

from cartes import COEUR, COULEURS, CarteBelote, CarteSetBelote, Couleur, Pli

class Annonce:
    VALID_SCORES = list(range(80, 170, 10)) + [0, 1000, 2000]

    def __init__(self, atout, score_a_faire, joueur):
        if int(score_a_faire) not in self.VALID_SCORES:
            raise ValueError("score_a_faire non valide")

        if not isinstance(atout, Couleur):
            raise ValueError("couleur non valide")

        self.atout = atout
        self.score_a_faire = score_a_faire
        self.joueur = joueur

    def __lt__(self, other):
        if self.score_a_faire < other.score_a_faire:
            return True

ANNONCE_NULLE = Annonce(atout=COEUR, score_a_faire=0, joueur=None)

def poser_question(question, reponses_possibles):
    reponse = None
    while reponse not in reponses_possibles:
        reponse = input(question)
    return reponse

class Joueur:
    def __init__(self, nom: str):
        self.nom: str = nom
        self.belote: bool = False
        self.main: CarteSetBelote = CarteSetBelote()
        self.doit_annoncer: bool = True
        self.equipe: Optional[Equipe] = None  # Défini lors de la création de l'équipe
        self.plis: list[Pli] = []

    def __repr__(self) -> str:
        return self.nom

    def _reinit(self):
        self.doit_annoncer = True
        self.belote = False
        self.main = CarteSetBelote()
        self.plis = []

    @property
    def _total_points(self):
        score = 0
        for pli in self.plis:
            score += pli._points
        if self.belote:
            score += 20
        return score

    def _afficher_main(self):
        print(self.main)

    def _annoncer(self, meilleure_annonce):
        self._afficher_main()
        reponse = poser_question(f"Souhaitez-vous annoncer {self} ?", ["o", "n"])

        if reponse == "o":
            couleur = poser_question("Couleur ?", list(map(lambda x: x.nom, COULEURS)))
            couleur = list(filter(lambda x: x.nom == couleur, COULEURS))[0]

            scores_possibles = list(
                map(
                    lambda x: str(x),
                    filter(
                        lambda x: x > meilleure_annonce.score_a_faire,
                        Annonce.VALID_SCORES,
                    ),
                )
            )

            score = poser_question("Score ?", scores_possibles)
            annonce = Annonce(atout=couleur, score_a_faire=int(score), joueur=self)
        else:
            annonce = None

        self.doit_annoncer = False
        return annonce

    def _demander_melanger(self) -> bool:
        reponse = input(f"Souhaitez-vous mélanger {self.nom}? [o/n]: ")
        if reponse == "o":
            return True
        return False

    def _ajouter_carte_en_main(self, carte):
        self.main.append(carte)

    def _faire_annonce(self) -> bool:
        reponse = input(f"Souhaitez-vous faire une annonce {self.nom}? [o/n]: ")
        if reponse == "o":
            return True
        return False

    def _couleur_demandee_en_main(self, couleur_demandee) -> bool:
        return (
            len(
                list(
                    filter(
                        lambda x: x.couleur.forme == couleur_demandee.couleur.forme,
                        self.main,
                    )
                )
            )
            > 0
        )

    def _atout_en_main(self) -> bool:
        return True if True in [carte.atout for carte in self.main] else False

    def _meilleur_atout_en_main(self, other_atout: CarteBelote) -> bool:
        if other_atout.atout is False:
            raise ValueError("Vous devez appeler cette fonction avec un autre atout")
        meilleurs_atouts_en_main = list(
            filter(lambda x: x.atout and x > other_atout, self.main)
        )
        if meilleurs_atouts_en_main:
            return True

        return False

    def _jouer_carte(self, pli: Pli, couleur_atout: Couleur):
        premiere_carte_jouee = pli[0] if pli else None

        print(f"À toi de jouer {self}")
        self._afficher_main()

        while True:
            index_carte = 999
            while index_carte not in range(0, len(self.main)):
                index_carte = input("Index de la carte à jouer: ")
                try:
                    index_carte = int(index_carte)
                except Exception:
                    ...

            carte_a_jouer = self.main[index_carte]

            if premiere_carte_jouee is not None:
                carte_gagnante = pli._carte_la_plus_forte(couleur_atout=couleur_atout)

                if carte_a_jouer.couleur.forme != premiere_carte_jouee.couleur.forme:
                    couleur_en_main: bool = self._couleur_demandee_en_main(
                        couleur_demandee=premiere_carte_jouee
                    )
                    if couleur_en_main:
                        print(
                            f"Vous possèder du {premiere_carte_jouee.couleur.forme} "
                            "en main. "
                            f"Vous ne pouvez pas jouer du {carte_a_jouer.couleur.forme}"
                        )

                        continue

                    else:
                        if self._atout_en_main():
                            if (
                                carte_a_jouer.atout is False
                                and carte_gagnante.joueur.equipe != self.equipe
                            ):
                                print("Vous devez couper !")
                                continue

                else:
                    if premiere_carte_jouee.atout and carte_a_jouer.atout:
                        if carte_a_jouer < premiere_carte_jouee:
                            if self._meilleur_atout_en_main(other_atout=carte_gagnante):
                                print("Vous avez un atout supérieur en main")

            if self.belote and carte_a_jouer.atout:
                if carte_a_jouer.valeur in ("D", "R"):
                    atouts_en_main = list(filter(lambda x: x.atout is True, self.main))
                    valeurs_atouts = list(map(lambda x: x.valeur, atouts_en_main))
                    if "D" in valeurs_atouts or "R" in valeurs_atouts:
                        print("Belote")
                    else:
                        print("Rebelote")
            self.main.pop(index_carte)
            return carte_a_jouer

class Equipe:
    def __init__(self, joueur1, joueur2):
        self.joueur1: Joueur = joueur1
        self.joueur2: Joueur = joueur2

        joueur1.equipe = self
        joueur2.equipe = self
        self.score: int = 0

    def __repr__(self) -> str:
        return f"{self.joueur1.nom} & {self.joueur2.nom}"

    @property
    def joueurs(self) -> tuple:
        return (self.joueur1, self.joueur2)


import os, sys, time, datetime

# Additional packages
import numpy as np

# ARL Env
from dVRK.PSM_cartesian_ddpg_env import PSMCartesianDDPGEnv

# Stable baselines algorithms
from stable_baselines.ddpg.policies import MlpPolicy
from stable_baselines import HER, DDPG
from stable_baselines.common.noise import NormalActionNoise
from stable_baselines.common.noise import OrnsteinUhlenbeckActionNoise, AdaptiveParamNoiseSpec
from stable_baselines.common.callbacks import CheckpointCallback

def redirect_stdout(filepath: str = None):
  """Redirect the ouput stream to a file. Also redirect error output stream
  """
  cdir = os.getcwd()
  basepath = os.path.join(cdir, '.logs')

  if not os.path.exists(basepath):
    os.makedirs(basepath)

  if filepath is None:
    now = datetime.datetime.now()
    filepath = 'log_' + now.strftime("%Y_%m_%d-%H_%M_%S.txt")
    filepath = os.path.join(basepath, filepath)

  err_filepath = filepath[:-4] + '_err.txt'

  if os.path.exists(filepath):
    filepath = filepath[:-4]
    filepath += now.strftime("_%H_%M_%S") + '.txt'

  sys.stdout = open(filepath, 'w')
  sys.stderr = open(err_filepath, 'w')
  print("Began logging")
  return

def main(env: PSMCartesianDDPGEnv):
  # the noise objects for DDPG
  n_actions = env.action.action_space.shape[0]
  param_noise = None
  action_noise = OrnsteinUhlenbeckActionNoise(
    mean=np.zeros(n_actions),
    sigma=float(0.5) * np.ones(n_actions)
  )

  model = DDPG(
    MlpPolicy,
    env,
    gamma=0.95,
    verbose=1,
    nb_train_steps=300,
    nb_rollout_steps=150,
    param_noise=param_noise,
    batch_size=128,
    action_noise=action_noise,
    random_exploration=0.05,
    normalize_observations=True,
    tensorboard_log="./ddpg_dvrk_tensorboard/",
    observation_range=(-1.5,
                       1.5),
    critic_l2_reg=0.01
  )

  model.learn(
    total_timesteps=4000000,
    log_interval=100,
    callback=CheckpointCallback(save_freq=100000,
                                save_path="./ddpg_dvrk_tensorboard/")
  )
  model.save("./ddpg_robot_env")

  # NOTE:
  # If continuing learning from previous checkpoint,
  # Comment above chunk of code {model=DDPG(''') till model.save("./her_robot_env")} and uncomment below lines:
  # Replace the XXXXX below with the largest number present in (rl_model_) directory ./ddpg_dvrk_tensorboard/
  # remaining_training_steps = 4000000 - XXXXX
  # model_log_dir = './ddpg_dvrk_tensorboard/rl_model_XXXXX_steps.zip'
  # model = DDPG.load(model_log_dir, env=env)
  # # Reset the model
  # env.reset()
  # model.learn(remaining_training_steps, log_interval=100,
  #             callback=CheckpointCallback(save_freq=100000, save_path="./ddpg_dvrk_tensorboard/"))
  # model.save("./ddpg_robot_env")

def load_model(eval_env):
  model = DDPG.load('./ddpg_robot_env', env=eval_env)
  count = 0
  step_num_arr = []
  for _ in range(20):
    number_steps = 0
    obs = eval_env.reset()
    for _ in range(400):
      action, _ = model.predict(obs)
      obs, reward, done, _ = eval_env.step(action)
      number_steps += 1
      if done:
        step_num_arr.append(number_steps)
        count += 1
        print("----------------It reached terminal state -------------------")
        break
  print(
    "Robot reached the goal position successfully ",
    count,
    " times and the Average step count was ",
    np.average(np.array(step_num_arr))
  )

if __name__ == '__main__':
  # redirect_stdout()
  root_link_name = 'baselink'
  env_kwargs = {
    'action_space_limit': 0.05,
    'goal_position_range': 0.05,
    'position_error_threshold': 0.01,
    'goal_error_margin': 0.0075,
    'joint_limits':
      {
        'lower_limit': np.array([-0.2,
                                 -0.2,
                                 0.1,
                                 -1.5,
                                 -1.5,
                                 -1.5,
                                 -1.5]),
        'upper_limit': np.array([0.2,
                                 0.2,
                                 0.24,
                                 1.5,
                                 1.5,
                                 1.5,
                                 1.5])
      },
    'workspace_limits':
      {
        'lower_limit': np.array([-0.04,
                                 -0.03,
                                 -0.2]),
        'upper_limit': np.array([0.03,
                                 0.04,
                                 -0.091])
      },
    'enable_step_throttling': False,
    'steps_to_print': 10000
  }
  # Training
  ambf_env = PSMCartesianDDPGEnv(**env_kwargs)
  time.sleep(5)
  ambf_env.make(root_link_name)
  ambf_env.reset()
  main(env=ambf_env)
  ambf_env.ambf_client.clean_up()
  # Evaluate learnt policy
  eval_env = PSMCartesianDDPGEnv(**env_kwargs)
  time.sleep(5)
  eval_env.make(root_link_name)
  eval_env.reset()
  load_model(eval_env=eval_env)
  eval_env.ambf_client.clean_up()


# Control flow allows you to build logic into your programs
# Your program can run block of code based on a given condition
phone_balance = 3
bank_balance = 0
if phone_balance < 5:
    phone_balance += 10
    bank_balance -= 10

season = ''
# Example of if, elif and else statements
if season == 'spring':
    print('plant the garden!')
elif season == 'summer':
    print('water the garden!')
elif season == 'fall':
    print('harvest the garden!')
elif season == 'winter':
    print('stay indoors!')
else:
    print('unrecognized season')

# Complex Boolean expression
# logical operators like and, or, not

weight = 55
height = 164

if 18.5 <= weight / height**2 < 25:
    print("BMI is considered 'normal'")

is_raining = True
is_sunny = True

if is_raining and is_sunny:
    print("Is there a rainbow?")

unsubscribed = False
location = 'USA'

if (not unsubscribed) and (location == "USA" or location == "CAN"):
    print("send email")

# Good and bad examples of boolean expressions
# Dont use True or False as conditions
# to check whether a boolean is true use the boolean itself without the boolean_varibale == True check
# Similarly, to check if a boolean is false, use the not boolean_variable combination
# example:
is_night = True
is_day = False

if is_night:
    print("Good night!")

if not is_day:
    print("Get some more sleep!")

# Truth Value Testing
# If a control flow contains a value to the right of the equal sign that is not a boolean expression,
# python will check for its Truth value to decide whether to run the code or not.
# Constants defined to be false:
    # None, False
    # Zero of any numeric type: 0 , 0.0, 0j
    # Empty sequences and collections: (), [], {}, ' ', set(0), range(0), etc.
# Anything else not listed above will contain a truth value of True


def isprime(n):
    for i in range(2,n):
        if(n%i==0):
            return False
    else:
        return True
def difference(n):
    if(n==0):
        return 0
    elif(n==1):
        return 1
    elif(isprime(n)):
        return 0
    a=0
    b=0
    n1=n+1
    while(True):
        if(isprime(n1)):
            a=n1
            break
        else:
            n1+=1
    n1=n-1
    while(True):
        if(isprime(n1)):
            b=n1
            break
        else:
            n1-=1
    diff1=a-n
    diff2=n-b
    return min(diff1,diff2)
n=int(input())
print(difference(n))


# You are given coins of different denominations and a total amount of money amount. Write a function to compute the fewest number of coins that you need to make up that amount. If that amount of money cannot be made up by any combination of the coins, return -1.

# Example 1:
# coins = [1, 2, 5], amount = 11
# return 3 (11 = 5 + 5 + 1)

# Example 2:
# coins = [2], amount = 3
# return -1.

# Note:
# You may assume that you have an infinite number of each kind of coin.
class Solution:
    def coinChange(self, coins, amount):
        """
        :type coins: List[int]
        :type amount: int
        :rtype: int
        """
        rt = [-1] * (amount + 1)
        rt[0] = 0
        for i in range(amount+1):
            for j in coins:
                if j > i:
                    continue
                if rt[i-j] != -1:
                    if rt[i] == -1:
                        rt[i] = rt[i-j] + 1
                    else:
                        rt[i] = min([rt[i], rt[i-j] + 1])

        return rt[-1]


import os
import torch
import genova
from torch import nn, optim
import torch.distributed as dist
from torch.utils.data import DataLoader
from torch.cuda.amp import autocast, GradScaler
from torch.nn.parallel import DistributedDataParallel as DDP

from .optimal_path_inference import optimal_path_infer
from .seq_generation_inference import seq_generation_infer

class Task:
    def __init__(self, cfg, serialized_model_path, distributed=True):
        self.cfg = cfg
        self.distributed = distributed
        self.serialized_model_path = serialized_model_path
        if cfg.mode == 'train':
            if self.distributed:
                dist.init_process_group(backend='nccl')
                self.local_rank = int(os.environ["LOCAL_RANK"])
                self.device = torch.device("cuda", self.local_rank)
                torch.cuda.set_device(self.local_rank)
            else: self.device = torch.device('cuda')
        else:
            if isinstance(cfg.infer.device, int):
                torch.cuda.set_device(cfg.infer.device)
                self.device = torch.device('cuda:'+str(cfg.infer.device))
            else:
                self.device = torch.device('cpu')

    def initialize(self, *, train_spec_header,train_dataset_dir,val_spec_header,val_dataset_dir):
        self.model = genova.models.Genova(self.cfg).to(self.device)

        if self.cfg.task == 'optimal_path':
            self.train_loss_fn = nn.KLDivLoss(reduction='batchmean')
            self.eval_loss_fn = nn.KLDivLoss(reduction='sum')
        elif self.cfg.task == 'node_classification':
            self.train_loss_fn = nn.BCEWithLogitsLoss()
            self.eval_loss_fn = nn.BCEWithLogitsLoss(reduction='sum')
        elif self.cfg.task == 'sequence_generation':
            self.train_loss_fn = nn.CrossEntropyLoss()
            self.eval_loss_fn = nn.CrossEntropyLoss(reduction='sum')
        else:
            raise NotImplementedError

        assert self.distributed==dist.is_initialized()
        if self.distributed: self.model = DDP(self.model, device_ids=[self.local_rank], output_device=self.local_rank)
        self.optimizer = optim.AdamW(self.model.parameters(), lr=self.cfg.train.lr)
        self.scaler = GradScaler()
        self.persistent_file_name = os.path.join(self.serialized_model_path,self.cfg.wandb.project+'_'+self.cfg.wandb.name+'.pt')
        if os.path.exists(self.persistent_file_name):
            checkpoint = torch.load(self.persistent_file_name)
            if self.distributed: self.model.module.load_state_dict(checkpoint['model_state_dict'])
            else: self.model.load_state_dict(checkpoint['model_state_dict'])
            self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])

        self.train_dl = self.train_loader(train_spec_header,train_dataset_dir)
        self.eval_dl = self.eval_loader(val_spec_header,val_dataset_dir)

    def test_initialize(self, *, test_spec_header=None, test_dataset_dir=None):
        assert not self.distributed
        self.model = genova.models.Genova(self.cfg).to(self.device)
        self.persistent_file_name = os.path.join(self.serialized_model_path,self.cfg.wandb.project+'_'+self.cfg.wandb.name+'.pt')
        print('checkpoint: ', self.persistent_file_name)
        assert os.path.exists(self.persistent_file_name)
        if isinstance(self.cfg.infer.device, int):
            checkpoint = torch.load(self.persistent_file_name)
        else:
            checkpoint = torch.load(self.persistent_file_name,map_location='cpu')
        self.model.load_state_dict(checkpoint['model_state_dict'])
        self.model.eval()
        self.test_dl = self.test_loader(test_spec_header,test_dataset_dir)
        self.test_spec_header = test_spec_header

    def train_loader(self,train_spec_header,train_dataset_dir):
        ds = genova.data.GenovaDataset(self.cfg,spec_header=train_spec_header,dataset_dir_path=train_dataset_dir)
        sampler = genova.data.GenovaBatchSampler(self.cfg,self.device,0.95,train_spec_header,[0,128,256,512],self.model)
        collate_fn = genova.data.GenovaCollator(self.cfg)
        if self.distributed:
            train_dl = DataLoader(ds,batch_sampler=sampler,collate_fn=collate_fn,pin_memory=True,num_workers=10)
        else:
            train_dl = DataLoader(ds,batch_sampler=sampler,collate_fn=collate_fn,pin_memory=True)
        train_dl = genova.data.DataPrefetcher(train_dl,self.device)
        return train_dl

    def eval_loader(self,val_spec_header,val_dataset_dir):
        ds = genova.data.GenovaDataset(self.cfg,spec_header=val_spec_header,dataset_dir_path=val_dataset_dir)
        sampler = genova.data.GenovaBatchSampler(self.cfg,self.device,2,val_spec_header,[0,128,256,512],self.model)
        collate_fn = genova.data.GenovaCollator(self.cfg)
        if self.distributed:
            eval_dl = DataLoader(ds,batch_sampler=sampler,collate_fn=collate_fn,pin_memory=True,num_workers=5)
        else:
            eval_dl = DataLoader(ds,batch_sampler=sampler,collate_fn=collate_fn,pin_memory=True)
        eval_dl = genova.data.DataPrefetcher(eval_dl,self.device)
        return eval_dl

    def test_loader(self,test_spec_header,test_dataset_dir):
        ds = genova.data.GenovaDataset(self.cfg,spec_header=test_spec_header,dataset_dir_path=test_dataset_dir)
        sampler = genova.data.GenovaSequentialSampler(test_spec_header)
        collate_fn = genova.data.GenovaCollator(self.cfg)
        test_dl = DataLoader(ds,batch_sampler=sampler,collate_fn=collate_fn,pin_memory=True)
        if isinstance(self.cfg.infer.device, int):
            test_dl = genova.data.DataPrefetcher(test_dl,self.device)
        return test_dl

    def model_save(self):
        if self.distributed:
            torch.save({'model_state_dict':self.model.module.state_dict(),
            'optimizer_state_dict':self.optimizer.state_dict()},self.persistent_file_name)
        else:
            torch.save({'model_state_dict':self.model.state_dict(),
            'optimizer_state_dict':self.optimizer.state_dict()},self.persistent_file_name)

    def train(self):
        total_step = 0
        loss_cum = 0
        if self.cfg.task =='node_classification':
            for epoch in range(0, self.cfg.train.total_epoch):
                for encoder_input, label, label_mask in self.train_dl:
                    total_step += 1
                    if total_step%self.cfg.train.detect_period == 1: loss_cum = 0
                    self.optimizer.zero_grad()
                    with autocast():
                        output = self.model(encoder_input=encoder_input).squeeze(-1)
                        loss = self.train_loss_fn(output[label_mask],label[label_mask])
                    loss_cum += loss.item()
                    self.scaler.scale(loss).backward()
                    self.scaler.step(self.optimizer)
                    self.scaler.update()
                    if total_step%self.cfg.train.detect_period == 0: yield loss_cum/self.cfg.train.detect_period, total_step, epoch
        else:
            for epoch in range(0, self.cfg.train.total_epoch):
                for encoder_input, decoder_input, tgt, label, label_mask, _ in self.train_dl:
                    total_step += 1
                    if total_step%self.cfg.train.detect_period == 1: loss_cum = 0
                    self.optimizer.zero_grad()
                    with autocast():
                        output = self.model(encoder_input=encoder_input, decoder_input=decoder_input, tgt=tgt)
                        if self.cfg.task == 'optimal_path': output = output.log_softmax(-1)
                        loss = self.train_loss_fn(output[label_mask],label[label_mask])
                    loss_cum += loss.item()
                    self.scaler.scale(loss).backward()
                    self.scaler.step(self.optimizer)
                    self.scaler.update()
                    if total_step%self.cfg.train.detect_period == 0: yield loss_cum/self.cfg.train.detect_period, total_step, epoch

    def eval(self) -> float:
        loss_cum = torch.Tensor([0]).to(self.device)
        total_seq_len = torch.Tensor([0]).to(self.device)
        if self.cfg.task =='node_classification':
            total_match = torch.Tensor([0]).to(self.device)
            true_positive = torch.Tensor([0]).to(self.device)
            total_positive = torch.Tensor([0]).to(self.device)
            total_true = torch.Tensor([0]).to(self.device)
            for encoder_input, label, label_mask in self.eval_dl:
                with torch.no_grad():
                    with autocast():
                        output = self.model(encoder_input=encoder_input)
                        output = output[label_mask].squeeze(-1)
                        label = label[label_mask]
                        loss = self.eval_loss_fn(output,label)
                    output = (output>0.5).float()
                    loss_cum += loss
                    total_seq_len += label_mask.sum()
                    total_match += (output == label).sum()
                    true_positive += ((output == label)[label == 1]).sum()
                    total_positive += (label == 1).sum()
                    total_true += (output == 1).sum()
            if self.distributed:
                dist.barrier()
                dist.all_reduce(loss_cum)
                dist.all_reduce(total_seq_len)
                dist.all_reduce(total_match)
                dist.all_reduce(true_positive)
                dist.all_reduce(total_positive)
                dist.all_reduce(total_true)
            return (loss_cum/total_seq_len).item(), \
                   (total_match/total_seq_len).item(), \
                   (true_positive/total_positive).item(), \
                   (true_positive/total_true).item()
        else:
            for encoder_input, decoder_input, tgt, label, label_mask, _ in self.eval_dl:
                with torch.no_grad():
                    with autocast():
                        output = self.model(encoder_input=encoder_input, decoder_input=decoder_input, tgt=tgt)
                        if self.cfg.task == 'optimal_path': output = output.log_softmax(-1)
                        loss = self.eval_loss_fn(output[label_mask],label[label_mask])
                    loss_cum += loss
                    total_seq_len += label_mask.sum()
            if self.distributed:
                dist.barrier()
                dist.all_reduce(loss_cum)
                dist.all_reduce(total_seq_len)
            return (loss_cum/total_seq_len).item()

    def inference(self) -> float:
        if self.cfg.task == 'optimal_path':
            optimal_path_infer(self.cfg, self.test_spec_header, self.test_dl, self.model, self.device)
        elif self.cfg.task == 'sequence_generation':
            seq_generation_infer(self.cfg, self.test_spec_header, self.test_dl, self.model, self.device)


"""
Projections in SQL
"""

def sql_proj(dbname, tbl, otbl, oepsg, cols=None, geomCol=None,
    newGeom=None, whr=None, new_pk=None):
    """
    Reproject geometric layer to another spatial reference system (srs)
    """

    from glass.pys      import obj_to_lst
    from glass.sql.q import q_to_ntbl

    geomCol = 'geom' if not geomCol else geomCol
    newGeom = 'geom' if not newGeom else newGeom

    if not cols:
        from glass.prop.sql import cols_name

        cols = cols_name(dbname, tbl)

        cols.remove(geomCol)

    else:
        cols = obj_to_lst(cols)

        if geomCol in cols and geomCol == newGeom:
            cols.remove(geomCol)
            cols.append('{c} AS old_{c}'.format(c=geomCol))

    Q = (
        "SELECT {}, ST_Transform({}, {}) AS {} "
        "FROM {}{}"
    ).format(
        ", ".join(cols), geomCol, str(oepsg), newGeom, tbl,
        "" if not whr else " WHERE {}".format(whr)
    )

    otbl = q_to_ntbl(dbname, otbl, Q, api='psql')

    if new_pk:
        from glass.sql.k  import create_pk

        create_pk(dbname, otbl, new_pk)

    return otbl


# 管理员登录session键名
SESSION_LOGIN = "SESSION_LOGIN"

# 用户头像地址
SESSION_HEAD_PORTRAIT = "SESSION_HEAD_PORTRAIT"

# 登录账号
SESSION_USER_ID = 'session_user_id'

# 用户是否为开发者
SESSION_IS_DEVELOPER = "SESSION_IS_DEVELOPER"

# 用户类型
SESSION_USER_TYPE = "SESSION_USER_TYPE"

# 登录重定向uri
SESSION_REDIRECT_URI = "SESSION_REDIRECT_URI"

# 登录次数(失败一次后需要输入验证码)
SESSION_LOGIN_TIME = "SESSION_LOGIN_TIME"

# 验证码
SESSION_LOGIN_VALIDATE = "SESSION_LOGIN_VALIDATE"

# 第三方登录标志（包括设备管理系统厂商帐号）
SESSION_LOGIN_THIRD = "SESSION_LOGIN_THIRD"

# 注册成功标志
SESSION_REGISTER_SUCCESS = "SESSION_REGISTER_SUCCESS"

#################################################################
# 记住登录账户的cookie key
COOKIE_USER_ACCOUNT = "COOKIE_USER_ACCOUNT"

# 自动登录token的cookie key
AUTO_LOGIN = "token"


# coding=utf-8

madlib_template = """I enjoy long, {0} walks on the beach, getting {1} in the rain and serendipitous encounters with {2}.
I really like pina coladas mixed with {3}, and romantic, candle-lit {4}.
I am looking for {5} and beauty in the form of a {6} goddess.
I would prefer if she knew how to cook, clean, and wash my {7}."""

def GetWord(prompt, completion):
	word = raw_input(prompt)
	print("You are {}% complete!".format(completion/8.0*100.0))
	return word

adjective = GetWord("Adjective: ", 1)
verb = GetWord("Verb Ending in 'ed': ", 2)
pluralNoun1 = GetWord("Plural Noun: ", 3)
liquid = GetWord("Liquid: ", 4)
pluralNoun2 = GetWord("Plural Noun: ", 5)
noun = GetWord("Noun: ", 6)
nationality = GetWord("Nationality: ", 7)
pluralNoun3 = GetWord("Plural Noun: ", 8)

print(madlib_template.format(adjective, verb, pluralNoun1, liquid, pluralNoun2, noun, nationality, pluralNoun3))


import pathlib2
import os
import wx

from option_loader import OptHandle

class EditFrame(wx.Frame):

    def __init__(self, opt_instance:OptHandle):
        super().__init__(parent = None, title="Edit Panel")
        self.opt_handle = opt_instance
        self.setup_panel()
        self.SetSize(0, 0, 500, 750)
        self.Center()

    def setup_panel(self):
        panel = wx.Panel(self)
        panel.BackgroundColour = wx.Colour(50, 100, 255)
        sizer = wx.BoxSizer(wx.VERTICAL)
        panel.SetSizer(sizer)
        flex_grid = wx.FlexGridSizer(2, len(self.opt_handle.dict), 2)
        sizer.Add(flex_grid, wx.SizerFlags().Center().Expand())

        for key_value_pair in self.opt_handle.dict.items():
            key, value = key_value_pair

            flex_grid.Add(wx.StaticText(panel, label = str(key)), wx.SizerFlags().Center())
            tmp_sizer = wx.BoxSizer()
            tmp_sizer.Add(wx.StaticText(panel, label = str(value)), wx.SizerFlags().Center())
            button = wx.Button(panel, label = "Change Hotkey")
            tmp_sizer.Add(button)
            flex_grid.Add(tmp_sizer, wx.SizerFlags().Right())
            """
            row_sizer = wx.BoxSizer()
            sizer.Add(row_sizer, wx.SizerFlags().Expand())

            left_sizer = wx.BoxSizer()
            left_label = wx.StaticText(panel, label = str(key))
            left_sizer.Add(left_label)
            row_sizer.Add(left_sizer, flags = wx.SizerFlags().Border(wx.LEFT, 20).Center())

            right_sizer = wx.BoxSizer()
            right_label = wx.StaticText(panel, label = str(value))
            right_button = wx.Button(panel, label = "Change Key")
            right_sizer.Add(right_label, flags = wx.SizerFlags())
            right_sizer.Add(right_button, flags = wx.SizerFlags())
            row_sizer.Add(right_sizer, flags = wx.SizerFlags().Right())
            """

if __name__ == "__main__":
    app = wx.App()
    frame = EditFrame(OptHandle())
    frame.Show()
    app.MainLoop()


from django.conf.urls import patterns, url
from django.core.urlresolvers import reverse_lazy
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from . import views
from models import Entry

urlpatterns = patterns(
    '',
    url(r'^(?P<slug>\S+)/copy/$', views.copy_blog, name='blog_copy'),
    url(r'^folders/info/', views.folderview, name='folder_view'),
    url(r'^folders/$', views.TagList.as_view(), name='folders'),
    url(r'^new-folder/$', views.TagCreate.as_view(), name='tag_new'),
    url(r'^(?P<slug>\S+)/edit/$', views.BlogUpdate.as_view(), name='blog_edit'),
    url(r'^new-user/$', views.UserCreate.as_view(), name='user_new'),
    url(r'^new-post/$', views.EntryCreate.as_view(), name='entry_new'),
    url(r'^(?P<slug>\S+)/delete/$', views.EntryDelete.as_view(), name='blog_delete'),
    url(r'^tag-delete/(?P<slug>\S+)$', views.TagDelete.as_view(), name='tag_delete'),
    url(r'^tag-edit/(?P<slug>\S+)$', views.TagUpdate.as_view(), name='tag_edit'),
    url(r'^search-form/$', views.search_form, name='search_form'),
    url(r'^search/$', views.search, name='search'),
    url(r'^login$', 'django.contrib.auth.views.login', {'template_name': 'login.html'}, name='user_login'),
    url(r'^logout$', 'django.contrib.auth.views.logout_then_login', {'login_url': reverse_lazy('user_login')}, name='user_logout'),
    url(r'^$', views.BlogIndex.as_view(), name="index"),
    url(r'^order_by_title/$', views.BlogIndex.as_view(queryset=Entry.objects.order_by('title')), name='by_title'),
    url(r'^order_by_slug/$', views.BlogIndex.as_view(queryset=Entry.objects.order_by('slug')), name='by_slug'),
    url(r'^order_by_body/$', views.BlogIndex.as_view(queryset=Entry.objects.order_by('body')), name='by_body'),
    url(r'^date_desc/$', views.BlogIndex.as_view(queryset=Entry.objects.order_by('created')), name='by_date_desc'),
    url(r'^date_ascd/$', views.BlogIndex.as_view(queryset=Entry.objects.order_by('-created')), name='index'),
    url(r'^(?P<slug>\S+)/decrypt_form/$', views.decrypt_form, name="decrypt_form"),
    url(r'/decrypt/$', views.decrypt, name="decrypt"),
    url(r'^(?P<slug>\S+)$', views.BlogDetail.as_view(), name="entry_detail"),

)


from torch.utils.data import Dataset, DataLoader
from torchvision import transforms
import os
import random
import matplotlib.pyplot as plt
from PIL import Image
import torch
from torchvision.models import resnet18, ResNet18_Weights
import torch.nn as nn
import numpy as np

class Mydata(Dataset):
    def __init__(self, data_path, transform, label):
        super(Mydata, self).__init__()
        self.transform = transform
        self.label = label
        self.data_path = data_path

    def __len__(self):
        return len(self.data_path)

    def __getitem__(self, idx):
        img = Image.fromarray(plt.imread(self.data_path[idx]))
        img = self.transform(img)
        label = self.label[idx]
        return img, label

def visualize(labels, data_path):
    fig = plt.figure(figsize=(9, 9))
    rows, cols = 4, 4

    d_label = {
        0: 'Cat',
        1: 'Dog',
    }

    for i in range(1, rows * cols + 1):
        random_idx = random.randint(0, len(data_path) - 1)
        img = plt.imread(data_path[random_idx])
        label = d_label[labels[random_idx]]
        fig.add_subplot(rows, cols, i)
        plt.imshow(img)
        plt.title(label)
        plt.axis(False)
    plt.show()

def get_model():
    model = resnet18(weights=ResNet18_Weights.IMAGENET1K_V1)
    model.fc = nn.Sequential(
        nn.Linear(512, 1),
        nn.Sigmoid()
    )
    for name, param in model.named_parameters():
        if 'fc' not in name:
            param.requires_grad = False
    return model

def get_data():
    train_path = [os.path.join("train", name) for name in os.listdir("train")]
    cat_train_path = [x for x in train_path if 'cat' in x]
    dog_train_path = [x for x in train_path if 'dog' in x]
    train_data_path = cat_train_path[:int(len(cat_train_path) * 0.8)] + dog_train_path[:int(len(dog_train_path) * 0.8)]
    val_data_path = cat_train_path[int(len(cat_train_path) * 0.8):] + dog_train_path[int(len(dog_train_path) * 0.8):]
    random.shuffle(train_data_path)
    random.shuffle(val_data_path)
    labels_train = [0 if 'cat' in name else 1 for name in train_data_path]
    labels_val = [0 if 'cat' in name else 1 for name in val_data_path]
    return train_data_path, val_data_path, labels_train, labels_val

def get_dataloader():
    train_data_path, val_data_path, labels_train, labels_val = get_data()
    transform = transforms.Compose([
        transforms.Resize(size=(224, 224)),
        transforms.ToTensor(),
        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
    ])

    train_data = Mydata(train_data_path, transform=transform, label=labels_train)
    val_data = Mydata(val_data_path, transform=transform, label=labels_val)
    train_loader = DataLoader(train_data, shuffle=True, batch_size=16, num_workers=2)
    val_loader = DataLoader(val_data, shuffle=True, batch_size=16, num_workers=2)
    return train_loader, val_loader

def train_step(model,
               dataloader,
               loss_fn,
               optimizer,
               device):
    model.train()

    train_loss, train_acc = 0, 0

    for batch, (X, y) in enumerate(dataloader):
        X, y = X.to(device), y.to(device)
        y_pred = model(X)
        y_pred = y_pred.squeeze(-1)

        loss = loss_fn(y_pred, y.float())
        train_loss += loss.item()
        optimizer.zero_grad()

        loss.backward()

        optimizer.step()

        y_pred_class = torch.tensor([0 if x < 0.5 else 1 for x in y_pred]).to(device)
        train_acc += (y_pred_class == y).sum().item() / len(y_pred)
        if batch % 100 == 0:
            print(y_pred.squeeze(-1), y)
            print()
            print(
                "Batch {} Loss {:.4f} Train Acc {}".format(batch, loss, (y_pred_class == y).sum().item() / len(y_pred)))
    train_loss = train_loss / len(dataloader)
    train_acc = train_acc / len(dataloader)
    return train_loss, train_acc

def val_step(model,
             dataloader,
             loss_fn: torch.nn.Module,
             device: torch.device):
    model.eval()

    test_loss, test_acc = 0, 0

    with torch.no_grad():
        for batch, (X, y) in enumerate(dataloader):
            X, y = X.to(device), y.to(device)

            test_pred_logits = model(X)
            test_pred_logits = test_pred_logits.squeeze(-1)
            loss = loss_fn(test_pred_logits, y.float())
            test_loss += loss.item()

            test_pred_labels = torch.tensor([0 if x < 0.5 else 1 for x in test_pred_logits]).to(device)
            test_acc += ((test_pred_labels == y).sum().item() / len(test_pred_labels))

    test_loss = test_loss / len(dataloader)
    test_acc = test_acc / len(dataloader)
    return test_loss, test_acc

if __name__ == '__main__':
    # Config
    model = get_model()
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    num_epoches = 10
    optimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=3e-5)
    loss = nn.BCELoss()
    model = model.to(device)
    # Data
    train_loader, val_loader = get_dataloader()
    train_losses = []
    train_accs = []
    test_losses = []
    test_accs = []
    for epoch in range(num_epoches):
        train_loss, train_acc = train_step(model, train_loader, loss, optimizer, device)
        print("Epoch {} Train_loss {} Train_acc {}".format(epoch, train_loss, train_acc))
        train_losses.append(train_loss)
        train_accs.append(train_acc)
        test_loss, test_acc = val_step(model, val_loader, loss, device)
        print("Epoch {} Test_loss {} Test_acc {}".format(epoch, test_loss, test_acc))
        test_losses.append(test_loss)
        test_accs.append(test_acc)
        print("Done Epoch {}".format(epoch))
        # print("________________________________________________________________")
    torch.save(model.state_dict(), "Weight/model.pt")
    np.save('Accuracy/train_losses.npy', np.array(train_losses))
    np.save('Accuracy/train_accs.npy', np.array(train_accs))
    np.save('Accuracy/test_losses.npy', np.array(test_losses))
    np.save('Accuracy/test_accs.npy', np.array(test_accs))


from __future__ import annotations
from dataclasses import dataclass
import bson.json_util as json

__all__ = ['Node', 'build_execution_tree']

@dataclass
class Node:
    """Represent SBE tree node."""

    stage: str
    plan_node_id: int
    total_execution_time: int
    n_returned: int
    n_processed: int
    children: list[Node]

    def get_execution_time(self):
        """Execution time of the SBE node without execuion time of its children."""
        return self.total_execution_time - sum(n.total_execution_time for n in self.children)

    def print(self, level=0):
        """Pretty print of the SBE tree."""
        print(
            f'{"| "*level}{self.stage}, plaNodeId: {self.plan_node_id}, totalExecutionTime: {self.total_execution_time:,}, nReturned: {self.n_returned}, nProcessed: {self.n_processed}'
        )
        for child in self.children:
            child.print(level + 1)

def build_execution_tree(execution_stats: dict[str, any]) -> Node:
    """Build SBE executioon tree from 'executionStats' field of query explain."""
    assert execution_stats['executionSuccess']
    return process_stage(execution_stats['executionStages'])

def process_stage(stage: dict[str, any]) -> Node:
    """Parse the given SBE stage."""
    processors = {
        'filter': process_filter,
        'cfilter': process_filter,
        'traverse': process_traverse,
        'project': process_inner_node,
        'limit': process_inner_node,
        'scan': process_seek,
        'coscan': process_leaf_node,
        'nlj': process_nlj,
        'hj': process_hash_join_node,
        'mj': process_hash_join_node,
        'seek': process_seek,
        'ixseek': process_seek,
        'limitskip': process_inner_node,
        'group': process_inner_node,
        'union': process_union_node,
        'unique': process_unique_node,
        'unwind': process_unwind_node,
    }

    processor = processors.get(stage['stage'])
    if processor is None:
        print(json.dumps(stage, indent=4))
        raise ValueError(f'Unknown stage: {stage}')

    return processor(stage)

def process_filter(stage: dict[str, any]) -> Node:
    """Process filter stage."""
    input_stage = process_stage(stage['inputStage'])
    return Node(**get_common_fields(stage), n_processed=stage['numTested'], children=[input_stage])

def process_traverse(stage: dict[str, any]) -> Node:
    """Process traverse, not used by Bonsai."""
    outer_stage = process_stage(stage['outerStage'])
    inner_stage = process_stage(stage['innerStage'])
    return Node(**get_common_fields(stage), n_processed=stage['nReturned'],
                children=[outer_stage, inner_stage])

def process_hash_join_node(stage: dict[str, any]) -> Node:
    """Process hj node."""
    outer_stage = process_stage(stage['outerStage'])
    inner_stage = process_stage(stage['innerStage'])
    n_processed = outer_stage.n_returned + inner_stage.n_returned
    return Node(**get_common_fields(stage), n_processed=n_processed,
                children=[outer_stage, inner_stage])

def process_nlj(stage: dict[str, any]) -> Node:
    """Process nlj stage."""
    outer_stage = process_stage(stage['outerStage'])
    inner_stage = process_stage(stage['innerStage'])
    n_processed = stage['totalDocsExamined']
    return Node(**get_common_fields(stage), n_processed=n_processed,
                children=[outer_stage, inner_stage])

def process_inner_node(stage: dict[str, any]) -> Node:
    """Process SBE stage with one input stage."""
    input_stage = process_stage(stage['inputStage'])
    return Node(**get_common_fields(stage), n_processed=input_stage.n_returned,
                children=[input_stage])

def process_leaf_node(stage: dict[str, any]) -> Node:
    """Process SBE stage without input stages."""
    return Node(**get_common_fields(stage), n_processed=stage['nReturned'], children=[])

def process_seek(stage: dict[str, any]) -> Node:
    """Process seek stage."""
    return Node(**get_common_fields(stage), n_processed=stage['numReads'], children=[])

def process_union_node(stage: dict[str, any]) -> Node:
    """Process union stage."""
    children = [process_stage(child) for child in stage['inputStages']]
    return Node(**get_common_fields(stage), n_processed=stage['nReturned'], children=children)

def process_unwind_node(stage: dict[str, any]) -> Node:
    """Process unwind stage."""
    input_stage = process_stage(stage['inputStage'])
    return Node(**get_common_fields(stage), n_processed=input_stage.n_returned,
                children=[input_stage])

def process_unique_node(stage: dict[str, any]) -> Node:
    """Process unique stage."""
    input_stage = process_stage(stage['inputStage'])
    n_processed = stage['dupsTested']
    return Node(**get_common_fields(stage), n_processed=n_processed, children=[input_stage])

def get_common_fields(json_stage: dict[str, any]) -> dict[str, any]:
    """Exctract common field from json representation of SBE stage."""
    return {
        'stage': json_stage['stage'], 'plan_node_id': json_stage['planNodeId'],
        'total_execution_time': json_stage['executionTimeNanos'],
        'n_returned': json_stage['nReturned']
    }


import json
from pprint import pprint

with open('rpPurchases.json') as f:
    data = json.load(f)

l = []

for idx in data:
    l.append((idx["amount"],idx["paymentType"]))

rp = 0

for a in l:
	rp += a[0]

print(rp)


from typing import List, Tuple, Union

import lightgbm as lgb
import numpy as np
import pandas as pd

import mlrun.errors

from .._ml_common import AlgorithmFunctionality, MLTypes, MLUtils

class LGBMTypes(MLTypes):
    """
    Typing hints for the LightGBM framework.
    """

    # A union of all LightGBM model base classes:
    ModelType = Union[lgb.LGBMModel, lgb.Booster]

    # A type for all the supported dataset types:
    DatasetType = Union[MLTypes.DatasetType, lgb.Dataset]

    # An evaluation result as packaged by the training in LightGBM:
    EvaluationResultType = Union[
        Tuple[str, str, float, bool],  # As packaged in `lightgbm.train`
        Tuple[str, str, float, bool, float],  # As packaged in `lightgbm.cv`
    ]

    # Detailed type for the named tuple `CallbackEnv` passed during LightGBM's training for the callbacks:
    CallbackEnvType = Tuple[
        lgb.Booster, dict, int, int, int, List[EvaluationResultType]
    ]

class LGBMUtils(MLUtils):
    """
    Utilities functions for the LightGBM framework.
    """

    @staticmethod
    def to_array(dataset: LGBMTypes.DatasetType) -> np.ndarray:
        """
        Convert the given dataset to np.ndarray.

        :param dataset: The dataset to convert. Must be one of {lgb.Dataset, pd.DataFrame, pd.Series,
                        scipy.sparse.base.spmatrix, list, tuple, dict}.

        :return: The dataset as a ndarray.

        :raise MLRunInvalidArgumentError: If the dataset type is not supported.
        """
        if isinstance(dataset, lgb.Dataset):
            x = LGBMUtils.to_array(dataset=dataset.data)
            if dataset.label is None:
                return x
            y = LGBMUtils.to_array(dataset=dataset.label)
            return LGBMUtils.to_array(LGBMUtils.concatenate_x_y(x=x, y=y)[0])
        try:
            return MLUtils.to_array(dataset=dataset)
        except mlrun.errors.MLRunInvalidArgumentError:
            raise mlrun.errors.MLRunInvalidArgumentError(
                f"Could not convert the given dataset into a numpy ndarray. Supporting conversion from: "
                f"{LGBMUtils.get_union_typehint_string(LGBMTypes.DatasetType)}. The given dataset was of type: "
                f"'{type(dataset)}'"
            )

    @staticmethod
    def to_dataframe(dataset: LGBMTypes.DatasetType) -> pd.DataFrame:
        """
        Convert the given dataset to pd.DataFrame.

        :param dataset: The dataset to convert. Must be one of {lgb.Dataset, np.ndarray, pd.Series,
                        scipy.sparse.base.spmatrix, list, tuple, dict}.

        :return: The dataset as a DataFrame.

        :raise MLRunInvalidArgumentError: If the dataset type is not supported.
        """
        if isinstance(dataset, lgb.Dataset):
            x = LGBMUtils.to_dataframe(dataset=dataset.data)
            if dataset.label is None:
                return x
            y = LGBMUtils.to_dataframe(dataset=dataset.label)
            return LGBMUtils.concatenate_x_y(x=x, y=y)[0]
        try:
            return MLUtils.to_dataframe(dataset=dataset)
        except mlrun.errors.MLRunInvalidArgumentError:
            raise mlrun.errors.MLRunInvalidArgumentError(
                f"Could not convert the given dataset into a pandas DataFrame. Supporting conversion from: "
                f"{LGBMUtils.get_union_typehint_string(LGBMTypes.DatasetType)}. The given dataset was of type: "
                f"'{type(dataset)}'"
            )

    @staticmethod
    def get_algorithm_functionality(
        model: MLTypes.ModelType = None,
        y: MLTypes.DatasetType = None,
        objective: str = None,
    ) -> AlgorithmFunctionality:
        """
        Get the algorithm functionality of the LightGBM model. If SciKit-Learn API is used, pass the LGBBMModel and a y
        sample. Otherwise, training API is used, so pass the objective of the params dictionary.

        The objectives here are taken from the official docs of LightGBBM at:
        https://lightgbm.readthedocs.io/en/latest/Parameters.html#core-parameters

        :param model:     The model to check if its a regression model or a classification model (SciKit-Learn API).
        :param y:         The ground truth values to check if its multiclass and / or multi output (SciKit-Learn API).
        :param objective: The objective string (Training API).

        :return: The objective's algorithm functionality.
        """
        # Check if LightGBM is being used with SciKit-Learn API:
        if objective is None:
            return super().get_algorithm_functionality(model=model, y=y)

        # Declare the conversion map according to the LightGBM docs:
        objective_to_algorithm_functionality_map = {
            # regression application:
            "regression": AlgorithmFunctionality.REGRESSION,
            "regression_l2": AlgorithmFunctionality.REGRESSION,
            "l2": AlgorithmFunctionality.REGRESSION,
            "mean_squared_error": AlgorithmFunctionality.REGRESSION,
            "mse": AlgorithmFunctionality.REGRESSION,
            "l2_root": AlgorithmFunctionality.REGRESSION,
            "root_mean_squared_error": AlgorithmFunctionality.REGRESSION,
            "rmse": AlgorithmFunctionality.REGRESSION,
            "regression_l1": AlgorithmFunctionality.REGRESSION,
            "l1": AlgorithmFunctionality.REGRESSION,
            "mean_absolute_error": AlgorithmFunctionality.REGRESSION,
            "mae": AlgorithmFunctionality.REGRESSION,
            "huber": AlgorithmFunctionality.REGRESSION,
            "fair": AlgorithmFunctionality.REGRESSION,
            "poisson": AlgorithmFunctionality.REGRESSION,
            "quantile": AlgorithmFunctionality.REGRESSION,
            "mape": AlgorithmFunctionality.REGRESSION,
            "mean_absolute_percentage_error": AlgorithmFunctionality.REGRESSION,
            "gamma": AlgorithmFunctionality.REGRESSION,
            "tweedie": AlgorithmFunctionality.REGRESSION,
            # binary classification application:
            "binary": AlgorithmFunctionality.BINARY_CLASSIFICATION,
            # multi-class classification application:
            "multiclass": AlgorithmFunctionality.MULTICLASS_CLASSIFICATION,
            "softmax": AlgorithmFunctionality.MULTICLASS_CLASSIFICATION,
            "multiclassova": AlgorithmFunctionality.MULTICLASS_CLASSIFICATION,
            "multiclass_ova": AlgorithmFunctionality.MULTICLASS_CLASSIFICATION,
            "ova": AlgorithmFunctionality.MULTICLASS_CLASSIFICATION,
            "ovr": AlgorithmFunctionality.MULTICLASS_CLASSIFICATION,
            # cross-entropy application
            "cross_entropy": AlgorithmFunctionality.BINARY_CLASSIFICATION,
            "xentropy": AlgorithmFunctionality.BINARY_CLASSIFICATION,
            "cross_entropy_lambda": AlgorithmFunctionality.BINARY_CLASSIFICATION,
            "xentlambda": AlgorithmFunctionality.BINARY_CLASSIFICATION,
            # ranking application
            "lambdarank": AlgorithmFunctionality.MULTICLASS_CLASSIFICATION,
            "rank_xendcg": AlgorithmFunctionality.MULTICLASS_CLASSIFICATION,
            "xendcg": AlgorithmFunctionality.MULTICLASS_CLASSIFICATION,
            "xe_ndcg": AlgorithmFunctionality.MULTICLASS_CLASSIFICATION,
            "xe_ndcg_mart": AlgorithmFunctionality.MULTICLASS_CLASSIFICATION,
            "xendcg_mart": AlgorithmFunctionality.MULTICLASS_CLASSIFICATION,
        }

        # Return unknown if the objective is not in the map and otherwise return its functionality:
        if objective not in objective_to_algorithm_functionality_map:
            raise AlgorithmFunctionality.UNKNOWN
        return objective_to_algorithm_functionality_map[objective]


import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import fsolve

a = 2
#b = 1

def plotter(b):
	func = lambda x : a - b*x - np.exp(-x)
	guess = a/b
	max_x = fsolve(func, guess)
	x = np.arange(0.0, max_x*1.05, 0.01)
	y1 = a - b*x
	y2 = np.exp(-x)
	y3 = y1 - y2
	null = 0*x
	plt.figure()
	plt.fill_between(x, y1, y2)
	plt.plot(x, y3, color="yellow", label="difference")
	plt.plot(x, null, "--b")
	plt.title('b = ' + str(b))
	plt.legend(loc="best")
	plt.xlabel('time')
	plt.ylabel('P(t)')
	plt.savefig("fig/SIR_b_" + str(b) + ".pdf")

def stability_plotter(b):
	func = lambda x : a - b*x - np.exp(-x)
	guess = a/b
	max_x = fsolve(func, guess)
	x = np.arange(0.0, max_x*1.2, 0.01)
	y = a - b*x - np.exp(-x)
	null = 0*x
	# plt.figure()
	# plt.fill_between(x, y1, y2)
	plt.plot(x, y, label='b = ' + str(b))
	plt.plot(x, null, "--b")

b_list = [0.2, 0.5, 0.8, 1, 1.2, 1.5, 2] #, 5, 10]
for b in b_list:
	plotter(b)

plt.figure()
for b in b_list:
	stability_plotter(b)
plt.title("Stability")
plt.legend(loc="best")
plt.xlabel('$u(\\tau)$')
plt.ylabel('$du/d\\tau$')
# plt.xscale("log")
plt.savefig("fig/SIR_stability.pdf")

# plot u* as func of b
u_list = []
b_max = 2
b_list = np.arange(0.01, b_max, 0.01)
for b in b_list:
	func = lambda x : a - b*x - np.exp(-x)
	guess = a/b
	u_eq = fsolve(func, guess)
	u_list.append(u_eq)
plt.figure()
plt.plot(b_list, u_list)
plt.title("equilibrium of u, parametrized by 'b'")
plt.legend(loc="best")
plt.xlabel('$b$')
plt.ylabel('$u*$')
# plt.xscale("log")
plt.savefig("fig/SIR_equilibrium_u_b.pdf")


import logging

class CustomFormatter(logging.Formatter):
    """Logging Formatter to add colors and count warning / errors"""

    cyan = "\u001b[36m"
    green = "\u001b[32m"
    yellow = "\u001b[33m"
    red = "\u001b[35m"
    bold_red = "\u001b[31m"
    reset = "\u001b[0m"
    debug_format = "%(asctime)s - %(name)s - {colour}%(levelname)s - %(message)s\u001b[0m (%(filename)s:%(lineno)d)"
    norm_format = "%(asctime)s - %(name)s - {colour}%(levelname)s - %(message)s\u001b[0m"

    FORMATS = {
        logging.DEBUG: debug_format.format(colour=cyan),
        logging.INFO: norm_format.format(colour=green),
        logging.WARNING: debug_format.format(colour=yellow),
        logging.ERROR: debug_format.format(colour=red),
        logging.CRITICAL: debug_format.format(colour=bold_red)
    }

    def format(self, record):
        log_fmt = self.FORMATS.get(record.levelno)
        formatter = logging.Formatter(log_fmt, datefmt="%Y-%m-%d %H:%M:%S")
        return formatter.format(record)

def get_logger(module_name, output_file, terminal_level=logging.DEBUG, file_level=logging.DEBUG):
    logger = logging.getLogger(module_name)
    logger.setLevel(logging.DEBUG)

    terminal_handler = logging.StreamHandler()
    terminal_handler.setLevel(terminal_level)
    terminal_handler.setFormatter(CustomFormatter())
    logger.addHandler(terminal_handler)

    file_handler = logging.FileHandler(output_file)
    file_handler.setLevel(file_level)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s', datefmt="%Y-%m-%d %H:%M:%S")
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)
    return logger