from torch_geometric.nn.conv import MessagePassing
from torch_geometric.nn.dense.linear import Linear
from torch_geometric.typing import Adj, OptPairTensor, OptTensor, Size
from torch_geometric.utils.repeat import repeat
import torch
from torch_sparse import SparseTensor
from torch import Tensor, nn
from torch.nn import Parameter

from cutlass import cutlass

import math
import numpy as np
from typing import Any
from typing import List, Tuple, Union

def uniform(size: int, value: Any):
    if isinstance(value, Tensor):
        bound = 1.0 / math.sqrt(size)
        value.data.uniform_(-bound, bound)
    else:
        for v in value.parameters() if hasattr(value, 'parameters') else []:
            uniform(size, v)
        for v in value.buffers() if hasattr(value, 'buffers') else []:
            uniform(size, v)

def constant(value: Any, fill_value: float):
    if isinstance(value, Tensor):
        value.data.fill_(fill_value)
    else:
        for v in value.parameters() if hasattr(value, 'parameters') else []:
            constant(v, fill_value)
        for v in value.buffers() if hasattr(value, 'buffers') else []:
            constant(v, fill_value)
def zeros(value: Any):
    constant(value, 0.)

basestring = (str, bytes)
def is_list_of_strings(lst):
        if lst and isinstance(lst, list):
            return all(isinstance(elem, basestring) for elem in lst)
        else:
            return False
from cutlass import *
import scipy.optimize

MCache = None

def optimizeWeights2D(weights, basis, periodicity, nmc = 32 * 1024, targetIntegral = 1, windowFn = None, verbose = False):
    global MCache
    M = None
    numWeights = weights.shape[0] * weights.shape[1]

    # print(weights.shape, numWeights)
    normalizedWeights = (weights - torch.sum(weights) / weights.numel())/torch.std(weights)
    if not MCache is None:
        cfg, M = MCache
        w,b,n,p,wfn = cfg
        if not(w == weights.shape and np.all(b == basis) and n == nmc and np.all(p ==periodicity) and wfn == windowFn):
            M = None
    # else:
        # print('no cache')
    if M is None:
        r = torch.sqrt(torch.rand(size=(nmc,1)).to(weights.device).type(torch.float32))
        theta = torch.rand(size=(nmc,1)).to(weights.device).type(torch.float32) *2 * np.pi

        x = r * torch.cos(theta)
        y = r * torch.sin(theta)

        u = evalBasisFunction(weights.shape[0], x.T, which = basis[0], periodic = periodicity[0])[0,:].mT
        v = evalBasisFunction(weights.shape[1], y.T, which = basis[1], periodic = periodicity[1])[0,:].mT

    #     print('u', u.shape, u)
    #     print('v', v.shape, v)

        window = weights.new_ones(x.shape[0]) if windowFn is None else windowFn(torch.sqrt(x**2 + y**2))[:,0]

        nuv = torch.einsum('nu, nv -> nuv', u, v)
        nuv = nuv * window[:,None, None]

    #     print('nuv', nuv.shape, nuv)
        M = np.pi * torch.sum(nuv, dim = 0).flatten().detach().cpu().numpy() / nmc
#     print('M', M.shape, M)
        MCache = ((weights.shape, basis, nmc, periodicity, windowFn), M)

    w = normalizedWeights.flatten().detach().cpu().numpy()

    eps = 1e-2

    if 'chebyshev' in basis or 'fourier' in basis:
        res = scipy.optimize.minimize(fun = lambda x: (M.dot(x) - targetIntegral)**2, \
                                      jac = lambda x: 2 * M * (M.dot(x) - targetIntegral), \
                                      hess = lambda x: 2. * np.outer(M,M), x0 = w, \
                                      method ='trust-constr', constraints = None,\
                                      options={'disp': False, 'maxiter':100})
    else:
        sumConstraint = scipy.optimize.NonlinearConstraint(fun = np.sum, lb = -eps, ub = eps)
        stdConstraint = scipy.optimize.NonlinearConstraint(fun = np.std, lb = 1 - eps, ub = 1 + eps)

        res = scipy.optimize.minimize(fun = lambda x: (M.dot(x) - targetIntegral)**2, \
                                      jac = lambda x: 2 * M * (M.dot(x) - targetIntegral), \
                                      hess = lambda x: 2. * np.outer(M,M), x0 = w, \
                                      method ='trust-constr', constraints = [sumConstraint, stdConstraint],\
                                      options={'disp': False, 'maxiter':100})
    result = torch.from_numpy(res.x.reshape(weights.shape)).type(torch.float32).to(weights.device)
    if verbose:
        print('result: ', res)
        print('initial weights:', normalizedWeights)
        print('result weights:',result)
        print('initial:', M.dot(w))
        print('integral:', M.dot(res.x))
        print('sumConstraint:', np.sum(res.x))
        print('stdConstraint:', np.std(res.x))
    return result, res.constr, res.fun, M.dot(w), M.dot(res.x)

def mapToSpherical(positions):
    x = positions[:,0]
    y = positions[:,1]
    z = positions[:,2]
    r = torch.sqrt(x**2 + y**2 + z**2)
    theta = torch.atan2(y, x)
    phi = torch.acos(z / (r + 1e-7))

    return torch.vstack((r,theta,phi)).mT

def ballToCylinder(positions):
    r = torch.linalg.norm(positions, dim = 1)
    xy = torch.linalg.norm(positions[:,:2], dim = 1)
    absz = torch.abs(positions[:,2])

#     debugPrint(r)
#     debugPrint(xy)
#     debugPrint(absz)

    x = positions[:,0]
    y = positions[:,1]
    z = positions[:,2]

    termA = torch.zeros_like(positions)

    eps = 1e-7

    xB = x * r / (xy + eps)
    yB = y * r / (xy + eps)
    zB = 3 / 2 * z
    termB = torch.vstack((xB, yB, zB)).mT

    xC = x * torch.sqrt(3 * r / (r + absz + eps))
    yC = y * torch.sqrt(3 * r / (r + absz + eps))
    zC = torch.sign(z) * r
    termC = torch.vstack((xC, yC, zC)).mT

    mapped = torch.zeros_like(positions)

    maskA = r < eps
    maskB = torch.logical_and(torch.logical_not(maskA), 5/4 * z**2 <= x**2 + y**2)
    maskC = torch.logical_and(torch.logical_not(maskA), torch.logical_not(maskB))

    mapped[maskB] = termB[maskB]
    mapped[maskC] = termC[maskC]

#     debugPrint(mapped)
    return mapped
# debugPrint(cylinderPositions)

def cylinderToCube(positions):
    x = positions[:,0]
    y = positions[:,1]
    z = positions[:,2]
    xy = torch.linalg.norm(positions[:,:2], dim = 1)
    eps = 1e-7

    termA = torch.vstack((torch.zeros_like(x), torch.zeros_like(y), z)).mT
    # debugPrint(termA)

    xB = torch.sign(x) * xy
    yB = 4. / np.pi * torch.sign(x) * xy * torch.atan(y/(x+eps))
    zB = z
    termB = torch.vstack((xB, yB, zB)).mT

    xC = 4. / np.pi * torch.sign(y) * xy * torch.atan(x / (y + eps))
    yC = torch.sign(y) * xy
    zC = z
    termC = torch.vstack((xC, yC, zC)).mT

    maskA = torch.logical_and(torch.abs(x) < eps, torch.abs(y) < eps)
    maskB = torch.logical_and(torch.logical_not(maskA), torch.abs(y) <= torch.abs(x))
    maskC = torch.logical_and(torch.logical_not(maskA), torch.logical_not(maskB))

    # debugPrint(torch.sum(maskA))
    # debugPrint(torch.sum(maskB))
    # debugPrint(torch.sum(maskC))

    mapped = torch.zeros_like(positions)
    mapped[maskA] = termA[maskA]
    mapped[maskB] = termB[maskB]
    mapped[maskC] = termC[maskC]

    return mapped

def mapToSpherePreserving(positions):
    cylinderPositions = ballToCylinder(positions)
    cubePositions = cylinderToCube(cylinderPositions)
    return cubePositions

class RbfConv(MessagePassing):
    def __init__(
        self,
        in_channels: int,
        out_channels: int,
        dim: int = 2,
        size: Union[int, List[int]] = [4, 4],
        coordinateMapping : str = 'cartesian',
        rbf : Union[int, List[int]] = 'linear',
        aggr: str = 'sum',

        linearLayer: bool = False,
        feedThrough: bool = False,
        # biasOffset: bool = False,

        preActivation = None,
        postActivation = None,

        bias = True,

        # initializer = torch.nn.init.xavier_uniform_,

        initializer = torch.nn.init.uniform_,

        batch_size = [16,16],
        windowFn = None,
        normalizeWeights = False,
        normalizationFactor = None,
        **kwargs
    ):
        super().__init__(aggr=aggr, **kwargs)
        # self.aggr = aggr
        # assert self.aggr in ['add', 'mean', 'max', None]

        # self.flow = flow
        # assert self.flow in ['source_to_target', 'target_to_source']

        # self.node_dim = node_dim

        # self.inspector = Inspector(self)
        # self.inspector.inspect(self.message)
        # self.inspector.inspect(self.aggregate, pop_first=True)
        # self.inspector.inspect(self.message_and_aggregate, pop_first=True)
        # self.inspector.inspect(self.update, pop_first=True)

        self.__user_args__ = self.inspector.keys(
            ['message', 'aggregate', 'update']).difference(self.special_args)
        self.__fused_user_args__ = self.inspector.keys(
            ['message_and_aggregate', 'update']).difference(self.special_args)

        # Support for "fused" message passing.
        self.fuse = self.inspector.implements('message_and_aggregate')

        # Support for GNNExplainer.
        self.__explain__ = False
        self.__edge_mask__ = None
        self.in_channels = in_channels
        self.out_channels = out_channels

        self.dim = dim
        self.coordinateMapping = coordinateMapping
        # print('coordinate mapping', self.coordinateMapping)
        self.size = size if isinstance(size, list) else repeat(size, dim)
        self.rbfs = rbf if is_list_of_strings(rbf) else [rbf] * dim
        self.periodic = [False, False] if coordinateMapping != 'polar' else [False,True]
        self.initializer = initializer
        self.batchSize = batch_size
        self.feedThrough = feedThrough
        self.preActivation = None if preActivation is None else getattr(nn.functional, preActivation)
        self.postActivation = None if postActivation is None else getattr(nn.functional, postActivation)
        self.windowFn = windowFn
        self.use_bias = bias
        # print('Creating layer %d -> %d features'%( in_channels, out_channels))
        # print('For dimensionality: %d'% dim)
        # print('Parameters:')
        # print('\tRBF: ', self.rbfs)
        # print('\tSize: ', self.size)
        # print('\tPeriodic: ', self.periodic)

        if isinstance(in_channels, int):
            in_channels = (in_channels, in_channels)

        if self.use_bias:
            self.bias = Parameter(torch.zeros(out_channels))
        else:
            self.register_parameter('bias', None)

        self.K = torch.tensor(self.size).prod().item()
        if dim == 1:
            self.weight = Parameter(torch.Tensor(self.size[0], in_channels[0], out_channels))
        if dim == 2:
            self.weight = Parameter(torch.Tensor(self.size[0],self.size[1], in_channels[0], out_channels))
        if dim == 3:
            self.weight = Parameter(torch.Tensor(self.size[0],self.size[1], self.size[2], in_channels[0], out_channels))
        initializer(self.weight, -0.05, 0.05)
        with torch.no_grad():
            if self.rbfs[0] in ['chebyshev', 'fourier', 'gabor']:
                for i in range(self.dim):
                    if len(self.rbfs) == 1:
                        self.weight[i] *= np.exp(-i)
                    if len(self.rbfs) == 2:
                        self.weight[i,:] *= np.exp(-i)
                    if len(self.rbfs) == 3:
                        self.weight[i,:,:] *= np.exp(-i)
            if len(self.rbfs) > 1 and self.rbfs[1] in ['chebyshev', 'fourier', 'gabor']:
                for i in range(self.dim):
                    if len(self.rbfs) == 2:
                        self.weight[:,i] *= np.exp(-i)
                    if len(self.rbfs) == 3:
                        self.weight[:,i,:] *= np.exp(-i)
            if len(self.rbfs) > 2 and self.rbfs[2] in ['chebyshev', 'fourier', 'gabor']:
                for i in range(self.dim):
                    self.weight[:,:,i] = self.weight[:,:,i] * np.exp(-i)
            if normalizeWeights:
                if len(self.rbfs) == 2:
                    print('Starting normalization')
                    for i in range(in_channels[0]):
                        for j in range(out_channels):
                            newWeights, _, _, init, final = optimizeWeights2D(weights = self.weight[:,:,i,j].detach(),\
                                                                            basis = self.rbfs, periodicity = self.periodic, \
                                                                            nmc = 32*1024, targetIntegral = 1/in_channels[0], \
                                                                            windowFn = self.windowFn, verbose = False)
                            self.weight[:,:,i,j] = newWeights
                            print('Normalizing [%2d x %2d]: %1.4e => %1.4e (target: %1.4e)' %(i,j, init, final, 1/in_channels[0]))

                            # self.weight[:,:,i,j] /= in_channels[0]
                    print('Done with normalization\n------------------------------------------')

        self.root_weight = linearLayer
        if linearLayer:
            self.lin = Linear(in_channels[1], out_channels, bias=self.use_bias,
                              weight_initializer= 'uniform')

        # if biasOffset:
        #     self.bias = Parameter(torch.Tensor(out_channels))
        # else:
        #     self.register_parameter('bias', None)

        self.reset_parameters()

    def reset_parameters(self):
        # if not isinstance(self.weight, nn.UninitializedParameter):
            # size = self.weight.size(0) * self.weight.size(1)
            # self.initializer(self.weight)
        if self.root_weight:
            self.lin.reset_parameters()
        zeros(self.bias)

    def forward(self, x: Union[Tensor, OptPairTensor], edge_index: Adj,
                edge_attr: OptTensor = None, size: Size = None) -> Tensor:
        # print('x', x[0].shape, x)
        # print('edge_index', edge_index.shape, edge_index)
        # print('edge_attr', edge_attr.shape, edge_attr)
        # print('Size', Size)
        # if args.cutlad:
            # out = self.propagate(edge_index, x=x, edge_attr=edge_attr, size=size)
        # else:
        x_i, x_j = x
        edge_weights = None
        if not(self.windowFn is None):
            edge_weights = self.windowFn(torch.linalg.norm(edge_attr, axis = 1))

        positions = torch.hstack((edge_attr, torch.zeros(edge_attr.shape[0],1, device = edge_attr.device, dtype = edge_attr.dtype)))
        if self.coordinateMapping == 'polar':
            spherical = mapToSpherical(positions)
            mapped = torch.vstack((spherical[:,0] * 2. - 1.,spherical[:,1] / np.pi)).mT
        if self.coordinateMapping == 'cartesian':
            mapped = edge_attr
        if self.coordinateMapping == 'preserving':
            cubePositions = mapToSpherePreserving(positions)
            mapped = torch.vstack((cubePositions[:,0],cubePositions[:,1] / np.pi)).mT
        convolution = cutlass.apply
        out = convolution(edge_index, x_i, x_j, mapped, edge_weights, self.weight,
                                  x_i.shape[0], self.node_dim,
                              self.size , self.rbfs, self.periodic,
                              self.batchSize[0],self.batchSize[1])

        # out = self.propagate2(edge_index, x=x, edge_attr=edge_attr, size=size)

#         print('out: ', out.shape, out)

        x_r = x[1]
        if self.preActivation is not None:
            out = self.preActivation(out)

        if x_r is not None and self.root_weight:
            out = out + self.lin(x_r) if self.preActivation is not None else self.preActivation(self.lin(x_r))
        if self.bias is not None:
            out = out + self.bias
        if self.feedThrough:
            out = out + x_r if self.preActivation is not None else self.preActivation(x_r)
        if self.postActivation is not None:
            out = self.postActivation(out)
        return out

    def message(self, x_j: Tensor, edge_attr: Tensor) -> Tensor:
        if self.dim == 1:
            u = evalBasisFunction(self.size[0], edge_attr[:,0], which=self.rbfs[0], periodic = self.periodic[0]).T

            return torch.einsum('nu, uio,ni -> no',u,self.weight, x_j)
        if self.dim == 2:
            u = evalBasisFunction(self.size[0], edge_attr[:,0], which=self.rbfs[0], periodic = self.periodic[0]).T
            v = evalBasisFunction(self.size[1], edge_attr[:,1], which=self.rbfs[1], periodic = self.periodic[1]).T

            return torch.einsum('nu, nv, uvio,ni -> no',u,v,self.weight, x_j)
        if self.dim == 3:
            u = evalBasisFunction(self.size[0], edge_attr[:,0], which=self.rbfs[0], periodic = self.periodic[0]).T
            v = evalBasisFunction(self.size[1], edge_attr[:,1], which=self.rbfs[1], periodic = self.periodic[1]).T
            w = evalBasisFunction(self.size[2], edge_attr[:,1], which=self.rbfs[1], periodic = self.periodic[2]).T

            return torch.einsum('nu, nv, uvio,ni -> no',u,v,w,self.weight, x_j)

    def __repr__(self) -> str:
        return (f'{self.__class__.__name__}({self.in_channels}, '
                f'{self.out_channels}, dim={self.dim})')

    def __check_input__(self, edge_index, size):
        the_size: List[Optional[int]] = [None, None]

        if isinstance(edge_index, Tensor):
            assert edge_index.dtype == torch.long
            assert edge_index.dim() == 2
            assert edge_index.size(0) == 2
            if size is not None:
                the_size[0] = size[0]
                the_size[1] = size[1]
            return the_size

        elif isinstance(edge_index, SparseTensor):
            if self.flow == 'target_to_source':
                raise ValueError(
                    ('Flow direction "target_to_source" is invalid for '
                     'message propagation via `torch_sparse.SparseTensor`. If '
                     'you really want to make use of a reverse message '
                     'passing flow, pass in the transposed sparse tensor to '
                     'the message passing module, e.g., `adj_t.t()`.'))
            the_size[0] = edge_index.sparse_size(1)
            the_size[1] = edge_index.sparse_size(0)
            return the_size

        raise ValueError(
            ('`MessagePassing.propagate` only supports `torch.LongTensor` of '
             'shape `[2, num_messages]` or `torch_sparse.SparseTensor` for '
             'argument `edge_index`.'))

    def __collect__(self, args, edge_index, size, kwargs):
        i, j = (1, 0) if self.flow == 'source_to_target' else (0, 1)

        out = {}
        for arg in args:
            if arg[-2:] not in ['_i', '_j']:
                out[arg] = kwargs.get(arg, Parameter.empty)
            else:
                dim = 0 if arg[-2:] == '_j' else 1
                data = kwargs.get(arg[:-2], Parameter.empty)

                if isinstance(data, (tuple, list)):
                    assert len(data) == 2
                    if isinstance(data[1 - dim], Tensor):
                        self.__set_size__(size, 1 - dim, data[1 - dim])
                    data = data[dim]

                if isinstance(data, Tensor):
                    self.__set_size__(size, dim, data)
                    data = self.__lift__(data, edge_index,
                                         j if arg[-2:] == '_j' else i)

                out[arg] = data

        if isinstance(edge_index, Tensor):
            out['adj_t'] = None
            out['edge_index'] = edge_index
            out['edge_index_i'] = edge_index[i]
            out['edge_index_j'] = edge_index[j]
            out['ptr'] = None
        elif isinstance(edge_index, SparseTensor):
            out['adj_t'] = edge_index
            out['edge_index'] = None
            out['edge_index_i'] = edge_index.storage.row()
            out['edge_index_j'] = edge_index.storage.col()
            out['ptr'] = edge_index.storage.rowptr()
            out['edge_weight'] = edge_index.storage.value()
            out['edge_attr'] = edge_index.storage.value()
            out['edge_type'] = edge_index.storage.value()

        out['index'] = out['edge_index_i']
        out['size'] = size
        out['size_i'] = size[1] or size[0]
        out['size_j'] = size[0] or size[1]
        out['dim_size'] = out['size_i']

        return out

    def propagate2(self, edge_index: Adj, size: Size = None, **kwargs):
        decomposed_layers = 1 if self.explain else self.decomposed_layers

        for hook in self._propagate_forward_pre_hooks.values():
            res = hook(self, (edge_index, size, kwargs))
            if res is not None:
                edge_index, size, kwargs = res

        size = self.__check_input__(edge_index, size)

        if decomposed_layers > 1:
            user_args = self.__user_args__
            decomp_args = {a[:-2] for a in user_args if a[-2:] == '_j'}
            decomp_kwargs = {
                a: kwargs[a].chunk(decomposed_layers, -1)
                for a in decomp_args
            }
            decomp_out = []

        for i in range(decomposed_layers):
            # if decomposed_layers > 1:
                # for arg in decomp_args:
                    # kwargs[arg] = decomp_kwargs[arg][i]

            # coll_dict = self.__collect__(self.__user_args__, edge_index,
                                            # size, kwargs)

            # msg_kwargs = self.inspector.distribute('message', coll_dict)
            # for hook in self._message_forward_pre_hooks.values():
                # res = hook(self, (msg_kwargs, ))
                # if res is not None:
                    # msg_kwargs = res[0] if isinstance(res, tuple) else res
                    #
            # aggr_kwargs = self.inspector.distribute('aggregate', coll_dict)

            convolution = cutlass.apply

            inFeatures = kwargs['x'][0]

            edge_weights = None
            if not(self.windowFn is None):
                edge_weights = self.windowFn(torch.linalg.norm(kwargs['edge_attr'], axis = 1))
                # print(torch.linalg.norm(kwargs['edge_attr'], axis = 1))
                # print(edge_weights.shape)
                # print(edge_weights)
                # print(inFeatures.shape)
                # inFeatures = inFeatures * window[:,None]
                # print(inFeatures.shape)

            positions = torch.hstack((kwargs['edge_attr'], torch.zeros(kwargs['edge_attr'].shape[0],1, device = kwargs['edge_attr'].device, dtype = kwargs['edge_attr'].dtype)))
            if self.coordinateMapping == 'polar':
                spherical = mapToSpherical(positions)
                mapped = torch.vstack((spherical[:,0] * 2. - 1.,spherical[:,1] / np.pi)).mT
            if self.coordinateMapping == 'cartesian':
                mapped = kwargs['edge_attr']
            if self.coordinateMapping == 'preserving':
                cubePositions = mapToSpherePreserving(positions)
                mapped = torch.vstack((cubePositions[:,0],cubePositions[:,1] / np.pi)).mT

            out = convolution(edge_index, kwargs['x'][0], kwargs['x'][1], mapped, edge_weights, self.weight,
                                            size[0], self.node_dim,
                                        self.size , self.rbfs, self.periodic,
                                        self.batchSize[0],self.batchSize[1])

        #     for hook in self._aggregate_forward_hooks.values():
        #         res = hook(self, (aggr_kwargs, ), out)
        #         if res is not None:
        #             out = res

        #     update_kwargs = self.inspector.distribute('update', coll_dict)
        #     out = self.update(out, **update_kwargs)

        #     if decomposed_layers > 1:
        #         decomp_out.append(out)

        #     if decomposed_layers > 1:
        #         out = torch.cat(decomp_out, dim=-1)

        # for hook in self._propagate_forward_hooks.values():
        #     res = hook(self, (edge_index, size, kwargs), out)
        #     if res is not None:
        #         out = res

        return out


# test_cloud_client.py

import pytest
from datetime import datetime
from testfixtures import LogCapture
from cloud_server import initialize_server, add_database_entry
from database_definitions import Patient
import io
import os

initialize_server()

pathname = os.getcwd()
full_pathname = pathname + '/images/test_image.jpg'

def convert_file_to_b64_string():
    from cloud_client import convert_file_to_b64_string
    b64str = convert_file_to_b64_string(full_pathname)
    assert b64str[0:20] == "/9j/4AAQSkZJRgABAQEA"

def test_b64_to_ndarray():
    from cloud_client import convert_file_to_b64_string
    from cloud_client import b64_to_ndarray
    b64str = convert_file_to_b64_string("test_image.jpg")
    nd = b64_to_ndarray(b64str)
    answer = nd[0][0:5]
    expected = [[68, 115, 197],
                [68, 115, 197],
                [68, 115, 197],
                [68, 115, 197],
                [68, 115, 197]]
    assert (answer == expected).all

list1 = ['a', 'b', 'c', 'd']
list2 = [23, 98, 47, 24]

@pytest.mark.parametrize("listvar, val, expected", [(list1, 'c', 2),
                                                    (list2, 98, 1)])
def test_get_index(listvar, val, expected):
    from cloud_client import get_index
    answer = get_index(listvar, val)
    assert answer == expected

def test_resize_image():
    from cloud_client import convert_file_to_b64_string
    from cloud_client import b64_to_ndarray
    from cloud_client import resize_image
    b64str = convert_file_to_b64_string("test_image.jpg")
    nd = b64_to_ndarray(b64str)
    resized_nd = resize_image(nd)
    answer = resized_nd[0][0:5]
    expected = [[68, 115, 197],
                [68, 115, 197],
                [68, 115, 197],
                [68, 115, 197],
                [68, 115, 197]]
    assert (answer == expected).all

def test_b64_string_to_file():
    from cloud_client import convert_file_to_b64_string
    from cloud_client import b64_string_to_file
    import filecmp
    import os
    b64str = convert_file_to_b64_string("test_image.jpg")
    b64_string_to_file(b64str, open("test_image_output.jpg", "wb"))
    answer = filecmp.cmp("test_image.jpg",
                         "test_image_output.jpg")
    os.remove("test_image_output.jpg")
    assert answer is True

def test_process_b64():
    from cloud_client import convert_file_to_b64_string
    from cloud_client import b64_to_ndarray
    from cloud_client import resize_image
    b64str = convert_file_to_b64_string("test_image.jpg")
    nd = b64_to_ndarray(b64str)
    resized_nd = resize_image(nd)
    answer = resized_nd[0][0:5]
    expected = [[68, 115, 197],
                [68, 115, 197],
                [68, 115, 197],
                [68, 115, 197],
                [68, 115, 197]]
    assert (answer == expected).all


"""factRequestForDocuments

Revision ID: 489ca98de532
Revises: d6c3af32b13e
Create Date: 2022-01-26 22:39:14.403851

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '489ca98de532'
down_revision = 'd6c3af32b13e'
branch_labels = None
depends_on = None

def upgrade():
    op.create_table('factRequestForDocuments',
		sa.Column('actionid', sa.Integer(), nullable=False),
		sa.Column('foirequestid', sa.Integer(), nullable=False),
		sa.Column('runcycleid', sa.Integer(), nullable=False),
		sa.Column('actiontype', sa.Integer()),
		sa.Column('description', sa.VARCHAR(length=4000)),
		sa.Column('priority', sa.VARCHAR(length=3000)),
		sa.Column('emailaddress', sa.VARCHAR(length=4000)),
		sa.Column('createddate', sa.DateTime()),
		sa.Column('actiondate', sa.DateTime()),
		sa.Column('duedate', sa.DateTime()),
		sa.Column('responsedate', sa.DateTime()),
		sa.Column('parentactionid', sa.Integer()),
		sa.Column('createdby', sa.VARCHAR(length=200)),
		sa.Column('subject', sa.VARCHAR(length=150)),
		sa.Column('programofficeid', sa.Integer()),
		sa.Column('reqfordocstatusid', sa.Integer()),
		sa.Column('completeddate', sa.DateTime()),
		sa.Column('requestofficeid', sa.Integer()),
		sa.Column('visiblerequestid', sa.VARCHAR(length=50)),
		sa.Column('requestdescription', sa.VARCHAR(length=4000)),
		sa.Column('officeid', sa.Integer()),
		sa.Column('requesttypeid', sa.Integer()),
		sa.Column('overduedays', sa.Integer()),
		sa.Column('elapseddays', sa.Integer()),
		sa.Column('passduedays', sa.Integer()),
		sa.Column('rfdage', sa.Integer()),
		sa.Column('remainingdays', sa.Integer()),
		sa.Column('methodofdelivery', sa.VARCHAR(length=1)),
		sa.ForeignKeyConstraint(['reqfordocstatusid'], ['dimRequestForDocumentsStatus.reqfordocstatusid']),
		sa.PrimaryKeyConstraint('actionid', 'foirequestid', 'runcycleid')
    )

def downgrade():
    op.drop_table('factRequestForDocuments')


import os
import pandas as pd

def get_file_list(path):
    file_list = []
    for root, dirs, files in os.walk(path):
        for file in files:
            file_list.append([root, file])
    return file_list

def save_to_excel(file_list, excel_file):
    df = pd.DataFrame(file_list, columns=["Папка", "Имя файла"])
    df["Расширение файла"] = df["Имя файла"].str.split(".", expand=True)[1]
    df.index += 1
    df.to_excel(excel_file, index_label="Номер строки")

try:
    file_list = get_file_list(os.getcwd())
    save_to_excel(file_list, "result.xlsx")
except Exception as ex:
    print(ex)


import click
import csv
import itertools
from Bio import motifs
from Bio.Seq import Seq
from Bio.SeqUtils import GC
from Bio.Alphabet import generic_dna
from Bio.Alphabet.IUPAC import unambiguous_dna

def homer_parse(fstream):
    def build(name, freq):
        m = motifs.Motif(counts=freq)
        m.name = name
        return m
    nct  = "ACGT"
    name = ""
    mtx  = {a: [] for a in nct}

    for line in fstream:
        if line.startswith('>'):
            if name != '':
                yield build(name, mtx)
            name = line.rstrip().split()[1]
            mtx  = {a: [] for a in nct}
        else:
            score = [float(x) for x in line.rstrip().split("\t")]
            for i, a in enumerate(nct):
                mtx[a].append(score[i])
    if name != '':
        yield build(name, mtx)

def fasta_iter(istream):
    name = None
    sequence = ''
    for line in istream:
        if line.startswith('>'):
            if name is not None:
                yield(name, sequence)
            name = line.rstrip().replace('>', '')
            sequence = ''
        else:
            sequence += line.rstrip()
    if name is not None:
        yield (name, sequence)

def mutation_iter(istream):
    for line in istream:
        if line.startswith('#'):
            continue

        oligo, pos, name, ref, alt = line.rstrip().split()
        if ('N' in ref) or ('N' in alt):
            continue
        yield (oligo, int(pos), name, ref, alt.split(','))

def progress(iter, freq=100):
    count = 0
    rotator = 0
    label = ("|", "/", "-", "\\")
    for i in iter:
        yield(i)
        count += 1
        if count % freq == 0:
            rotator = (rotator + 1) % 4
            click.echo("[%s] %6d\r" % (label[rotator], count), nl=False, err=True)

@click.command()
@click.option('--motif', '-m', type=click.File('r'), required=True)
@click.option('--reference', '-r', type=click.File('r'), required=True)
@click.option('--mutation', '-M', type=click.File('r'), default='-')
@click.option('--out', '-o', type=click.File('w'), default='-')
def fit_motif(motif, reference, mutation, out):
    click.echo("Loading reference sequences.", err=True)
    oligos = { name: seq for (name, seq) in progress(fasta_iter(reference)) }

    click.echo("Loading motifs matrix.", err=True)
    motifs = { _name(m) : m for m in progress(homer_parse(motif)) }

    reader = mutation_iter(mutation)
    writer = csv.writer(out, delimiter='\t')
    writer.writerow(['oligo', 'rsid', 'ref', 'alt', 'tf', 'ref_score', 'alt_score',
                     'score', 'flank_gc', 'core_gc'])

    click.echo("Progessing mutations.", err=True)
    for ((oligo, pos, rsid, ref, alts), tf) in progress(itertools.product(reader, motifs)):
        sequence = oligos[oligo]
        motif = motifs[tf]
        refseq = sequence[:pos] + ref + sequence[(pos+1):]
        refat, refscore = _score(motif, refseq)
        refgc = GC(flank(refseq, refat, refat + len(motif)))
        refcore_gc = GC(sequence[refat:(refat+len(motif))])
        for alt in alts:
            altseq = sequence[:pos] + alt + sequence[(pos+1):]
            altat, altscore = _score(motif, altseq)
            if altscore > refscore:
                flank_gc = GC(flank(altseq, altat, altat + len(motif)))
                core_gc  = GC(sequence[altat:(altat+len(motif))])
            else:
                flank_gc = refgc
                core_gc = refcore_gc

            writer.writerow([oligo, rsid, ref, alt, tf, refscore, altscore,
                             refscore-altscore, flank_gc, core_gc])

def _score(motif, seq):
    seq = Seq(seq, unambiguous_dna)
    pssm = motif.pssm
    fw_scores = pssm.calculate(seq)
    rv_scores = pssm.calculate(seq.reverse_complement())

    fw_index = max(range(len(fw_scores)), key=fw_scores.__getitem__)
    rv_index = max(range(len(rv_scores)), key=rv_scores.__getitem__)
    if fw_scores[fw_index] > rv_scores[rv_index]:
        return fw_index, fw_scores[fw_index]
    else:
        index = len(seq) - len(motif) - rv_index
        return index, rv_scores[rv_index]

def flank(seq, start, stop, size=10):
    bgupper = max([0, start-size])
    endown  = min([len(seq), stop+size])
    return seq[bgupper:start] + seq[stop:endown]

def _name(motif):
    name = motif.name.split('/')[0]
    return name.upper().replace('-', '_')

if __name__ == '__main__':
    fit_motif()


"""Helper functions for iterables."""

import collections
from itertools import islice, zip_longest
from typing import Callable, Iterable, Iterator, Optional, TypeVar

T = TypeVar("T")

# Courtesy of https://docs.python.org/3/library/itertools.html
def grouper(
    iterable: Iterable[T],
    n: int,
    *,
    incomplete: str = "fill",
    fillvalue: Optional[T] = None,
) -> Iterable[Iterable[T]]:
    """Collect data into non-overlapping fixed-length chunks or blocks."""
    # grouper('ABCDEFG', 3, fillvalue='x') --> ABC DEF Gxx
    # grouper('ABCDEFG', 3, incomplete='strict') --> ABC DEF ValueError
    # grouper('ABCDEFG', 3, incomplete='ignore') --> ABC DEF
    args = [iter(iterable)] * n
    if incomplete == "fill":
        return zip_longest(*args, fillvalue=fillvalue)
    if incomplete == "strict":
        return zip(*args, strict=True)
    if incomplete == "ignore":
        return zip(*args)
    else:
        raise ValueError("Expected fill, strict, or ignore")

# Also thank to https://docs.python.org/3/library/itertools.html
def sliding_window(iterable: Iterable[T], n: int) -> Iterable[tuple[T, ...]]:
    """Iterate over an iterable as a sliding window."""
    # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG
    it = iter(iterable)
    window = collections.deque(islice(it, n), maxlen=n)
    if len(window) == n:
        yield tuple(window)
    for x in it:
        window.append(x)
        yield tuple(window)

def read_iter_until(
    data: Iterator[T],
    terminators: Optional[set[T]] = None,
    pred: Optional[Callable[[T], bool]] = None,
) -> tuple[list[T], Optional[T]]:
    """Read an iterator until a terminator, returning the read string and the terminator."""
    result: list[T] = []
    for i in data:
        if terminators is not None and i in terminators:
            return result, i
        if pred is not None and pred(i):
            return result, i
        result.append(i)
    return result, None


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import re

import requests

SNIPPET_TEMPLATE = """<snippet>
    <content><![CDATA[
{}
]]></content>
    <description>{}</description>
    <scope>source.python</scope>
    <tabTrigger>{}</tabTrigger>
</snippet>"""

def _request_code(target_url):
    """Request a code file."""
    r = requests.get(target_url)
    code = r.text.encode('utf-8')

    return code

def _find_functions(code):
    """Find functions from the code."""
    # find non-private functions and the first line of their doc-string
    function_pattern = '(@.*\n)?.*def ([^_].*)\((.*)\):\n.*?"""(.*)'

    # find and return all of the functions
    functions = re.findall(function_pattern, code)
    return functions

def main(args):
    """Create sublime text snippets automatically from code."""
    CLASS_PREFIX = args['<class_prefix>']

    # define the prefix used when saving a new snippet and for the snippet's tab trigger
    if args['--snippet_prefix']:
        SNIPPET_PREFIX = args['--snippet_prefix']
    else:
        SNIPPET_PREFIX = CLASS_PREFIX

    # if we are pulling the code from a location on the internet (e.g. a raw file on github), get the code
    if args['<target_file>'].startswith("http"):
        code = _request_code(args['<target_file>'])
    else:
        with open(args['<target_file>'], 'r') as f:
            code = f.read().encode('utf-8')

    # find all of the functions in the code
    functions = _find_functions(code)

    # create a snippet for each function (as appropriate)
    for function in functions:
        # remove the newline from the end of the decorator
        decorator = function[0].strip()

        # handle a property decorator
        if decorator == "@property":
            pass
        # handle functions
        else:
            # split up the arguments for the function
            arguments = function[2].split(", ")
            # remove 'self' as an argument to the function (just move on if 'self' isn't an argument)
            try:
                arguments.remove('self')
            except ValueError as e:
                pass

            # create a string with the arguments to the function formatted for a sublime text snippet
            argument_string = ""
            count = 0

            for argument in arguments:
                count += 1
                argument_string += "${"
                argument_string += "{}:{}".format(count, argument)
                argument_string += "}"

                # if there are more arguments, add a trailing comma
                if count < len(arguments):
                    argument_string += ", "

            # create a name for the snippet based on the name of the function
            snippet_name = SNIPPET_PREFIX + function[1].replace("_", "")
            # create a description for the snippet based on the function's doc string
            snippet_description = function[3].replace('"""', '')

            # create snippet code (with the class prefix in front of it)
            snippet_code = CLASS_PREFIX + "." + function[1] + "(" + argument_string + ")"

            # create a snippet
            new_snippet = SNIPPET_TEMPLATE.format(snippet_code, snippet_description, snippet_name)

            # write the new snippet
            with open(args['<output_directory>'] + '/{}.sublime-snippet'.format(snippet_name), 'w') as f:
                f.write(new_snippet)

if __name__ == '__main__':
    main()


import requests
import numpy as np
from phe import paillier
import math

api_address = "http://localhost:8000"
N_FEATURES = 10
PRECISION = 2**(-16)
EXP = -8

TEST_DATA = [0.48555949, 0.29289251, 0.63463107, 0.41933057, 0.78672205, 0.58910837, 0.00739207, 0.31390802, 0.37037496, 0.3375726]
TEST_PRED = 0.44812144746653826

def encrypt_object(datapoint, public_key):
    return [public_key.encrypt(x, precision=PRECISION).ciphertext() for x in datapoint]

def generate_stealing_data():
    stealing_data = []
    for _ in range(N_FEATURES + 1):
        stealing_data.append(np.random.uniform(0, 1, N_FEATURES).tolist())
    return stealing_data

def discover_model(data, preds):
    N = len(data)
    X = np.ones((N, N_FEATURES + 1))
    X[:, 1:] = np.array(data)
    y = np.array(preds)
    coeffs = np.linalg.solve(X,y)
    bias = coeffs[0]
    weights = coeffs[1:]
    return weights, bias

def main():
    # create a session
    session = requests.session()
    public_key, private_key = paillier.generate_paillier_keypair()

    stealing_data = generate_stealing_data()
    preds = []
    for _, datapoint in enumerate(stealing_data):
        encrypted_list = encrypt_object(datapoint, public_key)
        r = session.post(f"{api_address}/prediction", json={"pub_key_n": public_key.n , "enc_feature_vector": encrypted_list}).json()
        encrypted_pred = paillier.EncryptedNumber(public_key, int(r["enc_prediction"]), EXP)
        res = private_key.decrypt(encrypted_pred)
        preds.append(res)
    weights, bias = discover_model(stealing_data, preds)
    print(f'weights:\n{weights}')
    print(f'bias: {bias:.3f}')

    stolen_pred = np.dot(TEST_DATA, weights) + bias
    assert 2**(-16) > abs(stolen_pred - TEST_PRED), "Prediction is not correct"
    print('All good!')

if __name__ == '__main__':
    main()


from qtdataflow.Qt import QtCore
QObject = QtCore.QObject
Signal = QtCore.Signal

class Node(object):
    """
    Logical Representation of a node.
    """

    def __init__(self):
        #self.schema = schema
        self.node_type = 'BaseNode'
        self.accepts_input = False
        self.generates_output = False
        self.out_conn = []
        self.in_conn = []

    def accept_type(self, node):
        return True

    def get_view(self):
        """
        Which view-class to use, has to return a QGraphicsItem
        """
        raise NotImplemented

    def get_toolbar_view(self):
        """
        Which view-class is used in the Toolbar, defaults to
        the standard view.
        """
        return self.get_view()

    def new_connection_out(self, node):
        """
        Called if a new connection (out) was made.
        """
        pass

    def new_connection_in(self, node):
        pass

class MultiTerminalNode(Node):
    """
    Node which can have more than one input/output Terminal.
    """

    def __init__(self):
        Node.__init__(self)
        self.input_terminals = {}
        self.output_terminals = {}

    @property
    def accepts_input(self):
        return len(self.input_terminals) > 0

    @property
    def generates_output(self):
        return len(self.output_terminals) > 0

class Schema(QObject):
    """
    Model a Schema, which includes all Nodes and connections.
    """
    node_created = Signal(Node)
    node_deleted = Signal(Node)
    nodes_connected = Signal(list)
    nodes_disconnected = Signal(list)

    def __init__(self):
        super(Schema, self).__init__()
        self.nodes = []
        self.connections = []

    def add_node(self, node):
        """
        Add given Node to the Schema.
        """
        if node not in self.nodes:
            self.nodes.append(node)
            self.node_created.emit(node)
        else:
            raise ValueError('Node already in Schema.')

    def delete_node(self, node):
        """
        Deletes given Node from the Schema, calls node_deleted event.
        """
        to_delete = [(o, i) for (o, i) in self.connections
                     if o == node or i == node]

        for o, i in to_delete:
            self.disconnect_nodes(o, i)

        self.nodes.remove(node)
        self.node_deleted.emit(node)

    def connect_nodes(self, out_node, in_node):
        if out_node is in_node:
            raise ValueError("Node can't connect to itself")
        out_node.out_conn.append(in_node)
        in_node.in_conn.append(out_node)

        self.connections.append((out_node, in_node))
        out_node.new_connection_in(in_node)
        in_node.new_connection_out(out_node)
        self.nodes_connected.emit([out_node, in_node])

    def disconnect_nodes(self, out_node, in_node):
        if (out_node, in_node) not in self.connections:
            raise ValueError("Nodes are not connected")
        self.nodes_disconnected.emit([out_node, in_node])
        out_node.out_conn.remove(in_node)
        in_node.in_conn.remove(out_node)
        self.connections.remove((out_node, in_node))

    def to_disk(self, file):
        import pickle
        to_pickle = (self.nodes, self.connections)
        return pickle.dump(to_pickle, file)

    def from_disk(self, file):
        import pickle
        nodes, connections = pickle.load(file)
        for n in nodes:
            self.add_node(n)
        for c in connections:
            self.connect_nodes(*c)


import os
from dotenv import load_dotenv
import discord
from discord import app_commands
from langchain.llms import OpenAI

# Load the .env file
load_dotenv()
token = os.getenv('DISCORD_TOKEN')
guild_id = os.getenv('DISCORD_GUILD')
openai_key = os.getenv('OPENAI_KEY')

class Client(discord.Client):
    def __init__(self):
        super().__init__(intents=discord.Intents.default())
        self.synced = False

    async def on_ready(self):
        await self.wait_until_ready()
        if not self.synced:
            await tree.sync(guild = discord.Object(id = guild_id))
            self.synced = True
        print(f"Logged in as {self.user} (ID: {self.user.id}).")
        print("-----")

client = Client()
tree = app_commands.CommandTree(client)

@tree.command(name = "test", description = "testing", guild = discord.Object(id = guild_id))
async def self(interaction: discord.Interaction, name: str):
    await interaction.response.send_message(f"Hello {name}! I was made with Discord.py!")

@tree.command(name="llm", description="Language Model", guild = discord.Object(id = guild_id))
async def self(interaction: discord.Interaction, prompt: str):
    llm = OpenAI(openai_api_key=openai_key, temperature=0.9)
    await interaction.response.send_message(llm.predict(prompt))

if __name__ == "__main__":
    client.run(token)


import sys
input = lambda : sys.stdin.readline().rstrip()

def main(s1, s2):
    l1, l2, answer = len(s1), len(s2), 0
    dp = [0]*l2
    for i1 in range(l1):
        dp_sub = [0]*l2
        for i2 in range(l2):
            dp_sub[i2] = (min(i2,1)*dp[i2-1] + 1)*(s1[i1]==s2[i2])
        answer = max(answer, max(dp_sub))
        dp = dp_sub
    print(answer)

if __name__ == "__main__":
    s2, s1 = sorted([input() for _ in range(2)], key=len)
    main(s1, s2)

# LCS, Longest Common Substring

4월_4주/BOJ_공통부분문자열/BOJ_공통부분문자열_강태훈.py

BOJ_공통부분문자열_강태훈.py


#!/usr/bin/env python

from setuptools import setup, find_packages

tests_require = [
    'nose',
    'unittest2',
]

setup(
    name='quickunit',
    version='0.6.0',
    author='David Cramer',
    author_email='dcramer@gmail.com',
    description='A discovery plugin for Nose which relies on sane structure.',
    url='http://github.com/dcramer/quickunit',
    packages=find_packages(exclude=["tests"]),
    zip_safe=False,
    entry_points={
        'nose.plugins.0.10': [
            'quickunit = quickunit.plugin:QuickUnitPlugin'
        ],
        'console_scripts': [
            'quickunit-finder = quickunit.scripts.finder:main',
        ],
    },
    license='Apache License 2.0',
    tests_require=tests_require,
    extras_require={
        'tests': tests_require,
    },
    include_package_data=True,
    classifiers=[
        'Intended Audience :: Developers',
        'Intended Audience :: System Administrators',
        'Operating System :: OS Independent',
        'Topic :: Software Development'
    ],
)


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, x):
#         self.val = x
#         self.left = None
#         self.right = None

class Solution:
    def twoSumBSTs(self, root1: TreeNode, root2: TreeNode, target: int) -> bool:
        if not root1 or not root2:
            return False

        tree1_dict = {}
        tree2_dict = {}
        self.inorder_traversal(root1, tree1_dict)
        self.inorder_traversal(root2, tree2_dict)
        #print(tree1_dict, tree2_dict)
        for key in tree1_dict:
            if target - key in tree2_dict:
                return True

        return False

    def inorder_traversal(self, root, tree_dict):
        if root is None:
            return
        self.inorder_traversal(root.left, tree_dict)
        tree_dict[root.val] = 1
        self.inorder_traversal(root.right, tree_dict)


from django.test import TestCase
from home.models import UserProfile

class UserProfileModelTestCase(TestCase):
    @classmethod
    def setUpTestData(cls):
        # Create test data before running the tests
        UserProfile.objects.create(
            name='John Doe',
            email='johndoe@example.com',
            bio='Lorem ipsum dolor sit amet',
            profile_picture='path/to/profile_picture.jpg'
        )

    def test_create_user_profile(self):
        # Retrieve the created user profile from the database
        user_profile = UserProfile.objects.get(email='johndoe@example.com')

        # Perform assertions to check if the created user profile matches the provided data
        self.assertEqual(user_profile.name, 'John Doe')
        self.assertEqual(user_profile.email, 'johndoe@example.com')
        self.assertEqual(user_profile.bio, 'Lorem ipsum dolor sit amet')
        self.assertEqual(user_profile.profile_picture,
                         'path/to/profile_picture.jpg')


__all__ = [
    "__title__", "__summary__", "__uri__", "__version__", "__author__",
    "__email__", "__license__", "__copyright__",
]

__name__ = "influxdump"
__summary__ = 'InfluxDB data backup tool'
__uri__ = "https://github.com/gams/influxdump"

__version__ = "1.0.6"

__author__ = 'Stefan Berder <stefan@measureofquality.com>'
__contact__ = 'code+influxdump@measureofquality.com'

__license__ = "Apache License, Version 2.0"
__copyright__ = "Copyright 2016-2020 %s" % __author__


from collections import deque

def bfs():
    dx = [0,1,0,-1]
    dy = [1,0,-1,0]

    while loc:
        x_y = loc.popleft()
        y, x = x_y[0], x_y[1]

        for i in range(4):
            nX = x + dx[i]
            nY = y + dy[i]

            if nX < 0 or nX >= m or nY < 0 or nY >= n: continue
            if maze[nY][nX] != 1: continue

            loc.append((nY, nX))
            maze[nY][nX] = maze[y][x] + 1

if __name__ == "__main__":

    n, m = map(int, input().split())
    maze = [[int(i) for i in input()] for j in range(n)]

    loc = deque([])
    loc.append((0,0))

    bfs()

    print(maze[n-1][m-1])


# EDGE LIST GRAPH
from queue import Queue

MAX = 100
V = None
E = None
visited = [False for i in range(MAX)]
path = [0 for i in range(MAX)]
graph = [[] for i in range(MAX)]
dist = [-1 for i in range(MAX)]

def BFS(s):
    dist[s] = 0
    visited[s] = True
    q = Queue()
    q.put(s)
    while q.empty() == False:
        u = q.get()
        for v in graph[u]:
            if visited[v] == False:
                visited[v] = True
                dist[v] = dist[u]
                q.put(v)

def printPath(s, f):
    b = []
    if f == s:
        return f

    while True:
        b.append(f)
        f = path[f]
        if f == s:
            b.append(s)
            break
    for i in range(len(b)-1, -1, -1):
        print(b[i], end = " ")

if __name__ == '__main__':
    V, E = map(int, input().split())
    for i in range(E):
        u, v = map(int, input().split())
        graph[u].append(v)
        graph[v].append(u)
    s = 1
    f = 5
    BFS(s)
    printPath(s, f)


# 23. Merge k Sorted Lists

# Definition for singly-linked list.
# class ListNode:
#     def __init__(self, x):
#         self.val = x
#         self.next = None

class Solution:
    def mergeKLists(self, lists: 'List[ListNode]') -> 'ListNode':
        newhead = ListNode(None)
        curr = newhead
        import heapq
        heap = []
        for i in range(len(lists)):
            root = lists[i]
            if root:
                heapq.heappush(heap, (root.val, i))
                lists[i] = lists[i].next

        while heap:
            # Pop the smallest value from the heap
            popped_val, idx = heapq.heappop(heap)

            curr.next = ListNode(popped_val)
            curr = curr.next

            root = lists[idx]
            if root:
                heapq.heappush(heap, (root.val, idx))
                lists[idx] = lists[idx].next

        return newhead.next


import os
import time
import datetime
from pathlib import Path
from datetime import datetime
import psutil
import math

class Location(str):
    def __call__(self, *args) -> 'Location':
        return Location(self.format(*args))

    def exists(self):
        return Path(self).exists()

    def read(self):
        with open(self, 'r') as f:
            return f.read()

    def write(self, value):
        with open(self, 'w') as f:
            f.write(value)

class Locations:
    PROC_CMDLINE = Location("/proc/{}/cmdline")
    PROC_CGROUP = Location("/proc/{}/cgroup")
    PROC_SMAP_ROLLUP = Location("/proc/{}/smaps_rollup")

    CGROUP_DIR = Location("/sys/fs/cgroup{}")
    MEM_CURRENT = Location("{}/memory.current")
    MEM_STAT = Location("{}/memory.stat")
    MEM_PRESSURE = Location("{}/memory.pressure")
    MEM_HIGH = Location("{}/memory.high")

    MEMINFO = Location("/proc/meminfo")

class ProcessList:
    def __init__(self):
        self.pids = {}

    def refresh(self):
        for process in os.listdir("/proc/"):
            if (not process.isdigit()) or (process in self.pids):
                continue

            pid = int(process)

            if not Locations.PROC_CMDLINE(pid).exists():
                continue

            try:
                self.pids[pid] = {
                    "pid": pid,
                    "cmdline": Locations.PROC_CMDLINE(pid).read().replace('\0', ' ').strip(),
                }
            except:
                continue

    def find(self, name):
        return [process for process in self.pids.values() if name == process["cmdline"]]

    def find_in(self, name):
        return [process for process in self.pids.values() if name in process["cmdline"]]

    def __str__(self):
        return "\n".join(
            "pid: {}\t cmdline: {}".format(pid['pid'], pid['cmdline'])
            for pid in self.pids.values()
        )

#  func found  on https://sleeplessbeastie.eu/2019/06/10/how-to-display-memory-used-by-processes-in-human-readable-form-using-python-script/
def pretty(nbytes):
  metric = ("B", "kB", "MB", "GB", "TB")
  if nbytes == 0:
    return "%s %s" % ("0", "B")
  nunit = int(math.floor(math.log(nbytes, 1024)))
  nsize = round(nbytes/(math.pow(1024, nunit)), 2)
  return '%s %s' % (format(nsize, ".2f"), metric[nunit])

if __name__ == "__main__":
    process_list = ProcessList()
    process_list.refresh()
    container  = process_list.find("/controller /")

    import sys
    output_file = sys.argv[1]

    with open(output_file, 'w') as file_object:
        file_object.write('time;uss;pss\n')

    for i in range(100000000000000):

        p = psutil.Process(container[0]["pid"])
        meminfo = p.memory_full_info()
        uss = meminfo.uss
        pss  =  meminfo.pss

        print(f"uss {pretty(p.memory_full_info().uss)}, pss={pretty(p.memory_full_info().pss)} ")
        with open(output_file, 'a') as file_object:
            file_object.write(f'{datetime.now()};{uss};{pss}\n'.replace(".", ","))

        time.sleep(0.2)


from re import A
import time, random
from Yinsh.yinsh_model import YinshGameRule
from template import Agent
from copy import deepcopy
from collections import deque
import numpy as np

THINKTIME = 0.5
C_PARAM = 1.414

class Node():
    def __init__(self, state, game_rule=None, agent_id=None, parent=None, parent_action=None):
        self.state = state
        self.parent = parent
        self.id = agent_id
        self.parent_action = parent_action
        self.children = []
        self.game_rule = game_rule

        self._untried_actions = self.get_legal_actions(self.state)
        self._number_of_visits = 0
        self._results = {}
        self._results[1] = 0
        self._results[-1] = 0
        self._score = self.state.agents[self.id].score

    # Current node, victory count minus failure count
    def q(self):
        wins = self._results[1]
        loses = self._results[-1]
        return wins - loses

    # Number of visits to the current node
    def n(self):
        return self._number_of_visits

    # Expand child nodes
    def expand(self):
        action = self._untried_actions.pop()
        current_state = deepcopy(self.state)

        next_state = self.game_rule.generateSuccessor(current_state, action, self.id)
        child_node = Node(next_state, parent=self, parent_action=action, agent_id=self.id, game_rule=self.game_rule)
        self.children.append(child_node)
        return child_node

    # The logic of generating the tree is to first determine whether the node is the final state
    # if not then determine whether it is fully expanded
    # if not then continue to expand the children of the node
    # otherwise choose a random node from the child nodes as the next node to be expanded
    def run_tree_policy(self):
        current_node=self
        while not current_node.is_terminal_node():
            if not current_node.is_fully_expanded():
                return current_node.expand()
            else:
                current_node=current_node.get_random_child()
        return current_node

    def is_terminal_node(self):
        if not self.parent:
            return False
        elif len(self.get_legal_actions(self.state)) == 1:
            return True
        else:
            return self._score == 3 or self.parent._score == 3

    def is_fully_expanded(self):
        return len(self._untried_actions) == 0

    # Select an optimal node from all the child nodes (next state)
    def choose_best_child(self):
        # UCT algorithm
        try:
            choices_weights = [(c.q() / c.n()) + C_PARAM * np.sqrt((2*np.log(self.n()) / c.n())) for c in self.children]
            if self.id == 0:
                # If the current player is the first player,
                # the child node with the greatest weight is selected as the optimal action
                print("mct best action so far")
                return self.children[np.argmax(choices_weights)]
            else:
                # If the current player is a backhand,
                # the child node with the smallest weight (the state with the lowest first-hand win rate)
                # is selected as the optimal action
                print("mct found")
                return self.children[np.argmin(choices_weights)]
        except:
            # error process
            return self.get_random_child()

    def get_random_child(self):
        return random.choice(self.children)

    def get_legal_actions(self, current_state):
        return deepcopy(self.game_rule.getLegalActions(current_state, self.id))

    # Self-play simulation, random selection of actions for child nodes until the endgame
    def rollout(self):
        current_rollout_state = deepcopy(self.state)
        final_result = 0
        while final_result == 0:
            possible_moves = self.get_legal_actions(current_rollout_state)
            action = random.choice(possible_moves)
            current_rollout_state = deepcopy(self.game_rule.generateSuccessor(current_rollout_state, action, self.id))
            # reward?
            new_score = current_rollout_state.agents[self.id].score
            if new_score == 3 or self._score == 3:
                final_result = 1. if new_score > self._score else -1.
                break
            if len(self.get_legal_actions(current_rollout_state)) == 1:
                break
        return final_result

    # Goes back up and passes the win/loss information to the parent nodes
    def backpropagate(self, result):
        self._number_of_visits += 1.
        if result == 0:
            return
        self._results[result] += result
        if self.parent:
            self.parent.backpropagate(result)

    # Each node calculates the best action by playing itself
    # and records the result, from which the best child node is selected
    def calc_best_action(self,stime):

        cnt = 0
        while time.time()-stime < THINKTIME:
            cnt += 1
            node = self.run_tree_policy()
            reward = node.rollout()
            node.backpropagate(reward)
        print("mct state",cnt)

        return self.choose_best_child()

# Agent class
class myAgent(Agent):
    def __init__(self, _id):
        super().__init__(_id)
        self.id = _id
        self.game_rule = YinshGameRule(2)

    # BFS search algorithm for len(actions) > 70
    def SelectAction_BFS(self, actions, rootstate):
        start_time = time.time()
        queue      = deque([ (deepcopy(rootstate),[]) ])
        count =0
        # Conduct BFS starting from rootstate.
        while len(queue) and time.time()-start_time < THINKTIME:
            count +=1
            state, path = queue.popleft()
            new_actions = self.game_rule.getLegalActions(state, self.id)

            for a in new_actions:
                next_state = deepcopy(state)
                next_path  = path + [a]
                score = state.agents[self.id].score
                new_state = self.game_rule.generateSuccessor(next_state, a, self.id)
                reward = new_state.agents[self.id].score > score

                if reward:
                    print("BFS found",count)
                    return next_path[0]
                else:
                    queue.append((next_state, next_path))
        print("BFS random",count)
        return random.choice(actions)

    # MCTS algorithm for len(actions) <= 70
    def SelectAction(self, actions, rootstate):
        start_time = time.time()
        while  time.time()-start_time < THINKTIME:
            if len(actions) > 70:

                return self.SelectAction_BFS(actions, rootstate)
            else:

                tree = Node(rootstate, game_rule=self.game_rule, agent_id=self.id)

                return tree.calc_best_action(start_time).parent_action
        print('mct random')
        return random.choice(actions)


"""
Spyral, an awesome library for making games.
"""

__version__ = '0.2'
__license__ = 'MIT'
__author__ = 'Robert Deaton'

from types import ModuleType
import sys

import compat
import pygame

# import mapping to objects in other modules
all_by_module = {
    'spyral.sprite' : ['Sprite', 'Group', 'AggregateSprite'],
    'spyral.scene' : ['Scene', 'director'],
    'spyral.image' : ['Image'],
    'spyral.vector' : ['Vec2D'],
    'spyral.signal' : ['Signal'],
    'spyral.rect' : ['Rect'],
    'spyral.animation' : ['Animation'],
    'spyral.core' : ['init', 'quit'],
    'spyral.font' : ['Font'],
    'spyral.camera' : ['Camera'],
    'spyral.clock' : ['GameClock'],
    'spyral.event' : ['keys']
}

# modules that should be imported when accessed as attributes of werkzeug
attribute_modules = frozenset(['memoize', 'point', 'camera', 'animator', 'event', '_lib', 'color', 'font'])

object_origins = {}
for module, items in all_by_module.iteritems():
    for item in items:
        object_origins[item] = module

class module(ModuleType):
    """Automatically import objects from the modules."""

    def __getattr__(self, name):
        if name in object_origins:
            module = __import__(object_origins[name], None, None, [name])
            for extra_name in all_by_module[module.__name__]:
                setattr(self, extra_name, getattr(module, extra_name))
            return getattr(module, name)
        elif name in attribute_modules:
            __import__('spyral.' + name)
        return ModuleType.__getattribute__(self, name)

    def __dir__(self):
        """Just show what we want to show."""
        result = list(new_module.__all__)
        result.extend(('__file__', '__path__', '__doc__', '__all__',
                       '__docformat__', '__name__', '__path__',
                       '__package__', '__version__'))
        return result

# keep a reference to this module so that it's not garbage collected
old_module = sys.modules['spyral']

# setup the new module and patch it into the dict of loaded modules
new_module = sys.modules['spyral'] = module('spyral')
new_module.__dict__.update({
    '__file__': __file__,
    '__package__': 'spyral',
    '__path__': __path__,
    '__doc__': __doc__,
    '__version__': __version__,
    '__all__': tuple(object_origins) + tuple(attribute_modules),
    '__docformat__': 'restructuredtext en'
})


from rest_framework.serializers import ModelSerializer
from rest_framework import exceptions
from api.models import VDO
from django_celery_results.models import TaskResult

class VDOSerializer(ModelSerializer):

    def create(self, validated_data):
        newvdo_record = self.Meta.model.objects.create(**validated_data)
        return newvdo_record

    class Meta:
        model = VDO
        fields = '__all__'

class StatusSerializer(ModelSerializer):

    def create(self, validated_data):
        newvdo_record = self.Meta.model.objects.create(**validated_data)
        return newvdo_record

    class Meta:
        model = TaskResult
        fields = 'task_id','status','result'


debug_queue = "queue", "inter"
""" How to select the debug queue.

    First part of the tuple is the keyword argument to modify when calling
    the pbs job, and the second is its value.
"""

accounts = ["BES000"]
""" List of slurm or pbs accounts allowed for use.

    This is used by ipython's %launch magic function.
    It is not required for slurm systems.
    If empty, then %launch will not have a queue option.
"""

qsub_exe = "sbatch"
""" Qsub executable. """

default_pbs = { 'account': accounts[0], 'walltime': "06:00:00", 'nnodes': 1, 'ppn': 8}
""" Defaults parameters filling the pbs script. """

pbs_string =  "#! /bin/bash\n"\
              "#SBATCH --account={account}\n"\
              "#SBATCH --time={walltime}\n"\
              "#SBATCH -N {nnodes}\n"\
              "#SBATCH -e \"{err}.%j\"\n"\
              "#SBATCH -o \"{out}.%j\"\n"\
              "#SBATCH -J {name}\n"\
              "#SBATCH -D {directory}\n\n"\
              "python {scriptcommand}\n"
""" Default slurm script. """

default_comm = { 'n': 2, 'ppn': default_pbs['ppn'] }
""" Default mpirun parameters. """

mpirun_exe = "mpirun -np {n} {placement} numa_wrapper -ppn={ppn} {program}"
""" Command-line to launch external mpi programs. """

def ipython_qstat(self, arg):
  """ squeue --user=`whoami` -o "%7i %.3C %3t  --   %50j" """
  from subprocess import Popen, PIPE
  from IPython.utils.text import SList
  from getpass import getuser

  # finds user name.
  whoami = getuser()
  squeue = Popen(["squeue", "--user=" + whoami, "-o", "\"%7i %.3C %3t    %j\""], stdout=PIPE)
  result = squeue.stdout.read().rstrip().split('\n')
  result = SList([u[1:-1] for u in result[1:]])
  return result.grep(str(arg[1:-1]))


import pandas as pd
import json
import requests

# # ----------from fred---------------
# api_keys = ["36211f27396765eca92b93f01dca74db", "4808384bf945022005347fcf2f6957fb",
#             "a66fc3e61d360c6088b022f2c06c831c", "1836f996f9157acd994d59547bb0f65c",
#             "4e2f7a3a68190b6584419017414974d5", "bc4c30a690776f1662cd6e8f5d30f3ce",
#             "3cd3452ab1a374f571db4822fd6f359a", "ad8557a75ffc5f367492cb34b67539e8"]
#
# api_endpoint = "https://api.stlouisfed.org/fred/series/observations"
# params = {
#     "series_id": 'GNPCA',
#     "api_key": api_keys[1],
#     "file_type": "json"
# }
# response = requests.get(api_endpoint, params=params)
# data_dict = response.json()
# print(data_dict)
# # Extract observations
# observations = data_dict['observations']
#
# # Create DataFrame
# df = pd.DataFrame(observations)
# df_final = pd.DataFrame()
#
# # Convert columns to appropriate data types
# df_final['date'] = pd.to_datetime(df['date'])
# df_final['value'] = pd.to_numeric(df['value'])
#
# print(df_final)
# df_final.to_csv('economic_data.csv', index=False)
# loaded_df_csv = pd.read_csv('economic_data.csv')
# print("Loaded DataFrame from CSV:")
# print(loaded_df_csv)

# ----------from nasdaq---------------
import nasdaqdatalink
nasdaqdatalink.read_key(filename="../api_keys/nasdaq1.txt")
# nasdaqdatalink.bulkdownload('FRED')

df1 = pd.read_csv('../../files/data/FRED_metadata_plus_categories.csv')
codes = df1['code']
# print(df1['code'])

nasdaq_code = []
for code in codes:
    nasdaq_code.append('FRED/'+code)

data = nasdaqdatalink.get(nasdaq_code)#['FRED/00XALCATM086NEST', 'FRED/00XALCBEM086NEST'])
print(data.info())
print(data)
data.to_csv('economic_data2.csv', index=True)
loaded_df_csv = pd.read_csv('economic_data2.csv')
print("Loaded DataFrame from CSV:")
print(loaded_df_csv)


"""Display a rotomap."""

import enum
import functools
import sys

import cv2
import numpy

import mel.lib.common
import mel.lib.fullscreenui
import mel.lib.image
import mel.rotomap.detectmoles
import mel.rotomap.mask
import mel.rotomap.moles
import mel.rotomap.tricolour

DEFAULT_MASKER_RADIUS = 200

_WHITE = (255, 255, 255)
_BLACK = (0, 0, 0)

def draw_mole(image, x, y, colours):
    def circle(radius, col):
        cv2.circle(image, (x, y), radius, col, -1)

    circle(20, _WHITE)
    circle(18, _BLACK)

    radius = 16
    for index in range(3):
        circle(radius, colours[index])
        radius -= 4

def draw_non_canonical_mole(image, x, y, colours):
    def rect(size, col):
        top_left = (x - size, y - size)
        bottom_right = (x + size, y + size)
        cv2.rectangle(image, top_left, bottom_right, col, -1)

    rect(20, _WHITE)
    rect(18, _BLACK)

    draw_mole(image, x, y, colours)

def draw_crosshair(image, x, y):
    inner_radius = 16
    outer_radius = 24

    directions = [(1, 0), (0, -1), (-1, 0), (0, 1)]  # Right, down, left, up

    size_color_list = [(3, _WHITE), (2, _BLACK)]

    for size, color in size_color_list:
        for d in directions:
            cv2.line(
                image,
                (x + (inner_radius * d[0]), y + (inner_radius * d[1])),
                (x + (outer_radius * d[0]), y + (outer_radius * d[1])),
                color,
                size,
            )

class Display(mel.lib.fullscreenui.ZoomableMixin):
    def __init__(self, screen):
        super().__init__()
        self._image_display = screen

        self._rect = numpy.array((screen.width, screen.height))
        title_height, _ = mel.lib.image.measure_text_height_width("abc")
        self._spacer_height = 10
        self._image_rect = self._rect - numpy.array(
            (0, title_height + self._spacer_height)
        )

        self._title = ""

    def show_current(self, image, overlay):
        self.zoomable_transform_update(image, self._image_rect)
        image = self.zoomable_transform_render()

        if overlay is not None:
            image = overlay(image, self._transform)

        caption = mel.lib.image.render_text_as_image(self._title)
        image = mel.lib.image.montage_vertical(
            self._spacer_height, image, caption
        )
        self._image_display.show_opencv_image(image)

    def set_title(self, title):
        self._title = title

def make_composite_overlay(*overlays):
    """Return an overlay, which will composite the supplied overlays in turn.

    :*overlays: The overlay callables to composite.
    :returns: A function which will composite *overlays and return the image.
    """

    def do_overlay(image, transform):
        for o in overlays:
            image = o(image, transform)
        return image

    return do_overlay

class StatusOverlay:
    def __init__(self):
        self.text = ""

    def __call__(self, image, transform):
        if self.text:
            text_image = mel.lib.image.render_text_as_image(self.text)
            mel.lib.common.copy_image_into_image(text_image, image, 0, 0)

        return image

class MoleMarkerOverlay:
    def __init__(self, uuid_to_tricolour):
        self._is_showing_markers = True
        self._is_faded_markers = True
        self._highlight_uuid = None

        self._uuid_to_tricolour = uuid_to_tricolour
        if self._uuid_to_tricolour is None:
            self._uuid_to_tricolour = (
                mel.rotomap.tricolour.uuid_to_tricolour_first_digits
            )

        self.moles = None

    def toggle_markers(self):
        self._is_showing_markers = not self._is_showing_markers

    def set_highlight_uuid(self, highlight_uuid):
        self._highlight_uuid = highlight_uuid

    def toggle_faded_markers(self):
        self._is_faded_markers = not self._is_faded_markers

    def __call__(self, image, transform):
        if not self._is_showing_markers:
            return image

        highlight_mole = None
        if self._highlight_uuid is not None:
            for m in self.moles:
                if m["uuid"] == self._highlight_uuid:
                    highlight_mole = m
                    break

        marker_image = image
        if self._is_faded_markers:
            marker_image = image.copy()
        for mole in self.moles:
            x, y = transform.imagexy_to_transformedxy(mole["x"], mole["y"])
            if mole is highlight_mole:
                draw_crosshair(marker_image, x, y)
            colours = self._uuid_to_tricolour(mole["uuid"])
            if mole[mel.rotomap.moles.KEY_IS_CONFIRMED]:
                draw_mole(marker_image, x, y, colours)
            else:
                draw_non_canonical_mole(marker_image, x, y, colours)
        if self._is_faded_markers:
            image = cv2.addWeighted(image, 0.75, marker_image, 0.25, 0.0)

        return image

class MarkedMoleOverlay:
    """An overlay to make marked moles obvious, for checking mark positions."""

    def __init__(self):
        self.moles = None
        self._highlight_uuid = None
        self.is_accentuate_marked_mode = False

    def set_highlight_uuid(self, highlight_uuid):
        self._highlight_uuid = highlight_uuid

    def __call__(self, image, transform):
        if self.is_accentuate_marked_mode:
            return self._draw_accentuated(image, transform)
        else:
            return self._draw_markers(image, transform)

    def _draw_accentuated(self, image, transform):
        # Reveal the moles that have been marked, whilst still showing
        # markers. This is good for verifying that markers are actually
        # positioned on moles.

        mask_radius = 50

        image = image.copy() // 2
        mask = numpy.zeros((*image.shape[:2], 1), numpy.uint8)

        for mole in self.moles:
            x, y = transform.imagexy_to_transformedxy(mole["x"], mole["y"])
            cv2.circle(mask, (x, y), mask_radius, 255, -1)

        masked_faded = cv2.bitwise_and(image, image, mask=mask)
        image = cv2.add(masked_faded, image)

        highlight_mole = None
        if self._highlight_uuid is not None:
            for m in self.moles:
                if m["uuid"] == self._highlight_uuid:
                    highlight_mole = m
                    break

        for mole in self.moles:
            x, y = transform.imagexy_to_transformedxy(mole["x"], mole["y"])
            kind = mole.get("kind", None)
            looks_like = mole.get("looks_like", None)

            colour = (128, 0, 0)
            if mole[mel.rotomap.moles.KEY_IS_CONFIRMED]:
                colour = (255, 0, 0)
            if kind == "mole":
                if looks_like == "mole":
                    colour = (255, 255, 255)
                elif looks_like == "non-mole":
                    colour = (255, 255, 0)
                elif looks_like == "unsure":
                    colour = (255, 0, 128)
                else:
                    raise Exception(f"Mole looks_like is invalid: {mole}")
            elif kind == "non-mole":
                if looks_like == "mole":
                    colour = (0, 255, 255)
                elif looks_like == "non-mole":
                    colour = (0, 0, 255)
                elif looks_like == "unsure":
                    colour = (128, 0, 255)
                else:
                    raise Exception(f"Mole looks_like is invalid: {mole}")

            cv2.circle(image, (x, y), mask_radius, colour, 2)
            if mole is highlight_mole:
                draw_crosshair(image, x, y)

        return image

    def _draw_markers(self, image, transform):
        # Hide the moles that have been marked, showing markers
        # distinctly from moles. This is good for marking moles that
        # haven't been marked, without worrying about the ones that
        # have been marked.

        for mole in self.moles:
            x, y = transform.imagexy_to_transformedxy(mole["x"], mole["y"])
            draw_mole(image, x, y, [[255, 0, 0], [255, 128, 128], [255, 0, 0]])

        return image

class BoundingAreaOverlay:
    """An overlay to show the bounding area, if any."""

    def __init__(self):
        self.bounding_box = None

    def __call__(self, image, transform):
        image //= 2
        if self.bounding_box is not None:
            color = (0, 0, 255)
            size = 2
            space = mel.lib.ellipsespace.Transform(self.bounding_box)

            def toimage(point):
                point = space.from_space((point))
                point = transform.imagexy_to_transformedxy(*point)
                return point

            border = [
                toimage((-1, -1)),
                toimage((1, -1)),
                toimage((1, 1)),
                toimage((-1, 1)),
                toimage((-1, -1)),
            ]
            border = numpy.array(border)
            centre = [
                toimage((0, 0.1)),
                toimage((0, -0.1)),
                toimage((0.05, 0)),
                toimage((0.1, 0)),
                toimage((-0.1, 0)),
                toimage((0, 0)),
                toimage((0, 0.1)),
            ]
            centre = numpy.array(centre)

            cv2.drawContours(image, [border, centre], -1, color, size)

        return image

class EditorMode(enum.Enum):
    edit_mole = 1
    edit_mask = 2
    bounding_area = 3
    mole_mark = 4
    debug_automole = 0

class Editor:
    def __init__(self, directory_list, screen):
        self._uuid_to_tricolour = mel.rotomap.tricolour.UuidTriColourPicker()
        self.display = Display(screen)
        self.moledata_list = [MoleData(x.image_paths) for x in directory_list]

        self._mode = EditorMode.edit_mole

        self.moledata_index = 0
        self.moledata = self.moledata_list[self.moledata_index]
        self._follow = None
        self._mole_overlay = MoleMarkerOverlay(self._uuid_to_tricolour)
        self.marked_mole_overlay = MarkedMoleOverlay()
        self.bounding_area_overlay = BoundingAreaOverlay()
        self._status_overlay = StatusOverlay()
        self.show_current()

        self.masker_radius = DEFAULT_MASKER_RADIUS

    def set_smaller_masker(self):
        self.masker_radius //= 2

    def set_larger_masker(self):
        self.masker_radius *= 2

    def set_default_masker(self):
        self.masker_radius = DEFAULT_MASKER_RADIUS

    def set_automoledebug_mode(self):
        self._mode = EditorMode.debug_automole
        self.show_current()

    def set_editmole_mode(self):
        self._mode = EditorMode.edit_mole
        self.show_current()

    def set_editmask_mode(self):
        self._mode = EditorMode.edit_mask
        self.show_current()

    def set_boundingarea_mode(self):
        self._mode = EditorMode.bounding_area
        self.show_current()

    def set_molemark_mode(self):
        self._mode = EditorMode.mole_mark
        self.show_current()

    def set_status(self, text):
        self._status_overlay.text = text

    def visit(self, visit_target_str):
        # Expect a string formatted like this:
        #
        #   path/to/jpg:uuid
        #
        # Anything after the expected bits is ignored.
        #
        path, visit_uuid, *_ = visit_target_str.split(":")
        print(path, visit_uuid)

        for _ in range(len(self.moledata_list)):
            if self.moledata.try_jump_to_path(str(path)):
                for m in self.moledata.moles:
                    if m["uuid"] == visit_uuid:
                        self.moledata.get_image()
                        self._follow = visit_uuid
                        self._mole_overlay.set_highlight_uuid(self._follow)
                        self.marked_mole_overlay.set_highlight_uuid(
                            self._follow
                        )
                        self.show_zoomed_display(m["x"], m["y"])
                        return
                self.show_current()
                return

            self.moledata_index += 1
            self.moledata_index %= len(self.moledata_list)
            self.moledata = self.moledata_list[self.moledata_index]

        print("Could not find:", path, ":", visit_uuid, file=sys.stderr)
        self.show_current()

    def follow(self, uuid_to_follow):
        self._follow = uuid_to_follow
        self._mole_overlay.set_highlight_uuid(self._follow)
        self.marked_mole_overlay.set_highlight_uuid(self._follow)

        follow_mole = None
        for m in self.moledata.moles:
            if m["uuid"] == self._follow:
                follow_mole = m
                break

        if follow_mole is not None:
            self.show_zoomed_display(follow_mole["x"], follow_mole["y"])

    def skip_to_mole(self, uuid_to_skip_to):
        original_index = self.moledata.index()
        done = False
        while not done:
            for m in self.moledata.moles:
                if m["uuid"] == uuid_to_skip_to:
                    return
            self.moledata.increment()
            self.moledata.get_image()
            if self.moledata.index() == original_index:
                return

    def toggle_markers(self):
        self._mole_overlay.toggle_markers()
        self.show_current()

    def toggle_faded_markers(self):
        self._mole_overlay.toggle_faded_markers()
        self.show_current()

    def set_mask(self, mouse_x, mouse_y, enable):
        image_x, image_y = self.display.windowxy_to_imagexy(mouse_x, mouse_y)
        value = 255 if enable else 0
        radius = self.masker_radius
        cv2.circle(self.moledata.mask, (image_x, image_y), radius, value, -1)
        self.moledata.save_mask()
        self.show_current()

    def show_current(self):
        self.display.set_title(self.moledata.current_image_path())
        image = self.moledata.get_image()

        if self._mode is EditorMode.edit_mole:
            self._mole_overlay.moles = self.moledata.moles
            self.display.show_current(
                image,
                make_composite_overlay(
                    self._mole_overlay, self._status_overlay
                ),
            )
        elif self._mode is EditorMode.debug_automole:
            image = image[:]
            image = mel.rotomap.detectmoles.draw_debug(
                image, self.moledata.mask
            )
            self.display.show_current(image, None)
        elif self._mode is EditorMode.edit_mask:
            mask = self.moledata.mask
            gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
            gray_image = cv2.cvtColor(gray_image, cv2.COLOR_GRAY2BGR)
            gray_image[:, :, 2] = mask
            self.display.show_current(gray_image, None)
        elif self._mode is EditorMode.bounding_area:
            box = self.moledata.metadata.get("ellipse", None)
            self.bounding_area_overlay.bounding_box = box
            self.display.show_current(image, self.bounding_area_overlay)
        elif self._mode is EditorMode.mole_mark:
            self.marked_mole_overlay.moles = self.moledata.moles
            self.display.show_current(image, self.marked_mole_overlay)
        else:
            raise Exception("Unknown mode", self._mode)

    def show_fitted(self):
        self.display.set_fitted()
        self.show_current()

    def set_zoom_level(self, zoom_level):
        self.display.set_zoom_level(zoom_level)
        self.show_current()

    def show_zoomed(self, mouse_x, mouse_y, zoom_level=None):
        image_x, image_y = self.display.windowxy_to_imagexy(mouse_x, mouse_y)
        self.display.set_zoomed(image_x, image_y, zoom_level)
        self.show_current()

    def show_zoomed_display(self, image_x, image_y, zoom_level=None):
        self.display.set_zoomed(image_x, image_y, zoom_level)
        self.show_current()

    def show_prev_map(self):
        def transition():
            self.moledata_index -= 1
            self.moledata_index %= len(self.moledata_list)
            self.moledata = self.moledata_list[self.moledata_index]

        self._adjusted_transition(transition)
        self.show_current()

    def show_next_map(self):
        def transition():
            self.moledata_index += 1
            self.moledata_index %= len(self.moledata_list)
            self.moledata = self.moledata_list[self.moledata_index]

        self._adjusted_transition(transition)
        self.show_current()

    def show_prev(self):
        self._adjusted_transition(self.moledata.decrement)
        self.show_current()

    def show_next(self):
        self._adjusted_transition(self.moledata.increment)
        self.show_current()

    def _adjusted_transition(self, transition_func):
        if self.display.is_zoomed() and "ellipse" in self.moledata.metadata:
            pos = self.display.get_zoom_pos()
            ellipse = self.moledata.metadata["ellipse"]
            pos = mel.lib.ellipsespace.Transform(ellipse).to_space(pos)

            transition_func()
            self.moledata.ensure_loaded()

            if "ellipse" in self.moledata.metadata:
                ellipse = self.moledata.metadata["ellipse"]
                pos = mel.lib.ellipsespace.Transform(ellipse).from_space(pos)
                self.display.set_zoomed(pos[0], pos[1])
        else:
            transition_func()

    def show_next_n(self, number_to_advance):
        for i in range(number_to_advance):
            self.moledata.increment()
            self.moledata.get_image()
        self.show_current()

    def add_mole(self, mouse_x, mouse_y):
        image_x, image_y = self.display.windowxy_to_imagexy(mouse_x, mouse_y)
        mel.rotomap.moles.add_mole(self.moledata.moles, image_x, image_y)
        self.moledata.save_moles()
        self.show_current()

    def add_mole_display(self, image_x, image_y, mole_uuid=None):
        mel.rotomap.moles.add_mole(
            self.moledata.moles, image_x, image_y, mole_uuid
        )
        self.moledata.save_moles()
        self.show_current()

    def confirm_mole(self, mouse_x, mouse_y):
        image_x, image_y = self.display.windowxy_to_imagexy(mouse_x, mouse_y)
        mole_uuid = mel.rotomap.moles.get_nearest_mole_uuid(
            self.moledata.moles, image_x, image_y
        )
        mel.rotomap.moles.set_nearest_mole_uuid(
            self.moledata.moles, image_x, image_y, mole_uuid, is_canonical=True
        )
        self.moledata.save_moles()
        self.show_current()

    def confirm_all(self):
        for m in self.moledata.moles:
            m["is_uuid_canonical"] = True
        self.moledata.save_moles()
        self.show_current()

    def set_mole_uuid(self, mouse_x, mouse_y, mole_uuid, is_canonical=True):
        image_x, image_y = self.display.windowxy_to_imagexy(mouse_x, mouse_y)
        mel.rotomap.moles.set_nearest_mole_uuid(
            self.moledata.moles, image_x, image_y, mole_uuid, is_canonical
        )
        self.moledata.save_moles()
        self.show_current()

    def get_mole_uuid(self, mouse_x, mouse_y):
        image_x, image_y = self.display.windowxy_to_imagexy(mouse_x, mouse_y)
        return mel.rotomap.moles.get_nearest_mole_uuid(
            self.moledata.moles, image_x, image_y
        )

    def get_nearest_mole(self, mouse_x, mouse_y):
        image_x, image_y = self.display.windowxy_to_imagexy(mouse_x, mouse_y)
        nearest_index = mel.rotomap.moles.nearest_mole_index(
            self.moledata.moles, image_x, image_y
        )
        mole = None
        if nearest_index is not None:
            mole = self.moledata.moles[nearest_index]
        return mole

    def move_nearest_mole(self, mouse_x, mouse_y):
        image_x, image_y = self.display.windowxy_to_imagexy(mouse_x, mouse_y)
        mel.rotomap.moles.move_nearest_mole(
            self.moledata.moles, image_x, image_y
        )
        self.moledata.save_moles()
        self.show_current()

    def remove_mole(self, mouse_x, mouse_y):
        image_x, image_y = self.display.windowxy_to_imagexy(mouse_x, mouse_y)
        mel.rotomap.moles.remove_nearest_mole(
            self.moledata.moles, image_x, image_y
        )
        self.moledata.save_moles()
        self.show_current()

    def crud_mole(self, mole_uuid, mouse_x, mouse_y):
        image_x, image_y = self.display.windowxy_to_imagexy(mouse_x, mouse_y)

        i = mel.rotomap.moles.uuid_mole_index(self.moledata.moles, mole_uuid)
        if i is not None:
            self.moledata.moles[i]["x"] = image_x
            self.moledata.moles[i]["y"] = image_y
        else:
            mel.rotomap.moles.add_mole(
                self.moledata.moles, image_x, image_y, mole_uuid
            )

        self.moledata.save_moles()
        self.show_current()

    def remap_uuid(self, from_uuid, to_uuid):
        print(f"Remap globally {from_uuid} to {to_uuid}.")
        self.moledata.remap_uuid(from_uuid, to_uuid)
        self.show_current()

class MoleData:
    def __init__(self, path_list):
        # Make an instance-specific cache of images. Note that this means that
        # mel will need to be re-run in order to pick up changes to mole
        # images. This seems to be fine for use-cases to date, only the mole
        # data seems to change from underneath really.
        @functools.lru_cache()
        def load_image(image_path):
            return mel.lib.image.load_image(image_path)

        self._load_image = load_image

        self.moles = []
        self.metadata = {}
        self.image = None
        self.mask = None
        self._mask_path = None
        self._path_list = path_list
        self._list_index = 0
        self.image_path = self._path_list[self._list_index]
        self._num_images = len(self._path_list)
        self._loaded_index = None
        self.ensure_loaded()

    def get_image(self):
        self.ensure_loaded()
        return self.image

    def reload(self):
        self._loaded_index = None
        self.ensure_loaded()

    def ensure_loaded(self):
        if self._loaded_index == self._list_index:
            return

        image_path = self._path_list[self._list_index]
        self.image = self._load_image(image_path)
        self.image_path = image_path

        self.moles = mel.rotomap.moles.load_image_moles(image_path)
        self.metadata = mel.rotomap.moles.load_image_metadata(image_path)

        height, width = self.image.shape[:2]
        self._mask_path = mel.rotomap.mask.path(image_path)
        self.mask = mel.rotomap.mask.load_or_none(image_path)
        if self.mask is None:
            self.mask = numpy.zeros((height, width), numpy.uint8)

        self._loaded_index = self._list_index

    def remap_uuid(self, from_uuid, to_uuid):
        for image_path in self._path_list:
            moles = mel.rotomap.moles.load_image_moles(image_path)
            for m in moles:
                if m["uuid"] == from_uuid:
                    m["uuid"] = to_uuid
                    m[mel.rotomap.moles.KEY_IS_CONFIRMED] = True
            mel.rotomap.moles.save_image_moles(moles, image_path)

        image_path = self._path_list[self._list_index]
        self.moles = mel.rotomap.moles.load_image_moles(image_path)

    def decrement(self):
        new_index = self._list_index + self._num_images - 1
        self._list_index = new_index % self._num_images

    def increment(self):
        self._list_index = (self._list_index + 1) % self._num_images

    def index(self):
        return self._list_index

    def save_mask(self):
        mel.lib.common.write_image(self._mask_path, self.mask)

    def save_moles(self):
        image_path = self._path_list[self._list_index]
        mel.rotomap.moles.normalise_moles(self.moles)
        mel.rotomap.moles.save_image_moles(self.moles, image_path)

    def current_image_path(self):
        return self._path_list[self._list_index]

    def try_jump_to_path(self, path):
        for i, image_path in enumerate(self._path_list):
            if str(path) == str(image_path):
                if self._list_index != i:
                    self._list_index = i
                    self.ensure_loaded()
                return True
        return False

# -----------------------------------------------------------------------------
# Copyright (C) 2016-2018 Angelos Evripiotis.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ------------------------------ END-OF-FILE ----------------------------------


import tensorflow as tf
import utils  # local file import from baselines.toxic_comments

class UtilsTest(tf.test.TestCase):

  def test_make_cv_train_and_eval_splits(self):
    num_folds = 10
    train_fold_ids = ['2', '5']

    (train_split, eval_split, train_folds, eval_folds,
     eval_fold_ids) = utils.make_cv_train_and_eval_splits(
         num_folds, train_fold_ids, return_individual_folds=True)

    expected_train_folds = ['train[20%:30%]', 'train[50%:60%]']
    expected_eval_folds = [
        'train[0%:10%]', 'train[10%:20%]', 'train[30%:40%]', 'train[40%:50%]',
        'train[60%:70%]', 'train[70%:80%]', 'train[80%:90%]', 'train[90%:100%]'
    ]
    expected_eval_fold_ids = [0, 1, 3, 4, 6, 7, 8, 9]

    self.assertEqual(train_split, 'train[20%:30%]+train[50%:60%]')
    self.assertEqual(eval_split, '+'.join(expected_eval_folds))
    self.assertListEqual(train_folds, expected_train_folds)
    self.assertListEqual(eval_folds, expected_eval_folds)
    self.assertListEqual(eval_fold_ids, expected_eval_fold_ids)

if __name__ == '__main__':
  tf.test.main()


from pyspark.sql import SparkSession
from pyspark.sql.functions import lit

spark = (SparkSession
         .builder
         .appName("files_creator")
         .config("spark.sql.sources.partitionOverwriteMode", "dynamic")
         .config("hive.exec.dynamic.partition", "true")
         .config("hive.exec.dynamic.partition.mode", "nonstrict")
         .enableHiveSupport()
         .getOrCreate()
         )

df = spark.range(100000).cache()
df2 = df.withColumn("partitionCol", lit("p1"))
df2.repartition(200).write.partitionBy("partitionCol").saveAsTable("schema.table")