import pyro
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from copy import deepcopy
from pyro.optim import Adam
from pyro.infer import SVI, Trace_ELBO
from pyro.nn import PyroModule, PyroSample
from pyro.distributions import constraints
from pyro.distributions.transforms import Radial, Planar
from pyro.infer.autoguide import AutoMultivariateNormal
from pyro.distributions import Normal, MultivariateNormal, TransformedDistribution

class RegressionModel(nn.Module):
    """Simple neural network regression model with one hidden layer"""

    def __init__(self, n_features, noise_var=1, device='cuda'):
        super().__init__()
        self.n_features = n_features
        self.noise_var = noise_var
        self.device = device

        self.fc_1 = nn.Linear(in_features=self.n_features, out_features=25)
        self.fc_2 = nn.Linear(in_features=25, out_features=1)

        self.ordered_modules = nn.ModuleList([
            self.fc_1,
            nn.ReLU(),
            self.fc_2
        ])

        self.noise_scale = torch.sqrt(
            torch.tensor(self.noise_var, dtype=torch.float, device=self.device)
        )
        self.to(self.device)

    def forward(self, x):
        for module in self.ordered_modules:
            x = module(x)
        return x

    def loss(self, model_output, target):
        log_likelihood = Normal(
            loc=model_output, scale=self.noise_scale).log_prob(target).sum()
        return -log_likelihood

    def optimizer(self, lr=1e-3, weight_decay=0):
        return optim.Adam(self.parameters(), lr=lr, weight_decay=weight_decay)

class BayesianLastLayer:
    """Neural network with Bayesian last layer."""
    def __init__(self, base_model, noise_var=1):
        self.base_model = base_model
        self.noise_var = noise_var
        self.device = base_model.device

        # Feature extractor
        self.feature_extractor = nn.Sequential(
            *self.base_model.ordered_modules[:-1]
        )
        # Properties of last layer (assumed to be linear)
        self.in_features = self.base_model.ordered_modules[-1].in_features
        self.out_features = self.base_model.ordered_modules[-1].out_features
        self.n_weight = self.base_model.ordered_modules[-1].weight.numel()
        self.n_bias = self.base_model.ordered_modules[-1].bias.numel()

         # Distribution over parameters in last layer
        self.base_mean = torch.zeros(
            self.n_weight + self.n_bias, device=self.device
        )
        self.base_scale_tril = torch.eye(
            self.n_weight + self.n_bias, device=self.device
        )
        self.base_dist = MultivariateNormal(
            loc=self.base_mean,
            scale_tril=self.base_scale_tril
        )

        self.noise_scale = torch.sqrt(
            torch.tensor(self.noise_var, dtype=torch.float, device=self.device)
        )

    def model(self, x, y=None):
        # Sample parameters
        parameters = pyro.sample('parameters', self.base_dist)
        parameters = parameters.squeeze()

        # Extract weight and biases
        weight = parameters[...,:self.n_weight]
        weight = weight.reshape(
            *weight.shape[:-1], self.out_features, self.in_features
        )
        bias = parameters[...,self.n_weight:]

        # Compute model output
        x_encoded = self.feature_extractor(x)
        model_output = x_encoded @ weight.transpose(-2,-1) + bias.unsqueeze(-2)
        model_output = model_output.squeeze()

        with pyro.plate('data', size=len(x)):
            obs = pyro.sample(
                    'obs',
                    Normal(loc=model_output, scale=self.noise_scale),
                    obs=y
                )
            return model_output

    def guide_mvn(self, x, y=None):
        scale = pyro.param(
            'scale',
            0.1*torch.ones(self.n_weight + self.n_bias, device=self.device),
            constraint=constraints.softplus_positive
        )
        scale_tril = pyro.param(
            'scale_tril',
            torch.eye(self.n_weight + self.n_bias, device=self.device),
            constraint=constraints.unit_lower_cholesky
        )
        loc = pyro.param(
            'loc',
            torch.zeros(self.n_weight + self.n_bias, device=self.device),
        )

        parameters = pyro.sample(
            'parameters',
            MultivariateNormal(loc=loc, scale_tril=scale[..., None]*scale_tril)
        )

    def guide_nf(self, x,  y=None):
        num_flows = 5
        flows = [Planar(self.n_weight + self.n_bias) for _ in range(num_flows)]
        for i, flow in enumerate(flows):
            flow.to(self.device)
            pyro.module(f'flow_{i}', flow)

        parameters = pyro.sample(
            'parameters',
            TransformedDistribution(self.base_dist, flows)
        )

def set_seed(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    pyro.set_rng_seed(seed)

def train_bayesian_last_layer(X, y, model, num_particles, vectorize_particles):
    pyro.clear_param_store()
    print(f'Num particles: {num_particles}; vectorized: {vectorize_particles}')
    set_seed(0)
    for name, param in pyro.get_param_store().named_parameters():
        print(name, param)

    # Make last layer Bayesian
    model = deepcopy(model)
    model_bll = BayesianLastLayer(model)
    #guide = AutoMultivariateNormal(model_bll.model)

    elbo = Trace_ELBO(
        num_particles=num_particles, vectorize_particles=vectorize_particles
    )
    svi = SVI(
        model_bll.model,
        model_bll.guide_nf,
        Adam({"lr": 1e-4}),
        elbo
    )

    n_svi_steps = 10000
    for step in range(n_svi_steps):
        loss = svi.step(X, y)
        if step % 100 == 0:
            print(f'SVI step: {step}, Loss: {loss}')

    for name, param in pyro.get_param_store().named_parameters():
        print(name, param)

    print('\n')

# Data
set_seed(0)
device = 'cuda'
N, M = 1000, 15
X = torch.randn(N, M, device=device)
w = torch.randn(M, 1, device=device)
y = X @ w + torch.randn(N, 1, device=device)
y = y.squeeze(-1)
print(X.shape, y.shape)

# Train initial neural network
n_train_steps = 10000
model = RegressionModel(n_features=M, device=device)
optimizer = model.optimizer()
for step in range(n_train_steps):
    model_output = model(X)
    loss = model.loss(model_output, y)

    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    if step % 100 == 0:
        print(f'Train step: {step}; Loss: {loss.detach().item()}')
print('\n')

# Seems to work
train_bayesian_last_layer(
    X, y, model, num_particles=100, vectorize_particles=True
)

""""
# Not sure if this works, loss don't seem to decrease much
train_bayesian_last_layer(
    X, y, model, num_particles=1, vectorize_particles=True
)

# Seems to work
train_bayesian_last_layer(
    X, y, model, num_particles=7, vectorize_particles=False
)

# Does not seem to work, loss increases a lot
train_bayesian_last_layer(
    X, y, model, num_particles=7, vectorize_particles=True
)
"""


from collections import Counter
f = open("python.csv", encoding="gbk")
lines = f.readlines()
f.close()

city_list = []

for line in lines:
    line = line.strip()
    city = line.split('","')[2]
    city_list.append(city)

result = Counter(city_list)
city_info = result.most_common(20)
city_info = {city:count for city,count in city_info}

from pyecharts import Pie
columns = city_info.keys()
data1 = city_info.values()
pie = Pie("饼状图", "python在各城市工作数量", title_pos='center', width=900)
pie.add("工作数量", columns, data1, center=[75, 50], is_legend_show=False, is_label_show=True)
pie.render("饼状图.html")


import os
import logging

import torch
import torchvision
import numpy as np
from dataloader.dataloader import CIFAR10

## color augmentation
## https://github.com/kevin-ssy/FishNet/blob/master/utils/data_aug.py #4
class ColorAugmentation(object):
    def __init__(self):
        self.eig_vec = torch.Tensor([
            [0.4009, 0.7192, -0.5675],
            [-0.8140, -0.0045, -0.5808],
            [0.4203, -0.6948, -0.5836],
        ])
        self.eig_val = torch.Tensor([[0.2175, 0.0188, 0.0045]])

    def __call__(self, tensor):
        assert tensor.size(0) == 3
        alpha = torch.normal(mean=torch.zeros_like(self.eig_val)) * 0.1
        quatity = torch.mm(self.eig_val * alpha, self.eig_vec)
        tensor = tensor + quatity.view(3, 1, 1)
        return tensor

class BestSaver(object):

    def __init__(self, comment=None):
        exe_fname=os.path.basename(__file__)
        save_path = "{}".format(exe_fname.split(".")[0])

        if comment is not None and str(comment):
            save_path = save_path + "_" + str(comment)

        save_path = save_path + ".pth"

        self.save_path = save_path
        self.best = float('-inf')

    def save(self, metric, data):
        if metric >= self.best:
            self.best = metric
            torch.save(data, self.save_path)
            logging.info("Saved best model to {}".format(self.save_path))

class AverageMeter(object):

    def __init__(self):
        self.reset()

    def reset(self):
        self.val, self.avg, self.sum, self.count = 0.0, 0.0, 0.0, 0.0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def config_logging(comment=None):

    if comment is not None and str(comment):
        log_fname = str(comment)

    log_fname = log_fname + ".log"
    log_format = "%(asctime)s [%(levelname)-5.5s] %(message)s"

    logging.basicConfig(
        level=logging.INFO,
        format=log_format,
        handlers=[logging.FileHandler(log_fname), logging.StreamHandler()]
    )

def prepare_dataloader(
            batch_size=256,
            img_size=32):

    img_size = img_size

    train_transform = torchvision.transforms.Compose([
        torchvision.transforms.RandomCrop(img_size),
        torchvision.transforms.RandomHorizontalFlip(),
        torchvision.transforms.ToTensor(),
        ColorAugmentation(),
        torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                         std=[0.229, 0.224, 0.225])
    ])
    test_transform = torchvision.transforms.Compose([
        torchvision.transforms.ToTensor(),
        torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                         std=[0.229, 0.224, 0.225])
    ])

    train_dataset = CIFAR10(img_path='./data/train/',
                        json_path='./data/',
                        json_file='train_img2labels.json',
                        transform=train_transform)
    train_loader = torch.utils.data.DataLoader(
                            train_dataset,
                            batch_size=batch_size,
                            shuffle=True
    )

    valid_dataset = CIFAR10(img_path='./data/valid/',
                        json_path='./data/',
                        json_file='valid_img2labels.json',
                        transform=test_transform)
    valid_loader = torch.utils.data.DataLoader(
                            valid_dataset,
                            batch_size=batch_size,
                            shuffle=False
    )

    test_dataset = CIFAR10(img_path='./data/test/',
                        json_path='./data/',
                        json_file='test_img2labels.json',
                        transform=test_transform)
    test_loader = torch.utils.data.DataLoader(
                            test_dataset,
                            batch_size=batch_size,
                            shuffle=False
    )

    return train_dataset, train_loader, valid_dataset, valid_loader, test_dataset, test_loader


import math
import numpy as np

import torch
import torch.nn as nn
import torch.nn.functional as F

def mahalanobis_distance(X, Y, M):
    quad_term1 = torch.sum((X @ M) * X, dim=1, keepdim=True)
    quad_term2 = torch.sum((Y @ M) * Y, dim=1, keepdim=True).t()
    cross_term = 2 * (X @ M) @ Y.t()
    dist = quad_term1 + quad_term2 - cross_term
    return dist

def euclidean_distance(X, Y):
    quad_term1 = torch.sum(X * X, dim=-1, keepdim=True)
    quad_term2 = torch.sum(Y * Y, dim=-1, keepdim=True).transpose(-2, -1)
    cross_term = 2 * torch.matmul(X, Y.transpose(-2, -1))
    dist = quad_term1 + quad_term2 - cross_term
    return dist

def logmvnpdf(X, mu, Sigma):
    dim = X.shape[1]
    var = Sigma.view(1, -1)
    const = -0.5 * dim * math.log(2 * math.pi) - 0.5 * dim * torch.log(var)
    exp_term = (
        -0.5
        / var
        * (
            torch.sum(X ** 2, dim=1, keepdim=True)
            + torch.sum(mu ** 2, dim=1, keepdim=True).t()
            - 2 * torch.mm(X, mu.t())
        )
    )
    return const + exp_term

def logmvnpdf_noisy(X, mu, Sigma, variance_noise):
    dim = X.shape[1]
    var = Sigma.view(1, -1) + variance_noise.view(-1, 1)
    const = -0.5 * dim * math.log(2 * math.pi) - 0.5 * dim * torch.log(var)
    exp_term = (
        -0.5
        / var
        * (
            torch.sum(X ** 2, dim=1, keepdim=True)
            + torch.sum(mu ** 2, dim=1, keepdim=True).t()
            - 2 * torch.mm(X, mu.t())
        )
    )
    return const + exp_term


import bpy
from bpy.types import PropertyGroup
from bpy.props import BoolProperty, IntVectorProperty, StringProperty, IntProperty, EnumProperty
from .get_slot_info import get_slot_info
from .image_is_visible import image_is_visible
from .ImagePropertiesAccessors import *
from .multiple_image_objects import multiple_image_objects
from .all_objects_have_slots import all_objects_have_slots
from ..Common import image_object
from .get_or_make_animation_pattern import get_or_make_animation_pattern
default_animation_pattern = 'f"filename{iz(3)}.png"'

def set_visible(o, value):
    # type: (bpy.EmptyImageObjectTypeWithData, bool) -> None
    o.show_empty_image_orthographic = value
    o.keyframe_insert(
        "show_empty_image_orthographic",
        frame=bpy.context.scene.frame_current
    )

def set_is_a_slot(o, value):
    # type: (bpy.EmptyImageObjectTypeWithData, bool) -> None
    if not value:
        o["font_is_a_slot"] = False
        return
    o["font_is_a_slot"] = True
    if "font_slot_number" not in o:
        o['font_slot_number'] = 0
    if "font_slot_name" not in o:
        o['font_slot_name'] = ""

class ImageProperties(PropertyGroup):
    """Property for editing images"""

    def get_number_of_objects(self):
        # type: () -> int
        objects = multiple_image_objects(bpy.context)
        if objects is None:
            return 0
        return len(objects)

    def set_number_of_objects(self, value):
        # type: (int) -> None
        objects = multiple_image_objects(bpy.context)
        if objects is None:
            return
        num_objects = len(objects)
        if num_objects == value:
            return
        if num_objects < value:
            number_to_create = value - num_objects
            last_object = objects[num_objects - 1]
            for _ in range(number_to_create):
                o = bpy.data.objects.new(
                    name=last_object.name,
                    object_data=None
                )
                bpy.context.view_layer.active_layer_collection.collection.objects.link(
                    o)
                o.data = last_object.data
                o.empty_display_type = "IMAGE"
                o.parent = last_object.parent
                o.location[0] = last_object.location[0]
                o.location[2] = last_object.location[2]
                o.select_set(True)
            return
        for o in objects[value:]:
            o.parent = None
            o.select_set(False)
            bpy.context.view_layer.active_layer_collection.collection.objects.unlink(o)

    animation_number_of_objects: IntProperty(
        name="Number of Images",
        description="The number of images in this animation",
        min=2,
        get=get_number_of_objects,
        set=set_number_of_objects
    )

    animation_show_all_preview: BoolProperty(
        name="Preview all selected names",
        description="Preview all selected names",
        default=True
    )

    animation_number_of_preview_name: IntProperty(
        name="Number of images to preview",
        description="Number of images to preview",
        min=0,
        default=1
    )

    animation_types = [
        ("INCREMENT", "Increment", "Load files counting upwards", 1),
        ("DECREMENT", "Decrement", "Load files counting downwards", 2),
        ("PATTERN", "Pattern",
         "Load files in a pattern", 3),
        ("PYTHON", "Python", "Load files in a pattern given a python script", 4)
    ]

    def get_animation_pattern_type(self):
        # type: () -> int
        o = image_object(bpy.context.active_object)
        if o is None or "font_animation_pattern_info" not in o:
            return 1
        type = o["font_animation_pattern_info"]["type"]
        if type == "INCREMENT":
            return 1
        if type == "DECREMENT":
            return 2
        if type == "PATTERN":
            return 3
        if type == "PYTHON":
            return 4
        return 1

    def set_animation_pattern_type(self, value):
        # type: (int) -> None
        o = image_object(bpy.context.active_object)
        if o is None:
            return
        pattern = get_or_make_animation_pattern(o)

        if value == 1:
            pattern["type"] = "INCREMENT"
        elif value == 2:
            pattern["type"] = "DECREMENT"
        elif value == 3:
            pattern["type"] = "PATTERN"
        elif value == 4:
            pattern["type"] = "PYTHON"

    animation_pattern_type: EnumProperty(
        items=animation_types,
        name="Pattern Type",
        get=get_animation_pattern_type,
        set=set_animation_pattern_type)

    animation_num_digits: IntProperty(
        name="Number of Digits",
        get=lambda s: pattern_get_prop("num_digits", 2),
        set=lambda s, v: pattern_set_prop("num_digits", v)
    )

    animation_start_number: IntProperty(
        name="First Number",
        min=0,
        get=lambda s: pattern_get_prop("start_number", 0),
        set=lambda s, v: pattern_set_prop("start_number", v)
    )
    animation_end_number: IntProperty(
        name="Last Number",
        min=0,
        get=lambda s: pattern_get_prop("end_number", 0),
        set=lambda s, v: pattern_set_prop("end_number", v)
    )

    animation_wrap_around: BoolProperty(
        name="Wrap Around",
        description="After you go above end number, start again from the beginning",
        get=lambda s: pattern_get_prop("wrap_around", True),
        set=lambda s, v: pattern_set_prop("wrap_around", v)
    )

    animation_file_name: StringProperty(
        name="Filename pattern",
        description="The name of the file, don't include the number at the end. or the extension Example: if your file is image_000, then type in image_",
        get=lambda s: pattern_get_prop("filename", ""),
        set=lambda s, v: pattern_set_prop("filename", v)
    )
    animation_file_extension: StringProperty(
        name="File extension",
        description="The name of the file extension. Please include the .beforehand, like .png",
        get=lambda s: pattern_get_prop("extension", ".png"),
        set=lambda s, v: pattern_set_prop("extension", v)
    )

    animation_python_snippet: StringProperty(
        name="Python Snippet",
        description="A python expression to return the filename given the index i (starting at zero).\n Look at the docs for the list of helper functions",
        get=lambda s: pattern_get_prop(
            "python", ""),
        set=lambda s, v: pattern_set_prop("python", v)
    )

    animation_pattern_size: IntProperty(
        name="Pattern Size",
        min=1,
        max=14,
        get=lambda s: pattern_get_prop("pattern_size", 1),
        set=lambda s, v: pattern_set_prop("pattern_size", v)
    )

    animation_pattern1: IntVectorProperty(
        name="Pattern",
        min=0,
        description="A pattern of numbers like [0,3,2] will load filename00, filename03, filename02, filename00, filename003 and so on",
        size=1,
        get=lambda s: get_pattern_of_length(1),
        set=lambda s, v: set_pattern_of_length(v),
    )
    animation_pattern2: IntVectorProperty(
        name="Pattern",
        min=0,
        description="A pattern of numbers like [0,3,2] will load filename00, filename03, filename02, filename00, filename003 and so on",
        size=2,
        get=lambda s: get_pattern_of_length(2),
        set=lambda s, v: set_pattern_of_length(v),
    )
    animation_pattern3: IntVectorProperty(
        name="Pattern",
        min=0,
        description="A pattern of numbers like [0,3,2] will load filename00, filename03, filename02, filename00, filename003 and so on",
        size=3,
        get=lambda s: get_pattern_of_length(3),
        set=lambda s, v: set_pattern_of_length(v),
    )
    animation_pattern4: IntVectorProperty(
        name="Pattern",
        min=0,
        description="A pattern of numbers like [0,3,2] will load filename00, filename03, filename02, filename00, filename003 and so on",
        size=4,
        get=lambda s: get_pattern_of_length(4),
        set=lambda s, v: set_pattern_of_length(v),
    )
    animation_pattern5: IntVectorProperty(
        name="Pattern",
        min=0,
        description="A pattern of numbers like [0,3,2] will load filename00, filename03, filename02, filename00, filename003 and so on",
        size=5,
        get=lambda s: get_pattern_of_length(5),
        set=lambda s, v: set_pattern_of_length(v),
    )
    animation_pattern6: IntVectorProperty(
        name="Pattern",
        min=0,
        description="A pattern of numbers like [0,3,2] will load filename00, filename03, filename02, filename00, filename003 and so on",
        size=6,
        get=lambda s: get_pattern_of_length(6),
        set=lambda s, v: set_pattern_of_length(v),
    )
    animation_pattern7: IntVectorProperty(
        name="Pattern",
        min=0,
        description="A pattern of numbers like [0,3,2] will load filename00, filename03, filename02, filename00, filename003 and so on",
        size=7,
        get=lambda s: get_pattern_of_length(7),
        set=lambda s, v: set_pattern_of_length(v),
    )
    animation_pattern8: IntVectorProperty(
        name="Pattern",
        min=0,
        description="A pattern of numbers like [0,3,2] will load filename00, filename03, filename02, filename00, filename003 and so on",
        size=8,
        get=lambda s: get_pattern_of_length(8),
        set=lambda s, v: set_pattern_of_length(v),
    )
    animation_pattern9: IntVectorProperty(
        name="Pattern",
        min=0,
        description="A pattern of numbers like [0,3,2] will load filename00, filename03, filename02, filename00, filename003 and so on",
        size=9,
        get=lambda s: get_pattern_of_length(9),
        set=lambda s, v: set_pattern_of_length(v),
    )
    animation_pattern10: IntVectorProperty(
        name="Pattern",
        min=0,
        description="A pattern of numbers like [0,3,2] will load filename00, filename03, filename02, filename00, filename003 and so on",
        size=10,
        get=lambda s: get_pattern_of_length(10),
        set=lambda s, v: set_pattern_of_length(v),
    )
    animation_pattern11: IntVectorProperty(
        name="Pattern",
        min=0,
        description="A pattern of numbers like [0,3,2] will load filename00, filename03, filename02, filename00, filename003 and so on",
        size=11,
        get=lambda s: get_pattern_of_length(11),
        set=lambda s, v: set_pattern_of_length(v),
    )
    animation_pattern12: IntVectorProperty(
        name="Pattern",
        min=0,
        description="A pattern of numbers like [0,3,2] will load filename00, filename03, filename02, filename00, filename003 and so on",
        size=12,
        get=lambda s: get_pattern_of_length(12),
        set=lambda s, v: set_pattern_of_length(v),
    )
    animation_pattern13: IntVectorProperty(
        name="Pattern",
        min=0,
        description="A pattern of numbers like [0,3,2] will load filename00, filename03, filename02, filename00, filename003 and so on",
        size=13,
        get=lambda s: get_pattern_of_length(13),
        set=lambda s, v: set_pattern_of_length(v),
    )
    animation_pattern14: IntVectorProperty(
        name="Pattern",
        min=0,
        description="A pattern of numbers like [0,3,2] will load filename00, filename03, filename02, filename00, filename003 and so on",
        size=14,
        get=lambda s: get_pattern_of_length(14),
        set=lambda s, v: set_pattern_of_length(v),
    )

    animation_key_before: BoolProperty(
        name="Invisible before",
        get=lambda s: get_prop("font_animation_key_before", True),
        set=lambda s, v: set_prop("font_animation_key_before", v),
        description="Add a key, setting all frames to invisible, before the animation"
    )
    animation_key_after: BoolProperty(
        name="Invisible After",
        get=lambda s: get_prop("font_animation_key_after", True),
        set=lambda s, v: set_prop("font_animation_key_after", v),
        description="Add a key, setting all frames to invisible, after the animation"
    )

    animation_repeat_times: IntProperty(
        name="Number Of Times",
        min=1,
        description="Play the animation this number of times. If you put 1, it will play once.",
        get=lambda s: get_prop("font_animation_repeat_times", 1),
        set=lambda s, v: set_prop("font_animation_repeat_times", v),
    )

    animation_use_current_frame: BoolProperty(
        name="Use Current Frame",
        description="Use the current frame as the start of the animation",
        get=lambda s: get_prop(
            "font_animation_use_current_frame", default=True),
        set=lambda s, v: set_prop("font_animation_use_current_frame", v)
    )

    animation_frame_number: IntProperty(
        name="Frame Number",
        description="The frame number to start the animation",
        get=lambda s: get_prop(
            "font_animation_start_frame", 0),
        set=lambda s, v: set_prop("font_animation_start_frame", v)
    )

    slot_name: StringProperty(
        name="Slot Name",
        description="(Optional) The name of the slot, only for your convenience.\nThis name will appear in the scene tree editor",
        get=lambda s: get_prop("font_slot_name", ""),
        set=lambda s, v: set_prop(
            "font_slot_name", v)
    )

    slot_number: IntProperty(
        name="Slot Number",
        description="All slots that share this slot number will be replaced by the same image",
        get=lambda s: get_prop("font_slot_number", 0),
        set=lambda s, v: set_prop("font_slot_number", v)
    )

    def get_is_slot(self):
        # type: () -> bool
        o = image_object(bpy.context.active_object)
        if o is None:
            return False
        return get_slot_info(o) is not None

    def set_is_slot(self, value):
        # type: (bool) -> None
        o = image_object(bpy.context.active_object)
        if o is None:
            return
        set_is_a_slot(o, value=value)

    is_slot: BoolProperty(
        name="Slot",
        description="If this is image is a slot, then the template image will be replaced by 'slot' image when you export the scene",
        get=get_is_slot,
        set=set_is_slot)

    def get_is_visible(self):
        # type: () -> bool
        o = image_object(bpy.context.active_object)
        if o is None:
            return False
        return image_is_visible(o, bpy.context.scene.frame_current)

    def set_is_visible(self, value):
        # type: (bool) -> None
        o = image_object(bpy.context.active_object)
        if o is None:
            return
        set_visible(o, value=value)

    is_visible: BoolProperty(
        name="Visible",
        description="If the image is visible or invisible at this frame.",
        get=get_is_visible,
        set=set_is_visible)

    def get_all_are_visible(self):
        # type: () -> bool
        objects = multiple_image_objects(bpy.context)
        if objects is None:
            return False
        for o in objects:
            if not image_is_visible(o, bpy.context.scene.frame_current):
                return False
        return True

    def set_all_are_visible(self, value):
        # type: (bool) -> None
        objects = multiple_image_objects(bpy.context)
        if objects is None:
            return
        for o in objects:
            set_visible(o, value=value)

    all_visible: BoolProperty(
        name="All Visible",
        description="If all selected images are visible.",
        get=get_all_are_visible,
        set=set_all_are_visible)

    def get_all_are_slots(self):
        # type: () -> bool
        objects = multiple_image_objects(bpy.context)
        if objects is None:
            return False
        return all_objects_have_slots(objects)

    def set_all_are_slots(self, value):
        # type: (bool) -> None
        objects = multiple_image_objects(bpy.context)
        if objects is None:
            return
        for o in objects:
            set_is_a_slot(o, value=value)

    all_are_slots: BoolProperty(
        name="Are all slots",
        description="If these is images are slots, then the template image will be replaced by 'slot' image when you export the scene",
        get=get_all_are_slots,
        set=set_all_are_slots)

    def get_all_slot_numbers(self):
        # type: () -> int
        objects = multiple_image_objects(bpy.context)
        if objects is None:
            return -1
        slot_num = -1
        for o in objects:
            slot_info = get_slot_info(o)
            if slot_info is None:
                return -1
            if slot_info.number == slot_num:
                continue
            if slot_num != -1:
                return -1
            slot_num = slot_info.number
        return slot_num

    def set_all_slot_numbers(self, value):
        # type: (int) -> None
        objects = multiple_image_objects(bpy.context)
        if objects is None:
            return
        for o in objects:
            o["font_slot_number"] = value

    all_slot_numbers: IntProperty(
        name="All Slot Numbers (-1 if they disagree)",
        description="All slots that share this slot number will be replaced by the same image",
        get=get_all_slot_numbers,
        set=set_all_slot_numbers
    )

    def get_all_slot_names(self):
        # type: () -> str
        objects = multiple_image_objects(bpy.context)
        if objects is None:
            return "No names"
        names = {o['font_slot_name']
                 for o in objects if 'font_slot_name' in o and o['font_slot_name'] != ""}
        if len(names) <= 0:
            return "No names"
        return " & ".join(names)

    def set_all_slot_names(self, value):
        # type: (str) -> None
        objects = multiple_image_objects(bpy.context)
        if objects is None:
            return
        for o in objects:
            o["font_slot_name"] = value

    all_slot_names: StringProperty(
        name="All Slot Names",
        description="(Optional) The name of the slot, only for your convenience.\nThis name will appear in the scene tree editor",
        get=get_all_slot_names,
        set=set_all_slot_names
    )

    all_slot_start: IntProperty(name="Starting at slot number", default=0)
    all_slot_name_pattern: StringProperty(
        name="Pattern",
        description="A python expression to return the filename given the index i (starting at zero).",
        default="f\"slot{i}\""
    )


"""
Module for scaping reddit top (front page) posts and adding them to mongo database.
Still needs a method for excluding posts that are already in the database.
"""
import time
from bs4 import BeautifulSoup
from pymongo import MongoClient
import datetime
import random

from reddit_scraping import Reddit_Scrape

client = MongoClient('mongodb://localhost:27017/')
db = client.reddit_top_db
reddit_new_db = db.reddit_top

working = True
check_delay = 7200
html = 'http://www.reddit.com'

while working == True:
    reddit_posts = Reddit_Scrape(html)
    scrape_time = datetime.datetime.now()
    reddit_dict = reddit_posts.main_loop()
    for post,content in reddit_dict.items():
        reddit_new_db.insert_one({
            'post': post,
            'info': content,
            'time': scrape_time
        })
    time.sleep(check_delay + int(random.random()*100))


# -*- coding: utf-8 -*-
"""
Created on Wed Apr 15 15:46:29 2020

@author: digo
"""

import matplotlib.pyplot as plt
import numpy as np
# import hierarchical clustering libraries
import scipy.cluster.hierarchy as sch
from sklearn.cluster import AgglomerativeClustering

#Reading File
##Train data
f_train = open('training_set.txt', 'r')
data = f_train.read()
f_train.close()
data_train = data.split()
data_train = map(float, data_train)
data_train_m = np.array(data_train)
x = len(data_train_m)/3
y = 3
data_train_m = data_train_m.reshape(x,y)
data_train_t = data_train_m.transpose()
points = data_train_m[:,1:]

# create clusters
hc = AgglomerativeClustering(n_clusters=2, affinity = 'euclidean', linkage = 'ward')
# save clusters for chart
y_hc = hc.fit_predict(points)

#Calculate Error Rate
error = 0
for i in range(len(y_hc)):
    if(y_hc[i] != data_train_t[0][i]):
        error += 1
error_rate = float(error)/len(data_train_m)
print('Agglomerative Hierarchical Error:')
print(error_rate*100.00)
print((1-error_rate)*100.00)

plt.scatter(points[y_hc ==0,0], points[y_hc == 0,1], s=10, c='red')
plt.scatter(points[y_hc==1,0], points[y_hc == 1,1], s=10, c='blue')
plt.show()

# create dendrogram
dendrogram = sch.dendrogram(sch.linkage(points, method='ward'))


#

import pygame
import random

class Hero(object):
    def __init__(self, x, y):
        self.image = pygame.image.load('chgirl.png').convert_alpha()
        self.x = x
        self.y = y
        self.bullets = []
        self.frame = 0
        self.velocity = 0

    def draw(self, screen):
        screen.blit(self.image, (self.x, self.y))
        for bullet in self.bullets:
            pygame.draw.circle(screen, (255, 255, 255),
                               (bullet[0], bullet[1]), 6)

    def update(self, screen):
        # Update frame counter
        self.frame += 1

        remove = False
        # Move any existing bullets
        for bullet in self.bullets:
            bullet[0] += 1
            if bullet[0] > screen.get_width():
                remove = True

        # Delete any offscreen bullets
        if remove:
            del self.bullets[0]

        # Move hero
        self.y += self.velocity
        if self.y > screen.get_height() and self.velocity > 0:
            self.y = -self.image.get_height() + 70   # -101
        if self.y < -self.image.get_height() and self.velocity < 0:
            self.y = screen.get_height() - 70

    def shoot(self):
        if len(self.bullets) < 9:
            y_loc = self.y + 100
            self.bullets.append([170, y_loc])
            self.bullets.append([150, y_loc])
            self.bullets.append([130, y_loc])

    def move_down(self):
        self.velocity = 3

    def move_up(self):
        self.velocity = -3

class Villain(object):
    def __init__(self, x, y):
        self.image = pygame.image.load('pkgirl.png').convert_alpha()
        self.x = x
        self.y = y
        self.bullets = []
        self.frame = 0
        self.velocity = 0

    def draw(self, screen):
        screen.blit(self.image, (self.x, self.y))
        for bullet in self.bullets:
            pygame.draw.circle(screen, (192, 192, 192),
                               (bullet[0], bullet[1]), 6)

    def update(self, screen):
        if random.randint(1, 60) == 1:
            self.shoot()
        if random.randint(1, 100) == 1:
            self.velocity *= -1

        # Update frame counter
        self.frame += 1

        remove = False
        # Move any existing bullets
        for bullet in self.bullets:
            bullet[0] -= 1
            if bullet[0] < 0:
                remove = True

        # Delete any offscreen bullets
        if remove:
            del self.bullets[0]

        # Move hero
        self.y += self.velocity
        if self.y > screen.get_height() and self.velocity > 0:
            self.y = -self.image.get_height() + 70   # -101
        if self.y < -self.image.get_height() and self.velocity < 0:
            self.y = screen.get_height() - 70

    def shoot(self):
        if len(self.bullets) < 15:
            y_loc = self.y + 100
            self.bullets.append([self.x - 110, y_loc])
            self.bullets.append([self.x - 90, y_loc])
            self.bullets.append([self.x - 70, y_loc])

    def move_down(self):
        self.velocity = 3

    def move_up(self):
        self.velocity = -3

def init():
    width, height = 800, 600
    pygame.init()
    # Screen
    return pygame.display.set_mode((width, height))

def main(screen):
    clock = pygame.time.Clock()
    hero = Hero(60, 100)
    villain = Villain(640, 400)
    villain.move_up()
    running = True
    while running:
        screen.fill((0, 0, 0))          # Redraw background
        hero.draw(screen)
        villain.draw(screen)
        pygame.display.flip()           # Display screen in window
        hero.update(screen)
        villain.update(screen)

        for event in pygame.event.get():
            if event.type == pygame.QUIT or (event.type == pygame.KEYDOWN and event.key == pygame.K_q):
                # exit()
                running = False

            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_SPACE:
                    hero.shoot()
                if event.key == pygame.K_s:
                    hero.move_down()
                if event.key == pygame.K_w:
                    hero.move_up()
        clock.tick(40)

screen = init()
main(screen)


from rabbitpy import Connection, Exchange, Queue, Message

import sys
import os

sys.path.append(
    os.path.dirname(
        os.path.abspath(
            os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
        )
    )
)

from config.config_rabbitpy import URL

with Connection(URL) as conn:
    with conn.channel() as channel:
        # * 대체 Exchange 객체 생성
        alter_exchange = Exchange(
            channel=channel, name="alter_exchange", exchange_type="fanout"
        )
        # * 대체 Exchange 선언
        alter_exchange.declare()

        exchange = Exchange(
            channel=channel,
            name="main_exchange",
            exchange_type="topic",
            arguments={"alternate-exchange": alter_exchange.name},
        )

        exchange.declare()

        # * 라우팅될 수 없는 메시지는 아래 Queue에 저장됨.
        queue = Queue(channel=channel, name="unroutable-messages")
        queue.declare()

        if queue.bind(alter_exchange, "#"):
            print("Queue bound to alternate-exchange")

        msg = Message(
            channel=channel,
            body_value="test",
            properties={"content_type": "text/plain"},
        )
        msg.publish(exchange=exchange)
        # * Queue bound to alternate-exchange

"""
Fanout: 자신이 알고 있는 모든 Queue에 메시지를 전달함.

Topic: 라우팅 키를 기반으로 선택적으로 메시지를 라우팅할 수 있음.
"""


import sys
basket = ['apple', 'pear', 'cherry', 'banana', 'apricot']
def guess_check(guess):
    if guess in basket:
        print("You guessed RIGHT!")
    else:
        print("Sorry! I don't have that fruit.")

def main():
    guess = ""
    response = ""
    print("Welcome to the ruit basket guessing game.\n")
    while True:
        try:
            guess = raw_input("Guess the name of a fruit in my basket\n")
            print("You guessed: %s" % guess)
            guess_check(guess)
            response = raw_input("Want to play again? y/n \n")
            if response == "y":
                continue
            else:
                sys.exit()
        except ValueError:
            print("Sorry! Invalid Input")
            sys.exit()
main()


#go through all prism_uniprot files and make lists of which domains occur in which protein. The lists are per domain and list all proteins this domain occurs in. The domains are named with their pfam name
#Just the domains occuring in the most proteins. So if domain A occurs in p1, p2 and p3, count 3.
#all instances: if domain A appears twice in protein 4, count +2

import pandas as pd
import sys
import_path_base = '/storage1/hezscha/src/'
#import_path_base = '/home/henrike/Documents/PD_AS/src/'
sys.path.insert(1, import_path_base + 'PRISM/prism/scripts/')
from PrismData import PrismParser, VariantData#VariantParser, VariantData
from Bio import AlignIO
import numpy as np
import glob
import os
import math

def read_from_prism(primsfile):
	parser = PrismParser()
	dataframe = parser.read(primsfile).dataframe
	meta_data = parser.read_header(primsfile)
	return meta_data, dataframe

def write_prism(metadata, dataframe, prism_file, comment=''):
	variant_dataset = VariantData(metadata, dataframe)
	parser = PrismParser()
	parser.write(prism_file, variant_dataset, comment_lines=comment)

prism_dir = "/storage1/shared/data/prism/"
outdir = '/storage1/hezscha/genome_proteome_map/results/domain_analysis/domain_lists/'

for d1 in os.scandir(prism_dir):
	try:
		for d2 in os.scandir(d1.path):
			try:
				for d3 in os.scandir(d2.path):
					for prism_file in os.scandir(d3.path):
						if prism_file.name.endswith('_filled.txt') or prism_file.name.endswith('_SNPinv.txt'):
							continue

						elif prism_file.name.startswith('prism_uniprot_002_'):

							uniprot = prism_file.name.split('_')[-1].split('.')[0]
							print(uniprot)
							metadata,df = read_from_prism(prism_file.path)

							if 'pfam_name' in df.columns:
								domains = df.pfam_name.unique()

								for dom in domains:
									#print(dom)
									#skip nan
									#figured out nan is not a string. I can't test with isnan because that fails when dom is a string
									if not isinstance(dom, str):
										#print('Skipping', dom)
										continue

									list_file = os.path.join(outdir, dom+'.uniprot')
									#does a list already exist? if not, create it
									if not os.path.exists(list_file):
										with open(list_file, 'w') as OUT:
											print(uniprot, file = OUT)

									#else open in append mode
									else:
										with open(list_file, 'a') as OUT:
											print(uniprot, file = OUT)

			except NotADirectoryError:
				continue
	except NotADirectoryError:
		continue


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('code_based_auth', '0004_auto_20150104_1423'),
    ]

    operations = [
        migrations.AlterField(
            model_name='codecomponent',
            name='hash_algorithm',
            field=models.CharField(blank=True, max_length=16, null=True, choices=[(b'md5', b'md5'), (b'sha1', b'sha1'), (b'sha224', b'sha224'), (b'sha256', b'sha256'), (b'sha384', b'sha384'), (b'sha512', b'sha512'), (b'noop', b'No hash')]),
            preserve_default=True,
        ),
    ]


#!/usr/bin/env python3.2

_author_ = "Gabe Pike"
_email_ = "gpike@isecpartners.com"

"""

I'm not really sure the instructions are asking for. My first solution to
this problem was actually the solution for #20 because it was the obvious way
of doing it. I decided to do a chosen plaintext attack for this problem. Hope
that'll do.

"""

from sys import exit
from base64 import b64decode
from Crypto import Random
from Crypto.Cipher import AES
from mycrypto import aes_ctr_crypt

p19_key = Random.new().read(AES.block_size)

def generate_ciphertexts():
    pts = None
    with open('p19-input.txt') as fh:
        pts = [b64decode(pt.encode('utf8')) for pt in fh.readlines()]

    cts = []
    for pt in pts:
        cts.append(ctr_fixed_nonce(pt))

    return cts

def ctr_fixed_nonce(pt):
    nonce = b"\x00" * AES.block_size
    return aes_ctr_crypt(pt, p19_key, nonce)

def break_fail_ctr(cts):
    pts = []

    for ct in cts:
        pts.append(_break_fail_ctr(ct))

    return pts

def _break_fail_ctr(ct):
    pt = b''
    for i in range(len(ct)):
        for j in range(256):
            guess = bytes([j])
            if ct[i] == ctr_fixed_nonce(pt+guess)[i]:
                pt += guess
                break

    return pt

def main():
    ciphertexts = generate_ciphertexts()
    plaintexts = break_fail_ctr(ciphertexts)
    for pt in plaintexts:
        print(pt.decode("utf8"))

if __name__ == '__main__':
    exit(main())


# Definition for a binary tree node.
class TreeNode:
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

class Solution:
    def insertIntoMaxTree(self, root, val):
        """
        :type root: TreeNode
        :type val: int
        :rtype: TreeNode
        """
        if root and val < root.val:
            root.right = self.insertIntoMaxTree(root.right, val)
            return root
        else:
            new_root = TreeNode(val)
            new_root.left = root
            return new_root


import random
from paho.mqtt import client as mqtt_client
import tkinter as tk
from gui import myGUI
import sv_ttk

def con_mqtt_client():
    broker="localhost"
    port = 1883
    client_id = f'python-mqtt-{random.randint(0, 1000)}'

    def on_connect(client, userdata, flags, rc):
        if rc == 0:
            print("Connected to MQTT Broker!")
        else:
            print("Failed to connect, return code %d\n", rc)
    # Set Connecting Client ID
    client = mqtt_client.Client(client_id)
    client.on_connect = on_connect
    client.connect(broker, port)
    return client

def main():
    client = con_mqtt_client()

    client.loop_start()

    window = tk.Tk()
    window.title("Human interface")

    sv_ttk.set_theme("dark")
    gui = myGUI(window,client)
    window.mainloop()

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Apr 14 10:20:12 2023

@author: emgordy
"""

import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
import sys

sys.path.append('functions/')

# custom modules, les miennes
import experiment_settings
import metricplots
import importlib as imp
imp.reload(metricplots)

mpl.rcParams['figure.facecolor'] = 'white'
mpl.rcParams['figure.dpi']= 150
mpl.rcParams['font.family'] = 'sans-serif'
mpl.rcParams['font.sans-serif'] = ['Helvetica']
mpl.rcParams['font.size'] = 12

params = {"ytick.color" : "k",
          "xtick.color" : "k",
          "axes.labelcolor" : "k",
          "axes.edgecolor" : "k"}
plt.rcParams.update(params)

experiment_name = "predictSST_terciles20202050_singlelocation_nobias_betterpreprocessing"

# experiment_name = "predictT2M_terciles20202050_singlelocation"
experiment_dict = experiment_settings.get_experiment_settings(experiment_name)
filefront = experiment_dict["filename"]

#%%
lat = np.arange(-60,70,10)+5
lon = np.arange(0,360,10)+5

nlat = len(lat)
nlon = len(lon)

#%%

file = "metricsfiles/" + filefront + "test_2050.npz"
test_2050=np.load(file)

#%%

acc = test_2050["acc"]
acc = acc[:,:,0,:]

shuffle = test_2050["shuffleacc"]
shuffle = shuffle[:,:,0,:]

shuffle90 = test_2050["shuffle90"]
shuffle90 = shuffle90[:,:,0,:]

metricplots.compareaccuracy_IV_sig(acc,shuffle,lon,lat,shuffle90)
metricplots.compareaccuracy_IVSDP50_sig(acc,shuffle,lon,lat,shuffle90)
metricplots.compareaccuracy_IVSDP_sig(acc,shuffle,lon,lat,shuffle90)
metricplots.compareaccuracy_SDP_allish(acc,lon,lat)


class Element:
	def __init__(self,value):
		self.value=value

class Queue:
    def __init__(self):
    	self.head=None
    	self.tail=None
    	self.current=None
    	self.previous=None
    	self.next=None
    	self.length=0
    def enqueue(self, element):
    	if (self.head==None):
    		self.head=element
    		self.current=element
    		self.tail=element
    		self.length+=1
    	elif (self.head!=None and self.previous==None):
    		self.head=element
    		self.previous=self.current
    		self.current=element
    		self.tail=self.previous
    		self.length+=1
    	elif (self.head!=None and self.previous!=None):

    		self.head=element
    		self.previous=self.current
    		self.current=element
    		self.length+=1

    def dequeue(self):
    	if (self.head==None):
    		return "the queue is empty"
    	elif (self.head!=None and self.previous==None):
    		self.head=None
    		self.tail=None
    		self.current=None
    	elif (self.head!=None and self.previous!=None):
    		self.tail=self.previous
    		self.current=self.previous

    def len(self):
    	return self.length

    def is_empty(self):
    	if (self.head==None):
    		return True
    	else :
    		return False

    def get_front(self):
    	if (self.head==None):
    		return "the queue is empty"
    	else :
    		return self.head.value
    def get_end(self):
    	if (self.tail==None):
    		return "the queue is empty"
    	else:
    		return self.tail.value

   # def print_queue(self):

playlist1=Element("Playlist1")
playlist2=Element("Playlist2")
playlist3=Element("Playlist3")

playlist4=Element("Playlist4")
playlist5=Element("Playlist5")
playlist6=Element("Playlist6")
queue1=Queue()
queue1.enqueue(playlist6)
queue1.enqueue(playlist5)
queue1.enqueue(playlist4)
queue1.enqueue(playlist3)
queue1.enqueue(playlist2)
queue1.enqueue(playlist1)

print (queue1.is_empty())
print (queue1.get_front())
print (queue1.get_end())
print (queue1.len())


import pandas as pd
import mlflow
import os

# Setting the MLflow tracking server
#mlflow.set_tracking_uri('http://training.itu.dk:5000/')

# Setting the requried environment variables
# os.environ['MLFLOW_S3_ENDPOINT_URL'] = 'http://130.226.140.28:5000'
# os.environ['AWS_ACCESS_KEY_ID'] = 'training-bucket-access-key'
# os.environ['AWS_SECRET_ACCESS_KEY'] = 'tqvdSsEDnBWTDuGkZYVsRKnTeu'

#mlflow.set_experiment("<gebu> - a2")
from sklearn.preprocessing import OneHotEncoder
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import PolynomialFeatures
from sklearn.svm import SVR
from sklearn.model_selection import train_test_split, TimeSeriesSplit
from sklearn.metrics import r2_score
from sklearn.preprocessing import StandardScaler
from sklearn.compose import ColumnTransformer
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.model_selection import GridSearchCV
import json
import shutil
import warnings

def run(search, name):
    with mlflow.start_run(run_name= name) as run:
        df = pd.read_json("./dataset.json", orient="split")
        df = df.dropna()

        transformer = ColumnTransformer([
                ('standard_scaler', StandardScaler(), ['Speed']), #scales the data to mean 0 and std 1
                ('num_features', PolynomialFeatures(degree= 1), ['Speed']),
                ('Categorical', OneHotEncoder(), ['Direction'])
                ])

        pipeline = Pipeline(steps = [
                ('transformers', transformer),
                ('model', GradientBoostingRegressor())])

        X = df[["Speed","Direction"]]
        y = df["Total"]

        number_of_splits = 5

        #Splitting data intro training and test data
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, shuffle= False)

        tscv = TimeSeriesSplit(n_splits=5)
        grid = GridSearchCV(pipeline, search, scoring= 'r2',verbose=1, cv=tscv)
        pipe = grid.fit(X_train, y_train)
        r2_score_test = pipe.score(X_test, y_test)
        r2_score = pipe.best_score_
        params = pipe.best_params_
        mlflow.log_metric('r2', r2_score_test)
        mlflow.log_params(params)

        artifact_path= f"mlruns/1/{run.info.run_id}"
        mlflow.sklearn.log_model(pipe, artifact_path=artifact_path)

#Updating best model

        try:
          with open("best_regressor.json", "r") as f:
            best_model = json.loads(f.read())

          if r2_score_test > best_model["score"]:
            shutil.rmtree('best_model')
            print(f"New model outperforms old model")
            mlflow.sklearn.save_model(
                pipe, path = 'best_model', conda_env="conda.yml")

            model_details = {
                "model_name": name,
                "score": r2_score_test,
                }
            with open("best_regressor.json", "w") as f:
              json.dump(model_details, f)
          else:
            print(f"Model {name} does not perform better than current saved one")

        except:
          print(f"There is no model saved yet. Let's save model: {name}")
          mlflow.sklearn.save_model(
              pipe, path = 'best_model', conda_env="conda.yml")

          model_details = {
              "model_name": name,
              "score": r2_score_test,
              }
          with open("best_regressor.json", "w") as f:
            json.dump(model_details, f)

def main():
        warnings.filterwarnings("ignore")
        svr = [{'model': [SVR()],
         'model__gamma': ['scale', 'auto'],
         'model__C': [1,5,10]}]
        rfr = [{'model': [RandomForestRegressor()],
         'model__max_depth': [i for i in range(1,8)]}]
        gbr = [{'model': [GradientBoostingRegressor()],
         'model__learning_rate': [0.1,0.2,0.3],
         'model__n_estimators': [100,200,300]}]
        run(svr, 'SVR')
        run(rfr, 'RFR')
        run(gbr,'GBR')

if __name__ == "__main__":
  main()


from typing import Callable, Tuple

from einops import rearrange
import torch
from torch import Tensor

DEBUG = False  # activate debug mode when needed

def hardest_triplet(
    triplet: Tuple[Tensor, Tensor, Tensor],
    label: Tuple[Tensor, Tensor, Tensor],
    distance: Callable[[Tensor, Tensor], Tensor],
    mode: str = "min",
) -> Tuple[Tensor, Tensor]:
    """Return hardest negative distance of a batch for each anchor

    Args:
        triplet: the triplet input embedding (anchor, positive, negative) of
            shape (B, C) where C is embedding dimension
        label: the corresponding triplet of label, each of shape (B)
        distance: the distance to be used
        mode: if "min" hardest negative are supposed to have minimal distance
            to anchor, else maximum distance

    Returns:
        positive_distance: the distance between anchor and positive samples
            (shape B)
        hardest_negative_distance: the hardest distance between anchor and
            any other sample not belonging to the same label group
    """
    # triplet is a 3-tuple of anchor, positive, negative samples
    # with shape (B, C) and corresponding labels have shape (B)
    anchor = triplet[0]
    anchor_label = label[0]
    triplet = torch.cat(triplet)  # shape (3B, C)
    label = torch.cat(label)  # shape (3B)

    # compute pairwise distance between anchor and triplet
    anchor = rearrange(anchor, "b c -> b c 1")
    triplet = rearrange(triplet, "n c -> 1 c n")  # N == 3B
    dist = distance(anchor, triplet)  # shape (B, 3B)

    # compute pairwise positive distance
    positive_distance = torch.diag(
        rearrange(dist, "b (t b1) -> b t b1", t=3)[:, 1]  # B1 == B
    )

    # compute pairwise boolean for identifying samples belonging to same group
    anchor_label = rearrange(anchor_label, "b -> b 1")
    label = rearrange(label, "n -> 1 n")  # N == 3B
    same_label = anchor_label == label  # shape (B, 3B)

    # find hardest negative distance
    inf = torch.tensor(float("inf")).to(anchor.device)
    if mode == "min":
        updated_distance = torch.where(same_label, inf, dist)
        hardest_negative_distance = torch.min(updated_distance, dim=1)[0]
    else:
        updated_distance = torch.where(same_label, -inf, dist)
        hardest_negative_distance = torch.max(updated_distance, dim=1)[0]

    # used for checking
    if DEBUG:
        print(f"Original distance:\n{dist}")
        print(f"Distance after filtering positives:\n{updated_distance}")
        print(f"Positive distance:\n{positive_distance}")
        print(f"Hardest negative distance:\n{hardest_negative_distance}")

    return positive_distance, hardest_negative_distance

if __name__ == "__main__":
    # check return values on a simple case
    import torch.nn.functional as F

    device = "cuda" if torch.cuda.is_available() else "cpu"
    DEBUG = True

    B = 2
    C = 256
    triplet = tuple(torch.randn((B, C)).to(device) for i in range(3))
    label = tuple(
        torch.tensor(_label).to(device) for _label in [[0, 1], [0, 1], [2, 0]]
    )
    distance = lambda x, y: 1 - F.cosine_similarity(x, y)

    hardest_triplet(triplet, label, distance, mode="min")


#!/usr/bin/env python
import sys
import logging
import argparse
import pymongo
from pymongo import MongoClient
from pymongo.errors import ServerSelectionTimeoutError
from .argo_config import ArgoConfig
from .common import get_config_paths
from .common import get_log_conf

log = logging.getLogger(__name__)

class ArgoMongoClient(object):

    def __init__(self, args, config, cols):
        self.args = args
        self.config = config
        self.cols = cols

    def ensure_status_indexes(self, db):
        """Checks if required indexes exist in specific argo status-related collections
        in mongodb

        Args:
            db (obj): pymongo database object

        """

        log.info("Checking required indexes in status collections...")

        def is_index_included(index_set, index):
            """gets a set of mongodb indexes and checks if specified
            mongodb index exists in this set

            Args:
                index_set (dict): pymongo mongodb index object
                index (obj): pymongo index object

            Returns:
                bool: If index exists in set return true
            """

            for name in list(index_set.keys()):
                if index_set[name]['key'] == index:
                    return True
            return False
        # Used in all status collections
        index_report_date = [("report", pymongo.ASCENDING),
                             ("date_integer", pymongo.ASCENDING)]
        # Used only in status_metrics collection
        index_date_host = [("date_integer", pymongo.ASCENDING),
                           ("report", pymongo.ASCENDING)]
        status_collections = ["status_metrics", "status_endpoints",
                              "status_services", "status_endpoint_groups"]

        index_history = [("date_integer",pymongo.DESCENDING),
                         ("id",pymongo.ASCENDING)]

        index_downtimes = [("date_integer",pymongo.DESCENDING)]

        # Check indexes in sync collections
        for col_name in ["topology_endpoints", "topology_groups", "weights"]:
            col = db[col_name]
            indexes = col.index_information()
            if not is_index_included(indexes, index_history):
                # ensure index
                col.create_index(index_history, background=True)
                log.info("Created (date_integer,id) index in %s.%s",
                         col.database.name, col.name)

        # Check for index in downtimes
        col = db["downtimes"]
        indexes = col.index_information()
        if not is_index_included(indexes, index_downtimes):
            col.create_index(index_downtimes, background=True)
            log.info("Created (date_integer) index in %s.%s",
                     col.database.name, col.name)

        # Check first for index report,date
        for status_name in status_collections:
            col = db[status_name]
            indexes = col.index_information()
            if not is_index_included(indexes, index_report_date):
                # ensure index
                col.create_index(index_report_date, background=True)
                log.info("Created (report,date) index in %s.%s",
                         col.database.name, col.name)

        # Check for index date,host in status_metrics
        col = db["status_metrics"]
        indexes = col.index_information()
        if not is_index_included(indexes, index_date_host):
            col.create_index(index_date_host, background=True)
            log.info("Created (report,date) index in %s.%s",
                     col.database.name, col.name)

    def mongo_clean_ar(self, uri, dry_run=False):
        """Gets a mongo database reference as a uri string and performs
        a/r data removal for a specific date

        Args:
            uri (str.): uri string pointing to a specific mongodb database
            dry_run (bool, optional): Optional flag that specifies if the execution is in dry mode run.
                                      If yes no data removal is performed. Defaults to False.
        """

        tenant_report = None

        # if report is given check if report exists in configuration
        if self.args.report:
            report_name = self.args.report
            tenant_group = "TENANTS:" + self.args.tenant
            if report_name in self.config.get(tenant_group, "reports"):
                tenant_report = self.config.get(
                    tenant_group, "report_"+report_name)
            else:
                log.critical("Report %s not found", report_name)
                sys.exit(1)

        # Create a date integer for use in the database queries
        date_int = int(self.args.date.replace("-", ""))

        # set up the mongo client
        try:
            log.info("Trying to connect to: " + uri.geturl())
            client = MongoClient(uri.geturl())
            # force a connection to test the client
            client.server_info()
        except ServerSelectionTimeoutError as pse:
            log.fatal(pse)
            sys.exit(1)

        # specify the db we will be using. e.g argo_TENANTA
        # from the uri, take the path, which reprents the db, and ignore the / in the begging
        db = client[uri.path[1:]]

        # iterate over the specified collections
        for col in self.cols:
            if tenant_report is not None:
                num_of_rows = db[col].find(
                    {"date": date_int, "report": tenant_report}).count()
                log.info("Collection: " + col + " -> Found " + str(
                    num_of_rows) + " entries for date: " + self.args.date + " and report: " + self.args.report)
            else:
                num_of_rows = db[col].find({"date": date_int}).count()
                log.info("Collection: " + col + " -> Found " + str(
                    num_of_rows) + " entries for date: " + self.args.date + ". No report specified!")
            if dry_run:
                log.info("Results won't be removed to dry-run mode")
            else:
                if num_of_rows > 0:
                    if tenant_report is not None:
                        # response returned from the delete operation
                        res = db[col].delete_many(
                            {"date": date_int, "report": tenant_report})
                        log.info("Collection: " + col + " -> Removed " + str(res.deleted_count) +
                                 " entries for date: " + self.args.date + " and report: " + self.args.report)
                    else:
                        # response returned from the delete operation
                        res = db[col].delete_many(
                            {"date": date_int, "report": tenant_report})
                        log.info("Collection: " + col + " -> Removed " + str(
                            res.deleted_count) + " entries for date: " + self.args.date + ". No report specified!")
                    log.info("Entries removed successfully")
                else:
                    log.info("Zero entries found. Nothing to remove.")

        # close the connection with mongo
        client.close()

    def mongo_clean_status(self, uri, dry_run=False):
        """Gets a mongo database reference as a uri string and performs
        status data removal for a specific date

        Args:
            uri (str.): uri string pointing to a specific mongodb database
            dry_run (bool, optional): Optional flag that specifies if the execution is in dry mode run.
                                      If yes no data removal is performed. Defaults to False.
        """

        tenant_report = None

        # if report is given check if report exists in configuration
        if self.args.report:
            report_name = self.args.report
            tenant_group = "TENANTS:" + self.args.tenant
            if report_name in self.config.get(tenant_group, "reports"):
                tenant_report = self.config.get(
                    tenant_group, "report_"+report_name)
            else:
                log.critical("Report %s not found", report_name)
                sys.exit(1)

        # Create a date integer for use in the database queries
        date_int = int(self.args.date.replace("-", ""))

        # set up the mongo client
        try:
            log.info("Trying to connect to: " + uri.geturl())
            client = MongoClient(uri.geturl())
            # force a connection to test the client
            client.server_info()
        except pymongo.errors.ServerSelectionTimeoutError as pse:
            log.fatal(pse)
            sys.exit(1)

        # specify the db we will be using. e.g argo_TENANTA
        # from the uri, retrieve the path section, which reprents the db, and ignore the / in the begging
        db = client[uri.path[1:]]

        # ensure indexes for status collections
        self.ensure_status_indexes(db)

        # iterate over the specified collections
        for col in self.cols:
            if tenant_report is not None:
                num_of_rows = db[col].find(
                    {"date_integer": date_int, "report": tenant_report}).count()
                log.info("Collection: " + col + " -> Found " + str(
                    num_of_rows) + " entries for date: " + self.args.date + " and report: " + self.args.report)
            else:
                num_of_rows = db[col].find({"date": date_int}).count()
                log.info("Collection: " + col + " -> Found " + str(
                    num_of_rows) + " entries for date: " + self.args.date + ". No report specified!")

            if dry_run:
                log.info("Results won't be removed to dry-run mode")
            else:
                if num_of_rows > 0:
                    if tenant_report is not None:
                        # response returned from the delete operation
                        res = db[col].delete_many(
                            {"date_integer": date_int, "report": tenant_report})
                        log.info("Collection: " + col + " -> Removed " + str(res.deleted_count) +
                                 " entries for date: " + self.args.date + " and report: " + self.args.report)
                    else:
                        # response returned from the delete operation
                        res = db[col].delete_many(
                            {"date_integer": date_int, "report": tenant_report})
                        log.info("Collection: " + col + " -> Removed " + str(
                            res.deleted_count) + " entries for date: " + self.args.Date + ". No report specified!")
                    log.info("Entries removed successfully")
                else:
                    log.info("Zero entries found. Nothing to remove.")

        # close the connection with mongo
        client.close()

def main_clean(args=None):
    # stand alone method to be used whenever we want to call the mongo_clean_status method independently

    # Get configuration paths
    conf_paths = get_config_paths(args.config)

    # Get logger config file
    get_log_conf(conf_paths['log'])

    # Get main configuration and schema
    config = ArgoConfig(conf_paths["main"], conf_paths["schema"])

    # set up the mongo uri
    section_tenant = "TENANTS:" + args.tenant
    mongo_endpoint = config.get("MONGO", "endpoint")
    mongo_uri = config.get(section_tenant, "mongo_uri").fill(
        mongo_endpoint=mongo_endpoint, tenant=args.tenant)

    if args.job == "clean_ar":
        argo_mongo_client = ArgoMongoClient(
            args, config, ["endpoint_ar", "service_ar", "endpoint_group_ar"])
        argo_mongo_client.mongo_clean_ar(mongo_uri, args.dry_run)

    elif args.job == "clean_status":
        argo_mongo_client = ArgoMongoClient(args, config, ["status_metrics", "status_endpoints", "status_services",
                                                           "status_endpoint_groups"])
        argo_mongo_client.mongo_clean_status(mongo_uri, args.dry_run)

# Provide the ability to the script, to run as a standalone module
if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Mongo clean up script")
    parser.add_argument(
        "-t", "--tenant", metavar="STRING", help="Name of the tenant", required=True, dest="tenant")
    parser.add_argument(
        "-r", "--report", metavar="STRING", help="Tenant report to be used", required=True, dest="report")
    parser.add_argument(
        "-d", "--date", metavar="STRING", help="Date to run the job for", required=True, dest="date")
    parser.add_argument(
        "-c", "--config", metavar="STRING", help="Path for the config file", dest="config")
    parser.add_argument(
        "-j", "--job", metavar="STRING", help="Stand alone method we wish to run", required=True, dest="job")
    parser.add_argument("--dry-run", help="Runs in test mode without actually submitting the job",
                        action="store_true", dest="dry_run")

    # Parse the arguments
    sys.exit(main_clean(parser.parse_args()))


from module.exception import CampaignEnd
from module.logger import logger
from module.map.map import Map
from module.map.map_grids import SelectedGrids
from module.os.fleet import OSFleet
from module.os.globe_camera import GlobeCamera
from module.ui.assets import OS_CHECK

class OSMap(OSFleet, Map, GlobeCamera):
    def clear_all_objects(self, grid=None):
        """Method to clear all objects around specific grid.

        Args:
            grid (GridInfo):

        Returns:
            int: Cleared count
        """
        if grid is not None:
            logger.hr(f'clear_all_objects: {grid}', level=2)
        for count in range(0, 10):
            grids = self.map.select(is_resource=True) \
                .add(self.map.select(is_enemy=True)) \
                .add(self.map.select(is_meowfficer=True)) \
                .add(self.map.select(is_exclamation=True)).delete(self.map.select(is_interactive_only=True))
            if grid is not None:
                grids = SelectedGrids([g for g in grids if self.grid_is_in_sight(g, camera=grid)])

            if not grids:
                logger.info(f'OS object cleared: {count}')
                return count

            grids = grids.sort_by_camera_distance(self.fleet_current)
            logger.hr('Clear all resource')
            logger.info(f'Grids: {grids}')
            logger.info(f'Clear object {grids[0]}')
            self.goto(grids[0], expected=self._get_goto_expected(grids[0]))
            self.handle_meowfficer_searching()

        logger.warning('Too many objects to clear, stopped')
        return 10

    def handle_meowfficer_searching(self):
        """
        Move back and forth to handle meowfficer farming

        Returns:
            bool: If handled
        """
        if not self.is_meowfficer_searching():
            return False

        logger.hr('Meowfficer searching')
        back = self.fleet_current
        sea = self.get_sea_grids()[:4]
        logger.info(f'Sea grids: {sea}')
        forth = sea[0]

        # Meowfficer searching finishes in 8 steps at max
        # If found good items in current step, progress bar doesn't increase
        for grid in [forth, back] * 7:
            self.goto(grid)
            if self.is_meowfficer_searching():
                percent = self.get_meowfficer_searching_percentage()
                logger.attr('Meowfficer_searching', f'{int(percent * 100)}%')
            else:
                logger.hr('Meowfficer searching end')
                return True

        logger.warning('Too many meowfficer searching steps')
        return True

    def clear_remain_grids(self):
        logger.hr('Clear remain grids', level=2)
        self.clear_all_objects()

    def full_clear(self):
        """
        Clear the whole map.
        """
        logger.info(f'Full scan start')
        self.map.reset_fleet()

        queue = self.map.camera_data

        while len(queue) > 0:
            queue = queue.sort_by_camera_distance(self.camera)
            self.focus_to(queue[0])
            self.focus_to_grid_center(0.25)
            self.view.predict()
            self.map.update(grids=self.view, camera=self.camera)
            self.map.show()

            self.clear_all_objects(queue[0])
            queue = queue[1:]

        self.clear_remain_grids()
        self.clear_akashi()
        logger.info('Full clear end')

    def clear_akashi(self):
        """
        Handle Akashi's shop after auto search.
        After auto search, fleet will near akashi.
        This method detect where akashi stands, enter shop, buy items and exit.

        Returns:
            bool: If found and handled.
        """
        if not self.config.OpsiGeneral_BuyAkashiShop:
            return False
        if self.zone.is_port:
            logger.info('Current zone is a port, do not have akashi')
            return False

        grid = self.radar.predict_akashi(self.device.image)
        if grid is None:
            logger.info('No akashi on this map')
            return False

        logger.info(f'Found Akashi on {grid}')
        self.handle_info_bar()

        self.update_os()
        self.view.predict()
        self.view.show()

        grid = self.convert_radar_to_local(grid)
        self.handle_akashi_supply_buy(grid)
        return True

    def run(self):
        self.device.screenshot()
        self.handle_siren_platform()
        self.map_init()
        self.full_clear()

    _auto_search_battle_count = 0

    def os_auto_search_daemon(self):
        logger.hr('OS auto search', level=2)
        self._auto_search_battle_count = 0

        while 1:
            self.device.screenshot()

            if self.is_in_map():
                self.device.stuck_record_clear()
            if self.handle_os_auto_search_map_option():
                continue
            if self.handle_retirement():
                continue
            if self.combat_appear():
                self._auto_search_battle_count += 1
                logger.attr('battle_count', self._auto_search_battle_count)
                self.auto_search_combat()
            if self.handle_map_event():
                # Auto search can not handle siren searching device.
                continue

    def run_auto_search(self):
        self.handle_ash_beacon_attack()

        for _ in range(3):
            try:
                self.os_auto_search_daemon()
            except CampaignEnd:
                logger.info('Get OS auto search reward')
                self.wait_until_appear(OS_CHECK, offset=(20, 20))
                logger.info('OS auto search finished')

            if self.handle_ash_beacon_attack():
                continue
            else:
                break

        self.clear_akashi()


import sys
import time
import sched
import random
import re
import os
import csv
import random

import argparse
import torch

from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler

import train
from train import Module

buttons = ["A", "B", "X", "Y", "up", "Down", "Left", "Right"]

def add_output_to_file(filename, id, buttons):
    """Buttons: boolean list, see list of buttons"""
    if not os.path.exists(filename):
        file = open(filename, 'w')
        file.write("id,A,B,X,Y,Up,Down,Left,Right\n")
        file.close()

    file = open(filename, 'a')
    file.write(str(id) + ',' + ','.join([str(button) for button in buttons]) + '\n')
    file.close()

class FileChangeHandler(FileSystemEventHandler):
    def __init__(self, data_path, model_checkpoint_folder, args):
        self.data_path = data_path
        self.model_checkpoint_folder = model_checkpoint_folder
        self.objectives = args.objectives
        self.t = args.t

        self.latest_model_file = None
        self.iter = 0
        self.dir_name = None
        self.prev_id = None
        self.model = None
        self.use_weighted_dataloader = args.use_weighted_dataloader

        #Other args
        self.reload_model_every_n_iterations = args.reload

    def load_model(self):
        model_file = train.get_latest_model(self.model_checkpoint_folder)

        if not model_file == self.latest_model_file and not model_file == None:
            model_hparams = {"t": self.t, "objectives": self.objectives}
            optimizer_hparams={"lr": 0.1}

            self.model = Module(self.objectives, data_path, model_hparams, optimizer_hparams)
            state_dict = torch.load(model_file)["state_dict"]
            self.model.load_state_dict(state_dict)
            self.latest_model_file = model_file

        print(f'Iteration {self.iter}\tLoaded model {self.latest_model_file}\tDeterministic: {self.iter % 4 >= 2}')

    def on_created(self, event):
        if event.is_directory:
            output_file = os.path.join(event.src_path, 'outputFile.csv')
            if not os.path.exists(output_file):
                file = open(output_file, 'w')
                file.write("id,A,B,X,Y,Up,Down,Left,Right\n")
                file.close()
            if self.iter % self.reload_model_every_n_iterations == 0:
                self.load_model()
            self.iter += 1
        else:
            event_dir_name = os.path.dirname(event.src_path)
            event_base_name = os.path.basename(event.src_path)

            if not event.src_path.split('.')[1] == "csv":
                id = int(re.sub("[^0-9]", "", event_base_name))
                output_file = event_dir_name + "/outputFile.csv"

                #Sometimes FIleSystemWatcher triggers twice, so check if new id is higher than previous id,
                #or if it is a new directory (in that case, reset the previous id)
                new_dir = False
                if not event_dir_name == self.dir_name:
                    new_dir = True
                    self.prev_id = 0
                self.dir_name = event_dir_name

                if id <= self.prev_id:
                    return #ignore false screenshot
                else:
                    self.prev_id = id
                if self.model:
                    #Add previous 3 frames if possible
                    input = {}
                    if not os.path.isfile(output_file): #First frame
                        input["previous_points"] = []
                        input["screenshots"] = []
                    else:
                        with open(output_file, newline='') as state_file_data:
                            reader = csv.DictReader(state_file_data, delimiter=',')
                            points = []
                            for row in reader:
                                if id - int(row["id"]) <= self.t * 10 - 10:
                                    points.append(row)
                            input["previous_points"] = points
                        input["screenshots"] = [f"{event_dir_name}/screenshot_{point['id']}.png" for point in input["previous_points"]]

                    #Add current frame
                    input["screenshots"].append(f"{event_dir_name}/screenshot_{id}.png")

                    next_inputs = train.get_next_input(self.model, input, deterministic=self.iter % 4 >= 2, objectives=self.objectives, t=self.t)

                    add_output_to_file(output_file, id, next_inputs)
                else:
                    #If no model, return random input
                    i = random.randint(0, 63)
                    next_inputs = [int(i/32)%2==1, int(i/16)%2==1, int(i/8)%2==1, int(i/4)%2==1, i%4==0, i%4==1, i%4==2, i%4==3]
                    next_inputs = [int(x) for x in next_inputs]

                    add_output_to_file(output_file, id, next_inputs)

    def on_modified(self, event):
        pass

if __name__ == "__main__":
    print('Running file watch.py')
    parser = argparse.ArgumentParser()

    # Experiment number, required!
    parser.add_argument('-e', '--experiment', type=int, required=True)
    parser.add_argument('-r', '--reload', type=int, default=2)
    parser.add_argument('-o', '--objectives', nargs='+', default=[])
    parser.add_argument('-t', type=int, default=4,
                        help='How many frames to look for in the past')

    parser.add_argument('--use_weighted_dataloader', action='store_true',
                        help='Use a dataloader weighted to get a balanced amount of x_speed values')
    parser.add_argument('--use_full_dataloader', action='store_false', dest='use_weighted_dataloader',
                        help='Use the entire dataset for the dataloader')
    parser.set_defaults(use_weighted_dataloader=False)

    args = parser.parse_args()

    if len(args.objectives) == 0:
        args.objectives = ['speed', 'death']
    print("Training objectives: ", args.objectives)

    data_path = f'experiments\\experiment_{args.experiment}\\data'
    model_checkpoint_folder = f"experiments\\experiment_{args.experiment}\\models\\lightning_logs"

    event_handler = FileChangeHandler(data_path, model_checkpoint_folder, args)
    observer = Observer()
    observer.schedule(event_handler, data_path, recursive=True)
    observer.start()

    try:
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        observer.stop()

    observer.join()


import collections
import copy
import os
import re
import sys
from itertools import chain
from typing import Any, Dict, List, Optional, Union

from .syntax import SOS_WILDCARD
from .utils import env

__all__ = ["expand_pattern"]

#
# The following is adapted from snakemake so that the syntax of the pattern parameter
# would be the same as snakemake.
#
# https://bitbucket.org/snakemake/snakemake/src/22eff35627401d3bb243e068c5dce97107e7090b/snakemake/io.py?at=master&fileviewer=file-view-default
#
# I will re-implement it if there is any license issue with the code
#

def regex(filepattern: str) -> str:
    f = []
    last = 0
    wildcards = set()
    for match in SOS_WILDCARD.finditer(filepattern):
        f.append(re.escape(filepattern[last:match.start()]))
        wildcard = match.group("name")
        if wildcard in wildcards:
            if match.group("constraint"):
                raise ValueError(
                    "If multiple wildcards of the same name "
                    "appear in a string, eventual constraints have to be defined "
                    "at the first occurence and will be inherited by the others."
                )
            f.append(f"(?P={wildcard})")
        else:
            wildcards.add(wildcard)
            matched = match.group("constraint") if match.group("constraint") else ".+"
            f.append(f"(?P<{wildcard}>{matched})")
        last = match.end()
    f.append(re.escape(filepattern[last:]))
    f.append("$")  # ensure that the match spans the whole file
    return "".join(f)

def glob_wildcards(
    pattern: str,
    files: Optional[List[str]] = None
) -> Dict[str, Union[List[Any], List[str]]]:
    """
    Glob the values of the wildcards by matching the given pattern to the filesystem.
    Returns a named tuple with a list of values for each wildcard.
    """
    pattern = os.path.normpath(pattern)
    if sys.platform == "win32":
        # we perform path matching with / slash only
        pattern = pattern.replace("\\", "/")
    first_wildcard = re.search("{[^{]", pattern)
    dirname = (
        os.path.dirname(pattern[:first_wildcard.start()])
        if first_wildcard else os.path.dirname(pattern))
    if not dirname:
        dirname = "."

    names = [match.group("name") for match in SOS_WILDCARD.finditer(pattern)]
    res = {x: [] for x in names}
    pattern = re.compile(regex(pattern))

    if files is None:
        files = ((os.path.join(dirpath, f) if dirpath != "." else f)
                 for dirpath, dirnames, filenames in os.walk(dirname)
                 for f in chain(filenames, dirnames))

    for f in files:
        # we perform path matching with only / slash
        match = re.match(pattern, str(f).replace("\\", "/"))
        if match:
            for name, value in match.groupdict().items():
                res[name].append(value)
    return res

def apply_wildcards(
    pattern: str,
    wildcards: Dict[str, Union[int, str]],
    fill_missing: bool = False,
    fail_dynamic: bool = False,
    dynamic_fill: None = None,
    keep_dynamic: bool = False,
) -> str:

    def format_match(match):
        name = match.group("name")
        try:
            value = wildcards[name]
            if fail_dynamic and value == dynamic_fill:
                raise RuntimeError(name)
            return str(value)  # convert anything into a str
        except KeyError as ex:
            if keep_dynamic:
                return f"{{{name}}}"
            if fill_missing:
                return dynamic_fill
            raise RuntimeError(f"Wildcard apply error: {ex} ({wildcards})") from ex

    return SOS_WILDCARD.sub(format_match, pattern)

def extract_pattern(pattern: str, ifiles: List[str]) -> Dict[str, any]:
    """This function match pattern to a list of input files, extract and return
    pieces of filenames as a list of variables with keys defined by pattern."""
    res = glob_wildcards(pattern, [])
    for ifile in ifiles:
        matched = glob_wildcards(pattern, [ifile])
        for key in matched.keys():
            if not matched[key]:
                # env.logger.warning('Filename {} does not match pattern {}. None returned.'.format(ifile, pattern))
                res[key].append(None)
            else:
                res[key].extend(matched[key])
    return res

def expand_pattern(pattern: str) -> List[str]:
    """This function expand patterns against the current namespace
    and return a list of filenames"""
    ofiles = []
    sz = None
    res = glob_wildcards(pattern, [])
    sz = None
    wildcard = [{}]
    for key in res.keys():
        if key not in env.sos_dict:
            raise ValueError(f"Undefined variable {key} in pattern {pattern}")
        if not isinstance(env.sos_dict[key], str) and isinstance(
                env.sos_dict[key], collections.Sequence):
            if sz is None:
                sz = len(env.sos_dict[key])
                wildcard = [copy.deepcopy(wildcard[0]) for x in range(sz)]
            elif sz != len(env.sos_dict[key]):
                raise ValueError(
                    f"Variables in output pattern should have the same length (other={sz}, len({key})={len(env.sos_dict[key])})"
                )
            for idx, value in enumerate(env.sos_dict[key]):
                wildcard[idx][key] = value
        else:
            for v in wildcard:
                v[key] = env.sos_dict[key]
    #
    for card in wildcard:
        ofiles.append(
            apply_wildcards(
                pattern,
                card,
                fill_missing=False,
                fail_dynamic=False,
                dynamic_fill=None,
                keep_dynamic=False,
            ))
    return ofiles


from __future__ import absolute_import, unicode_literals

from django.utils import six
from django.db import models
from django.apps import apps

try:
    from django.channels import Channel
except ImportError:
    from channels import Channel

stratus_app = apps.get_app_config('stratus')
VM = stratus_app.get_model('VM')

GAZON_STATUS_ENUM = [ (u'WAITING', u'Gazon entry need complementary info'),
                      (u'TO_CREATE', u'Gazon entry need to be created'),
                      (u'TO_DELETE', u'Gazon entry need to be deleted'),
                      (u'CREATED', u'Gazon is created for the VM'),
                      (u'DELETED', u'Gazon is deleted for the VM'),
                      (u'NONE', u'None has to been done for gazon. Ever'), ]

class GazonEntry(models.Model):

    vm = models.OneToOneField(VM, default=None)

    gazon_status = models.CharField(choices=GAZON_STATUS_ENUM, default='WAITING', max_length=100)
    gazon_error = models.CharField(blank=True, max_length=100)

    def delete_vm(self):
        if self.gazon_status in [u'CREATED', u'TO_CREATE']:
            self.gazon_status = u'TO_DELETE'
            Channel(u'gazonentry').send({})
            self.save()