import math
import torch
import numpy as np
from torch import autograd, optim
from torch.distributions.multivariate_normal import MultivariateNormal
import scipy.optimize
from tqdm import tqdm

def get_schedule(num, rad=4):
    if num == 1:
        return np.array([0.0, 1.0])
    t = np.linspace(-rad, rad, num)
    s = 1.0 / (1.0 + np.exp(-t))
    return (s - np.min(s)) / (np.max(s) - np.min(s))

def differentiable_annealed_importance_sampling(s:torch.Tensor, log_likelihood, log_q, n_steps,
        step_size, partial=False, gamma=0.9, mass_matrix=None, clip_grad=None,
        lrates=None, betas=None, block_grad=False, is_train=True, rng:np.random.RandomState=None):
    """
    s: partcle state: n_particles x d
    """
    n_particles = s.shape[:-1]
    dim = s.shape[-1]

    if n_steps == 0:
        return - log_q(s) + log_likelihood(s), s

    # if lrates is None:
    #     lrates = step_size * torch.ones(n_steps+1, device=s.device)
    if (type(step_size) is torch.Tensor or type(step_size) is np.ndarray) and len(step_size.shape) > 0:
        assert step_size.shape == s.shape[:-1]
        step_size = step_size[..., None]

    if betas is None:
        # betas = torch.linspace(1.0/n_steps, 1.0, n_steps, device=s.device)
        betas = get_schedule(n_steps+1)

    if mass_matrix is None:
        mass_matrix = torch.eye(dim, device=s.device)
    pi_mean = torch.zeros(dim, device=s.device)
    pi = MultivariateNormal(pi_mean, mass_matrix)
    inverse_mass_matrix = torch.inverse(mass_matrix)

    # s.requires_grad = True

    def log_annealed_prob(beta, s: torch.Tensor):
        return (1 - beta) * log_q(s) + beta * log_likelihood(s)

    def grad_log_annealed_prob(beta, s):
        ''' it's important to set create_graph=True '''
        with torch.enable_grad():
            s.requires_grad_()
            grad = autograd.grad(log_annealed_prob(beta, s).sum(), s, create_graph=is_train)[0]
            if clip_grad is not None:
                max_ = torch.prod(torch.tensor(s.shape)) * clip_grad # last dimension of mu_z
                grad = torch.clamp(grad, -max_, max_)
        return grad

    # sample initial momentum
    def pi_sample(n_particles):
        if rng is None:
            eps = pi.sample(n_particles)
        else:
            eps = torch.tensor(rng.multivariate_normal(pi_mean, mass_matrix, n_particles),
                               dtype=s.dtype, device=s.device)
        return eps

    v = pi_sample(n_particles)

    with torch.set_grad_enabled(is_train):

        elbo = - log_q(s)
        for k in range(1, n_steps+1):
            assert not torch.any(s.isnan()), "Current state has nan values"
            elbo = elbo - pi.log_prob(v)

            # leapfrog
            s = s + step_size / 2 * v @ inverse_mass_matrix
            v = v + step_size * grad_log_annealed_prob(betas[k], s)
            s = s + step_size / 2 * v @ inverse_mass_matrix

            elbo = elbo + pi.log_prob(v)

            if partial:
                # partial_refreshment
                v = gamma * v + math.sqrt(1 - math.pow(gamma, 2)) * pi_sample(n_particles)
            else:
                v = pi_sample(n_particles)

        elbo = elbo + log_likelihood(s)

    return elbo, s

def leapfrogs_and_bounds_optlr(s, log_likelihood, log_q, n_steps,
        step_size, partial=False, gamma=0.9, mass_matrix=None):

    init_log_lrates = math.log(step_size) * np.ones(n_steps)
    bounds = scipy.optimize.Bounds(-np.infty, -1.0)
    # log_lrates = torch.tensor(llrates_np, dtype=s.dtype, device=s.device, requires_grad=True)

    def func_fn(log_lrates):
        log_lrates = torch.tensor(log_lrates, dtype=s.dtype, device=s.device, requires_grad=True)
        elbo, _ = differentiable_annealed_importance_sampling(s, log_likelihood, log_q, n_steps,
            step_size, partial, gamma, mass_matrix, lrates=torch.exp(log_lrates))
        return -elbo.sum().data.cpu().numpy()

    def grad_fn(log_lrates):
        log_lrates = torch.tensor(log_lrates, dtype=s.dtype, device=s.device, requires_grad=True)
        elbo, _ = differentiable_annealed_importance_sampling(s, log_likelihood, log_q, n_steps,
            step_size, partial, gamma, mass_matrix, lrates=torch.exp(log_lrates))
        loss = -elbo.sum()
        return autograd.grad(loss, log_lrates)[0].data.cpu().numpy().astype(np.float64)

    res = scipy.optimize.minimize(func_fn, init_log_lrates, jac=grad_fn, bounds=bounds)
    log_lrates = torch.tensor(res.x, dtype=s.dtype, device=s.device)

    return differentiable_annealed_importance_sampling(s, log_likelihood, log_q, n_steps,
        step_size, partial, gamma, mass_matrix, lrates=torch.exp(log_lrates))


import pandas as pd
import random as rd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib.cm as cm
import pickle
import random
import math
class neuron:
        def __init__(self,weights):
            self.weights = weights
            self.outputneuron=None
            self.s=None
            self.errorneuron=None
            self.deltaweights=[random.random()/2 for i in range(len(self.weights))].copy()
            self.deltaoldweights=[0 for i in range(len(self.weights))].copy()

        def S(self,inputs):
            return self.weights[0]+np.dot(self.weights[1:],inputs)

        def activation_function(self,inputs):
            outputneuron=self.f(self.S(inputs))
            return outputneuron
        #bacKpropagation
        def f(self,x):
            return 1/(1+math.exp(-x))
        def derivation_function(self,s):
            return self.f(s)*(1-self.f(s))

        def update_weights_ofsingle_neuronj(self):
            for k in range(len(self.deltaweights)):
                wjk=self.weights[k]
                self.weights[k]=wjk+self.deltaweights[k]

class layer:
    def __init__(self,nbrof_neurons,nbrof_inputs):
        self.numberof_neurons = nbrof_neurons
        self.numerof_inputs= nbrof_inputs
        self.my_neurons=layer.generate_my_neurons(nbrof_neurons,nbrof_inputs)
        self.inputstolayer=None
        self.outputsoflayer=None

    def generate_my_neurons(nbrof_neurons,nbrof_inputs):
        list_of_neurons=[]
        #a=math.sqrt(6)/math.sqrt(nbrof_inputs+nbrof_neurons)
        a=math.sqrt(3)/math.sqrt(nbrof_inputs)
        #a=1
        #a=math.sqrt(6)/math.sqrt(nbrof_inputs+nbrof_neurons)
        for i in range(nbrof_neurons):
            weights_of_neuron=[np.random.uniform(-a,a) for i in range(nbrof_inputs+1)].copy()
            #weights_of_neuron=[1 for i in range(nbrof_inputs+1)].copy()
            new_neuron=neuron(weights_of_neuron)
            list_of_neurons.append(new_neuron)
        return list_of_neurons

    def generate_my_outputs(self,nbrof_neurons):
        outputslayer=[]
        for i in range(nbrof_neurons):
            output=self.my_neurons[i].activation_function(self.inputstolayer)
            outputslayer.append(output)
            self.my_neurons[i].outputneuron=output
            self.my_neurons[i].s=self.my_neurons[i].S(self.inputstolayer)

        return outputslayer

class network:
    def __init__(self,nbrof_layers,nbrof_neuronsinlayer,Inputs,number_of_classes):
        self.number_of_layers = nbrof_layers
        self.number_of_classes=number_of_classes
        self.nbrof_neuronsinlayer=nbrof_neuronsinlayer
        self.input_layer=Inputs.copy()
        self.myhidden_layers=self.generatemylayers(nbrof_layers,nbrof_neuronsinlayer)
        self.output_layer=self.generete_output_layer(number_of_classes)
        self.desiredoutput=None
        #self.Activate_network(nbrof_neuronsinlayer)
    def generatemylayers(self,nbrof_layers,nbrof_neuronsinlayer):
        list_of_layers=[]
        for i in range (nbrof_layers):
            if len(list_of_layers)==0:
                nbrof_inputs=len(self.input_layer)
            else:
                nbrof_inputs=list_of_layers[i-1].numberof_neurons
            new_layer=layer(nbrof_neuronsinlayer[i],nbrof_inputs)
            list_of_layers.append(new_layer)
        return list_of_layers

    def generete_output_layer(self,number_of_classes):
        if len(self.myhidden_layers)==0:
            print("no hiddenlayer")
            nbrof_inputstooutlayer=len(self.input_layer)
        else:
            before_output_layer=self.myhidden_layers[len(self.myhidden_layers)-1]
            nbrof_inputstooutlayer=before_output_layer.numberof_neurons
        output_layer=layer(number_of_classes,nbrof_inputstooutlayer)
        return output_layer

    def Activate_network(self,nbrof_neuronsinlayer):
        if self.number_of_layers==0:
            print("no hidden layers")
            self.output_layer.inputstolayer=self.input_layer.copy()
            self.output_layer.outputsoflayer=self.output_layer.generate_my_outputs(self.number_of_classes).copy()
        else:
            self.myhidden_layers[0].inputstolayer=self.input_layer.copy()
            self.myhidden_layers[0].outputsoflayer=self.myhidden_layers[0].generate_my_outputs(nbrof_neuronsinlayer[0]).copy()
            for i in range(1,self.number_of_layers):
                self.myhidden_layers[i].inputstolayer= self.myhidden_layers[i-1].outputsoflayer.copy()
                self.myhidden_layers[i].outputsoflayer=self.myhidden_layers[i].generate_my_outputs(nbrof_neuronsinlayer[i]).copy()

            self.output_layer.inputstolayer=self.myhidden_layers[self.number_of_layers-1].outputsoflayer.copy()
            self.output_layer.outputsoflayer=self.output_layer.generate_my_outputs(self.number_of_classes).copy()

    #Backpropagation:
    def update_error_output_layer(self,desired_outputs):
        H=0.9
        L=0.1
        n=self.number_of_layers
        learning_rate=0.0008
        skiped=[]
        for i in range(self.number_of_classes):
            if desired_outputs[i]==1 and self.output_layer.my_neurons[i].outputneuron>H:
                skiped.append(i)
                self.output_layer.my_neurons[i].errorneuron=0
                continue
            elif desired_outputs[i]==0 and self.output_layer.my_neurons[i].outputneuron<L:
                skiped.append(i)
                self.output_layer.my_neurons[i].errorneuron=0
                continue
            else:
                ei=desired_outputs[i]-self.output_layer.my_neurons[i].outputneuron
                si=self.output_layer.my_neurons[i].s
                self.output_layer.my_neurons[i].errorneuron=self.output_layer.my_neurons[i].derivation_function(si)*ei
                self.calculatedeltaj(self.output_layer.my_neurons[i],self.myhidden_layers[n-1],learning_rate)
        return skiped

    def update_error_hidden_layers(self):
        if n==0:
            print("no hidden layers")
            return
        self.update_error_singlehidden_layer(self.myhidden_layers[n-1],self.output_layer,self.nbrof_neuronsinlayer[n-1],n-1)
        for l in range(n-2,-1,-1):
            self.update_error_singlehidden_layer(self.myhidden_layers[l],self.myhidden_layers[l+1],self.nbrof_neuronsinlayer[l],l)
        return

    def update_error_singlehidden_layer(self,current_layer,following_layer,nbrofneuronsinlayer,l):
        learning_rate=0.0008
        for j in range(nbrofneuronsinlayer):
            dotprod=self.dot_weights_errors(j,following_layer)#decalge du bias
            sj=current_layer.my_neurons[j].s
            current_layer.my_neurons[j].errorneuron=current_layer.my_neurons[j].derivation_function(sj)*dotprod
            if l==0:
                self.calculatedeltaj_first_layer(current_layer.my_neurons[j],learning_rate)
            else:
                self.calculatedeltaj(current_layer.my_neurons[j],self.myhidden_layers[j-1],learning_rate)
        return

    def dot_weights_errors(self,j,following_layer):
        j=j+1#decalge du bias
        dotproduct=0
        for i in range(following_layer.numberof_neurons):
            wijerrori=following_layer.my_neurons[i].errorneuron*following_layer.my_neurons[i].weights[j]
            dotproduct=dotproduct+wijerrori
        return dotproduct
    #uppdate weights of neuron j:
    def calculatedeltaj(self,neuronj,previous_layer,learning_rate):
        alpha=0.8
        ej=neuronj.errorneuron#add TO SAVE OLD WEIGHTS
        neuronj.deltaweights[0]=learning_rate*ej+alpha*neuronj.deltaoldweights[0]
        neuronj.deltaoldweights[0]=neuronj.deltaweights[0]
        for k in range(0,previous_layer.numberof_neurons):
            hk=previous_layer.outputsoflayer[k]
            neuronj.deltaweights[k+1]=learning_rate*ej*hk+alpha*neuronj.deltaoldweights[k+1]
            neuronj.deltaoldweights[k+1]=neuronj.deltaweights[k+1]
        return
    def calculatedeltaj_first_layer(self,neuronj,learning_rate):
        alpha=0.8
        ej=neuronj.errorneuron#add TO SAVE OLD WEIGHTS
        neuronj.deltaweights[0]=learning_rate*ej+alpha*neuronj.deltaoldweights[0]
        neuronj.deltaoldweights[0]=neuronj.deltaweights[0]
        for k in range(0,len(self.input_layer)):
            hk=self.input_layer[k]
            neuronj.deltaweights[k+1]=learning_rate*ej*hk+alpha*neuronj.deltaoldweights[k+1]
            neuronj.deltaoldweights[k+1]=neuronj.deltaweights[k+1]
        return

    def backpropagate(self):
        skiped=self.update_error_output_layer(self.desiredoutput)
        self.update_error_hidden_layers()
        for i in range(self.number_of_classes):
            if (i in skiped):
                continue
            else:
                self.output_layer.my_neurons[i].update_weights_ofsingle_neuronj()
        for i in range(self.number_of_classes):
            self.output_layer.my_neurons[i].update_weights_ofsingle_neuronj()
        for layer in self.myhidden_layers:
            for neuron in layer.my_neurons:
                neuron.update_weights_ofsingle_neuronj()
        #print("finish backpropagation")
        return
    def classify(self):
        index=np.argmax(self.output_layer.outputsoflayer)
        return index

# In[2]:

##
#Import shuffled images
Inputsst=pd.read_csv("Imagestrainshuffled.txt",sep=' ',header=None)
teaching_input=pd.read_csv("Labelstrainshuffled.txt",sep=' ',header=None)
Inputsst=Inputsst.values
teaching_input=teaching_input.values.tolist()
lg_data=len(teaching_input)
##Import testing images
Inputstestu=pd.read_csv("Imagestestshuffled.txt",sep=' ',header=None)
testing_input=pd.read_csv("Labelstestshuffled.txt",sep=' ',header=None)
Inputstestu=Inputstestu.values.tolist()
testing_input=testing_input.values.tolist()
lg_test=len(testing_input)
#Imagestrainshuffled
#Labelstrainshuffled

####Testing the performance and loading the model
import pandas as pd
import random as rd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib.cm as cm
import pickle
import random
import math

with open('network2.pkl', 'rb') as inp:
    nettrained = pickle.load(inp)

with open('errors3.pkl', 'rb') as inp:
    errors = pickle.load(inp)

with open('errorstest3.pkl', 'rb') as inp:
    errorstest = pickle.load(inp)
with open('epochs3.pkl', 'rb') as inp:
    epochs = pickle.load(inp)
#

from sklearn import metrics
cm=metrics.confusion_matrix(teaching_input,Ypred)
print(cm)

# In[26]:

import seaborn as sns
df_cm = pd.DataFrame(cm, index = [i for i in "0123456789"],
                  columns = [i for i in "0123456789"])
plt.figure(figsize = (10,6))
heatmap=sns.heatmap(df_cm, annot=True,cmap='BrBG', fmt='g')
bottom, top = heatmap.get_ylim()
heatmap.set_ylim(bottom + 0.5, top - 0.5)
plt.xlabel("Predicted labels")
plt.ylabel("True labels")
plt.title("The confusion matrix of the training data")
plt.show()

# In[22]:

from sklearn import metrics
cmtest=metrics.confusion_matrix(testing_input,Ypredtest)
print(cmtest)

# In[25]:

import seaborn as sns
df_cmtest= pd.DataFrame(cmtest, index = [i for i in "0123456789"],
                  columns = [i for i in "0123456789"])
plt.figure(figsize = (10,7))
heatmap=sns.heatmap(df_cmtest, annot=True,cmap='BrBG', fmt='g')
bottom, top = heatmap.get_ylim()
heatmap.set_ylim(bottom + 0.5, top - 0.5)
plt.xlabel("Predicted labels")
plt.ylabel("True labels")
plt.title("The confusion matrix of the testing data")
plt.show()


#CRIANDO FUNÇÕES

def decorar():
    print(15*'-')
    print('MENU')
    print(15*'-')

def enfeitarNome(texto):
    print(15*'-')
    print(texto)
    print(15*'-')

def teste():
    return 'meu erro'

def calcularIMC(altura, peso):
    imc = peso/(altura**2)
    return imc

print(enfeitarNome(calcularIMC(2,99)))

soma = 0
for i in range(1):
    alt = float(input('digite sua altura'))
    pe = float(input('digite seu peso'))
    soma += calcularIMC(alt, pe)

media = soma/1
print(enfeitarNome(f'a media de IMC das pessoas foi {media}'))


import logging

import numpy as np

from src.MiniProjects.Maze.Maze import Maze

__author__ = 'frank.ma'

logger = logging.getLogger(__name__)

class MazeSolver(object):
    MOVES = dict(N=(-1, 0), W=(0, 1), S=(1, 0), E=(0, -1))
    found_solution = False

    def __init__(self,
                 maze: Maze):
        self.maze = maze
        self.maze_path = np.zeros(self.maze.maze.shape, dtype=bool)
        self.maze_parent_index = self.__init_parent_index(maze.maze)

    def solve(self):
        pass

    def print_solution(self,
                       output: str):
        if not self.found_solution:
            logger.info('No solution is found for the given maze.')
        else:
            maze_display = np.chararray(self.maze.maze.shape)
            maze_display.fill(str(' '))
            maze_display[self.maze.maze == 1] = '#'
            maze_display[self.maze_path] = 'X'
            maze_display[self.maze.start] = 'S'
            maze_display[self.maze.end] = 'E'
            np.savetxt(output, maze_display.decode(), fmt='%s', delimiter='')
            logger.info('Maze solution is printed in file %s.' % output)

    @staticmethod
    def __init_parent_index(maze: np.array):
        return np.array([[(x, y) for y in range(maze.shape[1])] for x in range(maze.shape[0])])


from __future__ import annotations

import itertools
from abc import ABC, abstractmethod
from collections import defaultdict
from collections.abc import Iterable, Iterator
from dataclasses import dataclass, field
from typing import cast

from cognite.client.data_classes import data_modeling as dm
from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType
from typing_extensions import Self

from cognite.pygen import config as pygen_config
from cognite.pygen.utils.text import create_name

_PRIMITIVE_TYPES = (dm.Text, dm.Boolean, dm.Float32, dm.Float64, dm.Int32, dm.Int64, dm.Timestamp, dm.Date, dm.Json)
_EXTERNAL_TYPES = (dm.TimeSeriesReference, dm.FileReference, dm.SequenceReference)

@dataclass(frozen=True)
class ViewSpaceExternalId:
    """
    This represents a view id.

    The motivation for this class is that API class and data classes are independent of the view version (given
    that all properties are equal). This enables reuse of data classes and APIs across view versions.

    Note this could be considered an edge case not worth supporting. However, when you publish data models
    from a .graphql format, the autogenerated view versions for identical types will be different when those types
    are part of two different data models.
    """

    space: str
    external_id: str

    @classmethod
    def from_(cls, view_id: dm.ViewId | dm.View) -> Self:
        return cls(space=view_id.space, external_id=view_id.external_id)

@dataclass(frozen=True)
class Field(ABC):
    """
    A field represents a pydantic field in the generated pydantic class.

    Args:
        name: The name of the field. This is used in the generated Python code.
        prop_name: The name of the property in the data model. This is used when reading and writing to CDF.
        pydantic_field: The name to use for the import 'from pydantic import Field'. This is used in the edge case
                        when the name 'Field' name clashes with the data model class name.

    """

    name: str
    prop_name: str
    pydantic_field: str

    @property
    def need_alias(self) -> bool:
        return self.name != self.prop_name

    @classmethod
    def from_property(
        cls,
        prop_name: str,
        prop: dm.MappedProperty | dm.ConnectionDefinition,
        data_class_by_view_id: dict[ViewSpaceExternalId, DataClass],
        field_naming: pygen_config.FieldNaming,
        view_name: str,
        pydantic_field: str = "Field",
    ) -> Field:
        name = create_name(prop_name, field_naming.name)
        if isinstance(prop, dm.SingleHopConnectionDefinition):
            variable = create_name(prop_name, field_naming.variable)

            edge_api_class_input = f"{view_name}_{prop_name}"
            edge_api_class = f"{create_name(edge_api_class_input, field_naming.edge_api_class)}API"
            edge_api_attribute = create_name(prop_name, field_naming.api_class_attribute)
            return EdgeOneToMany(
                name=name,
                prop_name=prop_name,
                prop=prop,
                data_class=data_class_by_view_id[ViewSpaceExternalId(prop.source.space, prop.source.external_id)],
                variable=variable,
                pydantic_field=pydantic_field,
                edge_api_class=edge_api_class,
                edge_api_attribute=edge_api_attribute,
            )
        elif isinstance(prop, dm.MappedProperty) and (
            isinstance(prop.type, _PRIMITIVE_TYPES) or isinstance(prop.type, _EXTERNAL_TYPES)
        ):
            type_ = _to_python_type(prop.type)
            if isinstance(prop.type, ListablePropertyType) and prop.type.is_list:
                return PrimitiveListField(
                    name=name,
                    prop_name=prop_name,
                    type_=type_,
                    is_nullable=prop.nullable,
                    prop=prop,
                    pydantic_field=pydantic_field,
                )
            elif isinstance(prop.type, dm.CDFExternalIdReference):
                # Note: these are only CDF External Fields that are not listable. Listable CDF External Fields
                # are handled above.
                edge_api_class_input = f"{view_name}_{prop_name}"
                edge_api_class = f"{create_name(edge_api_class_input, field_naming.edge_api_class)}"
                edge_api_attribute = create_name(prop_name, field_naming.api_class_attribute)
                return CDFExternalField(
                    name=name,
                    prop_name=prop_name,
                    type_=type_,
                    is_nullable=prop.nullable,
                    prop=prop,
                    pydantic_field=pydantic_field,
                    edge_api_class=edge_api_class,
                    edge_api_attribute=edge_api_attribute,
                )
            else:
                return PrimitiveField(
                    name=name,
                    prop_name=prop_name,
                    type_=type_,
                    is_nullable=prop.nullable,
                    default=prop.default_value,
                    prop=prop,
                    pydantic_field=pydantic_field,
                )
        elif isinstance(prop, dm.MappedProperty) and isinstance(prop.type, dm.DirectRelation):
            # For direct relation the source is required.
            view_id = cast(dm.ViewId, prop.source)
            target_data_class = data_class_by_view_id[ViewSpaceExternalId(view_id.space, view_id.external_id)]
            return EdgeOneToOne(
                name=name, prop_name=prop_name, prop=prop, data_class=target_data_class, pydantic_field=pydantic_field
            )

        else:
            raise NotImplementedError(f"Property type={type(prop)!r} is not supported")

    @abstractmethod
    def as_read_type_hint(self) -> str:
        raise NotImplementedError()

    @abstractmethod
    def as_write_type_hint(self) -> str:
        raise NotImplementedError()

    @property
    @abstractmethod
    def is_edge(self) -> bool:
        raise NotImplementedError()

    @property
    @abstractmethod
    def is_time_field(self) -> bool:
        raise NotImplementedError()

    @property
    @abstractmethod
    def is_text_field(self) -> bool:
        raise NotImplementedError()

@dataclass(frozen=True)
class PrimitiveFieldCore(Field, ABC):
    type_: str
    is_nullable: bool
    prop: dm.MappedProperty

    @property
    def is_edge(self) -> bool:
        return False

    @property
    def is_time_field(self) -> bool:
        return self.type_ in ("datetime.datetime", "datetime.date")

    @property
    def is_text_field(self) -> bool:
        return self.type_ == "str"

@dataclass(frozen=True)
class PrimitiveField(PrimitiveFieldCore):
    """
    This represents a basic type such as str, int, float, bool, datetime.datetime, datetime.date.
    """

    default: str | int | dict | None = None

    def as_read_type_hint(self) -> str:
        if self.need_alias:
            return f'Optional[{self.type_}] = {self.pydantic_field}({self.default}, alias="{self.prop_name}")'
        else:
            return f"Optional[{self.type_}] = {self.default}"

    def as_write_type_hint(self) -> str:
        out_type = self.type_
        if self.is_nullable:
            out_type = f"Optional[{out_type}] = {self.default}"
        return out_type

@dataclass(frozen=True)
class PrimitiveListField(PrimitiveFieldCore):
    """
    This represents a list of basic types such as list[str], list[int], list[float], list[bool],
    list[datetime.datetime], list[datetime.date].
    """

    def as_read_type_hint(self) -> str:
        if self.need_alias:
            return f'Optional[list[{self.type_}]] = {self.pydantic_field}(None, alias="{self.prop_name}")'
        else:
            return f"Optional[list[{self.type_}]] = None"

    def as_write_type_hint(self) -> str:
        if self.is_nullable:
            return f"Optional[list[{self.type_}]] = None"
        else:
            return f"list[{self.type_}]"

@dataclass(frozen=True)
class CDFExternalField(PrimitiveFieldCore):
    edge_api_class: str
    edge_api_attribute: str

    def as_read_type_hint(self) -> str:
        if self.need_alias:
            return f'Optional[{self.type_}] = {self.pydantic_field}(None, alias="{self.prop_name}")'
        else:
            return f"Optional[{self.type_}] = None"

    def as_write_type_hint(self) -> str:
        out_type = self.type_
        if self.is_nullable:
            out_type = f"Optional[{out_type}] = None"
        return out_type

@dataclass(frozen=True)
class EdgeField(Field, ABC):
    """
    This represents an edge field linking to another data class.
    """

    data_class: DataClass

    @property
    def is_edge(self) -> bool:
        return True

    @property
    def is_time_field(self) -> bool:
        return False

    @property
    def is_text_field(self) -> bool:
        return False

@dataclass(frozen=True)
class EdgeOneToOne(EdgeField):
    """
    This represents an edge field linking to another data class.
    """

    prop: dm.MappedProperty

    def as_read_type_hint(self) -> str:
        if self.need_alias:
            return f'Optional[str] = {self.pydantic_field}(None, alias="{self.prop_name}")'
        else:
            return "Optional[str] = None"

    def as_write_type_hint(self) -> str:
        return f"Union[{self.data_class.write_name}, str, None] = {self.pydantic_field}(None, repr=False)"

@dataclass(frozen=True)
class EdgeOneToMany(EdgeField):
    """
    This represents a list of edge fields linking to another data class.
    """

    variable: str
    edge_api_class: str
    edge_api_attribute: str
    prop: dm.SingleHopConnectionDefinition

    def as_read_type_hint(self) -> str:
        if self.need_alias:
            return f"Optional[list[str]] = {self.pydantic_field}(None, alias='{self.prop_name}')"
        else:
            return "Optional[list[str]] = None"

    def as_write_type_hint(self) -> str:
        return (
            f"Union[list[{self.data_class.write_name}], list[str], None]"
            f" = {self.pydantic_field}(default=None, repr=False)"
        )

@dataclass(frozen=True)
class DataClass:
    """
    This represents a data class. It is created from a view.
    """

    view_name: str
    read_name: str
    write_name: str
    read_list_name: str
    write_list_name: str
    variable: str
    variable_list: str
    file_name: str
    view_id: ViewSpaceExternalId
    fields: list[Field] = field(default_factory=list)

    @classmethod
    def from_view(cls, view: dm.View, data_class: pygen_config.DataClassNaming) -> Self:
        view_name = (view.name or view.external_id).replace(" ", "_")
        class_name = create_name(view_name, data_class.name)
        variable_name = create_name(view_name, data_class.variable)
        variable_list = create_name(view_name, data_class.variable_list)
        if variable_name == variable_list:
            variable_list = f"{variable_list}_list"
        file_name = f"_{create_name(view_name, data_class.file)}"
        return cls(
            view_name=view_name,
            read_name=class_name,
            write_name=f"{class_name}Apply",
            read_list_name=f"{class_name}List",
            write_list_name=f"{class_name}ApplyList",
            variable=variable_name,
            variable_list=variable_list,
            file_name=file_name,
            view_id=ViewSpaceExternalId.from_(view),
        )

    def update_fields(
        self,
        properties: dict[str, dm.MappedProperty | dm.ConnectionDefinition],
        data_class_by_view_id: dict[ViewSpaceExternalId, DataClass],
        field_naming: pygen_config.FieldNaming,
    ) -> None:
        pydantic_field = self.pydantic_field
        for prop_name, prop in properties.items():
            field_ = Field.from_property(
                prop_name, prop, data_class_by_view_id, field_naming, self.view_name, pydantic_field=pydantic_field
            )
            self.fields.append(field_)

    @property
    def text_field_names(self) -> str:
        return f"{self.read_name}TextFields"

    @property
    def field_names(self) -> str:
        return f"{self.read_name}Fields"

    @property
    def properties_dict_name(self) -> str:
        return f"_{self.read_name.upper()}_PROPERTIES_BY_FIELD"

    @property
    def pydantic_field(self) -> str:
        if any(
            name == "Field" for name in [self.read_name, self.write_name, self.read_list_name, self.write_list_name]
        ):
            return "pydantic.Field"
        else:
            return "Field"

    @property
    def init_import(self) -> str:
        import_classes = [self.read_name, self.write_name, self.read_list_name, self.write_list_name]
        if not self.has_only_one_to_many_edges:
            import_classes.append(self.field_names)
        if self.has_text_field:
            import_classes.append(self.text_field_names)
        return f"from .{self.file_name} import {', '.join(sorted(import_classes))}"

    def __iter__(self) -> Iterator[Field]:
        return iter(self.fields)

    @property
    def one_to_one_edges(self) -> Iterable[EdgeOneToOne]:
        return (field_ for field_ in self.fields if isinstance(field_, EdgeOneToOne))

    @property
    def one_to_many_edges(self) -> Iterable[EdgeOneToMany]:
        return (field_ for field_ in self.fields if isinstance(field_, EdgeOneToMany))

    @property
    def primitive_fields(self) -> Iterable[PrimitiveField]:
        return (field_ for field_ in self.fields if isinstance(field_, PrimitiveField))

    @property
    def primitive_core_fields(self) -> Iterable[PrimitiveFieldCore]:
        return (field_ for field_ in self.fields if isinstance(field_, PrimitiveFieldCore))

    @property
    def text_fields(self) -> Iterable[PrimitiveFieldCore]:
        return (field_ for field_ in self.primitive_core_fields if field_.is_text_field)

    @property
    def cdf_external_fields(self) -> Iterable[CDFExternalField]:
        return (field_ for field_ in self.fields if isinstance(field_, CDFExternalField))

    @property
    def single_timeseries_fields(self) -> Iterable[CDFExternalField]:
        return (field_ for field_ in self.cdf_external_fields if isinstance(field_.prop.type, dm.TimeSeriesReference))

    @property
    def has_one_to_many_edges(self) -> bool:
        return any(isinstance(field_, EdgeOneToMany) for field_ in self.fields)

    @property
    def has_edges(self) -> bool:
        return any(isinstance(field_, EdgeField) for field_ in self.fields)

    @property
    def has_primitive_fields(self) -> bool:
        return any(isinstance(field_, PrimitiveFieldCore) for field_ in self.fields)

    @property
    def has_only_one_to_many_edges(self) -> bool:
        return all(isinstance(field_, EdgeOneToMany) for field_ in self.fields)

    @property
    def fields_by_container(self) -> dict[dm.ContainerId, list[PrimitiveFieldCore | EdgeOneToOne]]:
        result: dict[dm.ContainerId, list[PrimitiveFieldCore | EdgeOneToOne]] = defaultdict(list)
        for field_ in self:
            if isinstance(field_, (PrimitiveFieldCore, EdgeOneToOne)):
                result[field_.prop.container].append(field_)
        return dict(result)

    @property
    def has_time_field(self) -> bool:
        return any(field_.is_time_field for field_ in self.fields)

    @property
    def has_text_field(self) -> bool:
        return any(field_.is_text_field for field_ in self.fields)

    @property
    def _field_type_hints(self) -> Iterable[str]:
        return (hint for field_ in self.fields for hint in (field_.as_read_type_hint(), field_.as_write_type_hint()))

    @property
    def use_optional_type(self) -> bool:
        return any("Optional" in hint for hint in self._field_type_hints)

    @property
    def use_pydantic_field(self) -> bool:
        pydantic_field = self.pydantic_field
        return any(pydantic_field in hint for hint in self._field_type_hints)

    @property
    def import_pydantic_field(self) -> str:
        if self.pydantic_field == "Field":
            return "from pydantic import Field"
        else:
            return "import pydantic"

    @property
    def dependencies(self) -> list[DataClass]:
        unique: dict[ViewSpaceExternalId, DataClass] = {}
        for field_ in self.fields:
            if isinstance(field_, EdgeField):
                # This will overwrite any existing data class with the same view id
                # however, this is not a problem as all data classes are uniquely identified by their view id
                unique[field_.data_class.view_id] = field_.data_class
        return sorted(unique.values(), key=lambda x: x.write_name)

    @property
    def has_single_timeseries_fields(self) -> bool:
        return any(
            isinstance(field_.prop.type, dm.TimeSeriesReference) and not isinstance(field_, PrimitiveListField)
            for field_ in self.single_timeseries_fields
        )

    @property
    def primitive_fields_literal(self) -> str:
        return ", ".join(
            f'"{field_.prop_name}"' for field_ in self if isinstance(field_, (PrimitiveField, CDFExternalField))
        )

    @property
    def text_fields_literals(self) -> str:
        return ", ".join(f'"{field_.name}"' for field_ in self.text_fields)

    @property
    def fields_literals(self) -> str:
        return ", ".join(f'"{field_.name}"' for field_ in self if isinstance(field_, PrimitiveFieldCore))

@dataclass(frozen=True)
class APIClass:
    client_attribute: str
    name: str
    file_name: str
    view_id: ViewSpaceExternalId

    @classmethod
    def from_view(cls, view: dm.View, api_class: pygen_config.APIClassNaming) -> APIClass:
        raw_name = view.name or view.external_id

        raw_name = raw_name.replace(" ", "_")

        return cls(
            client_attribute=create_name(raw_name, api_class.client_attribute),
            name=f"{create_name(raw_name, api_class.name)}API",
            file_name=create_name(raw_name, api_class.file_name),
            view_id=ViewSpaceExternalId.from_(view),
        )

@dataclass(frozen=True)
class MultiAPIClass:
    """
    This represents a set of APIs which are generated from a single data model.

    The motivation for having this class is the case when you want to create one SDK for multiple data models.
    """

    sub_apis: list[APIClass]
    client_attribute: str
    name: str
    model: dm.DataModel[dm.View]

    @property
    def model_id(self) -> dm.DataModelId:
        return self.model.as_id()

    @classmethod
    def from_data_model(
        cls,
        data_model: dm.DataModel[dm.View],
        api_class_by_view_id: dict[ViewSpaceExternalId, APIClass],
        multi_api_class: pygen_config.MultiAPIClassNaming,
    ) -> MultiAPIClass:
        sub_apis = sorted(
            [api_class_by_view_id[ViewSpaceExternalId.from_(view)] for view in data_model.views],
            key=lambda api: api.name,
        )

        data_model_name = data_model.name or data_model.external_id

        return cls(
            sub_apis=sub_apis,
            client_attribute=create_name(data_model_name, multi_api_class.client_attribute),
            name=f"{create_name(data_model_name, multi_api_class.name)}APIs",
            model=data_model,
        )

@dataclass
class FilterParameter:
    name: str
    type_: str
    default: None = None
    space: str | None = None

    @property
    def annotation(self) -> str:
        return f"{self.type_} | None"

    @property
    def is_time(self) -> bool:
        return self.type_ in ("datetime.datetime", "datetime.date")

@dataclass
class FilterCondition:
    filter: type[dm.Filter]
    prop_name: str
    keyword_arguments: dict[str, FilterParameter]

    @property
    def condition(self) -> str:
        if self.filter is dm.filters.In:
            parameter = next(iter(self.keyword_arguments.values())).name
            return f"{parameter} and isinstance({parameter}, list)"
        elif self.filter is dm.filters.Equals:
            parameter = next(iter(self.keyword_arguments.values())).name
            return f"{parameter} and isinstance({parameter}, str)"

        return " or ".join(arg.name for arg in self.keyword_arguments.values())

    @property
    def arguments(self) -> str:
        if self.prop_name == "externalId":
            property_ref = '["node", "externalId"], '
        else:
            property_ref = f'view_id.as_property_ref("{self.prop_name}"), '

        filter_args = self._create_filter_args()

        return f"{property_ref}{', '.join(filter_args)}"

    def _create_filter_args(self) -> list[str]:
        filter_args: list[str] = []
        for keyword, arg in self.keyword_arguments.items():
            if arg.is_time:
                filter_args.append(f"{keyword}={arg.name}.isoformat() if {arg.name} else None")
            else:
                filter_args.append(f"{keyword}={arg.name}")
        return filter_args

    @property
    def filter_call(self) -> str:
        return f"dm.filters.{self.filter.__name__}"

@dataclass
class FilterConditionOnetoOneEdge(FilterCondition):
    instance_type: type

    @property
    def condition(self) -> str:
        if self.filter is dm.filters.In:
            parameter = next(iter(self.keyword_arguments.values())).name
            return (
                f"{parameter} and isinstance({parameter}, list) and "
                f"isinstance({parameter}[0], {self.instance_type.__name__})"
            )
        elif self.filter is dm.filters.Equals:
            parameter = next(iter(self.keyword_arguments.values())).name
            return f"{parameter} and isinstance({parameter}, {self.instance_type.__name__})"
        raise NotImplementedError(f"Unsupported filter {self.filter} for Direct Relation")

    def _create_filter_args(self) -> list[str]:
        filter_args: list[str] = []
        for keyword, arg in self.keyword_arguments.items():
            if self.instance_type is str and self.filter is dm.filters.Equals:
                filter_args.append(f'{keyword}={{"space": "{arg.space}", "externalId": {arg.name}}}')
            elif self.instance_type is tuple and self.filter is dm.filters.Equals:
                filter_args.append(f'{keyword}={{"space": {arg.name}[0], "externalId": {arg.name}[1]}}')
            elif self.instance_type is str and self.filter is dm.filters.In:
                filter_args.append(f'{keyword}=[{{"space": "{arg.space}", "externalId": item}} for item in {arg.name}]')
            elif self.instance_type is tuple and self.filter is dm.filters.In:
                filter_args.append(f'{keyword}=[{{"space": item[0], "externalId": item[1]}} for item in {arg.name}]')
            else:
                raise NotImplementedError(f"Unsupported filter {self.filter} for Direct Relation")
        return filter_args

# This field is used when creating the list method.
_EXTERNAL_ID_FIELD = PrimitiveField(
    name="external_id",
    prop_name="externalId",
    type_="str",
    is_nullable=False,
    default=None,
    prop=dm.MappedProperty(
        container_property_identifier="externalId",
        type=dm.Text(),
        nullable=False,
        auto_increment=False,
        container=dm.ContainerId("dummy", "dummy"),
    ),
    pydantic_field="Field",
)

@dataclass
class ListMethod:
    parameters: list[FilterParameter]
    filters: list[FilterCondition]

    @classmethod
    def from_fields(cls, fields: Iterable[Field], config: pygen_config.Filtering) -> Self:
        parameters_by_name: dict[str, FilterParameter] = {}
        list_filters: list[FilterCondition] = []

        for field_ in itertools.chain(fields, (_EXTERNAL_ID_FIELD,)):
            # Only primitive and edge one-to-one fields supported for now
            if isinstance(field_, PrimitiveField):
                for selected_filter in config.get(field_.prop.type, field_.prop_name):
                    if selected_filter is dm.filters.Equals:
                        if field_.name not in parameters_by_name:
                            parameter = FilterParameter(name=field_.name, type_=field_.type_)
                            parameters_by_name[parameter.name] = parameter
                        else:
                            # Equals and In filter share parameter, you have to extend the type hint.
                            parameter = parameters_by_name[field_.name]
                            parameter.type_ = f"{field_.type_} | {parameter.type_}"
                        list_filters.append(
                            FilterCondition(
                                filter=selected_filter,
                                prop_name=field_.prop_name,
                                keyword_arguments=dict(value=parameter),
                            )
                        )
                    elif selected_filter is dm.filters.In:
                        if field_.name not in parameters_by_name:
                            parameter = FilterParameter(field_.name, type_=f"list[{field_.type_}]")
                            parameters_by_name[parameter.name] = parameter
                        else:
                            # Equals and In filter share parameter, you have to extend the type hint.
                            parameter = parameters_by_name[field_.name]
                            parameter.type_ = f"{parameter.type_} | list[{field_.type_}]"
                        list_filters.append(
                            FilterCondition(
                                filter=selected_filter,
                                prop_name=field_.prop_name,
                                keyword_arguments=dict(values=parameter),
                            )
                        )
                    elif selected_filter is dm.filters.Prefix:
                        parameter = FilterParameter(name=f"{field_.name}_prefix", type_=field_.type_)
                        parameters_by_name[parameter.name] = parameter
                        list_filters.append(
                            FilterCondition(
                                filter=selected_filter,
                                prop_name=field_.prop_name,
                                keyword_arguments=dict(value=parameter),
                            )
                        )
                    elif selected_filter is dm.filters.Range:
                        min_parameter = FilterParameter(name=f"min_{field_.name}", type_=field_.type_)
                        max_parameter = FilterParameter(name=f"max_{field_.name}", type_=field_.type_)
                        parameters_by_name[min_parameter.name] = min_parameter
                        parameters_by_name[max_parameter.name] = max_parameter
                        list_filters.append(
                            FilterCondition(
                                filter=selected_filter,
                                prop_name=field_.prop_name,
                                keyword_arguments=dict(gte=min_parameter, lte=max_parameter),
                            )
                        )
                    else:
                        # This is a filter not supported by the list method.
                        continue
            elif isinstance(field_, EdgeOneToOne):
                for selected_filter in config.get(field_.prop.type, field_.prop_name):
                    if selected_filter is dm.filters.Equals:
                        if field_.name not in parameters_by_name:
                            parameter = FilterParameter(
                                name=field_.name, type_="str | tuple[str, str]", space=field_.data_class.view_id.space
                            )
                            parameters_by_name[parameter.name] = parameter
                        else:
                            # Equals and In filter share parameter, you have to extend the type hint.
                            parameter = parameters_by_name[field_.name]
                            parameter.type_ = f"str | tuple[str, str] | {parameter.type_}"
                        list_filters.extend(
                            [
                                FilterConditionOnetoOneEdge(
                                    filter=selected_filter,
                                    prop_name=field_.prop_name,
                                    keyword_arguments=dict(value=parameter),
                                    instance_type=condition_type,
                                )
                                for condition_type in (str, tuple)
                            ]
                        )
                    elif selected_filter is dm.filters.In:
                        if field_.name not in parameters_by_name:
                            parameter = FilterParameter(
                                name=field_.name,
                                type_="list[str] | list[tuple[str, str]]",
                                space=field_.data_class.view_id.space,
                            )
                            parameters_by_name[parameter.name] = parameter
                        else:
                            # Equals and In filter share parameter, you have to extend the type hint.
                            parameter = parameters_by_name[field_.name]
                            parameter.type_ = f"{parameter.type_} | list[str] | list[tuple[str, str]]"
                        list_filters.extend(
                            [
                                FilterConditionOnetoOneEdge(
                                    filter=selected_filter,
                                    prop_name=field_.prop_name,
                                    keyword_arguments=dict(values=parameter),
                                    instance_type=condition_type,
                                )
                                for condition_type in (str, tuple)
                            ]
                        )
                    else:
                        # This is a filter not supported.
                        continue

        return cls(parameters=list(parameters_by_name.values()), filters=list_filters)

def _to_python_type(type_: dm.DirectRelationReference | dm.PropertyType) -> str:
    if isinstance(type_, (dm.Int32, dm.Int64)):
        out_type = "int"
    elif isinstance(type_, dm.Boolean):
        out_type = "bool"
    elif isinstance(type_, (dm.Float32, dm.Float64)):
        out_type = "float"
    elif isinstance(type_, dm.Date):
        out_type = "datetime.date"
    elif isinstance(type_, dm.Timestamp):
        out_type = "datetime.datetime"
    elif isinstance(type_, dm.Json):
        out_type = "dict"
    elif isinstance(type_, (dm.Text, dm.DirectRelation, dm.CDFExternalIdReference, dm.DirectRelationReference)):
        out_type = "str"
    else:
        raise ValueError(f"Unknown type {type_}")

    return out_type


import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output

from app import app
from apps import app_individual, app_aggregate

app.layout = html.Div([
    dcc.Location(id='url', refresh=False),
    html.Div(id='page-content')
])

index_page  = html.Div([
    dcc.Link('Individual Analysis', href='/individual'),
    html.Br(),
    dcc.Link('Agregrate Analysis', href='/agregrate'),
])

@app.callback(Output('page-content', 'children'),
              [Input('url', 'pathname')])
def display_page(pathname):
    if pathname == '/individual':
        return app_individual.layout
    elif pathname == '/agregrate':
        return app_aggregate.layout
    else:
        return index_page

if __name__=="__main__":
    # app.run_server(debug=True, port=5001)
    app.run_server(
        host='0.0.0.0',
        port=8050,
        debug=True
    )


from datetime import timedelta

from airflow import DAG
from airflow.models import Param
from airflow.operators.dummy_operator import DummyOperator
from common.operators.gce import (
    StartGCEOperator,
    StopGCEOperator,
    CloneRepositoryGCEOperator,
    SSHGCEOperator,
)
from common.utils import get_airflow_schedule
from airflow.providers.google.cloud.transfers.bigquery_to_gcs import (
    BigQueryToGCSOperator,
)
from airflow.providers.google.cloud.operators.bigquery import (
    BigQueryExecuteQueryOperator,
)
from airflow.providers.slack.operators.slack_webhook import SlackWebhookOperator

from common import macros
from common.alerts import task_fail_slack_alert
from common.config import (
    DAG_FOLDER,
    ENV_SHORT_NAME,
    MLFLOW_BUCKET_NAME,
    SLACK_CONN_ID,
    SLACK_CONN_PASSWORD,
    MLFLOW_URL,
    BIGQUERY_RAW_DATASET,
    BIGQUERY_TMP_DATASET,
)

from dependencies.ml.utils import create_algo_training_slack_block
from datetime import datetime

from jobs.ml.constants import IMPORT_TRAINING_SQL_PATH

DATE = "{{ ts_nodash }}"

# Environment variables to export before running commands
dag_config = {
    "STORAGE_PATH": f"gs://{MLFLOW_BUCKET_NAME}/algo_training_{ENV_SHORT_NAME}/algo_training_two_towers_{DATE}",
    "BASE_DIR": "data-gcp/jobs/ml_jobs/algo_training",
    "MODEL_DIR": "two_towers_model",
    "TRAIN_DIR": "/home/airflow/train",
    "EXPERIMENT_NAME": f"algo_training_qpi_v1.1_{ENV_SHORT_NAME}",
}

# Params
train_params = {
    "config_file_name": "user-qpi-features",
    "batch_size": 8192,
    "validation_steps_ratio": 0.1 if ENV_SHORT_NAME == "prod" else 0.4,
    "embedding_size": 64,
    "train_set_size": 0.95 if ENV_SHORT_NAME == "prod" else 0.8,
    "event_day_number": {"prod": 14, "dev": 365, "stg": 14}[ENV_SHORT_NAME],
}
gce_params = {
    "instance_name": f"algo-training-qpi-{ENV_SHORT_NAME}",
    "instance_type": {
        "dev": "n1-standard-2",
        "stg": "n1-standard-8",
        "prod": "n1-standard-32",
    },
}

schedule_dict = {"prod": "0 12 * * 3", "dev": None, "stg": "0 12 * * 2"}

default_args = {
    "start_date": datetime(2023, 3, 17),
    "on_failure_callback": task_fail_slack_alert,
    "retries": 0,
    "retry_delay": timedelta(minutes=2),
}

with DAG(
    "algo_training_two_towers_qpi",
    default_args=default_args,
    description="Custom training job",
    schedule_interval=get_airflow_schedule(schedule_dict[ENV_SHORT_NAME]),
    catchup=False,
    dagrun_timeout=timedelta(minutes=1440),
    user_defined_macros=macros.default,
    template_searchpath=DAG_FOLDER,
    params={
        "branch": Param(
            default="production" if ENV_SHORT_NAME == "prod" else "master",
            type="string",
        ),
        "config_file_name": Param(
            default=train_params["config_file_name"],
            type="string",
        ),
        "batch_size": Param(
            default=str(train_params["batch_size"]),
            type="string",
        ),
        "validation_steps_ratio": Param(
            default=str(train_params["validation_steps_ratio"]),
            type="string",
        ),
        "embedding_size": Param(
            default=str(train_params["embedding_size"]),
            type="string",
        ),
        "train_set_size": Param(
            default=str(train_params["train_set_size"]),
            type="string",
        ),
        "event_day_number": Param(
            default=str(train_params["event_day_number"]),
            type="string",
        ),
        "input_type": Param(
            default="enriched_clicks",
            type="string",
        ),
        "instance_type": Param(
            default=gce_params["instance_type"][ENV_SHORT_NAME],
            type="string",
        ),
        "instance_name": Param(
            default=gce_params["instance_name"]
            + "-"
            + train_params["config_file_name"],
            type="string",
        ),
        "run_name": Param(
            default=train_params["config_file_name"], type=["string", "null"]
        ),
    },
) as dag:
    start = DummyOperator(task_id="start", dag=dag)

    import_tables = {}
    for table in [
        "recommendation_user_features",
        "recommendation_item_features",
        "training_data_enriched_clicks",
    ]:
        import_tables[table] = BigQueryExecuteQueryOperator(
            task_id=f"import_{table}",
            sql=(IMPORT_TRAINING_SQL_PATH / f"{table}.sql").as_posix(),
            write_disposition="WRITE_TRUNCATE",
            use_legacy_sql=False,
            destination_dataset_table=f"{BIGQUERY_RAW_DATASET}.{table}",
            dag=dag,
        )

    # The params.input_type tells the .sql files which table to take as input
    for dataset in ["training", "validation", "test"]:
        task = BigQueryExecuteQueryOperator(
            task_id=f"import_tmp_{dataset}_table",
            sql=(
                IMPORT_TRAINING_SQL_PATH / f"recommendation_{dataset}_data.sql"
            ).as_posix(),
            write_disposition="WRITE_TRUNCATE",
            use_legacy_sql=False,
            destination_dataset_table=f"{BIGQUERY_TMP_DATASET}.{DATE}_recommendation_{dataset}_data",
            dag=dag,
        )
        import_tables[dataset] = task

    gce_instance_start = StartGCEOperator(
        task_id="gce_start_task",
        preemptible=False,
        instance_name="{{ params.instance_name }}",
        instance_type="{{ params.instance_type }}",
        accelerator_types=[{"name": "nvidia-tesla-t4", "count": 1}],
        retries=2,
        labels={"job_type": "ml"},
    )

    fetch_code = CloneRepositoryGCEOperator(
        task_id="fetch_code",
        instance_name="{{ params.instance_name }}",
        python_version="3.10",
        command="{{ params.branch }}",
        retries=2,
    )

    install_dependencies = SSHGCEOperator(
        task_id="install_dependencies",
        instance_name="{{ params.instance_name }}",
        base_dir=dag_config["BASE_DIR"],
        command="pip install -r requirements.txt --user",
        dag=dag,
    )

    store_data = {}
    for split in ["training", "validation", "test"]:
        store_data[split] = BigQueryToGCSOperator(
            task_id=f"store_{split}_data",
            source_project_dataset_table=f"{BIGQUERY_TMP_DATASET}.{DATE}_recommendation_{split}_data",
            destination_cloud_storage_uris=f"{dag_config['STORAGE_PATH']}/raw_recommendation_{split}_data/data-*.parquet",
            export_format="PARQUET",
            dag=dag,
        )

    store_data["bookings"] = BigQueryToGCSOperator(
        task_id=f"store_bookings_data",
        source_project_dataset_table=f"{BIGQUERY_RAW_DATASET}.training_data_bookings",
        destination_cloud_storage_uris=f"{dag_config['STORAGE_PATH']}/bookings/data-*.parquet",
        export_format="PARQUET",
        dag=dag,
    )

    preprocess_data = {}
    for split in ["training", "validation", "test"]:
        preprocess_data[split] = SSHGCEOperator(
            task_id=f"preprocess_{split}",
            instance_name="{{ params.instance_name }}",
            base_dir=dag_config["BASE_DIR"],
            environment=dag_config,
            command=f"PYTHONPATH=. python {dag_config['MODEL_DIR']}/preprocess.py "
            "--config-file-name {{ params.config_file_name }} "
            f"--input-dataframe-file-name raw_recommendation_{split}_data "
            f"--output-dataframe-file-name recommendation_{split}_data",
            dag=dag,
        )

    train = SSHGCEOperator(
        task_id="train",
        instance_name="{{ params.instance_name }}",
        base_dir=dag_config["BASE_DIR"],
        environment=dag_config,
        command=f"PYTHONPATH=. python {dag_config['MODEL_DIR']}/train.py "
        "--config-file-name {{ params.config_file_name }} "
        f"--experiment-name {dag_config['EXPERIMENT_NAME']} "
        "--batch-size {{ params.batch_size }} "
        "--validation-steps-ratio {{ params.validation_steps_ratio }} "
        "--embedding-size {{ params.embedding_size }} "
        "--seed {{ ds_nodash }} "
        "--run-name {{ params.run_name }}",
        dag=dag,
    )

    evaluate = SSHGCEOperator(
        task_id="evaluate",
        instance_name="{{ params.instance_name }}",
        base_dir=dag_config["BASE_DIR"],
        environment=dag_config,
        command=f"PYTHONPATH=. python evaluate.py "
        f"--experiment-name {dag_config['EXPERIMENT_NAME']} "
        "--config-file-name {{ params.config_file_name }}",
        dag=dag,
    )

    gce_instance_stop = StopGCEOperator(
        task_id="gce_stop_task", instance_name="{{ params.instance_name }}"
    )

    send_slack_notif_success = SlackWebhookOperator(
        task_id="send_slack_notif_success",
        http_conn_id=SLACK_CONN_ID,
        webhook_token=SLACK_CONN_PASSWORD,
        blocks=create_algo_training_slack_block(
            dag_config["EXPERIMENT_NAME"], MLFLOW_URL, ENV_SHORT_NAME
        ),
        username=f"Algo trainer robot - {ENV_SHORT_NAME}",
        icon_emoji=":robot_face:",
    )

    (
        start
        >> [
            import_tables["recommendation_user_features"],
            import_tables["recommendation_item_features"],
        ]
        >> import_tables["training_data_enriched_clicks"]
        >> import_tables["training"]
        >> [import_tables["validation"], import_tables["test"]]
        >> gce_instance_start
        >> fetch_code
        >> install_dependencies
        >> [
            store_data["training"],
            store_data["validation"],
            store_data["test"],
            store_data["bookings"],
        ]
        >> preprocess_data["training"]
        >> preprocess_data["validation"]
        >> preprocess_data["test"]
        >> train
        >> evaluate
        >> gce_instance_stop
        >> send_slack_notif_success
    )


import argparse
import torch
import torch.nn as nn
from torch.utils import data
import numpy as np
from torch.autograd import Variable
import torch.optim as optim
import torch.backends.cudnn as cudnn
import os
import os.path as osp
from multiframe3_CoAttentionSTN import FtoFAttentionModel
import scipy.io as sio
from Porcine_Dataloader import PorcineDataset
import torch.nn.functional as F
import visdom
import math

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

def CrossEntropyLoss(pred, label):
    #label = Variable(label.long()).cuda()
    criterion = torch.nn.BCELoss().cuda()

    return criterion(pred, label)

def generalized_diceLoss(pred, label):
    smooth = 1
    inputs = pred.view(-1)
    targets = label.view(-1)

    intersection = (inputs * targets).sum()
    dice = (2. * intersection + smooth) / (inputs.sum() + targets.sum() + smooth)

    return 1 - dice

def RegistrationLoss(img_fixed, img_moving, dx, dy, dz):
    #num_bats = img_moving.shape[0]  #number of samples in a batch
    num_rows = img_moving.shape[2]
    num_cols = img_moving.shape[3]
    num_depth = img_moving.shape[4]
    zv, yv, xv = torch.meshgrid([torch.arange(0, num_depth),
                                torch.arange(0, num_rows),
                                torch.arange(0, num_cols)])
    xv = xv.float().to(device)
    yv = yv.float().to(device)
    zv = zv.float().to(device)

    xv_moved = xv + dx[0, :, :, :]
    yv_moved = yv + dy[0, :, :, :]
    zv_moved = zv + dz[0, :, :, :]
    xv_moved_normalized = 2 * xv_moved / (num_cols - 1) - 1
    yv_moved_normalized = 2 * yv_moved / (num_rows - 1) - 1
    zv_moved_normalized = 2 * zv_moved / (num_depth - 1) - 1
    grid = torch.stack((xv_moved_normalized, yv_moved_normalized, zv_moved_normalized), 3)
    grid = grid.unsqueeze(0).float().to(device)
    out = F.grid_sample(img_moving.float(), grid, align_corners=True)

    return ((img_fixed - out)**2).mean()

class SpatialTransformer(nn.Module):
    def __init__(self, mode='bilinear'):
        super(SpatialTransformer, self).__init__()

        # Create the grid
        size = [64, 64, 64]
        self.mode = mode

        vectors = [torch.arange(0, s) for s in size]
        mesh_1, mesh_2, mesh_3 = torch.meshgrid(vectors)
        grid = torch.stack((mesh_1, mesh_2, mesh_3), 3)
        grid = grid.unsqueeze(0).float().to('cuda')

        self.register_buffer('grid', grid)
        self.sigmoid = nn.Sigmoid()

    def forward(self, src, flow):
        shape = flow.shape[2:]
        flow = flow.permute(0, 2, 3, 4, 1)
        new_loc = self.grid + flow

        # Need to normalize grid values to [-1, 1] for resampler
        new_loc[:, :, :, :, 0] = 2 * (new_loc[:, :, :, :, 0] / (shape[0] - 1) - 0.5)
        new_loc[:, :, :, :, 1] = 2 * (new_loc[:, :, :, :, 1] / (shape[1] - 1) - 0.5)
        new_loc[:, :, :, :, 2] = 2 * (new_loc[:, :, :, :, 2] / (shape[2] - 1) - 0.5)

        # Need to flip the channels
        new_loc = new_loc[..., [2, 1, 0]]

        return F.grid_sample(src, new_loc, align_corners=True, mode=self.mode)

class NCC:
    """
    Local (over window) normalized cross correlation loss.
    """

    def __init__(self, win=None):
        self.win = win

    def loss(self, y_true, y_pred):

        Ii = y_true
        Ji = y_pred

        # get dimension of volume
        # assumes Ii, Ji are sized [batch_size, *vol_shape, nb_feats]
        ndims = len(list(Ii.size())) - 2
        assert ndims in [1, 2, 3], "volumes should be 1 to 3 dimensions. found: %d" % ndims

        # set window size
        win = [9] * ndims if self.win is None else self.win

        # compute filters
        sum_filt = torch.ones([1, 1, *win]).to("cuda")

        pad_no = math.floor(win[0] / 2)

        if ndims == 1:
            stride = (1)
            padding = (pad_no)
        elif ndims == 2:
            stride = (1, 1)
            padding = (pad_no, pad_no)
        else:
            stride = (1, 1, 1)
            padding = (pad_no, pad_no, pad_no)

        # get convolution function
        conv_fn = getattr(F, 'conv%dd' % ndims)

        # compute CC squares
        I2 = Ii * Ii
        J2 = Ji * Ji
        IJ = Ii * Ji

        I_sum = conv_fn(Ii, sum_filt, stride=stride, padding=padding)
        J_sum = conv_fn(Ji, sum_filt, stride=stride, padding=padding)
        I2_sum = conv_fn(I2, sum_filt, stride=stride, padding=padding)
        J2_sum = conv_fn(J2, sum_filt, stride=stride, padding=padding)
        IJ_sum = conv_fn(IJ, sum_filt, stride=stride, padding=padding)

        win_size = np.prod(win)
        u_I = I_sum / win_size
        u_J = J_sum / win_size

        cross = IJ_sum - u_J * I_sum - u_I * J_sum + u_I * u_J * win_size
        I_var = I2_sum - 2 * u_I * I_sum + u_I * u_I * win_size
        J_var = J2_sum - 2 * u_J * J_sum + u_J * u_J * win_size

        cc = cross * cross / (I_var * J_var + 1e-5)

        return 1 - torch.mean(cc)

def get_arguments():
    """Parse all the arguments provided from the CLI.

    Returns:
      A list of parsed arguments.
    """
    parser = argparse.ArgumentParser(description="Attention Network")

    # optimitization configuration
    parser.add_argument("--is-training", action="store_true",
                        help="Whether to update the running means and variances during the training.")
    parser.add_argument("--learning-rate", type=float, default=0.00001,
                        help="Base learning rate for training with polynomial decay.")  # 0.001
    parser.add_argument("--weight-decay", type=float, default=0.0005,
                        help="Regularization parameter for L2-loss.")  # 0.0005
    parser.add_argument("--momentum", type=float, default=0.9,
                        help="Momentum component of the optimiser.")
    parser.add_argument("--power", type=float, default=0.9,
                        help="Decay parameter to compute the learning rate.")
    parser.add_argument("--random-seed", type=int, default=1234,
                        help="Random seed to have reproducible results.")

    return parser.parse_args()

args = get_arguments()

def configure_dataset_init_model(args):
    args.batch_size = 1  # 1 card: 5, 2 cards: 10 Number of images sent to the network in one step, 16 on paper
    args.maxEpoches = 100  # 1 card: 15, 2 cards: 15 epoches, equal to 30k iterations, max iterations= maxEpoches*len(train_aug)/batch_size_per_gpu'),

def netParams(model):
    '''
    Computing total network parameters
    Args:
       model: model
    return: total network parameters
    '''
    total_parameters = 0
    for parameter in model.parameters():
        i = len(parameter.size())
        # print(parameter.size())
        p = 1
        for j in range(i):
            p *= parameter.size(j)
        total_parameters += p

    return total_parameters

def main():

    print("=====> Integrated Motion Tracking (Image Only) - Porcine")

    print("=====> Set GPU for training")
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

    cudnn.enabled = True

    print("=====> Building network")
    model = FtoFAttentionModel()
    model.to(device)
    spatial_transform = SpatialTransformer()
    #ncc = NCC().loss
    mseloss = nn.MSELoss()

    cudnn.benchmark = True

    print('=====> Computing network parameters')
    total_parameters = netParams(model)
    print('Total network parameters: ' + str(total_parameters))
    print('Learning Rate: ' + str(args.learning_rate))
    maxEpoch = 100

    print("=====> Preparing training data")

    # optimizer = optim.SGD(model.parameters(), lr=args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay)
    optimizer = optim.Adam(model.parameters(), lr=args.learning_rate)
    scheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.95)
    optimizer.zero_grad()

    vis = visdom.Visdom()

    Y1 = torch.zeros((1)).cpu()
    X1 = torch.zeros((1)).cpu()

    loss_window = vis.line(
        Y=np.column_stack((Y1, Y1)),
        X=np.column_stack((X1, X1)),
        opts=dict(xlabel='epoch', ylabel='Loss', title='CoAttentionSTN+Temporal (3frame)',
                  legend=['training', 'validation']))

    print("=====> Preparing Dataloader")
    snapshot_dir = '/data/CoAttentionSTN_MedIA22/full_cycle_results_only_pstrain_model/new_trained_model_results'
    train_dataset = PorcineDataset('/data/Training_Data/MedIA2022_CoAttentionSTN/Porcine/full_cycle/train')
    val_dataset = PorcineDataset('/data/Training_Data/MedIA2022_CoAttentionSTN/Porcine/full_cycle/validation')
    params = {'batch_size': 1,
              'shuffle': True}
    training_generator = torch.utils.data.DataLoader(train_dataset, **params)
    validation_generator = torch.utils.data.DataLoader(val_dataset, **params)
    print('Total number of training images: ' + str(len(training_generator)))
    print('Total number of validation images: ' + str(len(validation_generator)))

    lowest_loss = 1e6
    print("=====> Begin to train")
    for epoch in range(0, maxEpoch):

        train_loss = 0.0
        val_loss = 0.0

        for phase in ['train', 'val']:
            if phase == 'train':
                model.train()
                np.random.seed(args.random_seed + epoch)

                for img, filename in validation_generator:
                    # Transfer to GPU
                    local_batch = img.to(device)#, myocardium1.to(device), myocardium2.to(device)

                    x_batch = local_batch.float()
                    #myo1 = myo1.float()
                    #myo2 = myo2.float()

                    x_batch /= 255

                    #myocardium1 = myo1.unsqueeze(1)
                    #myocardium2 = myo2.unsqueeze(1)

                    x_batch.requires_grad_()
                    image = Variable(x_batch)

                    optimizer.zero_grad()

                    pred_disp_es_ed, frame1, frame2, frame_mid, exemplar_mask, query_mask, interm_mask, pred_disp12, pred_disp23 = model(image)

                    deformed_img1 = spatial_transform(image[:, 0, :, :, :].unsqueeze(1), pred_disp_es_ed)
                    deformed_img2 = spatial_transform(frame1, pred_disp_es_ed)
                    deformed_img3 = spatial_transform(frame1, pred_disp12)
                    deformed_img4 = spatial_transform(frame_mid, pred_disp23)
                    deformed_int1 = spatial_transform(image[:, 0, :, :, :].unsqueeze(1), pred_disp12)
                    deformed_int2 = spatial_transform(image[:, 1, :, :, :].unsqueeze(1), pred_disp23)
                    deformed_attention_map1 = spatial_transform(exemplar_mask, pred_disp_es_ed)
                    deformed_attention_map2 = spatial_transform(exemplar_mask, pred_disp12)
                    deformed_attention_map3 = spatial_transform(interm_mask, pred_disp23)

                    loss_raw_img1 = mseloss(deformed_img1, image[:, 2, :, :, :].unsqueeze(1))
                    loss_raw_img2 = mseloss(deformed_int1, image[:, 1, :, :, :].unsqueeze(1))
                    loss_raw_img3 = mseloss(deformed_int2, image[:, 2, :, :, :].unsqueeze(1))

                    loss_att_img1 = mseloss(deformed_img2, frame2)
                    loss_att_img2 = mseloss(deformed_img3, frame_mid)
                    loss_att_img3 = mseloss(deformed_img4, frame2)

                    loss_attention1 = NCC().loss(deformed_attention_map1, query_mask)
                    loss_attention2 = NCC().loss(deformed_attention_map2, interm_mask)
                    loss_attention3 = NCC().loss(deformed_attention_map3, query_mask)

                    raw_img_loss = loss_raw_img1 + loss_raw_img2 + loss_raw_img3
                    att_img_loss = loss_att_img1 + loss_att_img2 + loss_att_img3
                    att_loss = loss_attention1 + loss_attention2 + loss_attention3
                    temp_cons = pred_disp12 + pred_disp23
                    temp_loss = mseloss(temp_cons, pred_disp_es_ed)

                    total_loss = raw_img_loss + att_img_loss + 0.02 * temp_loss + 0.05 * att_loss
                    total_loss.backward()
                    optimizer.step()
                    train_loss += total_loss.data

            else:
                model.eval()
                for img, filename in validation_generator:
                    # Transfer to GPU
                    local_batch = img.to(device)#, myocardium1.to(device), myocardium2.to(device)

                    x_batch = local_batch.float()
                    #myo1 = myo1.float()
                    #myo2 = myo2.float()

                    x_batch /= 255

                    #myocardium1 = myo1.unsqueeze(1)
                    #myocardium2 = myo2.unsqueeze(1)

                    x_batch.requires_grad_()
                    image = Variable(x_batch)

                    with torch.no_grad():
                        pred_disp_es_ed, frame1, frame2, frame_mid, exemplar_mask, query_mask, interm_mask, pred_disp12, pred_disp23 = model(image)

                    deformed_img1 = spatial_transform(image[:, 0, :, :, :].unsqueeze(1), pred_disp_es_ed)
                    deformed_img2 = spatial_transform(frame1, pred_disp_es_ed)
                    deformed_img3 = spatial_transform(frame1, pred_disp12)
                    deformed_img4 = spatial_transform(frame_mid, pred_disp23)
                    deformed_int1 = spatial_transform(image[:, 0, :, :, :].unsqueeze(1), pred_disp12)
                    deformed_int2 = spatial_transform(image[:, 1, :, :, :].unsqueeze(1), pred_disp23)
                    deformed_attention_map1 = spatial_transform(exemplar_mask, pred_disp_es_ed)
                    deformed_attention_map2 = spatial_transform(exemplar_mask, pred_disp12)
                    deformed_attention_map3 = spatial_transform(interm_mask, pred_disp23)

                    loss_raw_img1 = mseloss(deformed_img1, image[:, 2, :, :, :].unsqueeze(1))
                    loss_raw_img2 = mseloss(deformed_int1, image[:, 1, :, :, :].unsqueeze(1))
                    loss_raw_img3 = mseloss(deformed_int2, image[:, 2, :, :, :].unsqueeze(1))

                    loss_att_img1 = mseloss(deformed_img2, frame2)
                    loss_att_img2 = mseloss(deformed_img3, frame_mid)
                    loss_att_img3 = mseloss(deformed_img4, frame2)

                    loss_attention1 = NCC().loss(deformed_attention_map1, query_mask)
                    loss_attention2 = NCC().loss(deformed_attention_map2, interm_mask)
                    loss_attention3 = NCC().loss(deformed_attention_map3, query_mask)

                    raw_img_loss = loss_raw_img1 + loss_raw_img2 + loss_raw_img3
                    att_img_loss = loss_att_img1 + loss_att_img2 + loss_att_img3
                    att_loss = loss_attention1 + loss_attention2 + loss_attention3
                    temp_cons = pred_disp12 + pred_disp23
                    temp_loss = mseloss(temp_cons, pred_disp_es_ed)

                    total_loss = raw_img_loss + att_img_loss + 0.02 * temp_loss + 0.05 * att_loss
                    val_loss += total_loss.data

        scheduler.step()

        train_loss /= len(training_generator)
        val_loss /= len(validation_generator)
        vis.line(X=torch.ones((1, 1)).cpu() * epoch, Y=torch.Tensor([train_loss]).unsqueeze(0).cpu(), win=loss_window,
                 name='training',
                 update='append')

        vis.line(X=torch.ones((1, 1)).cpu() * epoch, Y=torch.Tensor([val_loss]).unsqueeze(0).cpu(), win=loss_window,
                 name='validation',
                 update='append')
        os.chdir(snapshot_dir)
        temp_epoch = epoch

        if phase == 'val' and val_loss < lowest_loss:
            lowest_loss = val_loss
            torch.save(model.state_dict(), osp.join(snapshot_dir, 'CoAttentionSTN_3frame_w_att_loss_full_cycle.pth'))
            file_name = 'CoAttentionSTN_3frame_w_att_loss_full_cycle.mat'
            image = image.cpu().detach().numpy()
            exemplar_mask = exemplar_mask.cpu().detach().numpy()
            query_mask = query_mask.cpu().detach().numpy()
            #int_mask = int_mask.cpu().detach().numpy()
            #myocardium1 = myocardium1.cpu().detach().numpy()
            #myocardium2 = myocardium2.cpu().detach().numpy()
            pred_disp_es_ed = pred_disp_es_ed.cpu().detach().numpy()
            #pred_disp12 = pred_disp12.cpu().detach().numpy()
            #pred_disp23 = pred_disp23.cpu().detach().numpy()
            deformed_img1 = deformed_img1.cpu().detach().numpy()
            deformed_img2 = deformed_img2.cpu().detach().numpy()
            frame1 = frame1.cpu().detach().numpy()
            frame2 = frame2.cpu().detach().numpy()

            sio.savemat(file_name, {'image': image, 'pred_disp_es_ed': pred_disp_es_ed, 'deformed1': deformed_img1,
                                     'deformed2': deformed_img2, 'frame1':frame1, 'frame2':frame2, 'epoch':temp_epoch,
                                    'exemplar_mask':exemplar_mask, 'query_mask':query_mask})
                                     #'myocardium1': myocardium1, 'myocardium2': myocardium2})
            print('Epoch: {0:3d} | train_loss: {1:2f} | val_loss: {2:2f} | Model Saved'.format(epoch, train_loss, val_loss))
        else:
            print('Epoch: {0:3d} | train_loss: {1:2f} | val_loss: {2:2f} |'.format(epoch, train_loss, val_loss))

if __name__ == '__main__':
    main()


import sqlite3
import csv

get_sql = """
DELETE
    FROM 'table_fees'
    WHERE truck_number = ? AND timestamp = ?
   """

def delete_wrong_fees(cursor: sqlite3.Cursor, wrong_fees_file: str) -> None:
    with open(f'../{wrong_fees_file}') as csvfile:
        spamreader = csv.reader(csvfile, delimiter=',')
        for line in spamreader:
            cursor.execute(get_sql, (line[0], line[1],))

if __name__ == '__main__':
    with sqlite3.connect('../hw.db') as conn:
        cursor = conn.cursor()
        delete_wrong_fees(cursor, 'wrong_fees.csv')


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# ***************************************************************************
# Copyright (c) 2018 西安交通大学
# All rights reserved
#
# 文件名称：Mayavi_test.py
#
# 摘    要：数据可视化mayavi库测试文件
#
# 创 建 者：上官栋栋
#
# 创建日期：2018年11月9日
#
# 修改记录
# 日期  修改者   		版本     修改内容
# ------------- 		-------  ------------------------
# ***************************************************************************

from tvtk.tools import tvtk_doc,ivtk
from tvtk.api import tvtk
import numpy as np
from mayavi import mlab
from traits.api import HasTraits ,Delegate,Instance,Int,Str

class DataVisual(object):
    def test1(self):
        '''调用帮助文档T'''
        tvtk_doc.main()

    def test2(self):
        '''展开交互图像窗口的常用流程'''
        s=tvtk.CubeSource(x_length=1.0,y_length=2.0,z_length=3.0)
        m=tvtk.PolyDataMapper(input_connection=s.output_port)
        a=tvtk.Actor(mapper=m)
        r=tvtk.Renderer(background=(0.5,0.5,0.5))
        r.add_actor(a)
        w=tvtk.RenderWindow(size=(500,500))
        w.add_renderer(r)
        i=tvtk.RenderWindowInteractor(render_window=w)
        i.initialize()
        i.start()

    def test3(self):
        plot3d=tvtk.MultiBlockPLOT3DReader(xyz_file_name="combxyz.bin",
                                           q_file_name="combq,bin",
                                           scalar_function_number=100,
                                           vector_function_number=200)
        plot3d.update()
        grid=plot3d.output.get_block(0)

        con=tvtk.ContourFilter()
        con.set_input_data(grid)
        con.generate_values(10,grid.point_data.scalars.range)

        m=tvtk.PolyDataMapper(scalar_range=grid.point_data.scalars.range,
                              input_connection=con.output_port)
        a=tvtk.Actor(mapper=m)
        a.property.opacity=0.5

        win=self.ivtk_scene(a)
        win.scene.isometric_view()
        self.event_loop()

    # 命名为tvtkfunc.py
    def ivtk_scene(self,actors):
        from tvtk.tools import ivtk
        # 创建一个带Crust（Python Shell）的窗口
        win = ivtk.IVTKWithCrustAndBrowser()
        win.open()
        win.scene.add_actor(actors)
        # 修正窗口错误
        dialog = win.control.centralWidget().widget(0).widget(0)
        from pyface.qt import QtCore
        dialog.setWindowFlags(QtCore.Qt.WindowFlags(0x00000000))
        dialog.show()
        return win

    def event_loop(self):
        from pyface.api import GUI
        gui = GUI()
        gui.start_event_loop()

    def test4(self):
        s=tvtk.CubeSource(x_length=1.0,y_length=2.0,z_length=3.0)
        m=tvtk.PolyDataMapper(input_connection=s.output_port)
        a=tvtk.Actor(mapper=m)

        gui=GUI()
        # win=ivtk.IVTKWithCrust()
        # win=ivtk.IVTK()
        win=ivtk.IVTKWithCrustAndBrowser()
        win.open()
        win.scene.add_actor(a)

        dialog=win.control.centralWidget().widget(0).widget(0)
        from pyface.qt import QtCore
        dialog.setWindowFlags(QtCore.Qt.WindowFlags(0x00000000))
        dialog.show()

        gui.start_event_loop()

    # imageData数据类型 and RectilinearGrid数据类型
    def test5(self):
        # imageData数据类型
        img=tvtk.ImageData(spacing=(1,1,1),origin=(0,0,0),dimensions=(3,4,5))

        #RectilinearGrid数据类型
        x=np.array([0,3,5,6])
        y=np.array([3,5,9,12])
        z=np.array([4,5,7,9])
        r=tvtk.RectilinearGrid()
        r.x_coordinates=x
        r.y_coordinates=y
        r.z_coordinates=z
        r.dimensions=len(x),len(y),len(z)

        for i in range(64):
            print(r.get_point(i))

    #StructureGrid数据类
    def test6(self):

        def generate_annulus(r, theta, z):
            """ Generate points for structured grid for a cylindrical annular
                volume.  This method is useful for generating a unstructured
                cylindrical mesh for VTK.
            """
            # Find the x values and y values for each plane.
            x_plane = (np.cos(theta) * r[:, None]).ravel()
            y_plane = (np.sin(theta) * r[:, None]).ravel()

            # Allocate an array for all the points.  We'll have len(x_plane)
            # points on each plane, and we have a plane for each z value, so
            # we need len(x_plane)*len(z) points.
            points = np.empty([len(x_plane) * len(z), 3])

            # Loop through the points for each plane and fill them with the
            # correct x,y,z values.
            start = 0
            for z_plane in z:
                end = start + len(x_plane)
                # slice out a plane of the output points and fill it
                # with the x,y, and z values for this plane.  The x,y
                # values are the same for every plane.  The z value
                # is set to the current z
                plane_points = points[start:end]
                plane_points[:, 0] = x_plane
                plane_points[:, 1] = y_plane
                plane_points[:, 2] = z_plane
                start = end

            return points

        dims = (3, 4, 3)
        r = np.linspace(5, 15, dims[0])
        theta = np.linspace(0, 0.5 * np.pi, dims[1])
        z = np.linspace(0, 10, dims[2])
        pts = generate_annulus(r, theta, z)
        sgrid = tvtk.StructuredGrid(dimensions=(dims[1], dims[0], dims[2]))
        sgrid.points = pts
        s = np.random.random((dims[0] * dims[1] * dims[2]))
        sgrid.point_data.scalars = np.ravel(s.copy())
        sgrid.point_data.scalars.name = 'scalars'

    #读取STL文件
    def test7(self):
        s = tvtk.STLReader(file_name="E:/mesh.stl")  # 调用stl文件
        m = tvtk.PolyDataMapper(input_connection=s.output_port)
        a = tvtk.Actor(mapper=m)

        # win=ivtk.IVTKWithCrust()
        # win=ivtk.IVTK()
        win = ivtk.IVTKWithCrustAndBrowser()
        win.open()
        win.scene.add_actor(a)
        win.scene.isometric_view()

        dialog = win.control.centralWidget().widget(0).widget(0)
        from pyface.qt import QtCore
        dialog.setWindowFlags(QtCore.Qt.WindowFlags(0x00000000))
        dialog.show()

        gui = GUI()
        gui.start_event_loop()

    def test8(self):

        plot3d = tvtk.MultiBlockPLOT3DReader(
            xyz_file_name="combxyz.bin",  # 网格文件
            q_file_name="combq.bin",  # 空气动力学结果文件
            scalar_function_number=100,  # 设置标量数据数量
            vector_function_number=200  # 设置矢量数据数量
        )
        plot3d.update()

        grid = plot3d.output.get_block(0)
        con=tvtk.ContourFilter()
        con.set_input_data(grid)
        con.generate_values(10,grid.point_data.scalars.range)

        m=tvtk.PolyDataMapper(scalar_range=grid.point_data.scalars.range,
                              input_connection=con.output_port)
        a=tvtk.Actor(mapper=m)
        a.property.opacity=0.5

        win=self.ivtk_scene(a)
        win.scene.isometric_view()
        self.event_loop()

    #mayavi mlab example
    def test9(self):
        # x=[[-1,1,1,-1,-1],[-1,1,1,-1,-1]]
        # y=[[-1,-1,-1,-1,-1],[2,2,2,2,2]]
        # z=[[1,1,-1,-1,1],[1,1,-1,-1,1]]
        x=[[0,0],[0,1]]
        y=[[0,0],[1,1]]
        z=[[0,1],[0,0]]
        mlab.mesh(x,y,z)
        mlab.show()

    def test10(self):
        from numpy import pi,sin,cos
        dphi,dtheta=pi/30,pi/30
        [phi,theta]=np.mgrid[0:pi+dphi*1.5:dphi,0:2*pi+dtheta*1.5:dtheta]
        r=7
        x=r*sin(phi)*cos(theta)
        y=r*cos(phi)
        z=r*sin(phi)*sin(theta)
        mlab.mesh(x,y,z,representation='wireframe')
        mlab.show()

    def test11(self):
        t=np.linspace(0,4*np.pi,20)
        x=np.sin(2*t)
        y=np.cos(t)
        z=np.cos(2*t)
        s=np.zeros(20)+1
        mlab.points3d(x, y, z, s, colormap='Spectral', scale_factor=0.025)
        mlab.plot3d(x,y,z,s,colormap='Spectral',tube_radius=0.025)
        mlab.show()

    #imshow函数
    def test12(self):

        data = np.loadtxt('C:\\Users\\Administrator\\Documents\\GitHub\\DifferentialEvolution\\map.csv',
                          delimiter=',')
        mlab.imshow(data,colormap='gist_earth',interpolate=False)
        mlab.show()

    #surf函数
    def test13(self):

        data=np.loadtxt('./Data/map.csv', delimiter=',')
        row=data.shape[0]
        column=data.shape[1]
        x,y=np.mgrid[0:row:1,0:column:1]

        surf=mlab.surf(x,y,data)
        lut=surf.module_manager.scalar_lut_manager.lut.table.to_array()
        lut[:,-1]=np.linspace(0,125,256)
        surf.module_manager.scalar_lut_manager.lut.table=lut
        mlab.show()

    #contour函数
    def test14(self):
        data=np.loadtxt('./Data/map.csv', delimiter=',')
        row=data.shape[0]
        column=data.shape[1]
        x,y=np.mgrid[0:row:1,0:column:1]

        x,y=np.mgrid[0:row:1,0:column:1]
        mlab.contour_surf(x,y,data,contours=100)
        mlab.show()

    #向量场绘制
    def test15(self):
        from numpy import mgrid,sqrt,sin,zeros_like
        x, y, z = mgrid[-0:3:0.6, -0:3:0.6, 0:3:0.3]
        r = sqrt(x ** 2 + y ** 2 + z ** 4)
        u = y * sin(r) / (r + 0.001)
        v = -x * sin(r) / (r + 0.001)
        w = zeros_like(r)
        mlab.quiver3d(x, y, z, u, v, w)
        mlab.colorbar()
        mlab.show()

    #鼠标选取物体
    def test16(self):
        figure=mlab.gcf()
        figure.scene.disable_render=True
        x1,y1,z1=np.random.random((3,10))
        red_glyphs=mlab.points3d(x1,y1,z1,color=(1,0,0),resolution=10)

        x2,y2,z2=np.random.random((3,10))
        white_glyphs=mlab.points3d(x2,y2,z2,color=(0.9,0.9,0.9),resolution=10)

        outline=mlab.outline(line_width=3)
        outline.outline_mode='cornered'
        outline.bounds=(x1[0]-0.1,x1[0]+0.1,
                        y1[0] - 0.1, y1[0] + 0.1,
                        z1[0] - 0.1, z1[0] + 0.1)
        figure.scene.disable_render=False

        glyph_points=red_glyphs.glyph.glyph_source.glyph_source.output.points.to_array()

        def picker_callback(picker):
            if picker.actor in red_glyphs.actor.actors:
                point_id =int(picker.point_id/glyph_points.shape[0])
                if point_id!=-1:
                    x,y,z=x1[point_id],y1[point_id],z1[point_id]
                    outline.bounds=(x-0.1,x+0.1,
                        y - 0.1, y + 0.1,
                        z - 0.1, z + 0.1)

        picker=figure.on_mouse_pick(picker_callback)
        picker.tolerance=0.01
        mlab.title('Click on red balls')
        mlab.show()

    #标量数据可视化
    def test17(self):
        x,y,z=np.ogrid[-10:10:20j,-10:10:20j,-10:10:20j]
        s=np.sin(x*y*z)/(x*y*z)
        src=mlab.pipeline.scalar_field(s)
        # mlab.pipeline.image_plane_widget(src,plane_orientation='x_axes',slice_index=10)
        # mlab.pipeline.image_plane_widget(src, plane_orientation='y_axes', slice_index=10)
        # mlab.pipeline.iso_surface(src,contours=[s.min()+0.1*s.ptp()],opacity=0.1)
        # mlab.pipeline.iso_surface(src,contours=[s.max()-0.1*s.ptp()])
        # mlab.contour3d(s,contours=5,transparent=True)
        mlab.pipeline.volume(src)

        mlab.outline()
        mlab.show()

    #向量数据可视化
    def test18(self):
        x, y, z = np.mgrid[0:1:20j, 0:1:20j, 0:1:20j]
        u = np.sin(np.pi * x) * np.cos(np.pi * z)
        v = -2 * np.sin(np.pi * y) * np.cos(2 * np.pi * z)
        w = np.cos(np.pi * x) * np.sin(np.pi * z) + np.cos(np.pi * y) * np.sin(2 * np.pi * z)

        #以下是四种不同的向量场绘制函数，mlab.pipeline.vector_field是对原向量场的降采样

        # #1.绘制等势面，即曲面向量范数相同
        # src=mlab.pipeline.vector_field(u,v,w)
        # magnitude=mlab.pipeline.extract_vector_norm(src)
        # mlab.pipeline.iso_surface(magnitude,contours=[1.9,1.8])

        # #2.绘制整个向量场
        # src=mlab.pipeline.vector_field(u,v,w)
        # mlab.pipeline.vectors(src,mask_points=10,scale_factor=1)

        # #3.仅绘制一个平面中的向量，且这个平面可以移动
        # src=mlab.pipeline.vector_field(u,v,w)
        # mlab.pipeline.vector_cut_plane(src,mask_points=10,scale_factor=1)

        # #4.绘制流线图
        # src=mlab.pipeline.vector_field(u,v,w)
        # magnitude=mlab.pipeline.extract_vector_norm(src)
        # flow = mlab.pipeline.streamline(magnitude, seedtype='point',
        #                                 seed_visible=False,
        #                                 seed_scale=0.5,
        #                                 seed_resolution=5 )

        #以下的方法针对所有数据

        #显示所有数据
        # mlab.quiver3d(u,v,w)

        #绘制流线图
        mlab.flow(u,v,w,seed_scale=1,seed_resolution=3,integration_direction='both',seedtype='sphere')

        mlab.show()

def showKrigingData(path):

    graphData = []
    pointData = []

    with open(path,'r') as file:
        texts = file.readlines()
        pos = 0
        import re
        reg_int = re.compile(r'-?\d+')
        reg_float = re.compile(r'-?\d\.\d+e[\+,-]\d+')

        num = int(reg_int.search(texts[pos]).group(0))
        pos+=1

        for k in range(num//3):
            row = int(reg_int.search(texts[pos]).group(0))
            col = int(reg_int.search(texts[pos+1]).group(0))
            pos+=3

            x = np.zeros((row,col))
            for i in range(row):
                text_list = reg_float.findall(texts[pos+i])
                for j in range(col):
                    x[i,j] = float(text_list[j])
            graphData.append(x)
            pos+=row+1

            y = np.zeros((row,col))
            for i in range(row):
                text_list = reg_float.findall(texts[pos+i])
                for j in range(col):
                    y[i,j] = float(text_list[j])
            graphData.append(y)
            pos+=row+1

            v = np.zeros((row,col))
            for i in range(row):
                text_list = reg_float.findall(texts[pos+i])
                for j in range(col):
                    v[i,j] = float(text_list[j])
            graphData.append(v)
            pos+=row

        num = int(reg_int.search(texts[pos]).group(0))
        pos+=1

        for k in range(num//2):
            row = int(reg_int.search(texts[pos]).group(0))
            col = int(reg_int.search(texts[pos+1]).group(0))
            pos+=3

            x = np.zeros((row,col))
            for i in range(row):
                text_list = reg_float.findall(texts[pos+i])
                for j in range(col):
                    x[i,j] = float(text_list[j])
            pointData.append(x)
            pos+=row+1

            y = np.zeros(row)
            for i in range(row):
                y[i] = float(reg_float.search(text_list[j]).group(0))
            pointData.append(y)
            pos+=row+1

    #当前只针对len(graphData)==3&len(pointData)==2的情况，多余的数据不显示：
    mlab.figure(size=[1024,800])
    graph = mlab.imshow(graphData[0],graphData[1],graphData[2])
    # mlab.surf(x,y,s)

    if len(pointData)==2:
        x = pointData[0][:,0]
        y = pointData[0][:,1]
        z = np.zeros_like(x)

        mlab.points3d(x,y,z,scale_factor=0.03)

    # 点的大小和颜色随顺序变化
    # x = pointData[0][:,0]
    # y = pointData[0][:,1]
    # newPointNum = x.shape[0]
    # z = np.zeros(newPointNum)
    # c = np.linspace(1,2,newPointNum)
    # mlab.points3d(x,y,z,c,colormap="copper", scale_factor=.25)

    # mlab.outline()
    # mlab.axes(xlabel='x', ylabel='y', zlabel='z')
    mlab.colorbar(graph,'value',orientation='vertical',label_fmt='%.2f')
    mlab.view(0,0)
    photoPath = re.sub(r'.txt','.png',path)
    mlab.savefig(photoPath)
    mlab.close()
    # mlab.show()

def test1():
    '''G8函数显示'''
    # f = lambda x,y:-(np.sin(2*np.pi*x)**3*np.sin(2*np.pi*y))/(x**3*(x+y))
    f = lambda x:-(np.sin(2*np.pi*x[0])**3*np.sin(2*np.pi*x[1]))/(x[0]**3*(x[0]+x[1]))
    g1 = lambda x:x[0]**2-x[1]+1
    g2 = lambda x:1-x[0]+(x[1]-4)**2

    # min = [0.001,0.001]
    # max = [10,10]

    min = [1,3]
    max = [2,5]

    mesh_x ,mesh_y = np.mgrid[min[0]:max[0]:500j,min[1]:max[1]:500j]
    v = np.zeros_like(mesh_x)

    for i in range(mesh_x.shape[0]):
        for j in range(mesh_x.shape[1]):
            p = np.array([mesh_x[i,j],mesh_y[i,j]])
            v[i,j] = np.log(f(p)+10000)
            # if g1(p)<0 and g2(p)<0:
            #     v[i,j] = f(p)
            # else:
            #     v[i,j] = 0

    mlab.figure(size=[1024,800])
    mlab.imshow(mesh_x,mesh_y,v)
    mlab.show()

def test2():
    '''遍历文件集'''
    root_path = '/home/sgdd/Optimization-under-Constraint/Data/约束优化算法测试1/stepC_5'
    for k in range(11):
        path = root_path+'/Kriging_Predicte_Model_%d.txt'%k
        showKrigingData(path)
        path = root_path+'/Kriging_Varience_Model_%d.txt'%k
        showKrigingData(path)
        path = root_path+'/Kriging_EI_Model_%d.txt'%k
        showKrigingData(path)

def test3():
    '''显示特定文件'''
    showKrigingData('./Data/约束优化算法测试1/G8_Function.txt')
    # showKrigingData('./Data/Kriging_EI加点模型测试1/A_Kriging_Predicte_Model.txt')
    # showKrigingData('./Data/Kriging_EI加点模型测试1/A_Kriging_Varience_Model.txt')
    # showKrigingData('./Data/Kriging_EI加点模型测试1/A_Kriging_EI_Model.txt')

if __name__=='__main__':
    # test=DataVisual()
    # test.test13()

    test3()


from fastapi import FastAPI
from controllers.routes import router
from services.services import download_and_load_data, download_and_load, clear_db_data
from apscheduler.schedulers.background import BackgroundScheduler

def poll_and_load_datasets_again():
    clear_db_data()
    download_and_load_data()

app = FastAPI()
app.include_router(router)

@app.on_event("startup")
async def startup_event():
    scheduler = BackgroundScheduler()
    scheduler.add_job(poll_and_load_datasets_again, 'interval', hours=1)
    scheduler.start()
    clear_db_data()
    download_and_load_data()

if __name__ == "__main__":
    import uvicorn

    uvicorn.run(app, host="0.0.0.0", port=8080)


"""empty message

Revision ID: 88f77742bc2d
Revises:
Create Date: 2017-07-03 13:15:12.714598

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '88f77742bc2d'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('redemption',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('coupon_id', sa.Integer(), nullable=True),
    sa.Column('redemption_date', sa.DateTime(), nullable=True),
    sa.ForeignKeyConstraint(['coupon_id'], ['coupon.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('redemption')
    # ### end Alembic commands ###


import torchvision
from torchvision import transforms
from torch.utils import data

def load_data_fashion_mnist(batch_size, resize=None, workers=4):
    """
        自动下载FashionMNIST数据集，并返回数据加载器
    """
    trans = [transforms.ToTensor()]
    if resize:
        trans.insert(0, transforms.Resize(resize))
    trans = transforms.Compose(trans)
    mnist_train = torchvision.datasets.FashionMNIST(
        root="../data", train=True, transform=trans, download=True)
    mnist_test = torchvision.datasets.FashionMNIST(
        root="../data", train=False, transform=trans, download=True)
    return (data.DataLoader(mnist_train, batch_size, shuffle=True,
                            num_workers=workers),
            data.DataLoader(mnist_test, batch_size, shuffle=False,
                            num_workers=workers))

def get_fashion_mnist_labels(labels):
    """返回Fashion-MNIST数据集的文本标签
    Defined in :numref:`sec_fashion_mnist`"""
    text_labels = ['t-shirt', 'trouser', 'pullover', 'dress', 'coat',
                   'sandal', 'shirt', 'sneaker', 'bag', 'ankle boot']
    return [text_labels[int(i)] for i in labels]


from flask import jsonify, render_template
from app.device import bp, device
from app.models import DeviceModel, Project

@bp.route('/devices', methods=['GET', 'POST'])
def devices():
    connected = device.get_devices()
    registered = DeviceModel.query.all()
    return render_template('device/devices.html', title='All devices',
                           connected=connected, registered=registered)

# @bp.route('/test_devices', methods=['GET', 'POST'])
# def test_devices():
#     registered = Device.query.all()
#     return jsonify(Device.serialize_list(registered))

@bp.route('/devices/check', methods=['GET', 'POST'])
def check():
    return {}