import math
import os
import sys

import numpy as np
import torch
from torch import nn
from torch.nn import Conv2d
from torch.nn import functional as F
from torch.nn.utils import remove_weight_norm, spectral_norm, weight_norm

from . import commons, modules
from .commons import get_padding
from .modules import (ConvNext2d, HarmonicEmbedder, IMDCTSymExpHead,
                      LoRALinear1d, SnakeFilter, WaveBlock)

parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(parent_dir)

sr2sr = {
    "24k": 24000,
    "32k": 32000,
    "40k": 40000,
    "48k": 48000,
}

class GeneratorVoras(torch.nn.Module):
    def __init__(
        self,
        emb_channels,
        inter_channels,
        gin_channels,
        n_layers,
        sr,
        hop_length,
    ):
        super(GeneratorVoras, self).__init__()
        self.n_layers = n_layers
        self.emb_pitch = HarmonicEmbedder(768, inter_channels, gin_channels, 16, 15)  #   # pitch 256
        self.plinear = LoRALinear1d(inter_channels, inter_channels, gin_channels, r=8)
        self.glinear = weight_norm(nn.Conv1d(gin_channels, inter_channels, 1))
        self.resblocks = nn.ModuleList()
        self.init_linear = LoRALinear1d(emb_channels, inter_channels, gin_channels, r=4)
        for _ in range(self.n_layers):
            self.resblocks.append(WaveBlock(inter_channels, gin_channels, [9] * 2, [1] * 2, [1, 9], 2, r=4))
        self.head = IMDCTSymExpHead(inter_channels, gin_channels, hop_length, padding="center", sample_rate=sr)
        self.post = SnakeFilter(4, 8, 9, 2, eps=1e-5)

    def forward(self, x, pitchf, x_mask, g):
        x = self.init_linear(x, g) + self.plinear(self.emb_pitch(pitchf, g), g) + self.glinear(g)
        for i in range(self.n_layers):
            x = self.resblocks[i](x, x_mask, g)
        x = x * x_mask
        x = self.head(x, g)
        x = self.post(x)
        return torch.tanh(x)

    def remove_weight_norm(self):
        self.plinear.remove_weight_norm()
        remove_weight_norm(self.glinear)
        for l in self.resblocks:
            l.remove_weight_norm()
        self.init_linear.remove_weight_norm()
        self.head.remove_weight_norm()
        self.post.remove_weight_norm()

    def fix_speaker(self, g):
        self.plinear.fix_speaker(g)
        self.init_linear.fix_speaker(g)
        for l in self.resblocks:
            l.fix_speaker(g)
        self.head.fix_speaker(g)

    def unfix_speaker(self, g):
        self.plinear.unfix_speaker(g)
        self.init_linear.unfix_speaker(g)
        for l in self.resblocks:
            l.unfix_speaker(g)
        self.head.unfix_speaker(g)

class Synthesizer(nn.Module):
    def __init__(
        self,
        segment_size,
        n_fft,
        hop_length,
        inter_channels,
        n_layers,
        spk_embed_dim,
        gin_channels,
        emb_channels,
        sr,
        **kwargs
    ):
        super().__init__()
        if type(sr) == type("strr"):
            sr = sr2sr[sr]
        self.segment_size = segment_size
        self.n_fft = n_fft
        self.hop_length = hop_length
        self.inter_channels = inter_channels
        self.n_layers = n_layers
        self.spk_embed_dim = spk_embed_dim
        self.gin_channels = gin_channels
        self.emb_channels = emb_channels
        self.sr = sr

        self.dec = GeneratorVoras(
            emb_channels,
            inter_channels,
            gin_channels,
            n_layers,
            sr,
            hop_length
        )

        self.emb_g = nn.Embedding(self.spk_embed_dim, gin_channels)
        print(
            "gin_channels:",
            gin_channels,
            "self.spk_embed_dim:",
            self.spk_embed_dim,
            "emb_channels:",
            emb_channels,
        )
        self.speaker = None

    def remove_weight_norm(self):
        self.dec.remove_weight_norm()

    def change_speaker(self, sid: int):
        if self.speaker is not None:
            g = self.emb_g(torch.from_numpy(np.array(self.speaker))).unsqueeze(-1)
            self.dec.unfix_speaker(g)
        g = self.emb_g(torch.from_numpy(np.array(sid))).unsqueeze(-1)
        self.dec.fix_speaker(g)
        self.speaker = sid

    def forward(
        self, phone, phone_lengths, pitch, pitchf, ds
        ):
        g = self.emb_g(ds).unsqueeze(-1)
        x = torch.transpose(phone, 1, -1)
        x_mask = torch.unsqueeze(commons.sequence_mask(phone_lengths, x.size(2)), 1).to(phone.dtype)
        x_slice, ids_slice = commons.rand_slice_segments(
            x, phone_lengths, self.segment_size
        )
        pitchf_slice = commons.slice_segments2(pitchf, ids_slice, self.segment_size)
        mask_slice = commons.slice_segments(x_mask, ids_slice, self.segment_size)
        o = self.dec(x_slice, pitchf_slice, mask_slice, g)
        return o, ids_slice, x_mask, g

    def infer(self, phone, phone_lengths, pitch, nsff0, sid, max_len=None):
        g = self.emb_g(sid).unsqueeze(-1)
        x = torch.transpose(phone, 1, -1)
        x_mask = torch.unsqueeze(commons.sequence_mask(phone_lengths, x.size(2)), 1).to(phone.dtype)
        o = self.dec((x * x_mask)[:, :, :max_len], nsff0, x_mask, g)
        return o, x_mask, (None, None, None, None)

class DiscriminatorP(torch.nn.Module):
    def __init__(self, period, gin_channels, upsample_rates, final_dim=256, use_spectral_norm=False):
        super(DiscriminatorP, self).__init__()
        self.period = period
        self.use_spectral_norm = use_spectral_norm
        self.init_kernel_size = upsample_rates[-1] * 3
        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
        N = len(upsample_rates)
        self.init_conv = norm_f(Conv2d(1, final_dim // (2 ** (N - 1)), (self.init_kernel_size, 1), (upsample_rates[-1], 1)))
        self.convs = nn.ModuleList()
        for i, u in enumerate(upsample_rates[::-1][1:], start=1):
            self.convs.append(
                ConvNext2d(
                    final_dim // (2 ** (N - i)),
                    final_dim // (2 ** (N - i - 1)),
                    gin_channels,
                    (u*3, 1),
                    (u, 1),
                    4,
                    r=2 + i//2
                )
            )
        self.conv_post = weight_norm(Conv2d(final_dim, 1, (3, 1), (1, 1)))

    def forward(self, x, g):
        fmap = []

        # 1d to 2d
        b, c, t = x.shape
        if t % self.period != 0:  # pad first
            n_pad = self.period - (t % self.period)
            x = F.pad(x, (n_pad, 0), "reflect")
            t = t + n_pad
        x = x.view(b, c, t // self.period, self.period)

        x = torch.flip(x, dims=[2])
        x = F.pad(x, [0, 0, 0, self.init_kernel_size - 1], mode="constant")
        x = self.init_conv(x)
        x = F.leaky_relu(x, modules.LRELU_SLOPE)
        x = torch.flip(x, dims=[2])
        fmap.append(x)

        for i, l in enumerate(self.convs):
            x = l(x, g)
            fmap.append(x)

        x = F.pad(x, [0, 0, 2, 0], mode="constant")
        x = self.conv_post(x)
        x = torch.flatten(x, 1, -1)

        return x, fmap

class MultiPeriodDiscriminator(torch.nn.Module):
    def __init__(self, upsample_rates, gin_channels, periods=[2, 3, 5, 7, 11, 17], **kwargs):
        super(MultiPeriodDiscriminator, self).__init__()

        discs = [
            DiscriminatorP(i, gin_channels, upsample_rates, use_spectral_norm=False) for i in periods
        ]
        self.ups = np.prod(upsample_rates)
        self.discriminators = nn.ModuleList(discs)

    def forward(self, y, y_hat, g):
        fmap_rs = []
        fmap_gs = []
        y_d_rs = []
        y_d_gs = []
        for d in self.discriminators:
            y_d_r, fmap_r = d(y, g)
            y_d_g, fmap_g = d(y_hat, g)
            y_d_rs.append(y_d_r)
            y_d_gs.append(y_d_g)
            fmap_rs.append(fmap_r)
            fmap_gs.append(fmap_g)

        return y_d_rs, y_d_gs, fmap_rs, fmap_gs


import csv
import os
from keras.models import load_model
import numpy as np
import scipy.io as sio

model = load_model("models/trained_model_h5.h5")

def load_data(record_path, start_t, end_t):
    data = sio.loadmat(record_path)
    signal = data['ECG']['data'][0][0]
    signal = signal[:, start_t:end_t]

    features = np.empty(60)
    features[:12] = signal.max(axis=1)
    features[12:24] = signal.min(axis=1)
    features[24:36] = signal.mean(axis=1)
    features[36:48] = signal.std(axis=1)
    features[48:] = signal.max(axis=1) + signal.min(axis=1)

    return np.expand_dims(signal.T, 0), np.expand_dims(features, 0)

def get_data_reference_dict(reference_path) -> dict:
    reference_dict = {}
    with open(reference_path, "r") as f_obj:
        reader = csv.reader(f_obj)
        for row in reader:
            name = row[0]
            if name == 'Recording':
                continue
            del row[0]
            reference_dict[name] = [int(x) for x in row if x]
            reference_dict[name].sort()
    return reference_dict

def get_data_and_delin_list(data_path):
    data_list = os.listdir(data_path)
    data_list.sort()
    reference_path = os.path.join(data_path, "REFERENCE.csv")
    data_reference_dict = get_data_reference_dict(reference_path)
    if 'REFERENCE.csv' in data_list:
        data_list.remove('REFERENCE.csv')
    if '.DS_Store' in data_list:
        data_list.remove('.DS_Store')
    return data_list, data_reference_dict

def load_and_predict_data_for_one_slice(data_path, start_t, end_t):
    signal, features = load_data(data_path, start_t, end_t)
    prediction = model.predict([signal, features])
    result = prediction.tolist()
    return prediction, result[0]

def load_and_predict_data(data_path):
    prediction = []
    result = []
    data = sio.loadmat(data_path)
    signal = data['ECG']['data'][0][0]
    max_len = signal.shape[1]

    for start_t in range(0, max_len - 3000, 500):
        pred, res = load_and_predict_data_for_one_slice(data_path, start_t, start_t + 3000)
        prediction.append(pred)
        result.append(res)

    answer_list = []
    for i in range(len(result[0])):
        sum = 0
        for j in range(len(result)):
            sum = sum + result[j][i]
        answer_list.append(sum)
    answer = np.argmax(answer_list) + 1
    return prediction, answer, answer_list

def main():
    data_folder_path = '../DATA/validation_set'

    data_list, data_reference_dict = get_data_and_delin_list(data_folder_path)
    print('good data size:', len(data_list))
    results = {}
    for data in data_list:
        data_path = os.path.join(data_folder_path, data)
        prediction, result, result_list = load_and_predict_data(data_path)
        results[data] = result

    with open("results.csv", 'w') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(['№', 'Recording', 'Reference', 'Prediction', 'Correct'])
        correct_predictions = 0
        for idx, (key, value) in enumerate(results.items()):
            name = key.split('.')[0]
            reference = ", ".join(str(x) for x in data_reference_dict[name])

            if value in data_reference_dict[name]:
                correct_predictions = correct_predictions + 1

            writer.writerow([idx + 1, name, reference, value, value in data_reference_dict[name]])
        writer.writerow([len(results), '', '', '', correct_predictions / len(results)])
        csvfile.close()

if __name__ == "__main__":
    main()


import numpy as np
import matplotlib.pyplot as plt
from components.tests import (
    test_helium, test_helium_x, test_helium_xc, test_helium_xc_Vosko
)
from components.misc import get_n

## Command line arguments
arg_verbose = False
arg_plot = False

## r = radius in spherical coordinates
r_max = 15
r_step = 0.015
rr = np.arange(0, r_max, r_step)

## Start with wave function for hydrogen
psi_start = 1 / np.sqrt(np.pi) * np.exp(-rr)

## Conditions for exiting the program
tolerance = 1e-7
max_iterations = 12

def main(args):
    ## Parse command line arguments
    global arg_verbose
    global arg_plot
    for arg in args[1:]:
        if arg == "-v" or arg == "verbose" or arg == "--verbose":
            arg_verbose = True
        if arg == "-p" or arg == "plot" or arg == "--plot":
            arg_plot = True

    ## Calculate energies and wave functions
    #psi, E = test_helium(rr, psi_start, tolerance, max_iterations, arg_verbose)
    #print("E =", E)
    #print()
    #psi_x, E_x = test_helium_x(
    #    rr, psi_start, tolerance, max_iterations, arg_verbose
    #)
    #print("E_x =", E_x)
    #print()
    #psi_xc, E_xc = test_helium_xc(
    #    rr, psi_start, tolerance, max_iterations, arg_verbose
    #)
    #print("E_xc_Perdew =", E_xc)
    #print()
    psi_xc_Vosko, E_xc_Vosko, E_vec = test_helium_xc_Vosko(
        rr, psi_start, tolerance, max_iterations, arg_verbose
    )
    print("E_xc_vosko =", E_xc_Vosko)
    #print()

    ## Plot
    if arg_plot:
        ## Calculate electron densities
        #n = get_n(psi)
        #n_x = get_n(psi_x)
        #n_xc = get_n(psi_xc)
        n_xc_Vosko = get_n(psi_xc_Vosko)

        ## Plot electron densities
        #plt.plot(rr[1:], n[1:], label='No XC')
        #plt.plot(rr[1:], n_x[1:], label='X')
        #plt.plot(rr[1:], n_xc[1:], label='XC-Perdew')
        #plt.plot(rr[1:], n_xc_Vosko[1:], label='XC-Vosko')

        ## Plot settings
        #xmin = 0
        #xmax = 2
        #ymin = 0
        #ymax = n_xc_Vosko[1] + 0.2
        #plt.axis([xmin, xmax, ymin, ymax])
        plt.grid()
        plt.xlabel("Iteration number", fontsize='x-large')
        plt.ylabel("Energy [Ha]", fontsize='x-large')
        plt.xticks(fontsize='large')
        plt.yticks(fontsize='large')
        #plt.legend(loc='upper right', shadow=False, fontsize='x-large')
        x = np.arange(1, len(E_vec) + 1)
        plt.plot(x, E_vec)

        ## Display the plot
        plt.show()
    return 0

if __name__ == "__main__":
    import sys

    sys.exit(main(sys.argv))


import dash
from dash import dcc
from dash import html
import dash_daq as daq
from dash.dependencies import Input, Output

import pandas as pd
import numpy as np
import plotly.graph_objs as go
import plotly.express as px
import umap
from umap import UMAP

import dash_bootstrap_components as dbc

# Launch the application
app = dash.Dash(external_stylesheets=[dbc.themes.BOOTSTRAP])

# Step 2. Import the dataset
#df = pd.read_csv('finance-charts-apple.csv')

with open('/mnt/c/Users/Ebru-as-user/Downloads/Leukemia_GSE9476.csv', 'r') as f:
  df = pd.read_csv(f)
# saving the string version of the type column to use for labelling
df_type_copy = df['type']

df['type'] = pd.factorize(df['type'])[0] + 1

# UMAP projection of data
reducer = UMAP(n_neighbors=3, min_dist=0.6, n_components=3,
               init='random', random_state=0)
embedding = reducer.fit_transform(df)

# Labels for axes of each plot
x_col = 'UMAP X'
y_col = 'UMAP Y'
z_col = 'UMAP Z'

x_col2 = 'FLT3'
y_col2 = 'NPM1'
z_col2 = 'IDH1'

# 3D arrays for the first and second plot
plot1_axes = embedding

plot2_axes = np.array(
    df[['206674_at', '221923_s_at', '201193_at']].values.tolist())

#boxes = ['214983_at', '206082_at', '210794_s_at', '203591_s_at']

# default columns to use for the box-and-whisker plot if the user doesn't select anything
boxes = ['205780_at', '211725_s_at', '208478_s_at', '203728_at']

# storing the max and mins of each dataset for later use
max_x1 = max(plot1_axes[:, 0])
min_x1 = min(plot1_axes[:, 0])

max_y1 = max(plot1_axes[:, 1])
min_y1 = min(plot1_axes[:, 1])

max_z1 = max(plot1_axes[:, 2])
min_z1 = min(plot1_axes[:, 2])

max_x2 = max(plot2_axes[:, 0])
min_x2 = min(plot2_axes[:, 0])

max_y2 = max(plot2_axes[:, 1])
min_y2 = min(plot2_axes[:, 1])

max_z2 = max(plot2_axes[:, 2])
min_z2 = min(plot2_axes[:, 2])

box_plots = go.Figure()
#for col in boxes:
#  box_plots.add_trace(go.Box(y=df[col], name=col))

#box_plots.add_trace(go.Box(y=df['petal_width']))
#box_plots.add_trace(go.Box(y=df['petal_length']))

# Creating the figures: each have one selection cube and whatever the plot will be
fig = go.Figure()

fig.add_trace(go.Mesh3d(name="cube",
                        # 8 vertices of a cube
                        x=[0, 0, 1, 1, 0, 0, 1, 1],
                        y=[0, 1, 1, 0, 0, 1, 1, 0],
                        z=[0, 0, 0, 0, 1, 1, 1, 1],
                        # i, j and k give the vertices of triangles
                        i=[7, 0, 0, 0, 4, 4, 6, 6, 4, 0, 3, 2],
                        j=[3, 4, 1, 2, 5, 6, 5, 2, 0, 1, 6, 3],
                        k=[0, 7, 2, 3, 6, 7, 1, 1, 5, 5, 7, 6], opacity=0.20
                        ))

fig2 = go.Figure()

fig2.add_trace(go.Mesh3d(name="cube",
                        # 8 vertices of a cube
                         x=[0 + min_x2, 0 + min_x2, 1 + min_x2, 1 + min_x2,
                            0 + min_x2, 0 + min_x2, 1 + min_x2, 1 + min_x2],
                         y=[0 + min_y2, 1 + min_y2, 1 + min_y2, 0 +
                            min_y2, 0 + min_y2, 1 + min_y2, 1 + min_y2, 0 + min_y2],
                         z=[0 + min_z2, 0 + min_z2, 0 + min_z2, 0 + min_z2, 1 +
                            min_z2, 1 + min_z2, 1 + min_z2, 1 + min_z2],
                        # i, j and k give the vertices of triangles
                        i=[7, 0, 0, 0, 4, 4, 6, 6, 4, 0, 3, 2],
                        j=[3, 4, 1, 2, 5, 6, 5, 2, 0, 1, 6, 3],
                        k=[0, 7, 2, 3, 6, 7, 1, 1, 5, 5, 7, 6], opacity=0.20
                        ))

#fig.add_trace(go.Scatter3d(
#    name="Bone_Marrow_CD34", x=embedding[:, 0][0:8], y=embedding[:, 1][0:8], z=embedding[:, 2][0:8]))

#fig.add_trace(go.Scatter3d(
#    name="Bone_Marrow", x=embedding[:, 0][9:18], y=embedding[:, 1][9:18], z=embedding[:, 2][9:18]))

#fig.add_trace(go.Scatter3d(
#    name="AML", x=embedding[:, 0][19:44], y=embedding[:, 1][19:44], z=embedding[:, 2][19:44]))

#fig.add_trace(go.Scatter3d(
#    name="PB", x=embedding[:, 0][45:53], y=embedding[:, 1][45:53], z=embedding[:, 2][45:53]))

#fig.add_trace(go.Scatter3d(
#    name="PBSC_CD34", x=embedding[:, 0][54:63], y=embedding[:, 1][54:63], z=embedding[:, 2][54:63]))

fig.add_trace(go.Scatter3d(
    name="selected", x=plot1_axes[:, 0], y=plot1_axes[:, 1], z=plot1_axes[:, 2], customdata=df_type_copy, hovertemplate="Type: %{customdata}", marker=dict(opacity=1, color=df['type'])))

fig.add_trace(go.Scatter3d(
    name="unselected", x=plot1_axes[:, 0], y=plot1_axes[:, 1], z=plot1_axes[:, 2], customdata=df_type_copy, hovertemplate="Type: %{customdata}", marker=dict(opacity=0.5, color=df['type'])))

#fig2.add_trace(go.Scatter3d(
#    name="unselected", x=embedding[:, 0], y=embedding[:, 1], z=embedding[:, 2], customdata=df_type_copy, hovertemplate="Type: %{customdata}", marker=dict(opacity=0.5, color=df['type'])))

#fig2.add_trace(go.Scatter3d(
#    name="selected", x=embedding[:, 0], y=embedding[:, 1], z=embedding[:, 2], customdata=df_type_copy, hovertemplate="Type: %{customdata}", marker=dict(opacity=1, color=df['type'])))

fig2.add_trace(go.Scatter3d(
    name="selected", x=plot2_axes[:, 0], y=plot2_axes[:, 1], z=plot2_axes[:, 2], customdata=df_type_copy, hovertemplate="Type: %{customdata}",  marker=dict(opacity=1, color=df['type'])))

fig2.add_trace(go.Scatter3d(
    name="unselected", x=plot2_axes[:, 0], y=plot2_axes[:, 1], z=plot2_axes[:, 2], customdata=df_type_copy, hovertemplate="Type: %{customdata}",  marker=dict(opacity=0.5, color=df['type'])))

fig.update_layout(showlegend = True)

#fig.update_xaxes(range=[1.5, 4.5])
#fig.update_yaxes(range=[3, 9])
#fig.update_yaxes(range=[3, 9])

# Organizing the app layout
app.layout = html.Div([dbc.Row([html.H1("Dash for 3D Data Exploration", style={'textAlign': 'center', 'color': '#7FDBFF'}),
                                html.H4(f'Select Variable for Propagation', style={
                                        'color': '#7FDBFF'}),
                                dcc.Dropdown(list(df.columns),
                                             id="box_dropdown", multi=True),
                                html.H5("Note: Hover over the points to see their color-coded classifications")]),
                       html.Br(),
                       dbc.Row([
                            dbc.Col([
                                dbc.Row(
                                    html.H2("UMAP Projection of Leukemia gene expression")),
                                html.Br(),
                                dbc.Row(html.Div(children=[html.H4('Select on first graph:', style={
                                        'color': '#7FDBFF', 'display': 'inline-block'}), daq.BooleanSwitch(id='include1', on=True, style={'display': 'inline-block'})])),
                                html.H4('Exploration Box Size:',
                                        style={'color': '#7FDBFF'}),
                                dcc.Slider(1, 15, value=4, marks=None, id='size1',
                                           tooltip={"placement": "bottom", "always_visible": True}),
                                dbc.Row(dcc.Graph(id='plot1', figure=fig)),
                                dbc.Row([
                                    dbc.Col([html.H4(f'X axis: {x_col}', style={'color': '#7FDBFF'}),
                                             dcc.Slider(min_x1, max_x1, marks=None, value=0, id='xslider1',
                                        tooltip={"placement": "bottom", "always_visible": True})]),
                                    dbc.Col([html.H4(f'Y axis: {y_col}', style={'color': '#7FDBFF'}),
                                            dcc.Slider(min_y1, max_y1, marks=None, value=0, id='yslider1',
                                                       tooltip={"placement": "bottom", "always_visible": True})]),
                                    dbc.Col([html.H4(f'Z axis: {z_col}', style={'color': '#7FDBFF'}),
                                            dcc.Slider(min_z1, max_z1, marks=None, value=0, id='zslider1',
                                                       tooltip={"placement": "bottom", "always_visible": True})])
                                ])], style={'border': '3px solid black'}
                            ),
                            dbc.Col([
                                dbc.Row(
                                    html.H2("Expression of top markers according to article below")),
                                html.A(
                                    "Link to Paper", href="https://arupconsult.com/ati/acute-myeloid-leukemia-molecular-genetic-testing"),
                                html.Br(),
                                dbc.Row(html.Div(children=[html.H4('Select on second graph:', style={
                                        'color': '#7FDBFF', 'display': 'inline-block'}), daq.BooleanSwitch(id='include2', on=True, style={'display': 'inline-block'})])),
                                html.H4('Exploration Box Size:',
                                        style={'color': '#7FDBFF'}),
                                dcc.Slider(1, 15, marks=None, value=0, id='size2',
                                           tooltip={"placement": "bottom", "always_visible": True}),
                                dbc.Row(dcc.Graph(id='plot2', figure=fig2)),
                                dbc.Row([
                                    dbc.Col([
                                        html.H4(f'X axis: {x_col2}', style={
                                                'color': '#7FDBFF'}),
                                        dcc.Slider(min_x2, max_x2, marks=None, value=min_x2, id='xslider2',
                                            tooltip={"placement": "bottom", "always_visible": True})
                                    ]),
                                    dbc.Col([html.H4(f'Y axis: {y_col2}', style={'color': '#7FDBFF'}),
                                             dcc.Slider(min_y2, max_y2, marks=None, value=min_y2, id='yslider2',
                                                        tooltip={"placement": "bottom", "always_visible": True})
                                    ]),
                                    dbc.Col([html.H4(f'Z axis: {z_col2}', style={'color': '#7FDBFF'}),
                                             dcc.Slider(min_z2, max_z2, marks=None, value=min_z2, id='zslider2',
                                                        tooltip={"placement": "bottom", "always_visible": True})
                                    ])
                                ])], style={'border': '3px solid black'}
                            ),
                        dbc.Col(dcc.Graph(id='box', figure=box_plots))]
)])

# Step 4. Create a Dash layout

#list(df.columns)

# Step 5. Add callback functions

@app.callback(Output('plot1', 'figure'),
              [Input('xslider1', 'value'),
              Input('yslider1', 'value'),
              Input('zslider1', 'value'),
              Input('size1', 'value'),
              Input('include1', 'on')])
def update_figure1(X, Y, Z, size, on):

    # redraw the cube to the new coordinates

    if on == True:
        newX = [0 + X, 0 + X, 1 + X + size, 1 + X +
                size, 0 + X, 0 + X, 1 + X + size, 1 + X + size]
        newY = [0 + Y, 1 + Y + size, 1 + Y + size, 0 +
                Y, 0 + Y, 1 + Y + size, 1 + Y + size, 0 + Y]
        newZ = [0 + Z, 0 + Z, 0 + Z, 0 + Z, 1 + Z +
                size, 1 + Z + size, 1 + Z + size, 1 + Z + size]
        fig.update_traces(x=newX, y=newY, z=newZ, i=[7, 0, 0, 0, 4, 4, 6, 6, 4, 0, 3, 2],
                        j=[3, 4, 1, 2, 5, 6, 5, 2, 0, 1, 6, 3],
                        k=[0, 7, 2, 3, 6, 7, 1, 1, 5, 5, 7, 6], opacity = 0.2, selector=dict(name="cube"))
        #fig.update_xaxes(range=[1.5, 4.5])
        #fig.update_yaxes(range=[3, 9])
        #fig.update_yaxes(range=[3, 9])

        fig.update_xaxes(range=[min_x1, max_x1])
        fig.update_yaxes(range=[min_y1, max_y1])
        fig.update_yaxes(range=[min_z1, max_z1])

        # create a new dataframe that contains the filtered points (the points that the exploration box overlaps)
        em_df = pd.DataFrame(plot1_axes, columns=['0', '1', '2'])
        em_filtered = em_df[(((em_df['0'] >= X) & (em_df['0'] <= X+size))
                            & ((em_df['1'] >= Y) & (em_df['1'] <= Y+size))
                            & ((em_df['2'] >= Z) & (em_df['2'] <= Z+size)))].index.to_list()
        df_filtered = em_df.filter(items=em_filtered, axis=0)

        # If points have been selected, make those selected points 100% opacity, and the unselected points 50% opacity
        if not df_filtered.empty:
            fig.update_traces(x=df_filtered.iloc[:, 0], y=df_filtered.iloc[:, 1], z=df_filtered.iloc[:, 2], marker=dict(
                opacity=1), selector=dict(name="selected"))
        else:
            fig.update_traces(marker=dict(opacity=0.5), selector=dict(name="selected"))
    # If the user doesn't want to select, then hide the exploration box, and make all the points look unselected
    else:
        fig.update_traces(marker=dict(
            opacity=0.5), selector=dict(name="selected"))
        fig.update_traces(opacity=0, selector=dict(name="cube"))

    return fig

# same as the above callback, but to update the second figure
@app.callback(Output('plot2', 'figure'),
              [Input('xslider2', 'value'),
              Input('yslider2', 'value'),
              Input('zslider2', 'value'),
              Input('size2', 'value'),
              Input('include2', 'on')])
def update_figure2(X, Y, Z, size, on):

    if on == True:
        # redraw the cube to the new coordinates

        newX = [0 + X, 0 + X, 1 + X + size, 1 + X +
                size, 0 + X, 0 + X, 1 + X + size, 1 + X + size]
        newY = [0 + Y, 1 + Y + size, 1 + Y + size, 0 +
                Y, 0 + Y, 1 + Y + size, 1 + Y + size, 0 + Y]
        newZ = [0 + Z, 0 + Z, 0 + Z, 0 + Z, 1 + Z +
                size, 1 + Z + size, 1 + Z + size, 1 + Z + size]

        fig2.update_traces(x=newX, y=newY, z=newZ, i=[7, 0, 0, 0, 4, 4, 6, 6, 4, 0, 3, 2],
                        j=[3, 4, 1, 2, 5, 6, 5, 2, 0, 1, 6, 3],
                        k=[0, 7, 2, 3, 6, 7, 1, 1, 5, 5, 7, 6], opacity = 0.2, selector=dict(name="cube"))

        fig2.update_xaxes(range=[min_x2, max_x2])
        fig2.update_yaxes(range=[min_y2, max_y2])
        fig2.update_yaxes(range=[min_z2, max_z2])

        em_df = pd.DataFrame(plot2_axes, columns=['0', '1', '2'])
        em_filtered = em_df[(((em_df['0'] >= X) & (em_df['0'] <= X+size))
                            & ((em_df['1'] >= Y) & (em_df['1'] <= Y+size))
                            & ((em_df['2'] >= Z) & (em_df['2'] <= Z+size)))].index.to_list()
        df_filtered = em_df.filter(items=em_filtered, axis=0)
        if not df_filtered.empty:
            fig2.update_traces(x=df_filtered.iloc[:, 0], y=df_filtered.iloc[:, 1], z=df_filtered.iloc[:, 2], marker=dict(
                opacity=1), selector=dict(name="selected"))
        else:
            fig2.update_traces(marker=dict(opacity=0.5), selector=dict(name="selected"))
    else:
        fig2.update_traces(marker=dict(
            opacity=0.5), selector=dict(name="selected"))
        fig2.update_traces(opacity=0, selector=dict(name="cube"))

    return fig2
#df_filtered = df['petal_length'].between(
    #    X-1, X, inclusive=True)

    # add or delete any boxes based on the dropdown selections
    # filter the dataframe based on the selection
    #df_filtered = df[(((df[x_col] >= X) & (df[x_col] <= X+size+1))
    #                  & ((df[y_col] >= Y) & (df[y_col] <= Y+size+1))
    #                  & ((df[z_col] >= Z) & (df[z_col] <= Z+size+1)))]
    # make sure this applies to all box plot traces in the figure

# Callback to update the box-and-whisker plot with what has been selected
@app.callback(Output('box', 'figure'), [Input('xslider1', 'value'),
              Input('yslider1', 'value'),
              Input('zslider1', 'value'),
              Input('xslider2', 'value'),
              Input('yslider2', 'value'),
              Input('zslider2', 'value'),
              Input('size1', 'value'),
              Input('size2', 'value'),
              Input('include1', 'on'),
              Input('include2', 'on'),
              Input('box_dropdown', 'value')])
def update_box(X1, Y1, Z1, X2, Y2, Z2, size1, size2, on1, on2, values):
    em_df = pd.DataFrame(plot1_axes, columns=['0', '1', '2'])
    em_df2 = pd.DataFrame(plot2_axes, columns=['0', '1', '2'])
    if on1 == True:
        em_filtered1 = em_df[(((em_df['0'] >= X1) & (em_df['0'] <= X1+size1))
                    & ((em_df['1'] >= Y1) & (em_df['1'] <= Y1+size1))
                    & ((em_df['2'] >= Z1) & (em_df['2'] <= Z1+size1)))].index.to_list()
    else:
        em_filtered1 = []
    if on2 == True:
        em_filtered2 = em_df2[(((em_df2['0'] >= X2) & (em_df2['0'] <= X2+size2))
                            & ((em_df2['1'] >= Y2) & (em_df2['1'] <= Y2+size2))
                            & ((em_df2['2'] >= Z2) & (em_df2['2'] <= Z2+size2)))].index.to_list()
    if (on1 == True) and (on2==True):
        em_filtered = np.intersect1d(em_filtered1, em_filtered2)
    elif (on1==True) and (on2 == False):
        em_filtered = em_filtered1
    else:
        em_filtered = em_filtered2
    df_filtered = df.filter(items=em_filtered, axis=0)
    current_boxes = []
    for part in box_plots.data:
        if part.name is not None: current_boxes.append(part.name)
    if values is not None:
        copy_boxes = values
    else:
        copy_boxes = boxes
    for col in copy_boxes:
        if(col not in current_boxes):
            box_plots.add_trace(go.Box(y=df_filtered[col], name=col))
        else:
            box_plots.update_traces(
                y=df_filtered[col], selector=dict(name=col))
    return box_plots

def generate_box():
    return

# Step 6. Add the server clause
if __name__ == '__main__':
    app.run_server()


#!/usr/bin/env python3

from __future__ import absolute_import, division, print_function, unicode_literals
from . import invoke_rpc_builtin, invoke_rpc_python_udf

from . import ProcessGroupAgent
from .internal_rpc_utils import serialize, PythonUDF

import sys
import torch
from enum import Enum

_agent = None

def _require_initialized(func):
    def wrapper(*args, **kwargs):
        if _agent is None:
            raise RuntimeError("RPC has not been initialized. "
                               "Call init_rpc(name) first.")
        return func(*args, **kwargs)
    return wrapper

def join_rpc():
    r"""
    Block until all local and remote RPC processes reach this method, process
    (send and receive) all pending messages, and then destroy local RPC agent.
    Every RPC process must call this method before exit.
    """
    global _agent

    if _agent:
        _agent.join()
        _agent = None

@_require_initialized
def sync_rpc():
    r"""
    Block until all local and remote RPC processes reach this method and finish
    sending all pending RPCs. As this method synchronizes at the process
    level, if multiple threads are spawned, only one of them should call this
    method at a time.
    """

    _agent.sync()

class RpcBackend(Enum):
    PROCESS_GROUP = 1

# TODO: add a context manager to wrap _init_rpc and join_rpc
def _init_rpc(name, backend=RpcBackend.PROCESS_GROUP):
    if sys.version_info < (3, 0):
        raise RuntimeError("RPC package does not support Python2.")

    global _agent

    if _agent:
        raise RuntimeError("RPC is already initialized")

    if backend == RpcBackend.PROCESS_GROUP:
        from .distributed_c10d import _get_default_group
        group = _get_default_group()
        # TODO: add try-except and destroy _agent in all processes if any fails.
        _agent = ProcessGroupAgent(name, group)
    else:
        raise RuntimeError("Unrecognized RPC backend ", backend)

@_require_initialized
def get_worker_id(worker_name=None):
    r"""
    Get worker id of a given worker name. Use this worker id to avoid passing
    an expensive string to ``rpc`` on every invocation.

    Arguments:
        worker_name (str): the string name of a worker. If ``None``, return the
                           the id of the current worker. (default ``None``)
    """
    if worker_name:
        return _agent.get_worker_id(worker_name)
    else:
        return _agent.get_worker_id()

@_require_initialized
def rpc(to, func, args=None, kwargs=None, async_call=False):
    r"""
    Make an RPC call to run function ``func`` on worker ``to``. By default, it
    blocks until the return value is locally available. RPC messages are sent
    and received in parallel to execution of Python code. This method is
    thread-safe.

    Arguments:
        to (int or str): id or name of the destination worker.
        func (callable): any callable function. builtin functions (like
                         ``torch.add``) can be sent over RPC more efficiently.
        args (tuple): the argument tuple for the ``func`` invocation.
        kwargs (dict): is a dictionary of keyword arguments for the ``func``
                       invocation.
        async_call (bool): If set to ``True``, this will be an asynchronous RPC,
                           and returns a ``torch.distributed.FutureMessage``
                           object immediately. Otherwise, this RPC will block
                           until the return value is locally available.
                           (default: ``False``)

    Returns:
        If ``async_call`` is ``False``, returns the result of running ``func``
        on ``args`` and ``kwargs``. If ``async_call`` is ``True``, returns a
        ``torch.distributed.FutureMessage`` object that can be waited on. When
        completed, the return value of ``func`` on ``args`` and ``kwargs`` can
        be retrieved from the ``FutureMessage`` object.

    Example::

        Synchronous example:

        On worker 0:
        >>> import torch.distributed as dist
        >>> dist.init_process_group(backend='gloo', rank=0, world_size=2)
        >>> dist.init_model_parallel("worker0")
        >>> ret = dist.rpc("worker1", torch.add, args=(torch.ones(2), 3))
        >>> dist.join_rpc()

        One worker 1:
        >>> import torch.distributed as dist
        >>> dist.init_process_group(backend='gloo', rank=1, world_size=2)
        >>> dist.init_model_parallel("worker1")
        >>> dist.join_rpc()

        Asynchronous example:

        On worker 0:
        >>> import torch.distributed as dist
        >>> dist.init_process_group(backend='gloo', rank=0, world_size=2)
        >>> dist.init_model_parallel("worker0")
        >>> worker1 = dist.get_worker_id("worker1")
        >>> fut1 = dist.rpc(worker1, torch.add, args=(torch.ones(2), 3), async_call=True)
        >>> fut2 = dist.rpc(worker1, min, args=(1, 2), async_call=True)
        >>> result = fut1.wait() + fut2.wait()
        >>> dist.join_rpc()

        One worker 1:
        >>> import torch.distributed as dist
        >>> dist.init_process_group(backend='gloo', rank=1, world_size=2)
        >>> dist.init_model_parallel("worker1")
        >>> dist.join_rpc()
    """
    if not callable(func):
        raise TypeError("function should be callable.")

    qualified_name = torch.jit._find_builtin(func)

    args = args if args else ()
    kwargs = kwargs if kwargs else {}

    if isinstance(to, str):
        to = get_worker_id(to)

    if qualified_name is not None:
        fut = invoke_rpc_builtin(_agent, to, qualified_name, *args, **kwargs)
    else:
        fut = invoke_rpc_python_udf(_agent, to, serialize(PythonUDF(func, args, kwargs)))

    if async_call:
        return fut
    else:
        return fut.wait()


import sys

# Prevent spurious errors during `python setup.py test`, a la
# http://www.eby-sarna.com/pipermail/peak/2010-May/003357.html:
try:
    import multiprocessing
except ImportError:
    pass

from setuptools import setup, find_packages

extra_setup = {}
if sys.version_info >= (3,):
    extra_setup['use_2to3'] = True

setup(
    name='verifier_date_utils',
    version='1.0',
    description='Various utilities for operating on dates and times',
    long_description=open('README.rst').read(),
    author='Erik Rose',
    packages=find_packages(exclude=['ez_setup']),
    tests_require=['nose', 'mock'],
    test_suite='nose.collector',
    url='https://github.com/civiccc/verifier_date_utils',
    include_package_data=True,
    classifiers=[
        'Intended Audience :: Developers',
        'Natural Language :: English',
        'Development Status :: 5 - Production/Stable',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Topic :: Software Development :: Libraries',
        ],
    keywords=['date', 'parse', 'time'],
    **extra_setup
)


from __future__ import annotations

from contextlib import contextmanager
from datetime import datetime
from decimal import Decimal
from typing import ContextManager, List, Text
from uuid import UUID, uuid4

import sqlalchemy as sa
import sqlalchemy_utils as sa_utils
from injector import inject
from sqlalchemy import func
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import Session

from application.bus import Event, EventBus
from application.db import Base, Transaction
from currency import BTCRate, Currency

from .interface import BuyOrder, Repository
from .types import BtcAmountColumn, FiatAmountColumn

class DBBuyOrder(Base, BuyOrder):
    def __init__(
            self,
            request_id: UUID,
            paid: Decimal,
            bought: Decimal,
            exchange_rate: BTCRate,
    ) -> None:
        self.id = uuid4()
        self.request_id = request_id
        self.paid = paid
        self.bought = bought
        self.exchange_rate = exchange_rate

    __tablename__ = "buy_orders"

    _db_id: int = sa.Column("id", sa.Integer, primary_key=True)
    id: UUID = sa.Column("order_id", sa_utils.UUIDType, unique=True)
    request_id: UUID = sa.Column(sa_utils.UUIDType, unique=True)

    paid: Decimal = sa.Column(FiatAmountColumn)
    bought: Decimal = sa.Column(BtcAmountColumn)

    _currency: Currency = sa.Column(
        "currency", sa.Enum(Currency, naive_enum=False)
    )
    _price: Decimal = sa.Column("price", FiatAmountColumn)
    _rate_date: datetime = sa.Column("rate_date", sa.DateTime)

    @hybrid_property
    def exchange_rate(self) -> BTCRate:
        return BTCRate(
            price=self._price,
            currency=self._currency,
            on_date=self._rate_date,
        )

    @exchange_rate.setter
    def exchange_rate(self, value: BTCRate) -> None:
        self._price = value.price
        self._currency = value.currency
        self._rate_date = value.on_date

    when_created: datetime = sa.Column(
        "when_created", sa.DateTime, default=datetime.utcnow, index=True
    )
    _when_updated: datetime = sa.Column(
        "when_updated", sa.DateTime, onupdate=datetime.utcnow,
    )

    def __repr__(self) -> Text:
        return (
            "<"
            f"{self.__class__.__name__}"
            f" db_id={self._db_id}"
            f" order_id={self.id}"
            f" request_id={self.request_id}"
            f" amount={self.bought!s}BTC"
            ">"
        )

@inject
class ORMRepository(Repository):
    _session: Session
    _pending_events: List[Event]

    def __init__(self, transaction: Transaction, bus: EventBus) -> None:
        self._transaction = transaction
        self._bus = bus

    @contextmanager
    def lock(self) -> ContextManager[Decimal]:
        events = []
        with self._transaction() as session:
            self._session = session
            self._pending_events = events

            lock = f"LOCK TABLE {DBBuyOrder.__tablename__} IN EXCLUSIVE MODE;"
            session.execute(lock)

            yield self._calculate_balance()

            del self._session
            del self._pending_events

        for event in events:
            self._bus.emit(event)

    def _calculate_balance(self) -> Decimal:
        balance, = self._session.query(func.sum(DBBuyOrder.bought)).one()
        return Decimal(balance or 0)

    def create(
            self,
            request_id: UUID,
            paid: Decimal,
            bought: Decimal,
            with_rate: BTCRate,
    ) -> BuyOrder:
        entry = DBBuyOrder(
            request_id=request_id,
            paid=paid,
            bought=bought,
            exchange_rate=with_rate,
        )

        self._session.add(entry)
        self._session.flush()
        return entry

    def emit(self, event: Event) -> None:
        self._pending_events.append(event)

    def get_order_id(self, for_request_id: UUID) -> BuyOrder | None:
        with self._transaction() as session:
            result = (
                session.query(DBBuyOrder.id)
                .filter_by(request_id=for_request_id)
                .one_or_none()
            )
        return result and result[0]


from gl import *
from pyglet import resource

Mat4Floats = GLfloat*4

NOTEXTURES = False

class MDLdict(object):
    """Materials display lists"""
    def __init__(self):
        self.mat_dls = {}
        self.mat_textures = {}
        self.mat_trans = {}
    def __del__(self):
        if glDeleteLists:
            for dl in self.mat_dls.values():
                glDeleteLists(dl,2)
    def get(self,k):
        return self.mat_dls.get(k)
    def load(self,fname):
        mtllines = resource.file(fname,"ru")
        mname = None
        mat_dl = None
        mat_params = {'Ka':GL_AMBIENT, 'Kd': GL_DIFFUSE,
                      'Ks':GL_SPECULAR, 'Ke':GL_EMISSION}
        tname = None
        for line in mtllines:
            tokens = line.split()
            if not tokens or line[0] == '#':
                continue
            if tokens[0] == 'newmtl':
                if mname:
                    if not tname: glDisable(GL_TEXTURE_2D)
                    glEndList()
                tname = None
                mname = tokens[1]
                mat_dl = self.mat_dls.get(mname)
                if mat_dl is None:
                    mat_dl = self.mat_dls[mname] = glGenLists(2)
                glNewList(mat_dl, GL_COMPILE)
            elif tokens[0] == 'Ns':
                glMaterialf(GL_FRONT, GL_SHININESS, float(tokens[1]))
            elif tokens[0] in mat_params:
                params = map(float,tokens[1:])
                floats4 = Mat4Floats(1.0,1.0,1.0,1.0)
                for i,f in enumerate(params):
                    floats4[i] = f
                self.mat_trans[mname] = (floats4[3] < 1.0)
                glMaterialfv(GL_FRONT, mat_params[tokens[0]],floats4)
            elif tokens[0] == 'map_Kd' and not NOTEXTURES:
                # need a texture
                glEnable(GL_TEXTURE_2D)
                glCallList(mat_dl+1) # will bind texture
                glEndList()
                tname = tokens[1]
                tex = resource.texture(tokens[1])
                glNewList(mat_dl+1,GL_COMPILE)
                if tex:
                    self.mat_textures[tname] = tex
                    trans = self.mat_trans.get(mname,False)
                    self.mat_trans[mname] = trans
                    glEnable(GL_TEXTURE_2D)
                    glBindTexture(GL_TEXTURE_2D,tex.id)
                # will end list before starting next one, or at end
        if mname:
            if not tname: glDisable(GL_TEXTURE_2D)
            glEndList()
    def select(self,k):
        dl = self.get(k)
        if dl:
            glCallList(dl)
    def is_transparent(self,k):
        return self.mat_trans.get(k,False)

default_mdl_dict = MDLdict()
def load(fname):
    default_mdl_dict.load(fname)
def get(matname):
    return default_mdl_dict.get(matname)
def select(matname):
    default_mdl_dict.select(matname)
def is_transparent(matname):
    return default_mdl_dict.is_transparent(matname)


from __future__ import annotations

from typing import TYPE_CHECKING, Any

from ..network import ID_TOKEN_AUTHENTICATOR
from .by_plugin import AuthByPlugin, AuthType
from .webbrowser import AuthByWebBrowser

if TYPE_CHECKING:
    from ..connection import SnowflakeConnection

class AuthByIdToken(AuthByPlugin):
    """Internal IdToken Based Authentication.

    Works by accepting an id_toke and use that to authenticate. Only be used when users are using EXTERNAL_BROWSER_AUTHENTICATOR
    """

    @property
    def type_(self) -> AuthType:
        return AuthType.ID_TOKEN

    @property
    def assertion_content(self) -> str:
        return self._id_token

    def __init__(
        self,
        id_token: str,
        application: str,
        protocol: str | None,
        host: str | None,
        port: str | None,
        **kwargs,
    ) -> None:
        """Initialized an instance with an IdToken."""
        super().__init__(**kwargs)
        self._id_token: str | None = id_token
        self._application = application
        self._protocol = protocol
        self._host = host
        self._port = port

    def reset_secrets(self) -> None:
        self._id_token = None

    def prepare(self, **kwargs: Any) -> None:
        pass

    def reauthenticate(
        self,
        *,
        conn: SnowflakeConnection,
        **kwargs: Any,
    ) -> dict[str, bool]:
        conn.auth_class = AuthByWebBrowser(
            application=self._application,
            protocol=self._protocol,
            host=self._host,
            port=self._port,
            timeout=conn.login_timeout,
            backoff_generator=conn._backoff_generator,
        )
        conn._authenticate(conn.auth_class)
        conn._auth_class.reset_secrets()
        return {"success": True}

    def update_body(self, body: dict[Any, Any]) -> None:
        """Idtoken needs the authenticator and token attributes set."""
        body["data"]["AUTHENTICATOR"] = ID_TOKEN_AUTHENTICATOR
        body["data"]["TOKEN"] = self._id_token


# -*- coding: utf-8 -*-

import os
import tensorflow as tf
from keras.applications.vgg16 import VGG16
from keras.applications.resnet50 import ResNet50,preprocess_input,decode_predictions
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential,Model
from keras.layers import Input,Activation,Dropout,Flatten,Dense
from keras import optimizers
import numpy as np
import time
import glob
from PIL import Image
from pathlib import Path

tf.test.gpu_device_name()

def getdsample(cat,datadir):
    allfiles=[]
    for i,c in enumerate(cat):
        files=glob.glob(datadir+cat+"/*.png")
        print("category : {} sampledata : {}".format(cat,len(files)))
        for f in files:
            allfiles.append(f)

    return len(allfiles)

def main():
    classes=["ドライヤー","電気ケトル","オフィスチェア","ハサミ","コンセント"]
    classNums=len(classes)
    category=["dryer","kettle","officechair","scissors","outlet"]
    img_width,img_height=400,400

    traindatadir="./objdata/train/"
    testdatadir="./objdata/test/"

    trainSmp=getdsample(category,traindatadir)
    testSmp=getdsample(category,testdatadir)

    batchSize=100
    epoch=20

    trainDataset=ImageDataGenerator(
        rescale=1.0/224,
        zoom_range=0.2,
        horizontal_flip=True
        )
    validationDataset=ImageDataGenerator(rescale=1.0/224)

    traindata=trainDataset.flow_from_directory(
        traindatadir,
        target_size=(img_width,img_height),
        color_mode='rgb',
        classes=classes,
        class_mode='categorical',
        batch_size=batchSize,
        shuffle=True
    )

    validata=validationDataset.flow_from_directory(
        testdatadir,
        target_size=(img_width,img_height),
        color_mode='rgb',
        classes=classes,
        class_mode='categorical',
        batch_size=batchSize,
        shuffle=True
    )

    model=ResNet50(include_top=True,weights='imagenet',input_tensor=None,pooling=None)

    top_model=Sequential()
    top_model.add(Flatten(input_shape=model.output_shape[1:]))
    top_model.add(Dense(256,actvation='relu'))
    top_model.add(Dropout(0.6))
    top_model.add(Dense(classNums,activation='softmax'))

    resnetmodel=Model(input=model.input,output=top_model(model.output))

    for layer in resnetmodel.layers[:15]:
        layer.trainable=False

    resnetmodel.compile(
        loss='categorical_crossentropy',
        optimizer=optimizers.SGD(lr='1e-3',momentum=0.9),
        metrics=['accuracy']
    )

    history=resnetmodel.fit_generator(
        traindata,
        samples_per_epoch=trainSmp,
        nb_epoch=epoch,
        validation_data=validata,
        nb_val_samples=testSmp
    )

    print(history.history['acc'])
    print(history.history['val_acc'])


from sklearn.model_selection import train_test_split
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import pickle
import torch
import torch.nn as nn
import torch.optim as optim
from train_lgbm import *
import lightgbm as lgb
from  torch.utils.data import TensorDataset,DataLoader
import torch.nn.functional as F
from sklearn.model_selection import train_test_split
import math
import time
import random
from sklearn.preprocessing import *
from model_train_util_vectorized import *
import sys
import os
import os.path as osp
import time
import datetime
from itertools import repeat
from typing import Optional
import gc
import pickle
from sklearn.preprocessing import *
from sys import argv

Device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu')
print('using device:', Device)
dataset_name = argv[1]
use_gbm = argv[2]
use_botspotpp = argv[3]

ROOT_DIR = f'{dataset_name}/'

class BotSpotTrans(object):
    def __init__(self,train_filepath,test_filepath):

        train_file = train_filepath
        test_file = test_filepath
        print("Loading train & test file...")
        train_df = pd.read_csv(train_file) # ѵ����
        test_df = pd.read_csv(test_file) # ���Լ�
        total_df = pd.concat([train_df, test_df], axis=0)

        print("Graph Generating...")
        self.edge_index = total_df[["combin_index", "device_index", "target"]].astype(int).values
        self.edge_index_train = train_df[["combin_index", "device_index", "target"]].astype(int).values
        self.edge_index_test = test_df[["combin_index", "device_index", "target"]].astype(int).values

        # stat_columns��ʾ����ͳ����ص�������list
        # category_columns��ʾ����category��������list
        # stat_columns���������ctit��cvr_total, category_columns��һ����label encoder���channel_id

        stat_columns_file = osp.join(ROOT_DIR, "stat_columns.txt")
        category_columns_file = osp.join(ROOT_DIR, "category_columns.txt")
        stat_columns = self.pickle_load(stat_columns_file)
        category_columns = self.pickle_load(category_columns_file)

        # ����������
        feature_columns = stat_columns + category_columns

        # �������ֻ��ctit����normalization
        normalized_columns = [stat_columns[-2]]
        except_normalized_columns = [column for column in stat_columns if column not in normalized_columns]

        # channel-campagin��ص�����ѵ����������, ������ctit��ͳ������channel_id(���һ��)
        combin_feature_columns = except_normalized_columns + [category_columns[0]]
        # device�ڵ���ص�����ѵ���������У�����ctit + ʣ�µ�category��
        device_feature_columns = normalized_columns + category_columns[1:]
        #���е�channel-campaign��
        device_columns = ["device_index"] + device_feature_columns
        # ���е�device����
        combin_columns = ["combin_index"] + combin_feature_columns

        #��������Ǵ�total_df���ֳ�device_df,���������Ҫ�������train_dfҲһ��
        device_df = total_df[device_columns].sort_values(["device_index"])
        device_df.drop_duplicates(subset="device_index", keep="first", inplace=True)

        #ͬ��Ҳ�Ǵ�total_df���ֳ���combin_df,���������Ҫ���е���
        combin_df = total_df[combin_columns].sort_values(["combin_index"])
        combin_df.drop_duplicates(subset="combin_index", keep="first", inplace=True)

        # ��ctit�н��������С��һ��
        norm_data = RobustScaler().fit_transform(device_df.loc[:,normalized_columns[0]].values.reshape((-1,1)))
        device_df.loc[:,normalized_columns[0]] = norm_data.reshape(-1,)

        print("feature matrix generating...")
        self.device_matrix = device_df[device_feature_columns].values
        # �����folat�ľ���ת�����float16�����Ը��������Ҫ����
        self.combin_matrix = combin_df[combin_feature_columns].astype('float16').values

    def pickle_dump(self, data, filename):
        with open(filename, 'wb') as f:
            pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)

    def pickle_load(self, filename):
        with open(filename, 'rb') as f:
            return pickle.load(f)

class Self_Attention(nn.Module):
    # a classical self_attention module as depicted in BERT
    def __init__(self, emb_size, num_head):
        super(Self_Attention, self).__init__()
        self.Q = nn.ModuleList([])
        self.K = nn.ModuleList([])
        self.V = nn.ModuleList([])
        output_size = emb_size // num_head
        self.output_size = output_size
        self.num_head = num_head
        self.final_linear = nn.Linear(emb_size, emb_size)

        for i in range(num_head):
            self.Q.append(nn.Linear(emb_size, output_size))
            self.K.append(nn.Linear(emb_size, output_size))
            self.V.append(nn.Linear(emb_size, output_size))

    def calc_attention(self, X, Q, K, V):
        query = Q(X)
        key = K(X)
        value = V(X)
        key_ = key.transpose(2, 1).contiguous()
        attn_weights = torch.softmax(torch.bmm(query, key_) / math.sqrt(self.output_size), dim=-1)
        output = torch.bmm(attn_weights, value)
        return output

    def forward(self, X):
        outputs = []
        for i in range(self.num_head):
            q, k, v = self.Q[i], self.K[i], self.V[i]
            out = self.calc_attention(X, q, k, v)
            outputs.append(out)
        outputs = torch.cat(outputs, dim=-1)
        return self.final_linear(outputs).mean(dim=1)

class BotSpot(nn.Module):
    def __init__(self,edge_index_train,edge_index_test,channel_feats,device_feats,device_package = None,use_enhance_botspot = False,use_gbm = False,gbm_model = None):
        super(BotSpot,self).__init__()
        if use_gbm:
            leaf_dim = 20
            assert gbm_model is not None
            self.leaf_emb_models = nn.ModuleList()
            for n in range(gbm_model.n_estimators):
                self.leaf_emb_models.append(nn.Embedding(31, leaf_dim)) # 31 is the max depth of decision tree
        self.gbm_best_model = gbm_model
        self.use_gbm = use_gbm
        self.self_attn_module = Self_Attention(32, 2)
        self.use_enhanced_botspot = use_enhance_botspot
        self.edge_index_train = edge_index_train
        self.edge_index_test = edge_index_test
        self.edge_num_train = edge_index_train.shape[0]
        self.edge_num_test = edge_index_test.shape[0]
        self.channel_feats = torch.from_numpy(channel_feats).float() # feature matrix for channel-campaign nodes
        self.device_feats = torch.from_numpy(device_feats).float()  # package_name already removed in the device feature matrix
        N_chan = channel_feats.shape[1]-1 + 16 # remove one col and add embeeding_size of 16
        N_dev = 16*(device_feats.shape[1]-1)+1 # add one col of ctit, others are embeddings
        self.edge_matrix = self.gen_adj_matrix() # generate adj matrix of shape (N_device,N_channel_campaign)
        self.device_split_val = 1 # the first col is ctit for device
        self.channel_split_val = -1 # the last col is channel id
        self.channel_neibr_cache = {}  # cache device neighbors as a boolean array of channel_campaign nodes for training stage
        self.channel_neibr_test_cache = {}  # cache neighbors as boolean array of channel_campaign nodes for test stage
        self.device_idx_cache = {} # cache device indices for a channel-campaign node
        if use_enhance_botspot:
            assert device_package is not None
            self.device_package = device_package
            self.super_device_neibr_cache = {} # cache channel_campaign indices for each super device node for training set
            self.super_device_neibr_test_cache = {} # cache channel_campaign indices for each super device node for testing set

        # precompute channel neighbors and  cache it
        N = self.edge_matrix.shape[1]
        for i in range(N):
            self.channel_neibr_cache[i] = self.edge_matrix[:,i]
#         N_pack = len(device_package)
        # precompute the super device neighbors for training set if botspot++ is used
        if self.use_enhanced_botspot:
            for idx,d in enumerate(device_package):
                if d not in self.super_device_neibr_cache:
                    self.super_device_neibr_cache[d] = set()
                t_ = np.where(self.edge_matrix[idx]==1)[0]
                for v in t_:
                    self.super_device_neibr_cache[d].add(v)

            for d in self.super_device_neibr_cache:
                self.super_device_neibr_cache[d] = list(self.super_device_neibr_cache)

        self.set_adj_test_matrix(True) # modify edge_matrix to include test edges
        for i in range(N):
            self.channel_neibr_test_cache[i] = self.edge_matrix[:,i]
        # precompute super device neighbors for test set, cache them in super_device_neibr_test_cache
        if self.use_enhanced_botspot:
            for idx,d in enumerate(device_package):
                if d not in self.super_device_neibr_test_cache:
                    self.super_device_neibr_test_cache[d] = set()
                t_ = np.where(self.edge_matrix[idx]==1)[0]
                for v in t_:
                    self.super_device_neibr_test_cache[d].add(v)

            for d in self.super_device_neibr_test_cache:
                self.super_device_neibr_test_cache[d] = list(self.super_device_neibr_test_cache)
        self.set_adj_test_matrix(False) # modify the edge matrix to exclude test edges to proceed on training

        # initialze embedding matrix
        emb_size = 16
        channel_id_max = int(channel_feats[:, -1].max() + 1)
        temp = np.max(device_feats[:, 1:], axis=0) + 1
        temp = [int(i) for i in temp]
        lang, plat, os, country, carrier, device_brand, plat_os = temp  # bypass install city, be careful
        self.channel_id_emb = nn.Embedding(channel_id_max, emb_size)
        self.carrier_emb = nn.Embedding(carrier, emb_size)
        self.language_emb = nn.Embedding(lang, emb_size)
        self.device_brand_emb = nn.Embedding(device_brand, emb_size)
        self.plat_os_emb = nn.Embedding(plat_os, emb_size)
        self.plat_emb = nn.Embedding(plat, emb_size)
        self.os_emb = nn.Embedding(os, emb_size)
        self.country_emb = nn.Embedding(country, emb_size)

        # device modules if there is no super device convolution
        if not self.use_enhanced_botspot:
            self.dev_linear1 = nn.Linear(N_dev,int(0.6*N_dev)) # NOT also used in channel side for convolving device feats
            self.dev_relu1 = nn.ReLU()
            self.dev_dropout1 = nn.Dropout(0.2)
            self.dev_linear2 = nn.Linear(int(0.6*N_dev),int(0.75*0.6*N_dev))
            self.dev_relu2 = nn.ReLU()
        # channel linear and message passing modules
        self.channel_linear1 = nn.Linear(N_chan,int(0.6*N_chan))
        self.channel_msg_pass1 = nn.Linear(N_dev,int(0.6*N_dev))
        fusion_input = int(0.6*N_chan) + int(0.6*N_dev)
        self.fusion_linear1 = nn.Linear(fusion_input,int(0.6*fusion_input))
        self.fusion_relu1 = nn.ReLU()
        self.fusion_dropout1 = nn.Dropout(0.2)
        fusion_output_dim =  int(0.6*fusion_input)
        device_output_dim = int(0.75*0.6*N_dev)

        if not self.use_enhanced_botspot:
            # concat modules if no botspot++
            concat_input_dim = fusion_output_dim+device_output_dim if not self.use_gbm else fusion_output_dim+device_output_dim+leaf_dim
            self.concat_linear1 = nn.Linear(concat_input_dim,int(0.6*concat_input_dim))
            self.concat_relu1 = nn.ReLU()
            self.concat_linear2 = nn.Linear(int(0.6*concat_input_dim),int(0.5*0.6*concat_input_dim))
            self.concat_relu2 = nn.ReLU()
            self.concat_linear3 = nn.Linear(int(0.5*0.6*concat_input_dim),1)
        else:
            # device side gnn if botspot++ is used
            self.dev_linear1 = nn.Linear(N_dev,int(0.6*N_dev)) # NOT also used in channel side for convolving device feats
#             self.device_msg_passing = nn.Linear(N_chan,int(0.6*N_chan))
            in_dim = int(0.6*N_dev) + int(0.6*N_chan)
            self.sup_dev_fusion_linear1 = nn.Linear(in_dim,int(0.6*in_dim))
            self.sup_dev_fusion_relu1 = nn.ReLU()
            self.sup_dev_fusion_dropout1 = nn.Dropout(0.2)

            # concat layer for botspot++
            sup_dev_fusion_output_dim = int(0.6*in_dim)
            concat_input_dim = fusion_output_dim+sup_dev_fusion_output_dim if not self.use_gbm else fusion_output_dim+sup_dev_fusion_output_dim+ leaf_dim
            self.concat_linear1 = nn.Linear(concat_input_dim,int(0.6*concat_input_dim))
            self.concat_relu1 = nn.ReLU()
            self.concat_linear2 = nn.Linear(int(0.6*concat_input_dim),int(0.5*0.6*concat_input_dim))
            self.concat_relu2 = nn.ReLU()
            self.concat_linear3 = nn.Linear(int(0.5*0.6*concat_input_dim),1)

    def to_emb(self, arr, *models):
        '''
        :param arr: matrix for holding high-cardinality features, without one-hot encoding
        :param left: channel node if left is True else device node
        :param models: a list of embedding matrices to embed each high-cardinality feature to dense embeddings
        :return: 2-d tensor with dense embeddings for all the high-cardinality features.
        '''

        out_arr = []
#         arr = torch.from_numpy(arr)
        arr = arr.long().to(Device)
        # device node sparse features

        # N = arr.shape[0]
        num_models = len(models)
        for i in range(len(models)):
            # if num_models > 2 and i == 4:  # bypass install city, hardcoded
            #     continueed
            #             print (i,models[i])
            out_arr.append(models[i](arr[:, i]))
        return torch.cat(out_arr, dim=1)

    def concat_device_feats(self, dev_feats):  # NEED TO MODIFY IT
        '''
        this method invokes to_emb to embed device categorical features into dense embeddings
        :param dev_feats: normalized device features
        :param more_dev_feats: feature matrix with high-cardinality features
        :return: feature matrix with dense embeddings
        '''
        dev_feats = dev_feats.to(Device)
        cat_dev_feats = dev_feats[:, self.device_split_val:]
        emb_tensor = self.to_emb(cat_dev_feats, self.language_emb,
                                 self.plat_emb, self.os_emb, self.country_emb, self.carrier_emb,
                                 self.device_brand_emb, self.plat_os_emb)

        dev_emb_feats = torch.cat((dev_feats[:, :self.device_split_val], emb_tensor), dim=1).float().to(Device)
        return dev_emb_feats

    def concat_channel_feats(self, chan_feats):
        '''
        this method invokes to_emb to embed channel_campaign node's categorical feature into dense embeddings
        similar to concat_device_feats, to add dense embeddings
        '''
        chan_feats = chan_feats.to(Device)
        emb_tensor = self.to_emb(chan_feats[:, self.channel_split_val:], self.channel_id_emb)
#         print (chan_feats[:, :self.channel_split_val].shape)
#         print (emb_tensor.shape)
        return torch.cat((chan_feats[:, :self.channel_split_val], emb_tensor), dim=1).float().to(Device)

    def gen_adj_matrix(self):
        """
        this method build ajdacency matrix of shape (N_device,N_channel).
        adj_matrix[j,i] = True indicates device node j connects to channel-campaign node i
        """
        e = np.vstack((self.edge_index_train,self.edge_index_test))
        N_dev = np.max(e[:,1])+1
        N_channel = np.max(e[:,0])+1
        adj_matrix = np.zeros((N_dev,N_channel),dtype = bool)
        for i,j,_ in self.edge_index_train:
            adj_matrix[j,i] = True
        return adj_matrix

    def set_adj_test_matrix(self,set_test = True):
        """
        this method modifies edge_matrix
        when set_test = True, edge_matrix includes test edges in it, otherwise does not
        """
        count_not_in_train = 0
        for i,j,_ in self.edge_index_test:
            if set_test:
                self.edge_matrix[j,i] = True
            else:
                self.edge_matrix[j,i] = False
        return

    def sample_minibatch(self,channel_vertices,device_vertices,train_stage = True):
        """
        input:
        for a minitach of edges, channel_vertices is edge[:,0], device_vertices is edge[:,1]
        this method takes a minibatch of edges outputs:
        1) features for channel_campaign nodes
        2) features for device nodes
        3) neighboring device features for each channel_campaign node
        4) neighboring channel_campaign features for each super device node
        """
        #set number of neighbors for channel and device for different stages
        num_neibr = 50 if train_stage else 800
        sup_dev_num_neibr = 20 if train_stage else 50
        edge_matrix = self.edge_matrix

        # channel_vertices and device_vertices must be numpy array
        # original features
        neibr_feats_tensor = []
        sup_dev_neibr_feats_tensor = []
        minibatch_channel_feats = self.concat_channel_feats(self.channel_feats[channel_vertices]).to(Device)  # shape of (minibatch,feats_num_channel)
        minibatch_device_feats = self.concat_device_feats(self.device_feats[device_vertices]).to(Device) # shape of (minibatch,feats_num_device)
        channel_vertices = channel_vertices.cpu().numpy()

        # for each channel vertices, get neighbor devices and its features
        for i in channel_vertices:
            if train_stage:
                neibr_feats = self.adj_indice_to_feat_mat(self.channel_neibr_cache[i],num_neibr,i)
                neibr_feats_tensor.append(neibr_feats)
            else:
                neibr_feats = self.adj_indice_to_feat_mat(self.channel_neibr_test_cache[i],num_neibr,i)
                neibr_feats_tensor.append(neibr_feats)
        neibr_feats_tensor = torch.cat(neibr_feats_tensor,dim = 0).to(Device) # neibr_feats_tensor would be a tensor of shape: (Minibatch,num_neibr,feats_num_device)

        # if use botspot++, for each device node, retrieve its super device index and get its neighboring channel_campaign node features
        if self.use_enhanced_botspot:
            for i in device_vertices:
                sup_dev_neibr_feats = self.adj_indice_to_feat_mat_super_device(sup_dev_num_neibr,i,train_stage)
                sup_dev_neibr_feats_tensor.append(sup_dev_neibr_feats)
            sup_dev_neibr_feats_tensor = torch.cat(sup_dev_neibr_feats_tensor,dim = 0).to(Device)
            return minibatch_channel_feats,minibatch_device_feats,neibr_feats_tensor,sup_dev_neibr_feats_tensor

        return minibatch_channel_feats,minibatch_device_feats,neibr_feats_tensor,-1

    def adj_indice_to_feat_mat(self,col,neibr_num,indice):
        """
        input:
        indice: indice of channel_campaign node
        neibr_num: num of sample size
        col: a boolean array of a col given channel_campaign node of $indice
        return:
        neighboring device features
        """
        if indice in self.device_idx_cache:
            ind = self.device_idx_cache[indice]
        else:
            ind = np.where(col==True)[0]
            self.device_idx_cache[indice] = ind
        if len(ind)>neibr_num:
            np.random.shuffle(ind)
            ind_subset = ind[:neibr_num]
        else:
            ind_subset = np.random.choice(ind,size = neibr_num,replace = True)

        dev_feats = self.concat_device_feats(self.device_feats[ind_subset].to(Device))
        return dev_feats.unsqueeze(0)

    def adj_indice_to_feat_mat_super_device(self,neibr_num,device_idx,train_stage = True):
        """
        input:
        device_idx: index of device node
        neibr_num: num of sample size
        return:
        neighboring channel_campaign features
        """
        i = self.device_package[device_idx]
        if train_stage:
            sup_dev_neibr = list(self.super_device_neibr_cache[i])
        else:
            sup_dev_neibr = list(self.super_device_neibr_test_cache[i])
        if len(sup_dev_neibr)>neibr_num:
            random.shuffle(sup_dev_neibr)
            c = self.concat_channel_feats(self.channel_feats[sup_dev_neibr[:neibr_num]].to(Device))
        else:
            sup_dev_neibr = np.random.choice(np.asarray(sup_dev_neibr),size = neibr_num,replace = True)
            c = self.concat_channel_feats(self.channel_feats[sup_dev_neibr].to(Device))
        return c.unsqueeze(0)

    def get_leaf_from_light_gbm(self, left_vertices, right_vertices, use_self_attn=False):
        # get leaf indices from gbm model and embed into dense matrix
        output_leaf_emb = []
        chan_data = self.channel_feats[left_vertices]
        dev_data = self.device_feats[right_vertices]
        try:
            edge_data = np.hstack((chan_data, dev_data))
        except:
            edge_data = torch.cat((chan_data, dev_data),
                                  dim=1)  # edge feature is the concatenation of channel_node and device_node
            edge_data = edge_data.cpu().numpy()
        # N = len(left_vertices)
        if len(edge_data.shape)==1:
            edge_data = edge_data.reshape((1,-1))
        pred_leaf = self.gbm_best_model.predict_proba(edge_data, pred_leaf=True)
        pred_leaf = torch.from_numpy(pred_leaf).long().to(Device)

        for i in range(pred_leaf.shape[1]):
            # print (self.leaf_emb_models[i](pred_leaf[:, i]).shape)
            output_leaf_emb.append(self.leaf_emb_models[i](pred_leaf[:, i]).unsqueeze(1))
            # ret = torch.cat(output_leaf_emb, dim=1).to(Device)  # leaf node concatenation
        if not use_self_attn:
            ret = torch.cat(output_leaf_emb, dim=1).mean(axis=1).to(Device)  # leaf node mean pooling
            return ret
        else:
            ret = torch.cat(output_leaf_emb, dim=1).to(Device)
            out = self.self_attn_module(ret)
            return out

    def forward(self,edges,train_stage = True):
        minibatch_channel_feats,minibatch_device_feats,neibr_feats_tensor,sup_dev_neibr_feats_tensor = self.sample_minibatch(edges[:,0],edges[:,1],train_stage)
        # forward device feats
        if not self.use_enhanced_botspot:
            device_out = self.dev_relu2(self.dev_linear2(self.dev_dropout1(self.dev_relu1(self.dev_linear1(minibatch_device_feats)))))
            channel_conv = self.channel_linear1(minibatch_channel_feats)
            dev_conv = self.dev_linear1(neibr_feats_tensor).mean(dim=1) # share dev_linear1
            fuse_conv = self.fusion_dropout1(self.fusion_relu1(self.fusion_linear1(torch.cat((channel_conv,dev_conv),dim=1))))
            if not self.use_gbm:
                h = self.concat_linear3(self.concat_relu2(self.concat_linear2(self.concat_relu1(self.concat_linear1(torch.cat((fuse_conv,device_out),dim=1))))))
            else:
                leaf_out = self.get_leaf_from_light_gbm(edges[:,0],edges[:,1])
                h = self.concat_linear3(self.concat_relu2(self.concat_linear2(self.concat_relu1(self.concat_linear1(torch.cat((fuse_conv,device_out,leaf_out),dim=1))))))
            return torch.sigmoid(h)
        else:
            channel_conv = self.channel_linear1(minibatch_channel_feats)
            dev_conv = self.dev_linear1(neibr_feats_tensor).mean(dim=1) # share dev_linear1
            fuse_conv = self.fusion_dropout1(self.fusion_relu1(self.fusion_linear1(torch.cat((channel_conv,dev_conv),dim=1))))
            # device side conv:
            sup_dev_conv = self.dev_linear1(minibatch_device_feats)
            sup_channel_conv = self.channel_linear1(sup_dev_neibr_feats_tensor).mean(dim=1)
            sup_fuse_conv = self.sup_dev_fusion_dropout1(self.sup_dev_fusion_relu1(self.sup_dev_fusion_linear1(torch.cat((sup_channel_conv,sup_dev_conv),dim=1))))
            if not self.use_gbm:
                h = self.concat_linear3(self.concat_relu2(self.concat_linear2(self.concat_relu1(self.concat_linear1(torch.cat((fuse_conv,sup_fuse_conv),dim=1))))))
            else:
                leaf_out = self.get_leaf_from_light_gbm(edges[:,0],edges[:,1])
                h = self.concat_linear3(self.concat_relu2(self.concat_linear2(self.concat_relu1(self.concat_linear1(torch.cat((fuse_conv,sup_fuse_conv,leaf_out),dim=1))))))
            return torch.sigmoid(h)

if __name__ == '__main__':
    bot_preprocess = BotSpotTrans(f'{dataset_name}/train.csv', f'{dataset_name}/test.csv')
    edge_index_train = bot_preprocess.edge_index_train
    edge_index_test = bot_preprocess.edge_index_test
    chan_feats = bot_preprocess.combin_matrix
    device_feats = bot_preprocess.device_matrix
    device_package = list(device_feats[:, -1])
    device_feats = device_feats[:, :-1]
    if use_gbm:
        gbm_model = make_dataset(chan_feats,device_feats,-1,1,edge_index_train,edge_index_test)
    else:
        gbm_model  = None
    botspot_model = BotSpot(edge_index_train, edge_index_test, chan_feats, device_feats,device_package = device_package,use_enhance_botspot = use_botspotpp,use_gbm = use_gbm,gbm_model = gbm_model)

    tr_dset = TensorDataset(torch.from_numpy(edge_index_train))
    tr_dloader = DataLoader(tr_dset, batch_size=500, shuffle=True)
    test_dset = TensorDataset(torch.from_numpy(edge_index_test))
    test_dloader = DataLoader(test_dset, batch_size=500, shuffle=False)
    optimizer = optim.Adam(botspot_model.parameters(), lr=2e-4, weight_decay=3e-6)
    # model_states = torch.load('model_results_1014/botspotpp/model_checkpoints_4.pt')
    # botspot_model.load_state_dict(model_states)

    botspot_model.to(Device)
    try:
        os.mkdir('model_results')
    except:
        pass
    _ = train(botspot_model, tr_dloader, test_dloader, optimizer, 10, save_name='model_results')


# -*- coding: utf-8 -*-
"""
Created on Fri Jan 29 10:51:07 2021

@author: REMI DECOUTY, DAMIEN LU
"""

import pandas
import glob, os
import igraph as ig
import math

# récupération de la liste de tous les fichiers à étudier

os.chdir("data")
allCsvFiles = glob.glob("*")
os.chdir("..")

# coloration d'une arête, en fonction du type de liaison

def color_edge(node, index):

    arrayPairType = []

    if (index != -1):
        arrayPairType = node.split(',')

    cases = {
        "cWW": "blue",

        "tWW": "red",

        "cWH": "orange",
        "cHW": "orange",

        "tWH": "yellow",
        "tHW": "yellow",

        "cWS": "brown",
        "cSW": "brown",

        "tWS": "magenta",
        "tSW": "magenta",

        "cHH": "cyan",

        "tHH": "purple",

        "cHS": "pink",
        "cSH": "pink",

        "tHS": "green",
        "tSH": "green",

        "cSS": "maroon",

        "tSS": "gray",
    }

    if (index != -1):
        return (cases.get(arrayPairType[index]))

    else:
        return (cases.get(node))

# fonction d'affichage du graphe, à partir du nom d'un fichier au format CSV

def draw_graph_from_csv(file):

    global filename
    filename = file
    df = pandas.read_csv("data/" + filename)[['index_chain','paired','pair_type_LW']]

    global g                    # déclaration d'une variable g globale
    g = ig.Graph()              # initialisation du graphe

    if (df['index_chain'][0] != 1):         # index_chain pas au bon format (nucléotide numéroté de 1 à N)
        print ("\nLe fichier " + file + " n'est pas au bon format ! \n\n")
        return

    for node in df.iterrows():
        g.add_vertices(1)       # ajout des noeuds (1 nucléotide = 1 noeud)

    for i in range(len(g.vs)):
        g.vs[i]["label"]= str(i+1)  # label du noeud, correspondant à la valeur index_chain

    for i in range(len(g.vs)):

        currentNode = df['index_chain'][i]      # noeud parcouru actuellement
        pairedNode = str(df['paired'][i])       # liste des nucléotides possédant une interaction avec le noeud actuel

        # liaisons phosphodiester entre les nucléotides

        if i != len(g.vs)-1:
            g.add_edges([(i,i+1)])

        # interactions canoniques entre certains nucléotides

        if pairedNode.find(",") != -1:                       # la liste "paired" contient plus d'un élément
            arrayPairedNodes = pairedNode.split(',')         # on récupère chacun des nucléotides de la liste
            for node in arrayPairedNodes:
                if (int(node) in df['index_chain'].tolist()):         # le nucléotide apparié existe bien dans la chaîne
                     if not g.are_connected(int(node)-1,currentNode-1):  # si l'arête n'existe pas déjà dans l'autre sens, alors on la crée
                         g.add_edges([(currentNode-1,int(node)-1)])      # ajout dans le graphe de la liason canonique
                         res_edge = g.es.find(_source=currentNode-1, _target=int(node)-1)
                         res_edge["color"] = color_edge(df['pair_type_LW'][i], arrayPairedNodes.index(node))

        else:
            pairedNode = pandas.to_numeric(df['paired'][i])  # cast d'un string en int, afin de vérifier si le nucléotide n'est pas apparié (NaN)
            if not math.isnan(pairedNode) and int(pairedNode) in df['index_chain'].tolist():
                if not g.are_connected(int(pairedNode)-1,currentNode-1):
                     g.add_edges([(currentNode-1,int(pairedNode)-1)])
                     res_edge = g.es.find(_source=currentNode-1, _target=int(pairedNode)-1)
                     res_edge["color"] = color_edge(df['pair_type_LW'][i],-1)

    visual_style = {}
    visual_style["edge_width"] = 3
    g.vs["color"] = "white"

# find_subgraph : fonction de recherches des sous-graphes, à partir d'un motif issu de carnaval
# Le principe est le suivant : la fonction get_subisomorphisms_vf2 de la librairie igraph renvoie
# la liste des sous-graphes isomorphes entre deux graphes. De plus, la fonction de comparaison compare_edges
# va vérifier pour chaque arête 2 à 2 si leur couleur est identique ou non.
# Si tel est le cas, alors on aura trouvé un sous-graphe correspondant au motif
# Enfin, subgraph_list, qui contient la liste de tous les sous-graphes trouvés, peut contenir des doublons
# Ex: [0,2,4,1] et [4,2,0,1]
# Pour éviter cela, on va filter subgraph_list dans un set, pour supprimer tous ces doublons

def find_subgraph(graph, motif, motif_name):

    def compare_edges(g1, g2, i1, i2):
        try:
            result = (g1.es[i1]['color'] == g2.es[i2]['color'])
        except:
            return False
        else:
            return result

    subgraph_list = g.get_subisomorphisms_vf2(motif,edge_compat_fn=compare_edges)

    if len(subgraph_list) >= 1:

        results = [tuple(x) for x in set(map(frozenset, subgraph_list))]
        print ("Le motif " + motif_name + " est présent " + str(len(results)) + " fois dans la chaîne d'ARN " + filename + "\n")

        for result in sorted(results):
            print (result)
        print ("\n\n")

        return len(results)

    else:
        print ("Le motif " + motif_name + " est absent de la chaîne d'ARN " + filename + "\n\n")
        return 0

# initialisation des motifs RIN issus de Carnaval, sous forme de graphe
# la couleur des arêtes permet d'identiier le type de liaison pair_type_LW

def transorm_RIN_to_graph():

    global rin_23, rin_129

    rin_23 = ig.Graph()
    rin_129 = ig.Graph()

    rin_129.add_vertices(4)
    rin_129.add_edges([(0,1),(1,2),(2,3),(0,3)])
    rin_129.vs["color"] = "white"

    for i in range(len(rin_129.vs)):
        rin_129.vs[i]["label"]= str(i+1)

    res_edge = rin_129.es.find(_source=1, _target=2)
    res_edge["label"] = "tWW"        # liaison tWW
    res_edge["color"] = "red"

    res_edge = rin_129.es.find(_source=0, _target=3)
    res_edge["label"] = "tHS"        # liaison tHS    (d'après la nomenclature officielle)
    res_edge["color"] = "green"

    rin_23.add_vertices(8)
    rin_23.add_edges([(0,1),(1,2),(2,3),(3,4),(4,5),(5,7),(1,6),(6,7),(1,4),(0,5)])
    rin_23.vs["color"] = "white"

    for i in range(len(rin_23.vs)):
        rin_23.vs[i]["label"]= str(i+1)

    res_edge = rin_23.es.find(_source=0, _target=5)
    res_edge["label"] = "cWW"
    res_edge["color"] = "blue"

    res_edge = rin_23.es.find(_source=1, _target=4)
    res_edge["label"] = "cWW"
    res_edge["color"] = "blue"

    res_edge = rin_23.es.find(_source=2, _target=3)
    res_edge["label"] = "cWW"
    res_edge["color"] = "blue"

    res_edge = rin_23.es.find(_source=1, _target=6)
    res_edge["label"] = "cSS"
    res_edge["color"] = "maroon"

    res_edge = rin_23.es.find(_source=5, _target=7)
    res_edge["label"] = "tSS"
    res_edge["color"] = "gray"

transorm_RIN_to_graph()

# Le code ci-dessous permet de parcourir tous les fichiers pour compter le nombre total de motif.
# Il est déconseillé d'exécuter le programme ci-dessous si votre machine n'est pas très puissante,
# le programme mettra plusieurs dizaines de minutes à s'éxécuter.
# Il faudra alors privilégier l'analyse fichier par fichier, en utilisant la version projet.py.

# En effet, on crée ici un pool de threads, égal au maximum des capacités de notre machine
# (cpu_count = nombre max de threads par coeur), on va ainsi utiliser au maximum les capacités du processeur.
# Les threads crées vont exécuter de manière parallèle la fonction count_nb_occurences, qui renvoie le nombre
# de motif rin_23 et rin_129 pour un fichier CSV donné.

# Le gain de performance en terme de temps d'exécution est assez énorme.
# Par exemple, sur une machine assez ancienne disposant de 4 coeurs, avec 2 threads par coeur,
# le temps d'exécution multithreadé est de 20 minutes, contre plus d'1 heure et 20 minutes en monothread.
# Sur une machine plus récente et plus puissante, le programme devrait probablement s'éxécuter plus rapidement.
# Pour réussir à diminuer le temps d'éxécution, nous avons enlevé les affichages des graphes et les messages
# dans la console, qui auraient monopolisé les ressources systèmes (17120 graphes à afficher !)

import multiprocessing

def count_nb_occurences(csvFile):
    results = []
    draw_graph_from_csv(csvFile)
    results.append(find_subgraph(g,rin_23,"rin_23"))
    results.append(find_subgraph(g,rin_129,"rin_129"))
    return results

if __name__ == '__main__':

    # Make the Pool of workers
    pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
    count_rins = ((pool.map(count_nb_occurences, allCsvFiles)))

    # Close the pool and wait for the work to finish
    pool.close()
    pool.join()

    result = [sum(x) for x in zip(*count_rins)]
    print("Nombre d'occurences de RIN 23 : " + str(result[0]))
    print("Nombre d'occurences de RIN 129 : " + str(result[1]))


import logging
import requests

class Move(object):
    def __init__(self, url, full_obj=None):
        self.full_obj = full_obj or self.get_from_api(url)

        self._name = None
        self._power = 0

    @property
    def name(self):
        if self.full_obj is None:
            self.full_obj = self.get_move_from_api()
        self._name = self.full_obj['name']

        return self._name

    @property
    def power(self):
        if self.full_obj is None:
            self.full_obj = self.get_move_from_api()
        p = self.full_obj['power']

        if p is None:
            p = 0.0

        self._power = p

        return self._power

    def get_from_api(self, url):
        """ Retrieves move information from web API """
        if url is None:
            return { "name": None, "power": None }

        logging.info("Retrieving move from api url: {}".format(url))
        res = requests.get(url)
        res.raise_for_status()
        info = res.json()

        return info


#!/usr/bin/python3
#
# MEG sensor space analysis for visual LTP (vertical & horizontal gratings)
#
# Authors: Paul Sowman, Judy Zhu

#######################################################################################

import os
import mne
import meegkit # for TSPCA
import glob
import matplotlib.pyplot as plt
import numpy as np
import copy

from mne.preprocessing import find_bad_channels_maxwell
#from autoreject import get_rejection_threshold  # noqa
#from autoreject import Ransac  # noqa
#from autoreject.utils import interpolate_bads  # noqa

from scipy import stats

import my_preprocessing

# Make plots interactive when running in interactive window in vscode
#plt.switch_backend('TkAgg') You can use this backend if needed
#plt.ion()
# %matplotlib qt

# set up file and folder paths here
exp_dir = "/mnt/d/Work/analysis_ME197/"
subject_MEG = '230426_72956_S2' #'220112_p003'
meg_tasks = ['_ltp1', '_ltp2', '_ltp3'] #'_oddball' #''

# the paths below should be automatic
data_dir = exp_dir + "data/"
processing_dir = exp_dir + "processing/"
meg_dir = data_dir + subject_MEG + "/meg/"
save_dir = processing_dir + "meg/" + subject_MEG + "/"
os.system('mkdir -p ' + save_dir) # create the folder if needed

fname_elp = glob.glob(meg_dir + "*.elp")
fname_hsp = glob.glob(meg_dir + "*.hsp")
fname_mrk = glob.glob(meg_dir + "*.mrk")

#%% Loop over tasks: pre, 2min post, 30min post
for counter, task in enumerate(meg_tasks):
    fname_raw = glob.glob(meg_dir + "*" + task + ".con")

    ica_fname = save_dir + subject_MEG + task + "-ica.fif"
    epochs_fname = save_dir + subject_MEG + task + "-epo.fif"
    ERFs_fname = save_dir + subject_MEG + task + "-ave.fif"
    ERFs_figure_fname = save_dir + subject_MEG + task + ".png"

    raw = mne.io.read_raw_kit(
        fname_raw[0],
        mrk=fname_mrk[0],
        elp=fname_elp[0],
        hsp=fname_hsp[0],
        stim=[*range(177, 179)], # these triggers (177 and 178) indicate vertical or horizontal
        slope="+",
        stim_code="channel",
        stimthresh=1,  # 2 for adults
        preload=True,
        allow_unknown_format=False,
        verbose=True,
    )

    # Apply TSPCA for noise reduction
    noisy_data = raw.get_data(picks="meg").transpose()
    noisy_ref = raw.get_data(picks=[160,161,162]).transpose()
    data_after_tspca, idx = meegkit.tspca.tsr(noisy_data, noisy_ref)[0:2]
    raw._data[0:160] = data_after_tspca.transpose()

    # browse data to identify bad sections & bad channels
    raw.plot()

    # Filtering & ICA
    raw = my_preprocessing.reject_artefact(raw, 0.1, 40, True, ica_fname)

    #%% ch misc 23-29 are trigger channels
    # MISC 18 and 19 == horizontal / vertical gratings
    # MISC 6 or 10 == Photodetector

    #%% Finding events
    events = mne.find_events(
        raw,
        output="onset",
        consecutive=False,
        min_duration=0,
        shortest_event=1,  # 5 for adults
        mask=None,
        uint_cast=False,
        mask_type="and",
        initial_event=False,
        verbose=None,
    )

    for index, event in enumerate(events):
        if event[2] == 177: # ch177 == MISC 18
            events[index, 2] = 2 # horizontal
        elif event[2] == 178: # ch178 == MISC 19
            events[index, 2] = 3 # vertical

    # Find times of PD triggers
    # Ensure correct PD channel is entered here, might sometimes be 165
    events_PD = mne.find_events(
        raw, stim_channel=[raw.info["ch_names"][x] for x in [169]], output="onset"
    )

    combined_events = np.concatenate([events, events_PD])
    combined_events = combined_events[np.argsort(combined_events[:, 0])]

    #%% find the difference between PD time and trigger time
    pd_delta = []
    for index, event in enumerate(combined_events):
        if (
            index > 0  # PD can't be first event
            and combined_events[index, 2] == 1 # current trigger is PD trigger
            and combined_events[index - 1, 2] != 1 # previous trigger is not PD trigger
        ):
            pd_delta.append(
                combined_events[index, 0] - combined_events[index - 1, 0] # find the time difference
            )
    # show histogram of PD delays
    n, bins, patches = plt.hist(
        x=pd_delta, bins="auto", color="#0504aa", alpha=0.7, rwidth=0.85
    )
    plt.grid(axis="y", alpha=0.75)
    plt.xlabel("Delay (ms)")
    plt.ylabel("Frequency")
    plt.title("Photo Detector Delays")
    plt.text(
        70,
        50,
        r"$mean="
        + str(round(np.mean(pd_delta)))
        + ", std="
        + str(round(np.std(pd_delta)))
        + "$",
    )
    maxfreq = n.max()
    # Set a clean upper y-axis limit.
    plt.ylim(ymax=np.ceil(maxfreq / 10) * 10 if maxfreq % 10 else maxfreq + 10)

    # Use target events to align triggers & avoid outliers using z of 3
    z = np.abs(stats.zscore(pd_delta))
    #TODO: check this part works correctly when we do have outliers!
    if [pd_delta[i] for i in np.where(z > 3)[0]]:
        tmax = -max([pd_delta[i] for i in np.where(z > 3)[0]]) / 1000
    else:
        tmax = 0

    events_to_find = [2, 3] # target events
    sfreq = raw.info["sfreq"]  # sampling rate
    tmin = -0.4  # PD occurs after trigger, hence negative
    fill_na = None  # the fill value for non-target
    reference_id = 1  # PD

    # loop through events and replace PD events with event class identifier i.e. trigger number
    events_target = {}
    for event in events_to_find:
        new_id = 20 + event
        events_target["event" + str(event)], lag = mne.event.define_target_events(
            combined_events,
            reference_id,
            event,
            sfreq,
            tmin,
            tmax,
            new_id,
            fill_na,
        )
    events = np.concatenate((events_target["event2"], events_target["event3"]))

    event_ids = {
        "horizontal": 22,
        "vertical": 23,
    }

    #%% === Epoching === #

    if os.path.exists(epochs_fname):
        epochs_resampled = mne.read_epochs(epochs_fname)
    else:
        epochs = mne.Epochs(
            raw, events, event_id=event_ids, tmin=-0.1, tmax=0.5, preload=True
        )

        conds_we_care_about = ["horizontal", "vertical"]
        epochs.equalize_event_counts(conds_we_care_about)

        # sanity check - PD triggers occur at 0ms
        mne.viz.plot_evoked(
            epochs.average(picks="MISC 010")
        )

        # downsample to 100Hz
        print("Original sampling rate:", epochs.info["sfreq"], "Hz")
        epochs_resampled = epochs.copy().resample(100, npad="auto")
        print("New sampling rate:", epochs_resampled.info["sfreq"], "Hz")

        # save the epochs to file
        epochs_resampled.save(epochs_fname)

        # alternatively, save the evoked (i.e. ERF) here instead of epochs.
        # That will save quite a bit of space. We prob won't need to
        # read the actual epochs again as we are not doing source analysis.
        # Note though: write_evokeds() only accept a list, not a dict, of evokeds,
        # so when read back in, you won't be able to do evokeds[cond], instead you
        # need to enumerate the evokeds and find out the cond via evoked.comment
        evokeds = []
        for cond in epochs_resampled.event_id:
            evokeds.append(epochs_resampled[cond].average())
        mne.write_evokeds(ERFs_fname, evokeds)

    # plot ERFs
    mne.viz.plot_evoked(epochs_resampled.average(), gfp="only")
    fig = mne.viz.plot_compare_evokeds(
        [
            epochs_resampled["horizontal"].average(),
            epochs_resampled["vertical"].average(),
        ]
    )
    fig[0].savefig(ERFs_figure_fname)

# report = mne.Report(title=fname_raw[0])
# report.add_evokeds(
#     evokeds=evoked, titles=["VEP"], n_time_points=25  # Manually specify titles
# )
# report.save(fname_raw[0] + "_report_evoked.html", overwrite=True)


from typing import List
from collections import deque

class Solution:
    def nearestExit(self, maze: List[List[str]], entrance: List[int]) -> int:
        nei = [[-1, 0], [1, 0], [0, -1], [0, 1]]
        dq, seen = deque(), set()
        dq.append(entrance)

        row, col = len(maze), len(maze[0])

        steps = 0
        while dq:
            n = len(dq)
            for _ in range(n):
                i, j = dq.popleft()
                # print(f'popped: {i}, {j}')
                if (i, j) in seen:
                    continue

                seen.add((i, j))
                if (i == 0 or i == row - 1
                        or j == 0 or j == col - 1) \
                        and [i, j] != entrance:

                    return steps

                for di, dj in nei:
                    new_i, new_j = i + di, j + dj
                    if 0 <= new_i < row \
                            and 0 <= new_j < col \
                            and maze[new_i][new_j] == '.':
                        dq.append((new_i, new_j))

            steps += 1

        return -1

sol = Solution()
# print(sol.nearestExit(maze=[["+", "+", ".", "+"],
#                             [".", ".", ".", "+"],
#                             ["+", "+", "+", "."]],
#                       entrance=[1, 2],
#                       ))
# print(sol.nearestExit(maze=[["+", "+", "+"], [".", ".", "."], ["+", "+", "+"]],
#                       entrance=[1, 0],
#                       ))

# print(sol.nearestExit(maze=[["+", "+", ".", "+"], [".", ".", ".", "+"], ["+", "+", "+", "."]],
#                       entrance=[1, 2],
#                       ))


#-*-coding:utf-8-*-
"""
This is base modoul bank
"""
import imp
import json
import urllib2
import urllib
import os
import time
import sys
from thePath import rootPath
class BANK(object):
    """
    Base class
    """
    def __init__(self):
        self.useragent = (
                "Mozilla/5.0 (Linux; Android 6.0;"
                " Nexus 5 Build/MRA58N) AppleWebKit/537.36"
                " (KHTML, like Gecko) "
                "Chrome/60.0.3112.113 Mobile Safari/537.36"
                )
        self.headers={'User-Agent': self.useragent}
        self.time = time.strftime("%Y-%m-%d")

    def getHtml(self, url):
        """
        This is to get html code
        """
        try:
            request = urllib2.Request(url, headers = self.headers)
            response = urllib2.urlopen(request, timeout = 60)
            pageCode = response.read()
            return pageCode
        except urllib2.URLError as e:
            if hasattr(e, "reason"):
                return None

    def getOutFile(self, fileName):
        """
        The function is to get outfile
        """
        outFile = self.getDirectory() + fileName
        if os.path.exists(outFile):
            os.system("rm " + outFile)
        return outFile

    def getDirectory(self):
        """
        The function is to get directory
        """
        backdir = (rootPath + \
                "data/primaryData/BANK/")
        Datedir = backdir + self.time + "/"
        if not os.path.exists(Datedir):
            try:
                os.makedirs(Datedir)
            except OSError as e:
                return Datedir
        return Datedir

    def saveFile(self, item, outFile):
        """
        The function is to save file
        """
        fp = open(outFile, 'a+')
        li = json.dumps(item, ensure_ascii=False) + '\n'
        fp.write(li.encode('utf8').replace("\\r", "-").replace("\\n","-")\
                .replace("\\", "-").replace("\\t","-"))
        fp.close()