"""
This module implements the machine learning-based model for the qubit. It has three classes:
    HamiltonianConstruction: This is an internal class for constructing Hamiltonians
    QuantumCell            : This is an internal class required for implementing time-ordered evolution
    QuantumEvolution       : This is an internal class to implement time-ordered quantum evolution
    VoLayer                : This is an internal class to calculate Vo given the interaction unitary and the observable
    QuantumMeasurement     : This is an internal class to model coupling losses at the output.
    qubitMLmodel           : This is the main class that defines machine learning model for the qubit.
"""
###############################################################################
# Preamble
import numpy as np
import tensorflow as tf
from tensorflow.keras import layers,optimizers,Model,initializers,callbacks
import zipfile
import os
import pickle
##############################################################################
class HamiltonianConstruction(layers.Layer):
    """
    This class defines a custom tensorflow layer that takes the Hamiltonian parameters as input, and generates the
    Hamiltonain matrix as an output at each time step for each example in the batch
    """

    def __init__(self, dynamic_operators, static_operators, **kwargs):
        """
        Class constructor

        dynamic_operators: a list of all operators that have time-varying coefficients
        static_operators : a list of all operators that have constant coefficients
        """

        self.dynamic_operators = [tf.constant(op, dtype=tf.complex64) for op in dynamic_operators]
        self.static_operators  = [tf.constant(op, dtype=tf.complex64) for op in static_operators]

        # this has to be called for any tensorflow custom layer
        super(HamiltonianConstruction, self).__init__(**kwargs)

    def call(self, inputs):
        """
        This method must be defined for any custom layer, it is where the calculations are done.

        inputs: a tensor representing the inputs to the layer. This is passed automatically by tensorflow.
        """

        H = []
        # loop over the strengths of all dynamic operators

        for idx_op, op in enumerate(self.dynamic_operators):

            # select the particular strength of the operator
            h = tf.cast(inputs[:,:,:,idx_op:idx_op+1] ,dtype=tf.complex64)

            # construct a tensor in the form of a row vector whose elements are [d1,d2,d3, 1,1], where d1, d2, and d3 correspond to the
            # number of examples, number of time steps of the input, and number of realizations
            temp_shape = tf.concat( [tf.shape(inputs)[0:3],tf.constant(np.array([1,1],dtype=np.int32))],0 )

            # add two extra dimensions for batch, time, and realization
            operator = tf.expand_dims(op,0)
            operator = tf.expand_dims(operator,0)
            operator = tf.expand_dims(operator,0)

            # repeat the pauli operators along the batch and time dimensions
            operator = tf.tile(operator, temp_shape)

            # repeat the pulse waveform to as 2x2 matrix
            temp_shape = tf.constant(np.array([1,1,1,2,2],dtype=np.int32))
            h = tf.expand_dims(h,-1)
            h = tf.tile(h, temp_shape)

            # Now multiply each operator with its corresponding strength element-wise and add to the list of Hamiltonians
            H.append( tf.multiply(operator, h) )

        # loop over the strengths of all static operators
        for op in self.static_operators:
            # construct a tensor in the form of a row vector whose elements are [d1,d2,d3,1,1], where d1, d2, and d2 correspond to the
            # number of examples, number of time steps of the input, and number of realizations
            temp_shape = tf.concat( [tf.shape(inputs)[0:3],tf.constant(np.array([1,1],dtype=np.int32))],0 )

            # add two extra dimensions for batch and time
            operator = tf.expand_dims(op,0)
            operator = tf.expand_dims(operator,0)
            operator = tf.expand_dims(operator,0)

            # repeat the pauli operators along the batch and time dimensions
            operator = tf.tile(operator, temp_shape)

            # Now add to the list of Hamiltonians
            H.append( operator )

        # now add all componenents together
        H =  tf.add_n(H)

        return H
###############################################################################
class QuantumCell(layers.Layer):
    """
    This class defines a custom tensorflow layer that takes Hamiltonian as input, and produces one step forward propagator
    """

    def __init__(self, delta_T, **kwargs):
        """
        Class constructor.
        delta_T: time step for each propagator
        """

        # here we define the time-step including the imaginary unit, so we can later use it directly with the expm function
        self.delta_T= tf.constant(delta_T*-1j, dtype=tf.complex64)

        # we must define this parameter for RNN cells
        self.state_size = [1]

        # we must call thus function for any tensorflow custom layer
        super(QuantumCell, self).__init__(**kwargs)

    def call(self, inputs, states):
        """
        This method must be defined for any custom layer, it is where the calculations are done.

        inputs: The tensor representing the input to the layer. This is passed automatically by tensorflow.
        states: The tensor representing the state of the cell. This is passed automatically by tensorflow.
        """

        previous_output = states[0]

        # evaluate -i*H*delta_T
        Hamiltonian = inputs * self.delta_T

        #evaluate U = expm(-i*H*delta_T)
        U = tf.linalg.expm( Hamiltonian )

        # accuamalte U to to the rest of the propagators
        new_output  = tf.matmul(U, previous_output)

        return new_output, [new_output]
###############################################################################
class QuantumEvolution(layers.RNN):
    """
    This class defines a custom tensorflow layer that takes Hamiltonian as input, and produces the time-ordered evolution unitary as output
    """

    def __init__(self, delta_T, **kwargs):
        """
        Class constructor.

        delta_T: time step for each propagator
        """

        # use the custom-defined QuantumCell as base class for the nodes
        cell = QuantumCell(delta_T)

        # we must call thus function for any tensorflow custom layer
        super(QuantumEvolution, self).__init__(cell,  **kwargs)

    def call(self, inputs):
        """
        This method must be defined for any custom layer, it is where the calculations are done.

        inputs: The tensor representing the input to the layer. This is passed automatically by tensorflow.
        """

        # define identity matrix with correct dimensions to be used as initial propagtor
        dimensions = tf.shape(inputs)
        I          = tf.eye( dimensions[-1], batch_shape=[dimensions[0], dimensions[2]], dtype=tf.complex64 )

        return super(QuantumEvolution, self).call(inputs, initial_state=[I])
###############################################################################
class QuantumMeasurement(layers.Layer):
    """
    This class defines a custom tensorflow layer that takes the unitary as input,
    and generates the measurement outcome probability as output
    """

    def __init__(self, initial_state, measurement_operator, **kwargs):
        """
        Class constructor

        initial_state       : The inital density matrix of the state before evolution.
        Measurement_operator: The measurement operator
        """
        self.initial_state        = tf.constant(initial_state, dtype=tf.complex64)
        self.measurement_operator = tf.constant(measurement_operator, dtype=tf.complex64)

        # we must call thus function for any tensorflow custom layer
        super(QuantumMeasurement, self).__init__(**kwargs)

    def call(self, x):
        """
        This method must be defined for any custom layer, it is where the calculations are done.

        x: a tensor representing the inputs to the layer. This is passed automatically by tensorflow.
        """

        # extract the different inputs of this layer which are the Vo and Uc
        Vo, Uc = x

        Uc = Uc[:,0,:]
        # construct a tensor in the form of a row vector whose elements are [d1,1,1], where d1 correspond to the
        # number of examples of the input
        temp_shape = tf.concat( [tf.shape(Uc)[0:1],tf.constant(np.array([1,1],dtype=np.int32))],0 )

        # add an extra dimension for the initial state and measurement tensors to represent batch
        initial_state        = tf.expand_dims(self.initial_state,0)
        measurement_operator = tf.expand_dims(self.measurement_operator,0)

        # repeat the initial state and measurment tensors along the batch dimensions
        initial_state        = tf.tile(initial_state, temp_shape )
        measurement_operator = tf.tile(measurement_operator, temp_shape)

        # evolve the initial state using the propagator provided as input
        final_state = tf.matmul(tf.matmul(Uc, initial_state), Uc, adjoint_b=True )

        # calculate the probability of the outcome
        expectation = tf.linalg.trace( tf.matmul( tf.matmul( Vo, final_state), measurement_operator) )

        return tf.squeeze( tf.reshape( tf.math.real(expectation), temp_shape), axis=-1 )
###############################################################################
class VoLayer(layers.Layer):
    """
    This class defines a custom tensorflow layer that takes a vector of parameters represneting eigendecompostion and reconstructs a 2x2 Hermitian traceless matrix.
    """

    def __init__(self, O, **kwargs):
        """
        Class constructor

        O: The observable to be measaured
        """
        # this has to be called for any tensorflow custom layer
        super(VoLayer, self).__init__(**kwargs)

        self.O = tf.constant(O, dtype=tf.complex64)

    def call(self, x):
        """
        This method must be defined for any custom layer, it is where the calculations are done.

        x: a tensor representing the inputs to the layer. This is passed automatically by tensorflow.
        """

        # retrieve the two types of parameters from the input: 3 eigenvector parameters and 1 eigenvalue parameter
        if len(x)==2:
            UI,Uc = x
        else:
            UI,Uc,w = x
            temp_shape = tf.constant(np.array([1,1,2,2],dtype=np.int32))
            w  = tf.cast( tf.tile(w, temp_shape), dtype=tf.complex64)

        UI_tilde = tf.matmul(Uc, tf.matmul(UI,Uc, adjoint_b=True) )

        # expand the observable operator along batch and realizations axis
        O = tf.expand_dims(self.O, 0)
        O = tf.expand_dims(O, 0)

        temp_shape = tf.concat( [tf.shape(Uc)[0:2], tf.constant(np.array([1,1],dtype=np.int32))], 0 )
        O = tf.tile(O, temp_shape)

        # Construct Vo operator
        VO = tf.matmul(O, tf.matmul( tf.matmul(UI_tilde,O, adjoint_a=True), UI_tilde) )

        if len(x)==2:
            VO = tf.reduce_mean(VO, axis=1, keepdims=False)
        else:
            VO = tf.reduce_sum( tf.multiply(VO, w), axis=1, keepdims=False)

        return VO
###############################################################################
class Noise(layers.Layer):
    """
    This class defines a custom tensorflow layer that generates realizations of a random process simulating noise.

    """
    def __init__(self, dim, K=1, **kwargs):
        """
        class constructor

        dim : how many axes to generate the noise
        K   : number of realizations
        """
        # we must call thus function for any tensorflow custom layer
        super(Noise, self).__init__(**kwargs)

        # store the constructor parameters
        self.K = K
        self.dim = dim

    def build(self, input_shape):
        """
        This method must be defined for any custom layer, here you define the training parameters.

        input_shape: a tensor that automatically captures the dimensions of the input by tensorflow.
        """

        M = input_shape.as_list()[1] # retreive the number of samples from the input

        # define custom weights to be generate the realizations

        # 1) time-domain samples
        self.beta = self.add_weight(name = "beta", shape=(1, M, self.K, self.dim), initializer=initializers.glorot_uniform(), dtype=tf.float32, trainable=True)

        # 2) weight of each realizations to be used when calculating averages
        self.w = self.add_weight(name="w", shape=(1, self.K, 1, 1), dtype=tf.float32, initializer=initializers.glorot_uniform())

         # we must call thus function for any tensorflow custom layer
        super(Noise, self).build(input_shape)

    def call(self, x):
        """
        This method must be defined for any custom layer, it is where the calculations are done.

        x: a tensor representing the inputs to the layer. This is passed automatically by tensorflow.
        """
        # make sure the weights add up to 1
        w    =  tf.nn.softmax(self.w, axis=1)

        # repeat the weights and time-realization for each example in the batch [for fixing training issues]
        temp_shape = tf.concat( [tf.shape(x)[0:1], tf.constant(np.array([1,1,1],dtype=np.int32))], 0 )
        beta       = tf.tile(self.beta, temp_shape)
        w          = tf.tile(w, temp_shape)

        # return the weights
        return [beta,w]
###############################################################################
class qubitMLmodel():
    """
    This is the main class that defines machine learning model of the qubit.
    """

    def __init__(self, T, M, dynamic_operators, static_operators, noise_operators, measurement_operators, initial_states, K=1):
        """
        Class constructor.

        T                : Evolution time
        M                : Number of time steps
        dynamic_operators: A list of arrays that represent the terms of the control Hamiltonian (that depend on pulses)
        static_operators : A list of arrays that represent the terms of the drifting Hamiltonian (that are constant)
        noise_operators  : A list of arrays that represent the terms of the classical noise Hamiltonians
        measurement_operators: A list of arrays representing the measuremetion operators
        initial_states   : A list of arrays representing the initial states of the quantum system
        K                : Number of noise realizations
        """

        # store the constructor arguments
        self.K       = K
        self.T       = T
        self.M       = M
        self.delta_T = T/M

       # define lists for stroring the training history
        self.training_history      = []
        self.val_history           = []

        # define tensorflow input layers for the pulse sequence and noise realization in time-domain
        pulse_time_domain = layers.Input(shape=(M,1,len(dynamic_operators)) , name="control")
        noise_time_domain,w = Noise(len(noise_operators), K=K, name="noise")(pulse_time_domain)

        # define the custom defined tensorflow layer that constructs the H0 part of the Hamiltonian from parameters at each time step
        H0 = HamiltonianConstruction(dynamic_operators=dynamic_operators, static_operators=static_operators, name="H0")(pulse_time_domain)

        # define the custom defined tensorflow layer that constructs the H1 part of the Hamiltonian from parameters at each time step
        H1 = HamiltonianConstruction(dynamic_operators=noise_operators, static_operators=[], name="H1")(noise_time_domain)

        # define the custom defined tensorflow layer that constructs the time-ordered evolution of H0
        U0 = QuantumEvolution(self.delta_T, return_sequences=True, name="U0")(H0)

        # define Uc which is U0(T)
        Uc = layers.Lambda(lambda u0: u0[:,-1,:,:,:], name="Uc")(U0)

        # define custom tensorflow layer to calculate HI
        U0_ext = layers.Lambda(lambda x: tf.tile(x, tf.constant([1,1,K,1,1], dtype=tf.int32) ) )(U0)
        HI = layers.Lambda(lambda x: tf.matmul( tf.matmul(x[0],x[1], adjoint_a=True), x[0] ), name="HI" )([U0_ext, H1])

        # define the custom defined tensorflow layer that constructs the time-ordered evolution of HI
        UI = QuantumEvolution(self.delta_T, return_sequences=False, name="UI")(HI)

        # construct the Vo operators
        Uc_ext = layers.Lambda(lambda x: tf.tile(x, tf.constant([1,K,1,1], dtype=tf.int32) ) )(Uc)

        Vo = [VoLayer(O, name="V%d"%idx_O)([UI,Uc_ext,w]) for idx_O, O in enumerate(measurement_operators)]

        # add the custom defined tensorflow layer that calculates the measurement outcomes
        expectations = [
                [QuantumMeasurement(rho,X, name="rho%dM%d"%(idx_rho,idx_X))([Vo[idx_X],Uc]) for idx_X, X in enumerate(measurement_operators)]
                for idx_rho,rho in enumerate(initial_states)]

        # concatenate all the measurement outcomes
        expectations = layers.Concatenate(axis=-1)(sum(expectations, [] ))

        # construct the model
        self.model    = Model( inputs = pulse_time_domain,  outputs = expectations )

        # specify the optimizer and loss function for training
        self.model.compile(optimizer=optimizers.Adam(lr=0.01), loss='mse')

        # print a summary of the model showing the layers, their connections, and the number of training parameters
        self.model.summary()

    def train_model(self, training_x, training_y, epochs, batch_size):
        """
        This method is for training the model given the training set

        training_x: A numpy array that stores the time-domain represenation of control pulses of dimensions (number of examples, number of time steps, 1, number of axes)
        training_y: A numpy array that stores the measurement outcomes (number of examples, number of measurements).
        epochs    : The number of iterations to do the training
        batch_size: The batch size
        """

        # Train the model for "epochs" number of iterations using the provided training set, and store the training history
        self.training_history = self.model.fit(training_x, training_y, epochs=epochs, batch_size=batch_size,verbose=2).history["loss"]

    def train_model_val(self, training_x, training_y, val_x, val_y, epochs, batch_size):
        """
        This method is for training the model given the training set and the validation set

        training_x: A numpy array that stores the time-domain represenation of control pulses of dimensions (number of examples, number of time steps, 1, number of axes)
        training_y: A numpy array that stores the measurement outcomes (number of examples, number of measurements).
        val_x     : The validation input array [similar to training_x]
        val_y     : The valiation desired output array [similar to training_y]
        epochs    : The number of iterations to do the training
        batch_size: The batch size
        """

        # Train the model for "epochs" number of iterations using the provided training set, and store the training history
        h  =  self.model.fit(training_x, training_y, epochs=epochs, batch_size=batch_size,verbose=1,validation_data = (val_x, val_y))
        self.training_history  = h.history["loss"]
        self.val_history       = h.history["val_loss"]

    def predict_measurements(self, testing_x):
        """
        This method is for predicting the measurement outcomes using the trained model. Usually called after training.

        testing_x: A list of two  numpy arrays the first is of shape (number of examples,number of time steps, number of signal parameters), and the second is of dimensions (number of examples, number of time steps, 1)
        """
        return self.model.predict(testing_x)

    def predict_control_unitary(self,testing_x):
        """
        This method is for evaluating the control unitary. Usually called after training.

        testing_x: A numpy array of shape (number of examples, number of time steps, number of axes)
        """

        # define a new model that connects the input voltage and the "UC" later output
        unitary_model = Model(inputs=self.model.input, outputs=self.model.get_layer('Uc').output)

        # evaluate the output of this model
        return unitary_model.predict(testing_x)

    def predict_noise(self, testing_x):
        """
        This method is for predicting the noise parameters
        """
        #beta = np.sin(self.model.get_weights()[0])
        beta = self.model.get_weights()[0]
        w    = np.exp(self.model.get_weights()[1])/np.sum(np.exp(self.model.get_weights()[1]))

        return beta,w

    def predict_Vo(self, testing_x):
        """
        This method is for predicting the Vo operators. Usally called after training.

        testing_x: A numpy array of shape (number of examples, number of time steps, number of axes)
        """

        # define a new model that connects the inputs to each of the Vo output layers
        Vo_model = Model(inputs=self.model.input, outputs=[self.model.get_layer(V).output for V in ["V%d"%idx for idx in range(self.m)] ] )

        # predict the output of the truncated model. This physically represents <U_I' O U_I>. We still need to multiply by O to get Vo = <O U_I' O U_I>
        Vo = Vo_model.predict(testing_x)

        return Vo

    def save_model(self, filename):
        """
        This method is to export the model to an external .mlmodel file

        filename: The name of the file (without any extensions) that stores the model.
        """

        # first save the ml model
        self.model.save_weights(filename+"_model.h5")

        # second, save all other variables
        data = {'training_history':self.training_history,
                'val_history'     :self.val_history,
                }
        f = open(filename+"_class.h5", 'wb')
        pickle.dump(data, f, -1)
        f.close()

        # zip everything into one zip file
        f = zipfile.ZipFile(filename+".mlmodel", mode='w')
        f.write(filename+"_model.h5")
        f.write(filename+"_class.h5")
        f.close()

        # now delete all the tmp files
        os.remove(filename+"_model.h5")
        os.remove(filename+"_class.h5")

    def load_model(self, filename):
        """
        This method is to import the models from an external .mlmodel file

        filename: The name of the file (without any extensions) that stores the model.
        """
        #unzip the zipfile
        f = zipfile.ZipFile(filename+".mlmodel", mode='r')
        f.extractall()
        f.close()

        # first load the ml model
        self.model.load_weights(filename+"_model.h5")

        # second, load all other variables
        f = open(filename+"_class.h5", 'rb')
        data = pickle.load(f)
        f.close()
        self.training_history  = data['training_history']
        self.val_history       = data['val_history']

        # now delete all the tmp files
        os.remove(filename+"_model.h5")
        os.remove(filename+"_class.h5")
###############################################################################


import logging.config
import os
import uuid

import pandas as pd
import pyspark.sql.functions as F
from pyspark.ml import PipelineModel
from pyspark.sql import SparkSession

from examples_utils import get_dataset_attrs, prepare_test_and_train, get_spark_session
from lightautoml.spark.automl.presets.tabular_presets import SparkTabularAutoML
from lightautoml.spark.dataset.base import SparkDataset
from lightautoml.spark.tasks.base import SparkTask
from lightautoml.spark.utils import log_exec_timer, logging_config, VERBOSE_LOGGING_FORMAT

logging.config.dictConfig(logging_config(level=logging.INFO, log_filename='/tmp/slama.log'))
logging.basicConfig(level=logging.DEBUG, format=VERBOSE_LOGGING_FORMAT)
logger = logging.getLogger(__name__)

# NOTE! This demo requires datasets to be downloaded into a local folder.
# Run ./bin/download-datasets.sh to get required datasets into the folder.

def main(spark: SparkSession, dataset_name: str, seed: int):
    # Algos and layers to be used during automl:
    # For example:
    # 1. use_algos = [["lgb"]]
    # 2. use_algos = [["lgb_tuned"]]
    # 3. use_algos = [["linear_l2"]]
    # 4. use_algos = [["lgb", "linear_l2"], ["lgb"]]
    use_algos = [["lgb", "linear_l2"], ["lgb"]]
    cv = 5
    path, task_type, roles, dtype = get_dataset_attrs(dataset_name)

    with log_exec_timer("spark-lama training") as train_timer:
        task = SparkTask(task_type)
        train_data, test_data = prepare_test_and_train(spark, path, seed)

        test_data_dropped = test_data

        # optionally: set 'convert_to_onnx': True to use onnx-based version of lgb's model transformer
        automl = SparkTabularAutoML(
            spark=spark,
            task=task,
            general_params={"use_algos": use_algos},
            lgb_params={'use_single_dataset_mode': True, 'convert_to_onnx': False, 'mini_batch_size': 1000},
            reader_params={"cv": cv, "advanced_roles": False}
        )

        oof_predictions = automl.fit_predict(
            train_data,
            roles=roles
        )

    logger.info("Predicting on out of fold")

    score = task.get_dataset_metric()
    metric_value = score(oof_predictions)

    logger.info(f"score for out-of-fold predictions: {metric_value}")

    transformer = automl.make_transformer()

    automl.release_cache()

    with log_exec_timer("spark-lama predicting on test (#1 way)") as predict_timer:
        te_pred = automl.predict(test_data_dropped, add_reader_attrs=True)

        score = task.get_dataset_metric()
        test_metric_value = score(te_pred)

        logger.info(f"score for test predictions: {test_metric_value}")

    with log_exec_timer("spark-lama predicting on test (#2 way)"):
        te_pred = automl.make_transformer().transform(test_data_dropped)

        pred_column = next(c for c in te_pred.columns if c.startswith('prediction'))
        score = task.get_dataset_metric()
        test_metric_value = score(te_pred.select(
            SparkDataset.ID_COLUMN,
            F.col(roles['target']).alias('target'),
            F.col(pred_column).alias('prediction')
        ))

        logger.info(f"score for test predictions: {test_metric_value}")

    base_path = "/tmp/spark_results"
    automl_model_path = os.path.join(base_path, "automl_pipeline")
    os.makedirs(base_path, exist_ok=True)

    with log_exec_timer("saving model") as saving_timer:
        transformer.write().overwrite().save(automl_model_path)

    with log_exec_timer("Loading model time") as loading_timer:
        pipeline_model = PipelineModel.load(automl_model_path)

    with log_exec_timer("spark-lama predicting on test (#3 way)"):
        te_pred = pipeline_model.transform(test_data_dropped)

        pred_column = next(c for c in te_pred.columns if c.startswith('prediction'))
        score = task.get_dataset_metric()
        test_metric_value = score(te_pred.select(
            SparkDataset.ID_COLUMN,
            F.col(roles['target']).alias('target'),
            F.col(pred_column).alias('prediction')
        ))

    logger.info(f"score for test predictions via loaded pipeline: {test_metric_value}")

    logger.info("Predicting is finished")

    result = {
        "seed": seed,
        "dataset": dataset_name,
        "used_algo": str(use_algos),
        "metric_value": metric_value,
        "test_metric_value": test_metric_value,
        "train_duration_secs": train_timer.duration,
        "predict_duration_secs": predict_timer.duration,
        "saving_duration_secs": saving_timer.duration,
        "loading_duration_secs": loading_timer.duration
    }

    print(f"EXP-RESULT: {result}")

    train_data.unpersist()
    test_data.unpersist()

    return result

def multirun(spark: SparkSession, dataset_name: str):
    seeds = [1, 5, 42, 100, 777]
    results = [main(spark, dataset_name, seed) for seed in seeds]

    df = pd.DataFrame(results)

    with pd.option_context('display.max_rows', None, 'display.max_columns', None):
        print(df)

    df.to_csv(f"spark-lama_results_{dataset_name}_{uuid.uuid4()}.csv")

if __name__ == "__main__":
    spark_sess = get_spark_session()
    # One can run:
    # 1. main(dataset_name="lama_test_dataste", seed=42)
    # 2. multirun(spark_sess, dataset_name="lama_test_dataset")
    main(spark_sess, dataset_name="lama_test_dataset", seed=42)

    spark_sess.stop()


#web scrape anichart/MAL for anime
#https://myanimelist.net/anime/season
#https://anichart.net/Spring-2019
from animeClass import Anime
import requests
from bs4 import BeautifulSoup
url = 'https://myanimelist.net/anime/season'
page = requests.get(url)
soup = BeautifulSoup(page.text, 'html.parser')

def title_scraping():
    titlesoup = soup.find(class_='seasonal-anime-list js-seasonal-anime-list js-seasonal-anime-list-key-1 clearfix').find_all(class_='link-title')
    seasonal_titles = []
    for title in titlesoup:
        seasonal_titles.append(title.text)
    #print(seasonal_titles)
    return seasonal_titles

def episode_scraping():
    episodesoup = soup.find(class_='seasonal-anime-list js-seasonal-anime-list js-seasonal-anime-list-key-1 clearfix').find_all(class_='eps')
    #getting error about find vs find all, want the spans that contains episode #, although it seems pointless right now
    #spans = episodesoup.find_all('span')
    print(episodesoup)
    episode_count = []
    for episode in episodesoup:
        episode_count.append(episode.text)
    #print(episode_count)
    return episode_count
episode_scraping()
'''
#imported animeClass above and create an instance of Anime object with the values for one punch man
#testing the Anime object
OPM = Anime("One Punch Man", 12, "Action", 15)
print(OPM.get_Genre())
'''

#use soup to get all tags with titletext maybe?
#then put them in hashmap
#what do i want to do with the titles
#what other data do i want to store
#create an anime object maybe?

#One Punch Man 2nd Season
#class= "seasonal-anime-list js-seasonal-anime-list js-seasonal-anime-list-key-1 clearfix"

#endgame is graphing/visualization


suitcase = ['shirt', 'shirt', 'pants', 'pants', 'pajamas', 'books']

beginning = suitcase[0:4]
print(beginning)
middle_index = len(suitcase) / 2

# select middle two
middle = suitcase[int(middle_index - .5):int(1 + middle_index + .5)]

start = suitcase[:3]
end = suitcase[-2:]


import sys
import numpy as np
import meshio
import sys

original_stdout = sys.stdout #Original standard output

#·# PARSE INPUT DATA
inputfile= str(input('\nEnter .msh file name/path (include .msh extension): '))
inputmsh = meshio.read(inputfile)
N_COORD  = inputmsh.points                    #Nodal coordinates
CONECT   = inputmsh.cells_dict["quad9"]  #Conectivities
NPE = 9   #Nodes per element in hexahedron

#·# BUILD CONNECTIVITIES AND COORDINATES MATRICES
# Build Connectivities Matrix
element_nodes = 1 + CONECT #pyth indices start in 0 & nªnod must start in 1
print('Element nodes')
print(element_nodes)
num_elements  = np.shape(element_nodes)[0] #Number of elements
print('\n'+18*'-'+' {0} 9-NODED QUAD ELEMENTS DETECTED '.format(num_elements)+18*'-'+'\n')
Elements = np.zeros([num_elements , NPE+1]) #Add aditional column(mat type)
Elements[:,0]  = 1  #1st Column = Layer /Material number
Elements[:,1:] = element_nodes   #Connectivity

print(Elements)

# Build Coordinates Matrix
Nodes = N_COORD
print(Nodes)

#·# WRITE OUTPUT (.txt file)
with open('Connectivities_and_Coordinates_Eq2D.txt','w') as f:
    sys.stdout = f #Change the standard output to the file created

    print('\nMACRO_MODEL.Conectivity = [\n')
    #np.set_printoptions(threshold=sys.maxsize)
    #print(Elements)
    for row in range(0, np.shape(Elements)[0]):
        print('{0:>6}\t{1:>6}\t{2:>6}\t{3:>6}\t{4:>6}\t{5:>6}\t{6:>6}\t{7:>6}\t{8:>6}\t{9:>6}'.format(int(Elements[row,0]),int    (Elements[row,1]),int(Elements[row,2]),int(Elements[row,3]),int(Elements[row,4]),int(Elements[row,5]),int(Elements[row,6]),int(Elements[row,7]),int(Elements[row,8]),int(Elements[row,9])))

    print('\n];')
    print('\nMACRO_MODEL.Coordinates = [\n')
    for row in range(0, np.shape(Nodes)[0]):
        print('{0:>6}\t{1:>6}'.format(Nodes[row,0],Nodes[row,1]))
    print('\n];')


import logging

import matplotlib.pyplot as plt

from sklearn.metrics import precision_recall_curve

from code.metrics.binary_f1 import BinaryMetric

class BinaryPRCurve(BinaryMetric):
    def __init__(self):
        super(BinaryPRCurve, self).__init__()

    def get_metric(self, plot: bool = False):
        p, r, threshold = precision_recall_curve(self.goldens, self.predicts)

        format = lambda name, values: "%s: " % name + ",".join(map(str, values.tolist()))
        logging.info(format("p", p))
        logging.info(format("r", r))
        logging.info(format("threshold", threshold))

        if plot:
            plt.figure("P-R Curve")
            plt.title("Precision/Recall Curve")
            plt.xlabel("Recall")
            plt.ylabel("Precision")
            plt.plot(r, p)
            plt.show()
            plt.savefig("P-R Curve.png")

        max_threshold = 0.
        max_f1 = 0.
        for cur_p, cur_r, thre in zip(p, r, threshold):
            f1 = 2*cur_p*cur_r / (cur_p + cur_r)
            if f1 >= max_f1:
                max_f1 = f1
                max_threshold = thre
        logging.info("max_threshold: %f, max_f1: %f" % (max_threshold, max_f1))
        print("max_threshold: %f, max_f1: %f" % (max_threshold, max_f1))


#!/usr/bin/python3
import argparse
import configparser
import subprocess
import getpass
import sys

from datetime import datetime
from pathlib import Path
from pprint import pformat

_USER = getpass.getuser()

class RunConfig:
    def __init__(self, distance_metric, t1, t2, k_list):
        self.distance_metric = distance_metric
        self.t1 = t1
        self.t2 = t2
        self.k_list = k_list

    def __str__(self):
        return pformat(vars(self))

    def __repr__(self):
        return str(vars(self))

def run_command(label, command, interact):
    print(f"Running {label}...")
    print(f"Command:\n{command}")
    result = subprocess.run(
        command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT
    )
    output = str(result.stdout, encoding="utf-8")
    if interact and result.returncode:
        input(
            f"{output}\nCommand failed: {label}\nPress enter to continue or ctrl+c to abort.\n"
        )
    elif output:
        print(output)

def is_hdfs_file(filepath):
    result = subprocess.run(f"hdfs dfs -ls {filepath}/".split(), capture_output=True,)
    return not bool(result.returncode)

def make_directory(input_name):
    return f"hdfs dfs -mkdir {input_name}"

def copy_to_hdfs(input_name, input_dir):
    return f"hdfs dfs -put -f {input_dir} {input_name}/raw"

def make_seqfiles(input_name):
    return f"mahout seqdirectory -i {input_name}/raw -o {input_name}/seqfiles -c UTF-8 -chunk 5"

def make_sparse_vecs(input_name):
    return f"mahout seq2sparse -nv -i {input_name}/seqfiles -o {input_name}/vectors"

def clean_hdfs(input_name):
    return f"hdfs dfs -rm -r {input_name}"

def get_final_clusters(kmeans_output):
    result = subprocess.run(
        f"hdfs dfs -ls {kmeans_output}".split(), capture_output=True,
    )
    result_list = result.stdout.split()
    if result_list:
        result_str = str(result_list[-1], encoding="utf-8")
    else:
        print(f"WARN: No clusters found at {kmeans_output}!!")
        result_str = ""
    return result_str

def get_centroids(input_name, distance_metric, t1, t2, output):
    return f"""\
mahout canopy -i {input_name}/vectors/tfidf-vectors -ow \
-o {output} \
-dm org.apache.mahout.common.distance.{distance_metric} \
-t1 {t1} -t2 {t2}
"""

def run_kmeans_canopy(input_name, distance_metric, output):
    final_clusters = get_final_clusters(
        f"{input_name}/canopy-centroids-{distance_metric}"
    )
    return f"""\
mahout kmeans -i {input_name}/vectors/tfidf-vectors \
-c {final_clusters} \
-o hdfs://lena/user/{_USER}/{output} \
-dm org.apache.mahout.common.distance.{distance_metric} \
-cl -cd 0.1 -ow -x 20
"""

def run_kmeans(input_name, k, distance_metric, output):
    return f"""\
mahout kmeans -i {input_name}/vectors/tfidf-vectors \
-c {input_name}/kmeans-centroids-{k}-{distance_metric} \
-o hdfs://lena/user/{_USER}/{output} \
-dm org.apache.mahout.common.distance.{distance_metric} \
-cl -cd 0.1 -ow -x 20 -k {k}
"""

def get_cluster_info(
    input_name, k, distance_metric, kmeans_output, output_dir, output_format
):
    ext = output_format.replace("_", "").lower()
    ext = "txt" if ext == "text" else ext
    final_clusters = get_final_clusters(kmeans_output)
    return f"""\
mahout clusterdump \
-i {final_clusters} \
-d {input_name}/vectors/dictionary.file-0 \
-dt sequencefile \
-b 100 \
-n 20 \
-p {kmeans_output}/clusteredPoints \
-o {output_dir}/{input_name}-eval-{ext}/clusters-{k}-{distance_metric}.{ext} \
-dm org.apache.mahout.common.distance.{distance_metric} \
-of {output_format} \
--evaluate
"""

def main():
    time_start = datetime.now()

    argparser = argparse.ArgumentParser(
        description="Automatically run and manage mahout kmeans with simple user configuration files."
    )
    argparser.add_argument("conf", help="config file for choosing kmeans parameters")
    argparser.add_argument(
        "--force-upload",
        action="store_true",
        help="rerun and overwrite raw files that already exist",
    )
    argparser.add_argument(
        "--force-centroids",
        action="store_true",
        help="rerun and overwrite centroid files that already exist",
    )
    argparser.add_argument(
        "--force-canopy-eval",
        action="store_true",
        help="rerun and overwrite canopy cluster evaluation files that already exist",
    )
    argparser.add_argument(
        "--force-kmeans",
        action="store_true",
        help="rerun and overwrite cluster files that already exist",
    )
    argparser.add_argument(
        "--force-eval",
        action="store_true",
        help="rerun and overwrite evaluation files that already exist",
    )
    argparser.add_argument(
        "--skip-canopy",
        action="store_true",
        help="skip canopy clustering and evaluation",
    )
    argparser.add_argument(
        "-i",
        "--interactive",
        dest="interact",
        action="store_true",
        help="user prompt on task failure",
    )

    try:
        args = argparser.parse_args()
    except:
        argparser.print_help()
        sys.exit(0)

    print(f"Start time: {time_start}")
    config = configparser.ConfigParser()

    with open(args.conf, "r") as f:
        config.read_file(f)

    run_configs = []

    for key in config.sections():
        rc = RunConfig(
            distance_metric=config[key].get("distance_metric").strip(),
            t1=config[key].get("t1"),
            t2=config[key].get("t2"),
            k_list=[x.strip() for x in config[key].get("k_list").split(",")],
        )
        run_configs.append(rc)

    input_dir = config["DEFAULT"]["input_dir"].strip()
    input_name = input_dir.split("/")[-1]

    output_dir = config["DEFAULT"]["output_dir"].strip()

    if not is_hdfs_file(f"{input_name}/vectors") or args.force_upload:
        run_command("clean_hdfs", clean_hdfs(input_name), False)
        run_command("make_directory", make_directory(input_name), args.interact)
        run_command("copy_to_hdfs", copy_to_hdfs(input_name, input_dir), args.interact)
        run_command("make_seqfiles", make_seqfiles(input_name), args.interact)
        run_command("make_sparse_vecs", make_sparse_vecs(input_name), args.interact)

    for conf in run_configs:
        centroid_output = f"{input_name}/canopy-centroids-{conf.distance_metric}"
        canopy_output = f"{input_name}/kmeans-clusters-canopy-{conf.distance_metric}"
        if not is_hdfs_file(centroid_output) or args.force_centroids:
            run_command(
                f"get_centroids, dm: {conf.distance_metric}",
                get_centroids(
                    input_name, conf.distance_metric, conf.t1, conf.t2, centroid_output
                ),
                args.interact,
            )
        if not is_hdfs_file(canopy_output) or args.force_centroids:
            run_command(
                f"run_kmeans_canopy, dm: {conf.distance_metric}",
                run_kmeans_canopy(input_name, conf.distance_metric, canopy_output),
                args.interact,
            )
        canopy_eval_output_csv = Path(
            f"{output_dir}/{input_name}-eval-csv/clusters-canopy-{conf.distance_metric}.csv"
        )
        if (not canopy_eval_output_csv.is_file() or args.force_canopy_eval) and not args.skip_canopy:
            canopy_eval_output_csv.parent.mkdir(parents=True, exist_ok=True)
            run_command(
                f"get_cluster_info: csv, dm: {conf.distance_metric}, k: canopy",
                get_cluster_info(
                    input_name,
                    "canopy",
                    conf.distance_metric,
                    canopy_output,
                    output_dir,
                    "CSV",
                ),
                args.interact,
            )
        canopy_eval_output_txt = Path(
            f"{output_dir}/{input_name}-eval-txt/clusters-canopy-{conf.distance_metric}.txt"
        )
        if (not canopy_eval_output_txt.is_file() or args.force_canopy_eval) and not args.skip_canopy:
            canopy_eval_output_txt.parent.mkdir(parents=True, exist_ok=True)
            run_command(
                f"get_cluster_info: txt, dm: {conf.distance_metric}, k: canopy",
                get_cluster_info(
                    input_name,
                    "canopy",
                    conf.distance_metric,
                    canopy_output,
                    output_dir,
                    "TEXT",
                ),
                args.interact,
            )

        for k in conf.k_list:
            kmeans_output = f"{input_name}/kmeans-clusters-k-{k}-{conf.distance_metric}"
            if not is_hdfs_file(kmeans_output) or args.force_kmeans:
                run_command(
                    f"run_kmeans, dm: {conf.distance_metric}, k: {k}",
                    run_kmeans(input_name, k, conf.distance_metric, kmeans_output),
                    args.interact,
                )

            eval_output_txt = Path(
                f"{output_dir}/{input_name}-eval-txt/clusters-{k}-{conf.distance_metric}.txt"
            )
            if not eval_output_txt.is_file() or args.force_eval:
                eval_output_txt.parent.mkdir(parents=True, exist_ok=True)
                run_command(
                    f"get_cluster_info: txt, dm: {conf.distance_metric}, k: {k}",
                    get_cluster_info(
                        input_name,
                        k,
                        conf.distance_metric,
                        kmeans_output,
                        output_dir,
                        "TEXT",
                    ),
                    args.interact,
                )

            eval_output_csv = Path(
                f"{output_dir}/{input_name}-eval-csv/clusters-{k}-{conf.distance_metric}.csv"
            )
            if not eval_output_csv.is_file() or args.force_eval:
                eval_output_csv.parent.mkdir(parents=True, exist_ok=True)
                run_command(
                    f"get_cluster_info: csv, dm: {conf.distance_metric}, k: {k}",
                    get_cluster_info(
                        input_name,
                        k,
                        conf.distance_metric,
                        kmeans_output,
                        output_dir,
                        "CSV",
                    ),
                    args.interact,
                )

    time_end = datetime.now()
    time_elapsed = time_end - time_start

    print(f"End time: {time_end}")
    print(f"Total job time: {time_elapsed}")

if __name__ == "__main__":
    main()


# import packages
import sys
import pandas as pd
import torch
import torchvision
from torch.utils.data import DataLoader
from torchvision import transforms

# load the functions to do spatial CV
from analysis_utils.spatial_CV import split_lsms_spatial

# import the torch_framework package
from analysis_utils.torch_framework.CrossValidator import CrossValidator
from analysis_utils.torch_framework.ResNet18 import ResNet18
from analysis_utils.torch_framework.SatDataset import SatDataset
from analysis_utils.torch_framework.torch_helpers import get_agg_img_stats, get_feat_stats, get_target_stats
from analysis_utils.torch_framework.torch_helpers import standardise as torch_standardise
from analysis_utils.torch_framework.CvEvaluator import CvEvaluator

# import the helpers to demean the data and get the deltas etc.
from analysis_utils.analysis_helpers import demean_df, make_delta_df

# check if the number of command line arguments is correct
if len(sys.argv) != 4:
    print("Please provide the values for the following variables:")
    print("model_name, cv_object_name, target_var")
    print("Usage: python my_script.py model_name cv_object_name target_var")
    raise(ValueError("Incorrect number of command line arguments"))

print("Hello!")
print("Initialising Training for the Within model using Remote sensing images")
print(f"Target variable: {sys.argv[3]}")
print("This is the delta version of the target variable. Just keeps the same name.")
print("\n")

####################################################################################################
# Accept the command line arguments
####################################################################################################

# get the command line arguments
model_name = sys.argv[1]
cv_object_name = sys.argv[2]
delta_target_var = sys.argv[3]

####################################################################################################
# Set the hyper-parameters
####################################################################################################
data_type = 'RS_v2'
id_var = 'delta_id'
cv_evaluator_object_name = f"evaluator_demeaned_{cv_object_name}"

# set the random seed
spatial_cv_random_seed = 348
random_seed = 230897

# set the number of folds for k-fold CV
n_folds = 5

# share of data used
max_obs = 10000000

# set hyper-parameters
hyper_params = {
    'lr': [1e-2, 1e-3],
    'batch_size': [256],
    'alpha': [1e-1, 1e-2, 1e-3],
    'step_size': [1],
    'gamma': [0.96],
    'n_epochs': [200],
    'patience': [40]
}

# training device
device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
print(f"Training device: {device} \n")

####################################################################################################
# Set the data paths
####################################################################################################

# set the global file paths
root_data_dir = "/scratch/users/nschmid5/data_analysis/Data"
#root_data_dir = "../../Data"
# the lsms data
lsms_pth = f"{root_data_dir}/lsms/processed/labels_cluster_v1.csv"

# set the image directories
sat_img_dir = f"{root_data_dir}/satellite_imgs"

# median LS images at the cluster level
delta_img_dir = f"{sat_img_dir}/{data_type}/RS_v2_delta"
delta_stats_pth = f"{sat_img_dir}/{data_type}/RS_v2_delta_img_stats.pkl"

####################################################################################################
# Load the data
####################################################################################################
# load the LSMS data
lsms_df = pd.read_csv(lsms_pth).iloc[:max_obs, :]

cl_df = lsms_df[['cluster_id', 'country', 'lat', 'lon']].copy().drop_duplicates().reset_index(drop=True)
within_df = lsms_df[['cluster_id', 'unique_id', 'log_mean_pc_cons_usd_2017']]

# demean the df
demeaned_df = demean_df(within_df)
demeaned_df = demeaned_df.rename(columns={'unique_id': 'delta_id'})
demeaned_df = pd.merge(demeaned_df, cl_df, on='cluster_id', how='left')

# create delta df
print("Make Delta DF")
delta_df = make_delta_df(within_df)
delta_df = pd.merge(delta_df, cl_df, on='cluster_id', how='left')

# add the demeaned df to the delta df
delta_df = pd.concat([delta_df, demeaned_df]).reset_index(drop=True)

# get the fold ids from spatial CV
print("\nDividing the data into k different folds using spatial CV")
fold_ids = split_lsms_spatial(lsms_df, n_folds=n_folds, random_seed=spatial_cv_random_seed)

# extract the image statistics for the demeaned images
delta_img_stats = get_agg_img_stats(delta_stats_pth, delta_df, id_var='delta_id')
delta_feat_stats = get_feat_stats(delta_img_stats)

# get the target stats
delta_target_stats = get_target_stats(delta_df, delta_target_var)

# define the transforms
# get the data transforms for the target --> is used in the DataLoader object
delta_target_transform = transforms.Compose([
    torchvision.transforms.Lambda(
        lambda t: torch_standardise(t, delta_target_stats['mean'], delta_target_stats['std'])),
])

# get the data transform for the Landsat image (normalisation and random horizontal + vertical flips)
delta_feat_transform = torchvision.transforms.Compose(
    [torchvision.transforms.RandomVerticalFlip(.5),
     torchvision.transforms.RandomHorizontalFlip(.5),
     transforms.Normalize(delta_feat_stats['mean'], delta_feat_stats['std'])]
)

####################################################################################################
# Train the model
####################################################################################################
print("\n\n")
print("=====================================================================")
print("Initialise CROSS VALIDATION")
print("=====================================================================")

# load the data into RAM first
# this increases training times
_dat = SatDataset(delta_df,
                  delta_img_dir,
                  data_type,
                  delta_target_var,
                  id_var,
                  delta_feat_transform,
                  delta_target_transform,
                  random_seed)
_loader = DataLoader(_dat, batch_size=hyper_params['batch_size'][0], shuffle=False)
_, _ = next(iter(_loader))

# run model training
# initialise the model and the CrossValidator object
resnet18 = ResNet18(
    input_channels=4,
    use_pretrained_weights=False,
    scaled_weight_init=False,
    random_seed=random_seed
)

cv = CrossValidator(
    model_class=resnet18,
    lsms_df=delta_df,
    fold_ids=fold_ids,
    img_dir=delta_img_dir,
    data_type=data_type,
    target_var=delta_target_var,
    id_var=id_var,
    feat_transform=delta_feat_transform,
    target_transform=delta_target_transform,
    device=device,
    model_name=model_name,
    random_seed=random_seed
)

# run k-fold-cv
cv.run_cv(hyper_params=hyper_params, tune_hyper_params=True)

# save the cv object
cv.save_object(name=cv_object_name)

# output the overall performance of the model
print("\n")
print('=' * 100)
print('Cross-Validated performance:')
print('=' * 100)
print(cv.compute_overall_performance(use_fold_weights=True))
print('\n\n')

####################################################################################################
# Use the trained model to predict only the demeaned images
####################################################################################################
print("Evaluate the model on the demeaned images")
print("\n")
# get the statistics for the demeaned dataset

# extract the image statistics for the demeaned images
demeaned_img_stats = get_agg_img_stats(delta_stats_pth, demeaned_df, id_var='delta_id')
demeaned_feat_stats = get_feat_stats(demeaned_img_stats)

# get the target stats
demeaned_target_stats = get_target_stats(demeaned_df, delta_target_var)

# define the transforms
# get the data transforms for the target --> is used in the DataLoader object
demeaned_target_transform = transforms.Compose([
    torchvision.transforms.Lambda(
        lambda t: torch_standardise(t, demeaned_target_stats['mean'], demeaned_target_stats['std'])),
])

# get the data transform for the Landsat image (normalisation and random horizontal + vertical flips)
demeaned_feat_transform = torchvision.transforms.Compose([
     transforms.Normalize(demeaned_feat_stats['mean'], demeaned_feat_stats['std'])
])

# load the CvEvaluator object
cv_evaluator = CvEvaluator(
    cv_object=cv,
    lsms_df=demeaned_df,
    fold_ids=fold_ids,
    device=device
)

cv_evaluator.evaluate(
    feat_transform=demeaned_feat_transform,
    target_transform=demeaned_target_transform
)

cv_evaluator.save_object(name=cv_evaluator_object_name)

# output the overall performance of the model
print("\n")
print('=' * 100)
print('Cross-Validated performance on delta images:')
print('=' * 100)
print(cv.compute_overall_performance(use_fold_weights=True))
print('\n\n')

# output the overall performance of the model
print("\n")
print('=' * 100)
print('Cross-Validated performance on demeaned images:')
print('=' * 100)
print(cv_evaluator.compute_overall_performance(use_fold_weights=True))
print('\n\n')
print('BYE BYE')


import numpy as np

class K_Means:

	# Fonction d'initialisation : "k" : nombre de clusters, "tol" : seuil de tolérance permettant de savoir quand on arrête de mettre à jour les centres
	# "max_itération : nombre d'itération maximum pour la mise à jours des centres des clusters"
	def __init__(self, k, epoch):
		self.k = k
		self.epoch = epoch

	# Fonction d'optimisation prenant en entré le jeu de données
	def fit(self, data):

		# Initialisation du dictionnaire qui va contenir les centres, et les points associés à ces centres.
		self.centers = {}
		self.idx_centers = [[] for i in range(self.k)]

		# Initialisation des centres par défaut avant optimisation (ceux-ci auraient pû être choisis au hasard cela n'a pas d'importance)
		for i in range(self.k):
			self.centers[i] = data[i]

		# Début de la boucle d'optimisation
		for i in range(self.epoch):

			# Déclaration d'un dictionnaire auxiliaire "classifications" qui va contenir les différents point en fontions des clusters.
			self.classifications = {}

			# Initialisation du dictionnaire, la valeur d'un clé est une liste qui va contenir les points.
			for j in range(self.k):
				self.classifications[j] = []

			# Ici, pour chaque feature du jeu de donnée, on calcul la distance de cette feature par rapport à chaque centre de cluster.
			for i, feature in enumerate(data):
				distances = [np.linalg.norm(feature - self.centers[center]) for center in self.centers]
				classification = distances.index(min(distances))
				# On ajoute le point ayant la distance minimum au cluster correspondant.
				self.classifications[classification].append(feature)
				self.idx_centers[classification].append(i)

			# On garde un trace des centres des clusters avant de faire la mise à jour.
			prev_centers = dict(self.centers)

			# C'est ici que l'on met à jour la position des centres des clusters en faisant la moyen avec les coordonnées des points pour chacun des clusters.
			for classification in self.classifications:
				self.centers[classification] = np.average(self.classifications[classification], axis = 0)

			# Petit boucle permettant de décider de l'arrêt de la mise à jour à partir d'un certain seuil.
			# Ce code n'est pas indispensable au sens où l'on à déjà un nombre d'itération maximum.
			# optimized = True
			# for c in self.centers:
			# 	original_center = prev_centers[c]
			# 	new_center = self.centers[c]
			# 	if np.sum((new_center - original_center) / new_center*100.0) > self.tol:
			# 		optimized = False

			# if optimized:
			# 	break

	# Fonction de prédiction permettant de demandé au modèle une prédiction sur une donnée en particulier
	def predict(self, data):
		distances = [np.linalg.norm(data - self.centers[center]) for center in self.centers]
		classification = distances.index(min(distances))
		return classification

	def eval_k_means(self, labels):
		for cluster in self.classifications:

			classes = [0 for x in range(10)]
			# Calcul de la classe dominante
			for j in range(len(self.classifications[cluster])):
				classes[ labels[self.idx_centers[cluster][j]] ] += 1

			# On détermine quelle classe est la plus représentée
			index_of_bigger_class = 0
			max_value_of_class = classes[0]

			for j in range (1, len(classes)):
				if (classes[j] > max_value_of_class):
					index_of_bigger_class = j
					max_value_of_class = classes[j]

			error_rate = 0

			for j in range(len(self.classifications[cluster])):
				if (index_of_bigger_class != labels[self.idx_centers[cluster][j]]):
					error_rate += 1
			error_rate = error_rate / len(self.classifications[cluster])
			print("cluster [", cluster, "] (", len(self.classifications[cluster]), ") : taux erreurs = ", error_rate)

# classifier = K_Means(10)
# classifier.fit(data)
# classifier.eval_k_means(labels)


from django.db import models
from django.contrib.auth.models import User

class Profile(models.Model):
    name = models.CharField(max_length=20)
    phone = models.IntegerField(default=0)
    user = models.OneToOneField(User, on_delete=models.CASCADE, null=True, blank=True)
    user_type = models.IntegerField(choices=((1, ("Member")),(2, ("Trainer"))),default=1)

    def __str__(self):
        return self.name

class Trainer(models.Model):
    trainer= models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True,related_name="trainers")
    name = models.CharField(max_length=20,default=True)
    phone = models.IntegerField(default=0)
    age= models.IntegerField()
    GENDER_CHOICES = (
        ('M', 'Male'),
        ('F', 'Female'),
    )
    gender = models.CharField(default=1,max_length=1, choices=GENDER_CHOICES)
    experience=models.IntegerField()
    skills=models.TextField(max_length=150)
    def __str__(self):
        return self.name

class Member(models.Model):
    member= models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, related_name="member")
    trainer= models.ForeignKey(Trainer, on_delete=models.CASCADE, null=True, blank=True, related_name="trainers")
    name = models.CharField(max_length=20,default=True)
    phone = models.IntegerField(default=0)
    age= models.IntegerField()
    GENDER_CHOICES = (
        ('M', 'Male'),
        ('F', 'Female'),
    )
    gender = models.CharField(default=1,max_length=1, choices=GENDER_CHOICES)
    height=models.IntegerField()
    weight=models.IntegerField()
    def __str__(self):
        return self.gender

class Workout(models.Model):
    name = models.CharField(max_length=20)
    image = models.ImageField(upload_to="pics/", null=True, blank=True)
    def __str__(self):
        return self.name

class Session(models.Model):
    member= models.ForeignKey(Member, on_delete=models.CASCADE, null=True, blank=True)
    trainer= models.ForeignKey(Trainer, on_delete=models.CASCADE, null=True, blank=True)
    name = models.ForeignKey(Workout, on_delete=models.CASCADE, null=True, blank=True)
    date = models.DateField(default=1)
    time= models.TimeField(default=1)
    duration = models.IntegerField()
    def __str__(self):
        return self.name.name


from django.urls import path

from . import views

app_name = 'project2'

urlpatterns = [
    # path("<int:id>", views.index, name="index"),
    path('ebtda2y/', views.ebtda2y, name="ebtda2y"),

    path('ebtda2y/info/', views.info, name="info"),
    path('ebtda2y/info/add_student/', views.add_student, name="add_student"),
    path('ebtda2y/info/edit_student/<int:student_id>/', views.edit_student, name="edit_student"),

    path('ebtda2y/eftekad', views.eftekad, name="eftekad"),

    path('ebtda2y/attendance/', views.attendance_general, name="attendance_general"),
    path('ebtda2y/attendance/<str:class_number>/', views.attendance_specific, name="attendance_specific"),
    path('ebtda2y/attendance/edit/<str:form_id>', views.attendance_edit_specific, name="attendance_edit_specific"),

    path('ebtda2y/birthdays/', views.birthdays, name="birthdays"),
]


import pathlib

import pytest

import zntrack

class LoadFromDeps(zntrack.Node):
    data = zntrack.deps()
    file: pathlib.Path = zntrack.dvc.deps()

    result = zntrack.zn.outs()

    def get_data(self):
        if self.data is not None:
            return self.data
        elif self.file is not None:
            return self.file.read_text()
        else:
            raise ValueError("No data provided")

    def run(self):
        self.result = self.get_data()

class WriteData(zntrack.Node):
    data = zntrack.zn.params()
    outs = zntrack.zn.outs()

    def run(self):
        self.outs = self.data

@pytest.mark.parametrize("value", ["data", "file"])
def test_init(value):
    with pytest.raises(TypeError):
        # setting only one of the deps args raise a requirement TypeError
        LoadFromDeps(**{value: None})

@pytest.mark.parametrize("eager", [True, False])
def test_from_zn_deps(proj_path, eager):
    with zntrack.Project(proj_path) as proj:
        data = WriteData(data="Hello World")
        node = LoadFromDeps(data=data.outs, file=None)
    proj.run(eager=eager)
    if not eager:
        node.load()

    assert node.result == "Hello World"

@pytest.mark.parametrize("eager", [True, False])
def test_from_dvc_deps(proj_path, eager):
    file = proj_path / "data.txt"
    file.write_text("Hello World")
    with zntrack.Project(proj_path) as proj:
        node = LoadFromDeps(file=file, data=None)

    proj.run(eager=eager)
    if not eager:
        node.load()

    assert node.result == "Hello World"

class EmptyNodesNode(zntrack.Node):
    # we use dvc.outs to generate zntrack.json
    file = zntrack.dvc.outs(zntrack.nwd / "file.txt")
    nodes = zntrack.zn.nodes(None)
    outs = zntrack.zn.outs()

    def run(self):
        if self.nodes is None:
            self.outs = 42
        else:
            self.outs = self.nodes.value
        self.file.write_text("Hello World")

@pytest.mark.parametrize("eager", [True, False])
def test_EmptyNode(proj_path, eager):
    with zntrack.Project() as project:
        node = EmptyNodesNode()
    project.run(eager=eager)
    if not eager:
        node.load()
    assert node.outs == 42


from train import *
import matplotlib.pyplot as plt
import torch
from torch import nn
from torch import optim
from collections import OrderedDict
from torchvision import datasets, transforms, models
import numpy as np
from PIL import Image
import PIL
import json
import argparse
import seaborn as sns

def arg_parser():
    parser = argparse.ArgumentParser(description="Training Image Classifier Project")

    parser.add_argument('--save_dir', default="./checkpoint.pth", type = str)

    parser.add_argument('--gpu', action='store_true', default="gpu")

    parser.add_argument('--image', type= str, default = 'flowers/test/20/image_04910.jpg')

    parser.add_argument('--cat_file', type= str, default = 'cat_to_name.json')

    parser.add_argument('--top_k', type=int, default =5)

    args = parser.parse_args()

    return args

def load_checkpoint(filepath):

    checkpoint = torch.load(filepath)

    arch = checkpoint['architechture']

    model = models.__dict__[arch](pretrained=True)

    # stop gradients
    for param in model.parameters(): param.requires_grad = False

    model.class_to_idx = checkpoint['class_to_idx']
    model.classifier = checkpoint['classifier']
    model.load_state_dict(checkpoint['state_dict'])
    epochs = checkpoint['epochs']

    return model

def process_image(image):
    ''' Scales, crops, and normalizes a PIL image for a PyTorch model,
        returns an Numpy array
    '''
    # TODO: Process a PIL image for use in a PyTorch model
    img = PIL.Image.open(image)

    width, height = img.size
    ratio_h = float(height) / float(width)
    ratio_w = float(width) / float(height)

    if width > height:
        new_height = ratio_w * 256
        img = img.resize((256, int(new_height)), Image.ANTIALIAS)

    else:
        new_width = ratio_h * 256
        img = img.resize((int(new_width), 256), Image.ANTIALIAS)

    width, height = img.size

    left = (width/2 - 224/2)
    top = (height/2 - 224/2)
    right = (width/2 + 224/2)
    bottom = (height/2 + 224/2)

    img = img.crop((left, top, right, bottom))

    numpy_img = np.array(img)/255

    mean = [0.485, 0.456, 0.406]
    std = [0.229, 0.224, 0.225]
    numpy_img = (numpy_img-mean)/std

    numpy_img = numpy_img.transpose(2, 0, 1)

    return numpy_img

def predict(image_path, model, cat_file, top_k):
    ''' Predict the class (or classes) of an image using a trained deep learning model.

    image_path: string. Path to image, directly to image and not to folder.
    model: pytorch neural network.
    top_k: integer. The top K classes to be calculated

    returns top_probabilities(k), top_labels
    '''
    with open(cat_file, 'r') as f:
        cat_to_name = json.load(f)

    model.to("cpu")
    model.eval();

    # Convert image from numpy to torch
    image = process_image(image_path)

    torch_image = torch.from_numpy(image).type(torch.FloatTensor).to('cpu')

    torch_image = torch_image.unsqueeze (dim = 0)

    with torch.no_grad ():
        logps = model.forward(torch_image)

    outputs  = torch.exp(logps)

    top_probs, top_indices  = outputs.topk(top_k)

    top_probs = top_probs.numpy().tolist()[0]

    top_indices = top_indices.numpy().tolist()[0]

    # Convert to classes
    idx_to_class = {value: key for key, value in
                        model.class_to_idx.items()}

    top_labels = [idx_to_class[idx] for idx in top_indices]
    top_flowers = [cat_to_name[label] for label in top_labels]

    return top_probs, top_labels, top_flowers

def print_probs(top_probs, flowers):

    print("Top five classes are:\n")
    n = 0
    for x, y in zip(top_probs, flowers):
        n +=1
        print("{}. {:30} , Probability: {:.3f}%".format(n, y, x*100))

def main():

    args = arg_parser()

    device = check_gpu(args.gpu)

    model = load_checkpoint(args.save_dir)

    top_probs, top_classes, top_flowers = predict(args.image, model,
                args.cat_file, args.top_k)

    print_probs(top_probs, top_flowers)

if __name__ == "__main__":
    main()


import random
import gym
import numpy as np

env = gym.make("FrozenLake-v1")
random.seed(0)
np.random.seed(0)
env.seed(0)

print("## Frozen Lake ##")
print("Start state:")
env.render()

def play_episode(policy=None):
    policy = policy if policy else lambda _: random.randint(0, 3)
    state = env.reset()
    done = False
    states = [state]
    actions = []
    rewards = []
    while not done:
        action = policy(state)
        state, reward, done, _ = env.step(action)
        actions.append(action)
        states.append(state)
        rewards.append(reward)
    return states, actions, rewards

def main():
    q = np.zeros(shape=(env.env.nS, env.env.nA))
    n = np.zeros(shape=(env.env.nS, env.env.nA))

    successful_episodes = 1000
    ep = 1
    while ep <= successful_episodes:
        states, actions, rewards = play_episode()

        for i, a in enumerate(actions):
            s = states[i]
            g = sum(rewards[i:])
            n[s, a] += 1
            q[s, a] += 1 / n[s, a] * (g - q[s, a])

        if sum(rewards) > 0:
            print("episode:\t{}".format(ep))
            print("q-values:")
            print(q)

            #greedy_policy = lambda s: np.random.choice(np.flatnonzero(q[s, :] == q[s, :].max()))
            greedy_policy = lambda s: q[s, :].argmax()
            total_rewards = []
            for _ in range(100):
                states, actions, rewards = play_episode(policy=greedy_policy)
                total_rewards.append(sum(rewards))

            print("average greedy reward:\t{}".format(np.mean(total_rewards)))
            print()

            ep += 1

main()


import os
import httplib
from flask import Flask, request, render_template

app = Flask(__name__)

@app.route("/")
def index():
    return render_template('index.html')

@app.route("/add", methods=['POST'])
def add():
    #
    # from POST parameters
    #
    x = int(request.form['x'])
    y = int(request.form['y'])

    #
    # from Kubernetes Service(environment variables)
    #
    my_calc_host = os.environ['MY_CALC_SERVICE_SERVICE_HOST']
    my_calc_port = os.environ['MY_CALC_SERVICE_SERVICE_PORT']

    #
    # REST call to MicroService(my-calc)
    #
    client = httplib.HTTPConnection(my_calc_host, my_calc_port)
    client.request("GET", "/addition/%d/%d" % (x, y))
    response = client.getresponse()
    result = response.read()
    return render_template('index.html', add_x=x, add_y=y,
add_result=result)

if __name__ == "__main__":
    app.debug = True
    app.run(host='0.0.0.0')


import contextlib
from typing import Mapping, Sequence, Optional

from dm_env import specs as dm_env_specs
from google.protobuf import any_pb2
from google.rpc import code_pb2
from dm_env_rpc.v1 import connection as dm_env_rpc_connection
from dm_env_rpc.v1 import dm_env_rpc_pb2
from dm_env_rpc.v1 import dm_env_utils
from dm_env_rpc.v1 import error
from dm_env_rpc.v1 import tensor_utils
from dm_env_rpc.v1.extensions import properties_pb2

@contextlib.contextmanager
def _convert_dm_env_rpc_error():
  """Helper to convert DmEnvRpcError to a properties related exception."""
  try:
    yield
  except error.DmEnvRpcError as e:
    if e.code == code_pb2.NOT_FOUND:
      raise KeyError('Property key not found!') from e
    elif e.code == code_pb2.PERMISSION_DENIED:
      raise PermissionError('Property permission denied!') from e
    elif e.code == code_pb2.INVALID_ARGUMENT:
      raise ValueError('Property value error!') from e
    raise

class PropertySpec(object):
  """Class that represents a property's specification."""

  def __init__(self, property_spec_proto: properties_pb2.PropertySpec):
    """Constructs a property specification from PropertySpec proto message.

    Args:
      property_spec_proto: A properties_pb2.PropertySpec message.
    """
    self._property_spec_proto = property_spec_proto

  @property
  def key(self) -> str:
    """Return the property's key."""
    return self._property_spec_proto.spec.name

  @property
  def readable(self) -> bool:
    """Returns True if the property is readable."""
    return self._property_spec_proto.is_readable

  @property
  def writable(self) -> bool:
    """Returns True if the property is writable."""
    return self._property_spec_proto.is_writable

  @property
  def listable(self) -> bool:
    """Returns True if the property is listable."""
    return self._property_spec_proto.is_listable

  @property
  def spec(self) -> Optional[dm_env_specs.Array]:
    """Returns a dm_env spec if the property has a valid dtype.

    Returns:
      Either a dm_env spec or, if the dtype is invalid, None.
    """
    if self._property_spec_proto.spec.dtype != (
        dm_env_rpc_pb2.DataType.INVALID_DATA_TYPE):
      return dm_env_utils.tensor_spec_to_dm_env_spec(
          self._property_spec_proto.spec)
    else:
      return None

  @property
  def description(self) -> str:
    """Returns the property's description."""
    return self._property_spec_proto.description

  def __repr__(self):
    return (f'PropertySpec(key={self.key}, readable={self.readable}, '
            f'writable={self.writable}, listable={self.listable}, '
            f'spec={self.spec}, description={self.description})')

class PropertiesExtension(object):
  """Helper class for sending and receiving property requests and responses."""

  def __init__(self, connection: dm_env_rpc_connection.ConnectionType):
    """Construct extension with provided dm_env_rpc connection to the env.

    Args:
      connection: An instance of Connection already connected to a dm_env_rpc
        server.
    """
    self._connection = connection

  def __getitem__(self, key: str):
    """Alias for PropertiesExtension read function."""
    return self.read(key)

  def __setitem__(self, key: str, value) -> None:
    """Alias for PropertiesExtension write function."""
    self.write(key, value)

  def specs(self, key: str = '') -> Mapping[str, PropertySpec]:
    """Helper to return sub-properties as a dict."""
    return {
        sub_property.key: sub_property for sub_property in self.list(key)
    }

  def read(self, key: str):
    """Reads the value of a property.

    Args:
      key: A string key that represents the property to read.

    Returns:
      The value of the property, either as a scalar (float, int, string, etc.)
      or, if the response tensor has a non-empty `shape` attribute, a NumPy
      array of the payload with the correct type and shape. See
      tensor_utils.unpack for more details.
    """
    response = properties_pb2.PropertyResponse()
    packed_request = any_pb2.Any()
    packed_request.Pack(
        properties_pb2.PropertyRequest(
            read_property=properties_pb2.ReadPropertyRequest(key=key)))
    with _convert_dm_env_rpc_error():
      self._connection.send(packed_request).Unpack(response)

    return tensor_utils.unpack_tensor(response.read_property.value)

  def write(self, key: str, value) -> None:
    """Writes the provided value to a property.

    Args:
      key: A string key that represents the property to write.
      value: A scalar (float, int, string, etc.), NumPy array, or nested lists.
        See tensor_utils.pack for more details.
    """
    packed_request = any_pb2.Any()
    packed_request.Pack(
        properties_pb2.PropertyRequest(
            write_property=properties_pb2.WritePropertyRequest(
                key=key, value=tensor_utils.pack_tensor(value))))
    with _convert_dm_env_rpc_error():
      self._connection.send(packed_request)

  def list(self, key: str = '') -> Sequence[PropertySpec]:
    """Lists properties residing under the provided key.

    Args:
      key: A string key to list properties at this location. If empty, returns
        properties registered at the root level.

    Returns:
      A sequence of PropertySpecs.
    """
    response = properties_pb2.PropertyResponse()
    packed_request = any_pb2.Any()
    packed_request.Pack(
        properties_pb2.PropertyRequest(
            list_property=properties_pb2.ListPropertyRequest(key=key)))
    with _convert_dm_env_rpc_error():
      self._connection.send(packed_request).Unpack(response)

    return tuple(
        PropertySpec(sub_property)
        for sub_property in response.list_property.values)


import cv2
from mediapipe.python.solutions import hands

# Crear el detector de manos
hands = hands.Hands(min_detection_confidence=0.5, min_tracking_confidence=0.5)

# Capturar el video
cap = cv2.VideoCapture(0)

while True:
    # Capturar un fotograma
    ret, frame = cap.read()

    # Detectar las manos
    results = hands.process(frame)

    # Dibujar las manos
    hands.draw(frame, results)

    # Mostrar el video
    cv2.imshow("Video", frame)

    # Esperar a que se presione una tecla
    key = cv2.waitKey(1)

    # Cerrar el programa si se presiona la tecla `q`
    if key == ord("q"):
        break

# Cerrar la cámara
cap.release()

# Destruir todas las ventanas abiertas
cv2.destroyAllWindows()


import sqlite3

connection = sqlite3.connect('data.bd')
cursor = connection.cursor()

create_table = "create table items (id INTEGER primary key, name text, price real)"
# Al usar la palabra INTEGER (con mayuscula) hace los id auto_increments
cursor.execute(create_table)

# Insertar uno
item = ('pelota', 6.12)
insert_query = "insert into items (name, price) values (?,?)"
cursor.execute(insert_query, item)

# Insertar varios items
items = [
    ('bicicleta', 63.56),
    ('laptop', 302.64),
    ('audifonos', 12.99)
]
cursor.executemany(insert_query, items)

#  Obtener items
"""select_query = "select * from users"
cursor.execute(select_query)"""

connection.commit()
connection.close()


#!/usr/bin/env python2

import psycopg2 as pg

# Open Connection
connection = pg.connect(database="news")
cursor = connection.cursor()

# Execute and Fetch Top 3. (Table View top3art)
cursor.execute("SELECT * FROM top3artic")
top3 = cursor.fetchall()

# Check for Empty Results
if top3 is not None:
  print("Top 3 Articles:")
  for elem in top3:
    print("Article: {0}, Views: {1}".format(elem[0], elem[1]))
else:
  print("No Articles have been accessed.")

# Close cursor and connection
cursor.close()
connection.close()


#%%
import time
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D
from tensorflow.keras.callbacks import TensorBoard
import pickle

# %%

X = pickle.load(open("data/kaggle_cats_and_dogs/processed_data/X.pickle", "rb"))
y = pickle.load(open("data/kaggle_cats_and_dogs/processed_data/y.pickle", "rb"))

X = X / 255

#%%
dense_layers = [0, 1, 2]
layer_sizes = [32, 64, 128]
conv_layers = [1, 2, 3]

for dense_layer in dense_layers:
    for layer_size in layer_sizes:
        for conv_layer in conv_layers:
            NAME = f"{conv_layer}-conv-{layer_size}-nodes-{dense_layer}-dnes-{int(time.time())}"
            print(NAME)

            tensorboard = TensorBoard(log_dir='logs/{}'.format(NAME))

            model = Sequential()
            model.add(Conv2D(64, (3, 3), input_shape=X.shape[1:]))
            model.add(Activation("relu"))
            model.add(MaxPooling2D(pool_size=(2, 2)))

            for l in range(conv_layer-1):
                model.add(Conv2D(64, (3, 3)))
                model.add(Activation("relu"))
                model.add(MaxPooling2D(pool_size=(2, 2)))

            model.add(Flatten())
            for l in range(dense_layer):
                model.add(Dense(layer_size))
                model.add(Activation("relu"))

            model.add(Dense(1))
            model.add(Activation('sigmoid'))

            model.compile(loss="binary_crossentropy",
                            optimizer="adam",
                            metrics=['accuracy'])

            model.fit(X, y, batch_size=32, validation_split=0.1, epochs=10, callbacks=[tensorboard])

# %%
model.save("cats_and_dogs_classifier.model")

# %%


import numpy as np
import math
import os
from matplotlib.colors import hsv_to_rgb, rgb_to_hsv
from snake_settings import *  #background colors, etc.
from database_routines import get_connector, make_tables

from PIL import Image

"""some (mostly numpy-based) utilities for wrangling images"""

def collect_image_data(original_image_dir=None,
                      hsv_image_dir=None,
                      original_image_file=None,
                      target_background_h=None,
                      target_background_s=None,
                      target_background_v=None,
                      target_background_tolerance=None,
                      new_background_hsv=None,
                      num_colors=None,
                      show_histo=True
                      ):
    """This routine gathers 'color swatches' from an image.  For every color found in > 1%
       of the pixels in a reduced form of the image, it returns the %pixels, and the color as
       RGB, HSV, and html.  The reduced form is a mapping to a color palatte consisting of
       num_colors colors (provided by setup.py)"""

    #replaces noisy background, reduces image to num_colors palatte
    im=reduce_image(original_image_dir=original_image_dir,
                          hsv_image_dir=hsv_image_dir,
                          original_image_file=original_image_file,
                          target_background_h=target_background_h,
                          target_background_s=target_background_s,
                          target_background_v=target_background_v,
                          target_background_tolerance=target_background_tolerance,
                          new_background_hsv=new_background_hsv,
                          num_colors=num_colors
                          )

    #this is the new, uniform background color (as RGB)
    background_rgb=hsv_to_rgb(np.array((new_background_hsv)))*255
    back_r, back_g, back_b = background_rgb

    #returns a list of the colors (RGB) and % of each
    color_list=get_color_list(im=im,
                                target_background_tolerance=target_background_tolerance,
                                back_r=back_r,
                                back_g=back_g,
                                back_b=back_b
                                )

    #The color list is a list of tuples.  Each tuple contains:
    # (%pixels represented, (r, g, b))
    #
    #We'll reconstruct this to be a list of dicts with these keys:
    # pct, r, g, b, h, s, v, html

    pik_data=[]

    for pct, rgb in color_list:
        #separates tuple of values [0,255]
        r, g, b= rgb
        #calculate HSV [0,1] (these raw HSV values could be fed to hsv_to_rgb to reverse the op)
        h_raw, s_raw, v_raw=rgb_to_hsv([ r/255, g/255, b/255])
        #calculate hsv values in natural units
        h = h_raw*360  #h is in terms of degrees
        s = s_raw*100  #s is in terms of percent
        v = v_raw*100  #v is in terms of percent

        html=RGBToHTMLColor(rgb)

        pik_data.append({'pct':pct,
                         'r':r, 'g':g, 'b':b,
                         'h':h, 's':s,'v':v,
                         'html':html})

    #return the color analysis and the image used
    return {'pik_data':pik_data, 'image': im}

def reduce_image(original_image_dir=None,
                      hsv_image_dir=None,
                      original_image_file=None,
                      target_background_h=None,
                      target_background_s=None,
                      target_background_v=None,
                      target_background_tolerance=None,
                      new_background_hsv=None,
                      num_colors=None,
                      show_histo=True
                      ):
    """This routine assumes accepts an image in RGB, RGBA, or PNG format,
       removes the (potentially noisy) background, replaces it with homogenous
       background, and reduces the color palatte.  It returns the reduced version"""

    #this returns an RGB image with background flattened to a known color
    im=filter_background(original_image_dir=original_image_dir,
                              hsv_image_dir=hsv_image_dir,
                              original_image_file=original_image_file,
                              target_background_h=target_background_h,
                              target_background_s=target_background_s,
                              target_background_v=target_background_v,
                              target_background_tolerance=target_background_tolerance,
                              new_background_hsv=new_background_hsv,
                              return_rgb=True
                              )

    im=im.convert("RGBA")

    #this reduces the image to a specified number of colors, then converts it
    # back to the RGBA color model
    im=im.convert('P', palette=Image.ADAPTIVE, colors=num_colors)
    im=im.convert("RGBA")
    return im

def make_png_thumb(img=None, background=None, hsv=True):
    "creates a transparent-background thumbnail for web page display"

    data=np.array(img)
    r, g, b, a = data.T

    #if background is hsv, convert it to rgb
    if hsv:
        background = hsv_to_rgb(np.array((background)))
        back_red, back_blue, back_green = background
        back_red=back_red*255
        back_blue=back_blue*255
        back_green=back_green*255
    else:
        back_red, back_blue, back_green = background
    #this sets the masking condition and replaces the background color
    replace = (r == back_red) & (b == back_blue) & (g == back_green)

    data[replace.T] = (0,0,0,0)

    #create a thumbnail (note - need to do masking ops first because thumbnail
    #routine does some color averaging).
    new_image=Image.fromarray(data, 'RGBA')
    new_image.thumbnail((100,100))
    return new_image

def filter_background(original_image_dir=None,
                      hsv_image_dir=None,
                      original_image_file=None,
                      target_background_h=None,
                      target_background_s=None,
                      target_background_v=None,
                      target_background_tolerance=None,
                      new_background_hsv=None,
                      return_rgb=None
                      ):
    """Routine for filtering the background of an image captured in the wild.
       Converts an image using the RGB color model to HSV.  Then, using the HSV
       model, identifies and converts all 'close enough' pixels to a uniform color.
       The resulting image can then easily be manipulated to reduce the new background
       to alpha, screen out the non-background pixels, etc.

       This should allow reasonable filtering, even when the background becomes soiled,
       worn, etc.

       Returns numpy array of the HSV image"""

    with Image.open(os.path.join(original_image_dir,original_image_file)) as im:
        #this converts to the HSV color model in stages
        #first lose the alpha (if required for RGBA images)

        #If the image came in as a .png file it'll convert to black
        #  because the transparency value is set to 0. We can screen out the
        #  background with a rigorous filter, but risk messing converting "real"
        #  black to the new background color. Will not be issue with other file formats.

        if os.path.splitext(original_image_file.lower())[1]=='.png':
            target_background_h=0
            target_background_s=0
            target_background_v=0
            target_background_tolerance=0
        im=im.convert("RGB")
        #im.show()
        #convert the image to a numpy array and transform to HSV
        #  thinking here that this will be better correct for uneven (dirty) background

        data_hsv=rgb_to_hsv(np.array(im)/255) #conversion for matplotlib routine

        #split image into h, s, v channels and find pixels that are 'close enough' to background
        h, s, v=data_hsv.T
        background_areas=mask(h, target_background_h,
                              s, target_background_s,
                              v, target_background_v,
                              slop=target_background_tolerance
                              )

        #substitute a single color for what may be a variagated background
        data_hsv[background_areas.T] = new_background_hsv

        #the hsv image, conveted back to RGB for display
        data_rgb=hsv_to_rgb(data_hsv)
        data_rgb_255=(data_rgb* 255).astype(np.int8)

        im2 = Image.fromarray(data_rgb_255, "RGB")
        #im2.show()
        if not return_rgb:
            return data_hsv
        else:
            return im2

def get_color_list(im=None,
                   target_background_tolerance=None,
                   back_r=None,
                   back_g=None,
                   back_b=None):

    """Ransacks an image and returns a list containing the palatte elements and what
         pecent of the image contains each color.

         ##No more:  Hard-coded logic screens out any
         color that's less than 1% of the total image"""

    #a sorted list of the most popular colors and the number of pixels of each
    colors=sorted(im.getcolors())

    #create a new list of sorted colors, counting up total pixel count
    draft_color_list=[]
    total_pixels=0
    #first pass count pixels, not including the background color

    for count, color in colors:
        #this checks whether this color is close to the background
        r, g, b, a = color
        if r >253:
            x=1

        is_back = mask(r, back_r, g, back_g, b, back_b, slop = target_background_tolerance)

        #if it's a "real" color, catelogue # of pixels and the RGB value
        if not is_back :
            total_pixels += count
            draft_color_list.append((count, (r, g, b)))

    #revise the count to reflect a decimal version of the % each color represents
    #   skip anything under 1%
    color_list=[]

    for count, color in draft_color_list:
        pct = int(100* count/total_pixels)
        #print(count, total_pixels, pct)
        if pct >=0:
            color_list.append((pct, color))
    return color_list

def create_histogram(original_image_dir=None,
                      hsv_image_dir=None,
                      original_image_file=None,
                      target_background_h=None,
                      target_background_s=None,
                      target_background_v=None,
                      target_background_tolerance=None,
                      new_background_hsv=None,
                      num_colors=None,
                      show_histo=True
                      ):
    """This routine creates a histogram of a reduced-palatte version of an original
        RBG, RGBA, or PNG image."""

    #replaces noisy background, reduces image to num_colors palatte
    im=reduce_image(original_image_dir=original_image_dir,
                          hsv_image_dir=hsv_image_dir,
                          original_image_file=original_image_file,
                          target_background_h=target_background_h,
                          target_background_s=target_background_s,
                          target_background_v=target_background_v,
                          target_background_tolerance=target_background_tolerance,
                          new_background_hsv=new_background_hsv,
                          num_colors=num_colors
                          )

    #this is the new, uniform background color (as RGB)
    background_rgb=hsv_to_rgb(np.array((new_background_hsv)))*255
    back_r, back_g, back_b = background_rgb

    #returns a list of the colors and % of each
    color_list=get_color_list(im=im,
                                target_background_tolerance=target_background_tolerance,
                                back_r=back_r,
                                back_g=back_g,
                                back_b=back_b
                                )

    #to visulize, we'll create a new np array.  We'll load each row
    # with color values - one column per percentage in image.  This results
    # in a crude histogram, mostly for debugging.

    default=np.array((255,255,255))  #the background color of the histo
    axis=np.array((255,0,0))         #the color of a bar at the bottom (to visualize scale)

    height=20  #height of each bar (pixels)
    width=10   #width of each bar (pixels)

    #create np array, pad it out w/ default color
    histo=np.zeros((len(color_list)*height +1, 100*width, 3   ))
    histo[:,:]=default

    #fill in color-coded bars (each the color of the pixels represented)
    rix=-1  #row index
    cix=-1  #column index
    for count, color in color_list:
        x=1
        for _ in range(height): #height of each bar
            rix+=1
            for _ in range(count):
                for _ in range(width):
                    cix+=1
                    histo[rix, cix,:]= color
            cix=-1
            a=1

    #add a bar to stake out 100%
    rix+=1
    for _ in range(100*width):
        histo[rix, cix,:]= axis
        cix+=1

    im3=Image.fromarray(histo.astype(np.int8), "RGB")
    if show_histo:
        im3.show()
    return im3

def mask(c1, c1_target, c2, c2_target, c3, c3_target, slop=0):
    "given three channels return a mask for nearly-matching values"
    #
    mask= ((c1 >= c1_target - slop) & (c1 <= c1_target + slop)) &\
          ((c2 >= c2_target - slop) & (c2 <= c2_target + slop)) &\
          ((c3 >= c3_target - slop) & (c3 <= c3_target + slop))
    return mask

def RGBToHTMLColor(rgb_tuple):
    """ convert an (R, G, B) tuple to #RRGGBB """
    #http://code.activestate.com/recipes/266466-html-colors-tofrom-rgb-tuples/
    hexcolor = '#%02x%02x%02x' % rgb_tuple
    # that's it! '%02x' means zero-padded, 2-digit hex values
    return hexcolor

def stg2tup(stg):
    "convert a sting like '123,456,789' to a tuple like (123, 456, 789)"
    return tuple([int(i) for i in stg.split(',')] )

def tup2stg(tup):
    "convert a tuple like (123, 456, 789) to a string like '123, 456, 789'"
    return  ', '.join([str(i) for i in tup])

def add_image_to_database(original_image_dir=None,
                          hsv_image_dir=None,
                          original_image_file=None,
                          target_background_h=None,
                          target_background_s=None,
                          target_background_v=None,
                          target_background_tolerance=None,
                          new_background_hsv=None,
                          num_colors=None,
                          db=None,
                          source='dummy_source',
                          species='monty_python',
                          identifying_authority='None',
                          ):

        '''Adds an image to the database, after first having removed the background
              color'''
        conn=get_connector(db=db)
        curs = conn.cursor()

        #get RGB, HSV, and html colors from a reduced-color form of
        #  the image (background, number of colors, etc come from snake_settings.py)
        img_dict=collect_image_data(original_image_dir=original_image_dir,
                          hsv_image_dir=hsv_image_dir,
                          original_image_file=original_image_file,
                          target_background_h=target_background_h,
                          target_background_s=target_background_s,
                          target_background_v=target_background_v,
                          target_background_tolerance=target_background_tolerance,
                          new_background_hsv=new_background_hsv,
                          num_colors=num_colors
                          )

        #convert Image object to thumbnail
        img=make_png_thumb(img_dict['image'], new_background_hsv)
        if not os.path.exists(hsv_image_dir):
                os.mkdir(hsv_image_dir)

        thumb_filename=os.path.join(hsv_image_dir,
                                  os.path.splitext(original_image_file)[0] +
                                     "_thumb.png"
                                     )
        img.save(thumb_filename)

        sql="""INSERT INTO snake_info  (thumb_file, species, img_file, identifying_authority, source, web_image_file)
                    VALUES('{}','{}','{}','{}','{}', '{}')
                    RETURNING id"""\
                .format(thumb_filename,
                        species,
                        os.path.join(original_image_dir, original_image_file),
                        identifying_authority,
                        source,
                        original_image_file)

        curs.execute(sql)
        snake_id = curs.fetchone()[0]

        ##TODO: local file name; make relative for web app
        for color in img_dict['pik_data']:
                sql="""INSERT INTO snake_colors  (pct, r, g, b, h, s, v, html, snake_id)
                            VALUES({}, {}, {},{},{},{},{}, '{}', {})""".format(
                            color['pct'],
                            color['r'],
                            color['g'],
                            color['b'],
                            color['h'],
                            color['s'],
                            color['v'],
                            color['html'],
                            snake_id)

                curs.execute(sql)
        conn.commit()

def add_directory_to_database(original_image_dir=None, hsv_image_dir=None, db='snakes') :
        """Adds all the images in a directory to the database (removing background, etc.)
           This includes removing the background, etc.  If employs specific file-naming
           conventions to parse the species out of a filename like:
           burmese_01.png"""
        for f in os.listdir(original_image_dir):
                #f='indigo_1.png'
                if os.path.isfile(os.path.join(original_image_dir, f)):
                        #species is the filename less the trailing '_##.png' bit
                        species=' '.join(os.path.splitext(f)[0].split('_')[:-1])
                        original_image_file=f
                        add_image_to_database(original_image_dir=original_image_dir,
                                                  hsv_image_dir=hsv_image_dir,
                                                  original_image_file=original_image_file,
                                                  target_background_h=target_background_h,
                                                  target_background_s=target_background_s,
                                                  target_background_v=target_background_v,
                                                  target_background_tolerance=target_background_tolerance,
                                                  new_background_hsv=new_background_hsv,
                                                  num_colors=num_colors,
                                                  db='snakes',
                                                  source='dummy_source',
                                                  species=species,
                                                  identifying_authority='None',
                                              )

if __name__=='__main__':
    assert stg2tup('123, 456, 789')==(123, 456, 789)
    assert stg2tup('123,456,789')==(123, 456, 789)
    assert tup2stg((123, 456, 789))=='123, 456, 789'
    print('Yea! Tests passed')


import math
fig, axes = plt.subplots(2, 2, figsize=(15, 10))
colors = ['blue', 'red', 'green', 'orange']

for i, loc in enumerate(hogan_daily.hogan_loc.unique()):
    if i < 2: ax = axes[0, i]
    else: ax = axes[1, i-2]
    n_invoices = hogan_daily.loc[hogan_daily.hogan_loc == loc, 'invoice_number']
    ax.hist(n_invoices, bins=int(n_invoices.max()), label='Number of Invoices',
           alpha=.7, color=colors[i])
    ax.set_xlabel('Bins of Number of Invoices')
    ax.set_ylabel('Observations in Bins')
    ax.set_title(str(loc))
    ax.grid(alpha=.2)
    xint = range(0, math.ceil(max(n_invoices)+1))
    ax.set_xticks(xint)
    sns.despine()
sns.set_style('whitegrid')
plt.show()

#hogan_daily.groupby('hogan_loc')['invoice_number'].hist(figsize=(13, 5))


import time
from adafruit_circuitplayground.express import cpx

# Maximum temperature, in degrees C
MAX_TEMP = 24

# Minimum temperature, in degrees C
MIN_TEMP = 14

# NeoPixel Colors
RED = (255, 0, 0)
GREEN = (0, 255, 0)
BLUE = (0, 0, 255)

while True:
    # Take the temperature
    temp = cpx.temperature

    # Evaluate the temperature
    if temp > MAX_TEMP:
        cpx.play_tone(587, 5)
        cpx.pixels.fill(RED)
    elif temp < MIN_TEMP:
        cpx.play_tone(494, 5)
        cpx.pixels.fill(BLUE)
    else:
        cpx.pixels.fill(GREEN)

    # Let's wait a second before taking the temp again
    time.sleep(1)