################################## Import Libraries ########################
from sklearn.model_selection import train_test_split
from tensorflow.keras.utils import get_file
from sklearn import metrics
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation
from tensorflow.keras.utils import get_file
from tensorflow import keras, data
from tensorflow.keras import layers, regularizers, activations
from tensorflow.keras.layers import Bidirectional
from tensorflow.keras import backend as K
import tensorflow_probability as tfp
import numpy as np
import pandas as pd
import os
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
import sys

os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
np.random.seed(0)
tf.random.set_seed(0)

#################
# Set Parameters
#################
row_mark = 225
batch_size = 1
time_step = 1
lstm_h_dim = 6
z_dim = 4
epoch_num = 32
threshold = 1

learning_rate = 1e-5
alpha = 0.8
beta = 0.2

mode = 'train'
model_dir = "./lstm_vae_model/"
image_dir = "./lstm_vae_images/"

#################
# Split Dataset
#################

def split_normalize_data(all_df):
    train_df = all_df[:row_mark]
    test_df = all_df[row_mark:]

    scaler = MinMaxScaler()
    scaler.fit(np.array(all_df)[:, 1:])
    train_scaled = scaler.transform(np.array(train_df)[:, 1:])
    test_scaled = scaler.transform(np.array(test_df)[:, 1:])
    return train_scaled, test_scaled

#################
# Reshape Function
#################

def reshape(da):
    return da.reshape(da.shape[0], time_step, da.shape[1]).astype("float32")

#################
# Reparametrization
#################

class Sampling(layers.Layer):
    def __init__(self, name='sampling_z'):
        super(Sampling, self).__init__(name=name)

    def call(self, inputs, **kwargs):
        mu, logvar = inputs
        print('mu: ', mu)
        sigma = K.exp(logvar * 0.5)
        epsilon = K.random_normal(shape=(mu.shape[0], z_dim), mean=0.0, stddev=1.0)
        z = mu + epsilon * sigma
        return z

    def get_config(self):
        config = super(Sampling, self).get_config()
        config.update({'name': self.name})
        return config

#################
# Encoder
#################

class Encoder(layers.Layer):
    def __init__(self, time_step, x_dim, lstm_h_dim, z_dim, name='encoder', **kwargs):
        super(Encoder, self).__init__(name=name, **kwargs)

        self.encoder_inputs = keras.Input(shape=(time_step, x_dim))
        self.encoder_lstm = Bidirectional(
            layers.LSTM(lstm_h_dim, activation='softplus', name='encoder_lstm', stateful=True))
        self.z_mean = layers.Dense(z_dim, name='z_mean')
        self.z_logvar = layers.Dense(z_dim, name='z_log_var')
        self.z_sample = Sampling()

    def call(self, inputs, **kwargs):
        self.encoder_inputs = inputs
        hidden = self.encoder_lstm(self.encoder_inputs)
        mu_z = self.z_mean(hidden)
        logvar_z = self.z_logvar(hidden)
        z = self.z_sample((mu_z, logvar_z))
        return mu_z, logvar_z, z

    def get_config(self):
        config = super(Encoder, self).get_config()
        config.update({
            'name': self.name,
            'z_sample': self.z_sample.get_config()
        })
        return config

#################
# Decoder
#################

class Decoder(layers.Layer):
    def __init__(self, time_step, x_dim, lstm_h_dim, z_dim, name='decoder', **kwargs):
        super(Decoder, self).__init__(name=name, **kwargs)

        self.z_inputs = layers.RepeatVector(time_step, name='repeat_vector')
        self.decoder_lstm_hidden = Bidirectional(layers.LSTM(lstm_h_dim, activation='softplus', return_sequences=True,
                                                             name='decoder_lstm'))
        self.x_mean = layers.Dense(x_dim, name='x_mean')
        self.x_sigma = layers.Dense(x_dim, name='x_sigma', activation='tanh')

    def call(self, inputs, **kwargs):
        z = self.z_inputs(inputs)
        hidden = self.decoder_lstm_hidden(z)
        mu_x = self.x_mean(hidden)
        sigma_x = self.x_sigma(hidden)
        x_dash = self.sample_x(mu_x, sigma_x)
        return mu_x, sigma_x, x_dash

    def get_config(self):
        config = super(Decoder, self).get_config()
        config.update({
            'name': self.name
        })
        return config

    def sample_x(self, mu_x, sigma_x):
        tfd = tfp.distributions
        x_dist = tfd.Normal(loc=mu_x, scale=tf.abs(sigma_x))
        x = x_dist.sample([1])
        x = tf.squeeze(x, axis=0)
        return x

#################
# VAE WITH ReEncoder
#################

loss_metric = keras.metrics.Mean(name='loss')
likelihood_metric = keras.metrics.Mean(name='log likelihood')

class VAER(keras.Model):
    def __init__(self, time_step, x_dim, lstm_h_dim, z_dim, name='vae_with_reencoder', **kwargs):
        super(VAER, self).__init__(name=name, **kwargs)

        self.encoder = Encoder(time_step, x_dim, lstm_h_dim, z_dim, **kwargs)
        self.decoder = Decoder(time_step, x_dim, lstm_h_dim, z_dim, **kwargs)

        #############
        # extension code
        #############
        self.re_encoder = Encoder(time_step, x_dim, lstm_h_dim, z_dim, name="re_encoder", **kwargs)

    def call(self, inputs, **kwargs):
        mu_z, logvar_z, z = self.encoder(inputs)
        mu_x, sigma_x, x_dash = self.decoder(z)
        mu_re_z, logvar_re_z, re_z = self.re_encoder(x_dash)

        recons_loss = self.l2_NORM(inputs, x_dash)
        kl_loss_1 = self.kl_loss(logvar_z,mu_z)
        kl_loss_2 = self.kl_loss(logvar_re_z,mu_re_z)
        latent_loss = self.l2_NORM(z, re_z)
        total_loss = recons_loss + kl_loss_1 + kl_loss_2 + latent_loss

        anomaly_score = self.anomaly_score(inputs, x_dash, z, re_z)
        dist = tfp.distributions.Normal(loc=mu_x, scale=tf.abs(sigma_x))
        log_px = -dist.log_prob(inputs)

        return anomaly_score,total_loss, log_px

    def get_config(self):
        config = {
            'encoder': self.encoder.get_config(),
            'decoder': self.decoder.get_config(),
            'name': self.name
        }
        return config

    def l2_NORM(self, x, x_dash):
        recons_loss = tf.norm(x - x_dash, axis=1)
        return K.mean(recons_loss)

    def kl_loss(self, logvar_z, mu_z):
        kl_loss = -0.5 * (1 + logvar_z - tf.square(mu_z) - tf.exp(logvar_z))
        kl_loss = tf.reduce_mean(tf.reduce_sum(kl_loss, axis=1))
        return kl_loss

    def mean_log_likelihood(self, log_px):
        log_px = K.reshape(log_px, shape=(log_px.shape[0], log_px.shape[2]))
        mean_log_px = K.mean(log_px, axis=1)
        return K.mean(mean_log_px, axis=0)

    def anomaly_score(self, x, x_dash, z_dash, re_z_dash):
        a1 = tf.squeeze(tf.norm(x - x_dash, ord=1, axis=1))
        a2 = tf.norm(z_dash - re_z_dash, ord=1, axis=0)
        ax = alpha * a1 + beta * a2
        ax_mean = K.mean(ax)
        return ax_mean

    def train_step(self, data):
        if isinstance(data, tuple):
            x = data[0]
        else:
            x = data

        with tf.GradientTape() as tape:
            anomaly_score,loss, log_px = self(x, training=True)
            mean_log_px = self.mean_log_likelihood(log_px)

        grads = tape.gradient(loss, self.trainable_variables)
        self.optimizer.apply_gradients(zip(grads, self.trainable_variables))

        loss_metric.update_state(loss)
        likelihood_metric.update_state(mean_log_px)
        return {'loss': loss_metric.result(), 'log_likelihood': likelihood_metric.result()}

def plot_loss_moment(history):
    _, ax = plt.subplots(figsize=(14, 6), dpi=80)
    ax.plot(history['loss'], 'blue', label='Loss', linewidth=1)
    ax.plot(history['log_likelihood'], 'red', label='Log likelihood', linewidth=1)
    ax.set_title('Loss and log likelihood over epochs')
    ax.set_ylabel('Loss and log likelihood')
    ax.set_xlabel('Epoch')
    ax.legend(loc='upper right')
    plt.savefig(image_dir + 'loss_lstm_vae_' + mode + '.png')

def plot_log_likelihood_train(df_log_px):
    plt.figure(figsize=(14, 6), dpi=80)
    plt.title("Log likelihood")
    sns.set_color_codes()
    sns.distplot(df_log_px, bins=40, kde=True, rug=True, color='blue')
    plt.savefig(image_dir + 'log_likelihood_train' + '.png')

def plot_log_likelihood_test(df_log_px):
    plt.figure(figsize=(14, 6), dpi=80)
    plt.title("Log likelihood")
    sns.set_color_codes()
    sns.distplot(df_log_px, bins=40, kde=True, rug=True, color='blue')
    plt.savefig(image_dir + 'log_likelihood_test' + '.png')

def plot_anomaly_train_score(df_anomaly):
    plt.figure(figsize=(14, 6), dpi=80)
    plt.title("Train Anomaly Score")
    plt.plot(df_anomaly)
    plt.savefig(image_dir + 'anomaly_score_train' + '.png')

def plot_anomaly_test_score(df_anomaly):
    plt.figure(figsize=(14, 6), dpi=80)
    plt.title("Test Anomaly Score")
    plt.plot(df_anomaly)
    plt.savefig(image_dir + 'anomaly_score_test' + '.png')

def save_model(model):
    with open(model_dir + 'lstm_vae.json', 'w') as f:
        f.write(model.to_json())
    model.save_weights(model_dir + 'lstm_vae_ckpt')

def load_model():
    lstm_vae_obj = {'Encoder': Encoder, 'Decoder': Decoder, 'Sampling': Sampling}
    with keras.utils.custom_object_scope(lstm_vae_obj):
        with open(model_dir + 'lstm_vae.json', 'r'):
            model = keras.models.model_from_json(model_dir + 'lstm_vae.json')
        model.load_weights(model_dir + 'lstem_vae_ckpt')
    return model

def main():
    try:
        dataset = pd.read_csv("./dataset/Bearing_dataset.csv")
        print("Dataset shape: ", dataset.shape)
    except Exception:
        print("Dataset not found")

    all_df = pd.DataFrame(dataset)
    train_scaled, test_scaled = split_normalize_data(all_df)
    x_dim = train_scaled.shape[1]
    print("train and test data shape after scaling: ", train_scaled.shape, test_scaled.shape)

    train_X = reshape(train_scaled)
    test_X = reshape(test_scaled)

    opt = keras.optimizers.Adam(learning_rate=learning_rate, epsilon=1e-6, amsgrad=True)

    if mode == "train":
        model = VAER(time_step, x_dim, lstm_h_dim, z_dim, dtype='float32')
        model.compile(optimizer=opt)
        train_dataset = data.Dataset.from_tensor_slices(train_X)
        train_dataset = train_dataset.shuffle(buffer_size=1024).batch(batch_size, drop_remainder=True)

        history = model.fit(train_dataset, epochs=epoch_num, shuffle=False).history
        model.summary()
        plot_loss_moment(history)
        save_model(model)
    elif mode == "infer":
        model = load_model()
        model.compile(optimizer=opt)
    else:
        print("Unknown mode: ", mode)
        exit(1)

    anomaly_score_train,loss_train,train_log_px = model.predict(train_X, batch_size=1)
    train_log_px = train_log_px.reshape(train_log_px.shape[0], train_log_px.shape[2])
    df_train_log_px = pd.DataFrame()
    df_train_log_px['log_px'] = np.mean(train_log_px, axis=1)
    plot_log_likelihood_train(df_train_log_px)

    df_train_anomaly = pd.DataFrame()
    df_train_anomaly['train_anomaly'] = anomaly_score_train
    plot_anomaly_train_score(df_train_anomaly)

    anomaly_score_test,loss_test,test_log_px = model.predict(test_X, batch_size=1)
    test_log_px = test_log_px.reshape(test_log_px.shape[0], test_log_px.shape[2])
    df_test_log_px = pd.DataFrame()
    df_test_log_px['log_px'] = np.mean(test_log_px, axis=1)
    plot_log_likelihood_test(df_test_log_px)

    df_log_px = pd.DataFrame()
    df_log_px['log_px'] = np.mean(test_log_px, axis=1)
    df_log_px = pd.concat([df_train_log_px, df_log_px])
    df_log_px['threshold'] = threshold
    df_log_px['anomaly'] = df_log_px['log_px'] > df_log_px['threshold']
    df_log_px.index = np.array(all_df)[:, 0]

    df_log_px.plot(logy=True, figsize=(16, 9), color=['blue', 'red'])
    plt.savefig(image_dir + 'anomaly_lstm_vae_train_and_test' + '.png')

    df_test_anomaly = pd.DataFrame()
    df_test_anomaly['test_anomaly'] = anomaly_score_test
    plot_anomaly_test_score(df_test_anomaly)

if __name__ == "__main__":
    main()


"""
作为一位web开发者， 懂得怎样去规划一个页面的尺寸是很重要的。 现给定一个具体的矩形页面面积，你的任务是设计一个长度为 L 和宽度为 W 且满足以下要求的矩形的页面。要求：

1. 你设计的矩形页面必须等于给定的目标面积。

2. 宽度 W 不应大于长度 L，换言之，要求 L >= W 。

3. 长度 L 和宽度 W 之间的差距应当尽可能小。
你需要按顺序输出你设计的页面的长度 L 和宽度 W。

示例：

输入: 4
输出: [2, 2]
解释: 目标面积是 4， 所有可能的构造方案有 [1,4], [2,2], [4,1]。
但是根据要求2，[1,4] 不符合要求; 根据要求3，[2,2] 比 [4,1] 更能符合要求. 所以输出长度 L 为 2， 宽度 W 为 2。
说明:

给定的面积不大于 10,000,000 且为正整数。
你设计的页面的长度和宽度必须都是正整数。
"""
from typing import List

class Solution:
    def constructRectangle(self, area: int) -> List[int]:
        for w in range(int(area ** 0.5), 0, -1):
            if area % w == 0:
                return [area // w, w]

if __name__ == '__main__':
    area = 4
    sol = Solution()
    print(sol.constructRectangle(area))


import pygame

NEGRO = (0, 0, 0)
BLANCO = (255, 255, 255)
FONDO = (24, 25, 30)
AZUL = (20, 80, 240)
SELECCIONA = (220, 220, 0)
DIMENCIONES = (600, 600)
LETRAS = ["A", "B", "C", "D", "E", "F", "G", "H"]

def dibujarTablero(screen, dimension, p_inicio, tamanio_fuente, fuente, seleccion):
    color = 0
    for i in range(8):
        for j in range(8):
            x = i * dimension + p_inicio[0]
            y = j * dimension + p_inicio[1]
            if color % 2 == 0:
                pygame.draw.rect(screen, BLANCO, [x, y, dimension, dimension], 0)
            else:
                pygame.draw.rect(screen, NEGRO, [x, y, dimension, dimension], 0)
            if seleccion[0] == LETRAS[i] and j == seleccion[1] - 1:
                pygame.draw.rect(screen, SELECCIONA, [x, y, dimension, dimension], 0)
            color += 1
        color += 1
        dibujarTexto(screen, LETRAS[i], [i * dimension + p_inicio[0], p_inicio[1] - tamanio_fuente], fuente)
        dibujarTexto(screen, str(i + 1), [p_inicio[0] - tamanio_fuente, i * dimension + p_inicio[1]], fuente)

def dibujarTexto(screen, texto, posicion, fuente):
    Texto = fuente.render(texto, 1, BLANCO)
    screen.blit(Texto, posicion)

def ajustarMedidas(tamanio_fuente):
    if DIMENCIONES[1] < DIMENCIONES[0]:
        ancho = int((DIMENCIONES[1] - (tamanio_fuente * 2)) / 8)
        inicio = ((DIMENCIONES[0] - DIMENCIONES[1]) / 2) + tamanio_fuente, tamanio_fuente
    else:
        ancho = int((DIMENCIONES[0] - (tamanio_fuente * 2)) / 8)
        inicio = tamanio_fuente, ((DIMENCIONES[0] - DIMENCIONES[1]) / 2) + tamanio_fuente
    return [inicio, ancho]

def obetnerPosicion(mouse, dimension, p_inicio, actual):
    rx, ry = mouse[0], mouse[1]
    for i in range(8):
        for j in range(8):
            x = j * dimension + p_inicio[0]
            y = j * dimension + p_inicio[1]
            if (rx >= x) and (rx <= x + dimension) and (ry >= y) and (ry <= y + dimension):
                actual = [LETRAS[i], j + 1]
    return actual

def main():
    pygame.init()
    screen = pygame.display.set_mode(DIMENCIONES)
    pygame.display.set_caption("MOBIBLE CHESS")
    game_over = False
    clock = pygame.time.Clock()
    tamanio_fuente = 30
    seleccion = ["Z", -1]
    fuente = pygame.font.Font(None, tamanio_fuente)
    puntoInicio, dimension = ajustarMedidas(tamanio_fuente)
    while game_over is False:
        for evento in pygame.event.get():
            if evento.type == pygame.QUIT:
                game_over = True
        botones = pygame.mouse.get_pressed()
        if botones[0]:
            pos = pygame.mouse.get_pos()
            seleccion = obetnerPosicion(pos, dimension, puntoInicio, seleccion)
        screen.fill(FONDO)
        dibujarTablero(screen, dimension, puntoInicio, tamanio_fuente, fuente, seleccion)
        pygame.display.flip()
        clock.tick(60)
    pygame.quit()

if __name__ == '__main__':
    main()


import math
import re
from decimal import Decimal

from benford.conf import DEFAULT_BASE
from benford.exceptions import NoSignificantDigitFound
from benford.utils import calc_percentage, round_decimal

EXPECTED_BENFORD_LAW_DISTRIBUTION = {
    1: Decimal('30.1'),
    2: Decimal('17.6'),
    3: Decimal('12.5'),
    4: Decimal('9.7'),
    5: Decimal('7.9'),
    6: Decimal('6.7'),
    7: Decimal('5.8'),
    8: Decimal('5.1'),
    9: Decimal('4.6'),
}

def get_expected_distribution_flat(base=DEFAULT_BASE):
    return [get_expected_distribution(i + 1) for i in range(base - 1)]

def get_expected_distribution(digit, base=DEFAULT_BASE):
    """
    Calculates probability of occurence of `digit` as first digit in a number
    for a given base (decimal as default).

    Implementation based on:
    https://en.wikipedia.org/wiki/Benford%27s_law#Benford's_law_in_other_bases

    :param digit: First significant digit to be checked.
    :param base: Base to calculate for.
    :return:
    """
    assert base >= 2, 'Base must be greater or equal 2'
    return round_decimal(100 * math.log(1 + 1 / digit, base), 1)

re_first_sig_digit = re.compile(r'[1-9]')

def get_first_significant_digit(value) -> int:
    string = str(value)
    match = re_first_sig_digit.search(string)
    if match and match.group():
        return int(match[0])
    raise NoSignificantDigitFound(value)

def map_significant_digits(samples: iter):
    return map(lambda x: get_first_significant_digit(x), samples)

def count_occurences(samples: list, search_for_values=None) -> dict:
    """
    Counts occurences of values in samples.
    """
    result = {}
    values_to_count = search_for_values if search_for_values else set(samples)
    for v in values_to_count:
        result[v] = samples.count(v)
    return result

def count_occurences_with_percentage(occurences: dict, decimal_places: int = 1):
    result = {}
    percent_remaining = Decimal('100')
    total_occurences = sum(occurences.values())
    k = None

    for k, v in occurences.items():
        v_percent = calc_percentage(v, total_occurences, decimal_places)
        if v_percent > percent_remaining:
            v_percent = percent_remaining
        result[k] = v_percent
        percent_remaining -= v_percent
    if percent_remaining != 0:
        result[k] += percent_remaining
        percent_remaining = 0

    assert percent_remaining == 0, 'Sum of occurence percentages is not 100%.'
    return result

def get_degrees_of_freedom_for_base(base):
    return base - 1


import re
def number1(x):
    return bool(re.match('^[+-]?\d*(\.)?\d*$',x))

def number2(y):
    return bool(re.match('^[+-]?\d*(\.)?\d*$',y))

def function(n):
    return bool(re.match('^[1-4]{1}$',n))

def input_func():
    n = input('Оберіть дію (1-додавання, 2-віднімання, 3-множення, 4-ділення): ')
    while not function(n):
        n = input('Помилка! Спробуйте знову: ')
    x = input('Введіть перше число: ')
    while not number1(x):
        x = input('Помилка! Спробуйте знову: ')
    y = input('Введіть друге число: ')
    while not number2(y):
        y = input('Помилка! Спробуйте знову: ')
    return calculator(n,x,y)

def calculator(n,x,y):
    n = int(n)
    x = float(x)
    y = float(y)
    if n == 1:
        return x + y
    elif n == 2:
        return x - y
    elif n == 3:
        return x * y
    else:
        if y == 0:
            return 'На нуль ділити не можна!'
        else:
            return x / y

print('Результат: ',input_func())


import json
import xml.etree.ElementTree as etree

def parse_by_key(filename_, k):
    """ Принимает имя файла и слово для ключа, в зависимости от расширения в имени файла парсит соответствующим образом.
    Возвращает список слов из значений ключей(тегов) по переданному слову для ключа"""
    ext = filename_.split('.')[1]
    lst = []
    if ext == 'json':
        with open(filename_, encoding='UTF-8') as fl:
            from_json = json.load(fl)
            search_recursive_and_split(from_json, k, lst)
    if ext == 'xml':
        tree = etree.parse(filename_.encode('cp1251'))
        from_xml = tree.findall(f'.//{k}')
        for item in from_xml:
            lst += item.text.split()
    if ext == 'txt':
        with open(filename_, encoding='UTF-8') as fl:
            for line in fl.readlines():
                lst += line.split()
    return lst

def search_recursive_and_split(obj, k, lst):
    """ Принимает словарь, имя для ключа, список. Рекурсивно ищет по словарям и спискам в исходном словаре
    значения с ключами по имени. К переданному списку добавляет все найденые значения разбытые на отдельные слова """
    if isinstance(obj, dict):
        for key, value in obj.items():
            if key == k:
                lst += value.split()
            search_recursive_and_split(value, k, lst)
    elif isinstance(obj, list):
        for item in obj:
            search_recursive_and_split(item, k, lst)

def extract_top_ten(list_for_extraction):
    """ Выбираем из списка слова длиннее 6 букв, сортируем по длине слов, выводим первые десять неповоторяющихся слов"""
    list_for_extraction = [word for word in list_for_extraction if len(word) > 6]
    list_for_extraction = sorted(list_for_extraction, reverse=True, key=lambda x: list_for_extraction.count(x))
    counter = 1
    top_ten = []
    for word in list_for_extraction:
        if word not in top_ten:
            top_ten.append(word)
            print(f'{counter}. {word}')
            counter += 1
        if counter > 10:
            break
    return top_ten

file_names = ['newsafr.txt', 'newsafr.json', 'newsafr.xml']
for filename in file_names:
    print(f'Топ 10 самых часто встречающихся в новостях слов длиннее 6 символов для {filename}')
    top_ten_list = parse_by_key(f'files/{filename}', 'title') + (
        parse_by_key(f'files/{filename}', 'description') if filename.split('.')[1] != 'txt' else [])
    extract_top_ten(top_ten_list)


import Read_utils as RU
from mlp import MLP
import numpy as np

Train_labels, Train_text_vec = RU.get_label_and_datavec('data/train_vec.txt')
val_labels, val_text_vec = RU.get_label_and_datavec('data/val_vec.txt')

if __name__ == '__main__':
    # tr_data = pd.read_csv('mnist_train.csv', header=None)
    # val_data = pd.read_csv('mnist_test.csv', header=None)
    # tr_data = tr_data.values
    # val_data = val_data.values
    # tr_x = tr_data[:, 1:]
    # tr_y = tr_data[:, 0]
    # val_x = val_data[:, 1:]
    # val_y = val_data[:, 0]
    # tr_x = tr_x / 255
    # val_x = val_x / 255
    tr_x = Train_text_vec
    tr_y = Train_labels
    # print(type(tr_x))
    # for i in range(5000):
    #     print(tr_x[5][i])
    val_x = val_text_vec
    val_y = val_labels

    print(tr_x.shape, tr_y.shape, val_x.shape, val_y.shape)

    # print(type(tr_x[0][0]))
    tr_x = tr_x.astype(float)
    # for i in range(200):
    #     print(tr_x[0][i])

    tr_y = tr_y.astype(int)
    val_x = val_x.astype(float)
    val_y = val_y.astype(int)

    tr_y_onehot = np.zeros((tr_y.shape[0], 2))
    # print(tr_y_onehot)
    for i, y in enumerate(tr_y):
        tr_y_onehot[i][y] = 1
    # print (tr_y_onehot)

    mlp = MLP(in_size=200, out_size=2, hidden_sizes=(200,))
    lr = 1e-2
    momentum = 0
    epochs = 20
    mlp.fit(tr_x, tr_y_onehot, val_x=val_x, val_y=val_y,
            batch_size=256, epochs=epochs, lr=lr, momentum=momentum)
    mlp.save_weights('model/mlp_weights.model')


# Copyright (c) Microsoft Corporation
# Licensed under the MIT License.

"""Defines the dashboard class."""

from rai_core_flask import FlaskHelper  # , environment_detector
from IPython.display import display, HTML
import json
import os
from html.parser import HTMLParser
import uuid
from .explanation_constants import ExplanationDashboardInterface

class InLineScript(HTMLParser):
    def __init__(self, load_widget_file):
        HTMLParser.__init__(self)
        self.content = ""
        self.load_widget_file = load_widget_file

    def handle_starttag(self, tag, attrs):
        if tag == "script":
            src = None
            for att in attrs:
                if att[0] == "src":
                    src = att[1]
                    break
            if src is not None:
                content = self.load_widget_file(src)
                self.content += f'<script>\r\n{content}\r\n'
                return
        self.content += self.get_starttag_text()

    def handle_endtag(self, tag):
        self.content += f'</{tag}>'
        pass

    def handle_data(self, data):
        self.content += data
        pass

class Dashboard(object):
    """The dashboard class, wraps the dashboard component."""

    def __init__(self, *,
                 dashboard_type,
                 model_data,
                 public_ip=None,
                 port=None,
                 add_local_url=False):
        """Initialize the dashboard."""

        if model_data is None or type is None:
            raise ValueError("Required parameters not provided")

        try:
            self._service = FlaskHelper(ip=public_ip, port=port)
        except Exception as e:
            self._service = None
            raise e

        self.id = uuid.uuid4().hex

        self.config = {
            "dashboardType": dashboard_type,
            "id": self.id,
            "baseUrl": self._service.env.base_url,
            'withCredentials': self._service.with_credentials
        }
        if add_local_url:
            local_url = ExplanationDashboardInterface.LOCAL_URL
            model_data[local_url] = self._service.env.base_url
        self.model_data = model_data
        self.add_route()

        html = self.load_index()
        print(f'{dashboard_type} started at {self._service.env.base_url}')
        display(HTML(html), metadata=dict(isolated=True))

    def add_route(self):
        # To enable multiple dashboards to run in the same notebook we need to
        # prevent them from using the same method names (in addition to using
        # dedicated ports). Below we rename the function for that purpose and
        # manually add the URL rule instead of using the route decorator.
        def visual():
            return self.load_index()
        self.add_url_rule(visual, '/', methods=["GET"])
        return

    def get_widget_path(path):
        script_path = os.path.dirname(os.path.abspath(__file__))
        return os.path.join(script_path, "widget", path)

    def load_index(self):
        index = self.load_widget_file("index.html")
        parser = InLineScript(self.load_widget_file)
        parser.feed(index)
        return parser.content

    def load_widget_file(self, path):
        js_path = Dashboard.get_widget_path(path)
        with open(js_path, "r", encoding="utf-8") as f:
            content = f.read()
            content = content.replace(
                "__rai_app_id__", f'rai_widget_{self.id}')
            content = content.replace(
                "__rai_config__", json.dumps(self.config))
            content = content.replace(
                "__rai_model_data__", json.dumps(self.model_data))
            return content

    def add_url_rule(self, func, route, methods):
        """To enable multiple dashboards to run in the same notebook we need to
        prevent them from using the same method names (in addition to using
        dedicated ports). We rename the function for that purpose and
        manually add the URL rule instead of using the route decorator.
        """
        func.__name__ = func.__name__ + str(id(self))
        self._service.app.add_url_rule(
            route,
            endpoint=func.__name__,
            view_func=func,
            methods=methods)


import numpy as np
# why use numpy ? eg -
#height ->
height = [123, 167, 175, 185, 154, 190]
# and their corrosponding weights ->
weight = [45, 56, 78, 76, 50, 80]

#calculate bmi - height/ weight**2
# this will error out height/ weight ** 2

np_height = np.array(height)
np_height_in_mtrs = np_height / 100
np_weight = np.array(weight)

bmi = np_weight/ np_height_in_mtrs ** 2

print(bmi)

#Remeber numpy arrays are homogeneous. can contains elemets of the same type only. eg here all ints
# access 1D numpy arrays - similar to lists
print(bmi[0])
print(bmi[1 : 3])

#you can also use >, < != , == operators directly with numpy. bmi < 24 will give a boolean array
# that u can use to filter the original array

low_bmi = bmi[bmi < 18]
print(low_bmi)

perfect_bmi = bmi[np.logical_and(bmi > 18, bmi < 25)]
print(perfect_bmi)


"""

编写一个函数来查找字符串数组中的最长公共前缀。

如果不存在公共前缀，返回空字符串 ""。

示例 1:

输入: ["flower","flow","flight"]
输出: "fl"
示例 2:

输入: ["dog","racecar","car"]
输出: ""
解释: 输入不存在公共前缀。
说明:

所有输入只包含小写字母 a-z 。

"""

class Solution(object):
    def longestCommonPrefix(self, strs):
        """
        :type strs: List[str]
        :rtype: str
        """
        result = ""

        try:
            small_length = len(strs[0])
            small_str = strs[0]
            for str in strs:
                if len(str) < small_length:
                    small_str = str
                    small_length = len(str)
        except:
            return ""

        for i in range(0, len(small_str)):
            for str in strs:
                if small_str[i] != str[i]:
                    return str[:i:]

        return small_str

if __name__ == '__main__':
    strs = ["aaa", "aa", "aaa"]

    solution = Solution()
    result = solution.longestCommonPrefix(strs)
    print(result)


import os
import shutil
from lyrebird import context
from . import android_helper

class DeviceService:
    """
    Background service : Android devices monitor

    Poll devices status per 1 sec with adb command
    """

    READY = 0
    RUNNING = 1
    STOP = 2

    def __init__(self):
        self.status = self.READY
        self.handle_interval = 1
        self.devices = {}
        self.reset_screenshot_dir()
        print('DeviceService OnCreate')

    def devices_to_dict(self):
        json_obj = {}
        for device_id in self.devices:
            json_obj[device_id] = self.devices[device_id].to_dict()
        return json_obj

    def run(self):
        self.status = self.RUNNING
        print('Android device listener start')
        while self.status == self.RUNNING:
            self.handle()
            context.application.socket_io.sleep(self.handle_interval)
        self.status = self.STOP
        print('Android device listener stop')

    def handle(self):
        devices = android_helper.devices()
        if len(devices) == len(self.devices):
            if len([k for k in devices if k not in self.devices]) == 0:
                return

        self.devices = devices
        context.application.socket_io.emit('device', namespace='/android-plugin')

    def start_log_recorder(self, device_id):
        for _device_id in self.devices:
            if _device_id == device_id:
                self.devices[_device_id].start_log()
            else:
                self.devices[_device_id].stop_log()

    def reset_screenshot_dir(self):
        if os.path.exists(android_helper.screenshot_dir):
            shutil.rmtree(android_helper.screenshot_dir)
            print('Android device log file reset')


from pkg import app, db
from databaseSetup import User, Lizard, Hobby, ChangeLog
from flask import request, jsonify
from werkzeug.contrib.atom import AtomFeed
import datetime

# JSON API Endpoints
@app.route('/lizard/<int:lizard_id>/hobby/JSON/')
def lizardHobbyJSON(lizard_id):
    hobbies = Hobby.query.filter_by(lizard_id=lizard_id).all()
    return jsonify(hobbies=[hobby.serialize for hobby in hobbies])

@app.route('/lizard/<int:lizard_id>/hobby/<int:hobby_id>/JSON/')
def individualHobbyJSON(lizard_id, hobby_id):
    hobby = Hobby.query.filter_by(id=hobby_id).all()
    if not hobby:
        hobby = Hobby()  # if no hobbies, populate with null and don't crash
    else:
        hobby = hobby[0]
    return jsonify(hobby=hobby.serialize)

@app.route('/lizard/JSON/')
def lizardJSON():
    lizards = Lizard.query.all()
    return jsonify(lizards=[lizard.serialize for lizard in lizards])

# ATOM API Endpoints
@app.route('/lizard.atom/')
def lizardATOM():
    # Get the last added lizard's creation date to populate as the
    # last updated date for the ATOM feed
    latest_update = Lizard.query.order_by(
        db.desc('creation_instant')).limit(1).all()

    updated = None
    if latest_update:
        updated = latest_update[0].creation_instant

    feed = AtomFeed(
        'All Lizards',
        feed_url=request.url,
        url=request.url_root,
        author={'name': 'Zach Attas', 'email': 'zach.attas@gmail.com'},
        id="http://localhost:8000/publicLizard/",
        updated=updated)

    lizards = Lizard.query.order_by(db.desc('creation_instant')).all()
    for lizard in lizards:
        user = User.query.filter_by(id=lizard.user_id).one()
        url = "http://localhost:8000/publicLizard/#%s" % (lizard.id)
        content = "Picture URL: %s" % \
            (lizard.picture_url, lizard.picture_url)
        feed.add(
            lizard.name,
            content,
            content_type='html',
            author={'name': user.name},
            url=url,
            id=url,
            updated=lizard.creation_instant,
            published=lizard.creation_instant)
    return feed.get_response()

@app.route('/hobby.atom/')
def hobbyATOM():
    latest_update = Hobby.query.order_by(
        db.desc('creation_instant')).limit(1).all()

    updated = None
    if latest_update:
        updated = latest_update[0].creation_instant

    feed = AtomFeed(
        'All Hobbies',
        feed_url=request.url,
        url=request.url_root,
        author={'name': 'Zach Attas', 'email': 'zach.attas@gmail.com'},
        id="http://localhost:8000/publicLizard/",
        updated=updated)

    hobbies = Hobby.query.order_by(db.desc('creation_instant')).all()
    for hobby in hobbies:
        lizard = Lizard.query.filter_by(id=hobby.lizard_id).one()
        user = User.query.filter_by(id=hobby.user_id).one()
        content = """
            Description: %s

            Lizard:
            %s

            Picture URL: %s""" % \
            (hobby.description, lizard.id, lizard.name,
                hobby.picture_url, hobby.picture_url)
        url = "http://localhost:8000/publicLizard/%s/publicHobby/#%s" % \
            (lizard.id, hobby.id)
        feed.add(
            hobby.name,
            content,
            content_type='html',
            author={'name': user.name},
            url=url,
            id=url,
            updated=hobby.creation_instant,
            published=hobby.creation_instant)
    return feed.get_response()

@app.route('/all.atom/')
def allATOM():
    # Query to perform union on all lizards and hobbies
    # Then sort results by creation instant
    results = db.engine.execute("""
        SELECT * FROM
            (SELECT
                id,
                name,
                user_id,
                NULL as description,
                creation_instant,
                picture_url,
                NULL as lizard_id,
                "lizard" AS type
            FROM Lizard
            UNION
            SELECT
                id,
                name,
                user_id,
                description,
                creation_instant,
                picture_url,
                lizard_id,
                "hobby" AS type
            FROM Hobby)
        ORDER BY creation_instant DESC""").fetchall()

    updated = None
    if results:
        # Executing the query with raw SQL returns unicode, not python date
        # Need to convert it back to a python date
        updated = datetime.datetime.strptime(
            results[0].creation_instant, '%Y-%m-%d %H:%M:%S.%f')

    feed = AtomFeed(
        'All Lizards and Hobbies',
        feed_url=request.url,
        url=request.url_root,
        author={'name': 'Zach Attas', 'email': 'zach.attas@gmail.com'},
        id="http://localhost:8000/publicLizard/",
        updated=updated)

    for result in results:
        user = User.query.filter_by(id=result.user_id).one()
        updated = datetime.datetime.strptime(
            result.creation_instant, '%Y-%m-%d %H:%M:%S.%f')

        if result.type == 'lizard':
            name = "Lizard %s" % (result.name)
            url = "http://localhost:8000/publicLizard/#%s" % (result.id)
            content = "Picture URL: %s" % \
                (result.picture_url, result.picture_url)

        if result.type == 'hobby':
            lizard = Lizard.query.filter_by(id=result.lizard_id).one()
            name = "Hobby %s" % (result.name)
            url = "http://localhost:8000/publicLizard/%s/publicHobby/#%s" % \
                (lizard.id, result.id)
            content = """
                Description: %s

                Lizard:
                %s

                Picture URL: %s""" % \
                (result.description, result.id, lizard.name,
                    result.picture_url, result.picture_url)

        feed.add(
            name,
            content,
            content_type='html',
            author={'name': user.name},
            url=url,
            id=url,
            updated=updated,
            published=updated)
    return feed.get_response()

@app.route('/changes.atom/')
def changesATOM():
    changes = ChangeLog.query.order_by(db.desc('update_instant')).all()

    updated = None
    if changes:
        updated = changes[0].update_instant

    feed = AtomFeed(
        'Changes to Lizard Database',
        feed_url=request.url,
        url=request.url_root,
        author={'name': 'Zach Attas', 'email': 'zach.attas@gmail.com'},
        id="http://localhost:8000/publicLizard/",
        updated=updated)

    for change in changes:
        user = User.query.filter_by(id=change.user_id).one()
        content = "Action: %s" % (change.action)

        if change.table == 'lizard':
            name = "Lizard %s" % (change.lizard_name)
            if change.action == 'delete':
                url = None
                unique_url = "http://localhost:8000/publicLizard/"
            if change.action == 'new' or change.action == 'update':
                url = "http://localhost:8000/publicLizard/#%s" % \
                    (change.lizard_id)
                unique_url = url

        if change.table == 'hobby':
            name = "Hobby %s" % (change.hobby_name)
            if change.action == 'delete':
                url = None
                unique_url = "http://localhost:8000/publicLizard/"
            if change.action == 'new' or change.action == 'update':
                url = "http://localhost:8000/publicLizard/%s/publicHobby/#%s" \
                    % (change.lizard_id, change.hobby_id)
                unique_url = url

        feed.add(
            name,
            content,
            content_type='html',
            author={'name': user.name},
            url=url,
            id=unique_url,
            updated=change.update_instant,
            published=change.update_instant)
    return feed.get_response()


from flask import Flask, flash, request, redirect, render_template, jsonify
import model_final,db

app = Flask(__name__)

@app.route('/old')
def upload_form():
    return render_template('TV.html')

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/predict', methods=['POST'])
def predict():
    data = request.json
    print("data", data)

    brand = data["brand"]
    hd = data["hd"]
    rating = int(data["rating"])
    speaker = int(data["speaker"])
    size = int(data["size"])
    hdmi = int(data["hdmi"])
    usb = int(data["usb"])
    result = model_final.output(brand, rating, speaker, size, hd, hdmi, usb)
    tv1,tv2,tv3 = db.output(speaker, size, hd, hdmi, usb)
    return jsonify({"price": result}) #tv1,tv2,tv3

@app.route('/result', methods=['POST'])
def result():
    if request.method == 'POST':
        brand = (request.form.get('brand'))
        hd = (request.form.get('hd'))
        rating = int(request.form.get('rating'))
        usb = int(request.form.get('usb'))
        size = float(request.form.get('size'))
        hdmi = int(request.form.get('hdmi'))
        speaker = int(request.form.get('speaker'))

        result = model_final.output(brand, rating, speaker, size, hd, hdmi, usb)
        return render_template("result.html", prediction=result)

if __name__ == "__main__":
    app.run(host="127.0.0.1", port=5000)


"""
Given a singly linked list, group all odd nodes together followed by the even nodes. Please note here we are talking about the node number and not the value in the nodes.
You should try to do it in place. The program should run in O(1) space complexity and O(nodes) time complexity.

=> Example 1:
Input: 1->2->3->4->5->NULL
Output: 1->3->5->2->4->NULL

=> Example 2:
Input: 2->1->3->5->6->4->7->NULL
Output: 2->3->6->7->1->5->4->NULL

=== Constraints ===
  The relative order inside both the even and odd groups should remain as it was in the input.
  The first node is considered odd, the second node even and so on ...
  The length of the linked list is between [0, 10^4].
"""

from singlyLinkedList import SinglyLinkedList
from usefulLinkedListMethods import printList

def getOddEvenList(head):
  if not head:
    return head
  cur = head
  pos = 0
  evens = []
  odds = []
  while cur:
    if pos % 2 == 0:
      evens.append(cur.value)
    else:
      odds.append(cur.value)
    pos += 1
    cur = cur.next
  LL = SinglyLinkedList()
  while evens:
    LL.addAtTail(evens[0])
    evens.pop(0)
  while odds:
    LL.addAtTail(odds[0])
    odds.pop(0)
  return LL.head

if __name__ == "__main__":
  LL = SinglyLinkedList()
  LL.addAtHead(1)
  LL.addAtTail(2)
  LL.addAtTail(3)
  LL.addAtTail(4)
  LL.addAtTail(5)
  head = getOddEvenList(LL.head)
  printList(head)

  LL_1 = SinglyLinkedList()
  LL_1.addAtHead(2)
  LL_1.addAtTail(1)
  LL_1.addAtTail(3)
  LL_1.addAtTail(5)
  LL_1.addAtTail(6)
  LL_1.addAtTail(4)
  LL_1.addAtTail(7)
  head = getOddEvenList(LL_1.head)
  printList(head)


#!/usr/bin/python

########################################
# module: cs3430_s19_hw11.py
# YOUR NAME
# YOUR A#
########################################

## add your imports here
import math
from PIL import Image
import sys
import numpy as np
import cv2

from tof import tof
from const import const
from var import var
from pwr import pwr
from maker import make_const, make_pwr, make_prod, make_quot
from maker import make_plus, make_ln, make_absv
from maker import make_pwr_expr, make_e_expr
from plus import plus
from prod import prod
from deriv import deriv

################# Problem 1 (1 point) ###################

def nra(poly_fexpr, g, n):

    der = deriv(poly_fexpr)
    func = tof(poly_fexpr)
    der_func = tof(der)
    x = g.get_val()
    for _ in range(n.get_val()):
        x = x - (func(x) / der_func(x))

    print(x)
    return make_const(x)

################# Unit Tests for Problem 1 ###################

def nra_ut_01():
    ''' Approximating x^2 - 2 = 0. '''
    fexpr = make_plus(make_pwr('x', 2.0),
                      make_const(-2.0))
    print(nra(fexpr, make_const(1.0), make_const(10000)))

def nra_ut_02():
    ''' Approximating x^2 - 3 = 0. '''
    fexpr = make_plus(make_pwr('x', 2.0),
                      make_const(-3.0))
    print(nra(fexpr, make_const(1.0), make_const(10000)))

def nra_ut_03():
    ''' Approximating x^2 - 5 = 0. '''
    fexpr = make_plus(make_pwr('x', 2.0),
                      make_const(-5.0))
    print(nra(fexpr, make_const(1.0), make_const(10000)))

def nra_ut_04():
    ''' Approximating x^2 - 7 = 0. '''
    fexpr = make_plus(make_pwr('x', 2.0),
                      make_const(-7.0))
    print(nra(fexpr, make_const(1.0), make_const(10000)))

def nra_ut_05():
    ''' Approximating e^-x = x^2. '''
    fexpr = make_e_expr(make_prod(make_const(-1.0),
                                  make_pwr('x', 1.0)))
    fexpr = make_plus(fexpr,
                      make_prod(make_const(-1.0),
                                make_pwr('x', 2.0)))
    print(nra(fexpr, make_const(1.0), make_const(10000)))

def nra_ut_06():
    ''' Approximating 11^{1/3}.'''
    fexpr = make_pwr('x', 3.0)
    fexpr = make_plus(fexpr,
                      make_const(-11.0))
    print(nra(fexpr, make_const(1.0), make_const(10000)))

def nra_ut_07():
    ''' Approximating 6^{1/3}.'''
    fexpr = make_pwr('x', 3.0)
    fexpr = make_plus(fexpr,
                      make_const(-6.0))
    print(nra(fexpr, make_const(1.0), make_const(10000)))

def nra_ut_08():
    ''' Approximating x^3 + 2x + 2. '''
    fexpr = make_pwr('x', 3.0)
    fexpr = make_plus(fexpr,
                      make_prod(make_const(2.0),
                                make_pwr('x', 1.0)))
    fexpr = make_plus(fexpr, make_const(2.0))
    print(nra(fexpr, make_const(1.0), make_const(10000)))

def nra_ut_09():
    ''' Approximating x^3 + x - 1. '''
    fexpr = make_pwr('x', 3.0)
    fexpr = make_plus(fexpr, make_pwr('x', 1.0))
    fexpr = make_plus(fexpr, make_const(-1.0))
    print(nra(fexpr, make_const(1.0), make_const(10000)))

def nra_ut_10():
    ''' Approximating e^(5-x) = 10 - x. '''
    fexpr = make_e_expr(make_plus(make_const(5.0),
                                  make_prod(make_const(-1.0),
                                            make_pwr('x', 1.0))))
    fexpr = make_plus(fexpr, make_pwr('x', 1.0))
    fexpr = make_plus(fexpr, make_const(-10.0))
    print(nra(fexpr, make_const(1.0), make_const(10000)))

# =================== Problem 2 (4 points) ===================

def luminosity(rgb, rcoeff=0.2126, gcoeff=0.7152, bcoeff=0.0722):
    return rcoeff*rgb[0]+gcoeff*rgb[1]+bcoeff*rgb[2]

def get_theta_grad(img, pix):
    x, y = pix

    dy = img.getpixel((x, y-1)) - img.getpixel((x, y+1))
    dx = img.getpixel((x-1, y)) - img.getpixel((x+1, y))

    if dy < 1: dy = 1.0
    if dx < 1: dx = 1.0

    return (math.atan(dy/dx), abs(math.sqrt(dy ** 2 + dx ** 2)))

def get_gray_lumin(img):
    width, height = img.size
    grayed = Image.new('L', (width, height))

    for i in range(width):
        for j in range(height):
            grayed.putpixel((i,j), math.floor(luminosity(img.getpixel((i,j)))))

    return grayed

def get_gradient_img(img, magn_thresh):
    width, height = img.size
    grads = Image.new("L", (width,height))

    for i in range(1,width-1):
        for j in range(1,height-1):
            #thetas[i,j], grads[i,j] = get_theta_grad(grayed, (i,j))
            theta, grad = get_theta_grad(img, (i,j))
            direction = abs(math.degrees(theta)) / 90
            if grad * theta > magn_thresh:
                grads.putpixel((i,j), 255)

    return grads

def get_x_and_y(theta, p):
    return (p * math.cos(theta), p * math.sin(theta))

def get_func(x1, y1, m):
    return lambda x: m * x - m * x1 + y1

def deg_to_rad(x):
    return x * (math.pi/180)

def ht_detect_lines(img_fp, magn_thresh=20, spl=20):
    print(spl)
    img  = Image.open(img_fp)
    blue = img.copy()

    width, height = img.size

    gray = 0

    if len(img.getpixel((0,0))) >= 3:
        gray = get_gray_lumin(img)
    else:
        gray = img

    edges = get_gradient_img(gray, magn_thresh)

    print(edges.getpixel((100,10)))

    ht_width = 180
    ht_height = int(math.sqrt(width**2 + height**2))

    ht = np.zeros((ht_width, ht_height)) ##need to figure out the first number

    for i in range(width):
        for j in range(height):
            if edges.getpixel((i,j)) == 255:
                for k in range(ht_width):
                    # k represents theta in degrees
                    theta = deg_to_rad(k)
                    p = int(i * math.cos(theta) + j * math.sin(theta))
                    if p >= 0 and p < ht_height:
                        ht[k, p] += 1

    for i in range(ht_width):
        for j in range(ht_height):
            if ht[i, j] >= spl:
                theta = deg_to_rad(i)
                x, y = get_x_and_y(theta, j)
                m = -1 * (math.cos(theta) / math.sin(theta))
                func = get_func(x, y, m)

                for x in range(width):
                    try:
                        blue.putpixel((x, int(func(x))), (0,0,255))
                    except Exception:
                        pass
                        #print('index out of bounds')

    return (img, np.array(blue), edges, ht)

################ Unit Tests for Problem 2 ####################
##
## I used Image for edge detection and numpy image representation
## to draw lines. Hence, I am using cv2.imwrite to save the
## image with drawn line (lnimg) and image.save to save the image
## with the edges. Feel free to modify but keep the signatures
## of these tests the same.

def ht_test_01(img_fp, magn_thresh=20, spl=20):
    img, lnimg, edimg, ht = ht_detect_lines(img_fp,
                                            magn_thresh=magn_thresh,
                                            spl=spl)
    cv2.imwrite('im01_ln.png', lnimg)
    edimg.save('im01_ed.png')
    del img
    del lnimg
    del edimg

def ht_test_02(img_fp, magn_thresh=20, spl=20):
    img, lnimg, edimg, ht = ht_detect_lines(img_fp,
                                            magn_thresh=magn_thresh,
                                            spl=spl)
    cv2.imwrite('im02_ln.png', lnimg)
    edimg.save('im02_ed.png')
    del img
    del lnimg
    del edimg

def ht_test_03(img_fp, magn_thresh=20, spl=20):
    img, lnimg, edimg, ht = ht_detect_lines(img_fp,
                                            magn_thresh=magn_thresh,
                                            spl=spl)
    cv2.imwrite('im03_ln.png', lnimg)
    edimg.save('im03_ed.png')
    del img
    del lnimg
    del edimg

def ht_test_04(img_fp, magn_thresh=20, spl=20):
    img, lnimg, edimg, ht = ht_detect_lines(img_fp,
                                            magn_thresh=magn_thresh,
                                            spl=spl)
    cv2.imwrite('im04_ln.png', lnimg)
    edimg.save('im04_ed.png')
    del img
    del lnimg
    del edimg

def ht_test_05(img_fp, magn_thresh=20, spl=20):
    img, lnimg, edimg, ht = ht_detect_lines(img_fp,
                                            magn_thresh=magn_thresh,
                                            spl=spl)
    cv2.imwrite('im05_ln.png', lnimg)
    edimg.save('im05_ed.png')
    del img
    del lnimg
    del edimg

def ht_test_06(img_fp, magn_thresh=20, spl=20):
    img, lnimg, edimg, ht = ht_detect_lines(img_fp,
                                            magn_thresh=magn_thresh,
                                            spl=spl)
    cv2.imwrite('im06_ln.png', lnimg)
    edimg.save('im06_ed.png')
    del img
    del lnimg
    del edimg

def ht_test_07(img_fp, magn_thresh=20, spl=20):
    img, lnimg, edimg, ht = ht_detect_lines(img_fp,
                                            magn_thresh=magn_thresh,
                                            spl=spl)
    cv2.imwrite('im07_ln.png', lnimg)
    edimg.save('im07_ed.png')
    del img
    del lnimg
    del edimg

def ht_test_08(img_fp, magn_thresh=20, spl=20):
    img, lnimg, edimg, ht = ht_detect_lines(img_fp,
                                            magn_thresh=magn_thresh,
                                            spl=spl)
    cv2.imwrite('im08_ln.png', lnimg)
    edimg.save('im08_ed.png')
    del img
    del lnimg
    del edimg

def ht_test_09(img_fp, magn_thresh=20, spl=20):
    img, lnimg, edimg, ht = ht_detect_lines(img_fp,
                                            magn_thresh=magn_thresh,
                                            spl=spl)
    cv2.imwrite('im09_ln.png', lnimg)
    edimg.save('im09_ed.png')
    del img
    del lnimg
    del edimg

def ht_test_10(img_fp, magn_thresh=20, spl=20):
    img, lnimg, edimg, ht = ht_detect_lines(img_fp,
                                            magn_thresh=magn_thresh,
                                            spl=spl)
    cv2.imwrite('im10_ln.png', lnimg)
    edimg.save('im10_ed.png')
    del img
    del lnimg
    del edimg

def ht_test_11(img_fp, magn_thresh=20, spl=20):
    img, lnimg, edimg, ht = ht_detect_lines(img_fp,
                                            magn_thresh=magn_thresh,
                                            spl=spl)
    cv2.imwrite('im11_ln.png', lnimg)
    edimg.save('im11_ed.png')
    del img
    del lnimg
    del edimg

def ht_test_12(img_fp, magn_thresh=20, spl=20):
    img, lnimg, edimg, ht = ht_detect_lines(img_fp,
                                            magn_thresh=magn_thresh,
                                            spl=spl)
    cv2.imwrite('im12_ln.png', lnimg)
    edimg.save('im12_ed.png')
    del img
    del lnimg
    del edimg

if __name__ == '__main__':
    ht_test_01('img/EdgeImage_01.jpg', 20, 60)
    #ht_test_01('img/kitchen.jpeg')
    #nra_ut_05()

    # x,y = get_x_and_y(deg_to_rad(60), 4)

    # func = get_func(x, y, -1 * math.cos(deg_to_rad(60))/math.sin(deg_to_rad(60)))
    # print(func(4))


# -*- coding: utf-8 -*-

import time
from datetime import datetime
from dateutil.relativedelta import relativedelta
from openerp import models, api, fields, exceptions, _
from openerp.exceptions import UserError

class HrTimesheetWiz(models.TransientModel):
    _name = 'hr.timesheet.wiz'
    _description = """Wizard who allow you to generate
    monthly timesheet"""

    @api.model
    def _default_date_start(self):
        user = self.env['res.users'].browse(self._uid)
        r = user.company_id and user.company_id.timesheet_range or 'month'
        if r=='month':
            return time.strftime('%Y-%m-01')
        elif r=='week':
            return (datetime.today() + relativedelta(weekday=0, days=-6)).strftime('%Y-%m-%d')
        elif r=='year':
            return time.strftime('%Y-01-01')
        return fields.date.context_today()

    @api.model
    def _default_date_stop(self):
        user = self.env['res.users'].browse(self._uid)
        r = user.company_id and user.company_id.timesheet_range or 'month'
        if r=='month':
            return (datetime.today() + relativedelta(months=+1,day=1,days=-1)).strftime('%Y-%m-%d')
        elif r=='week':
            return (datetime.today() + relativedelta(weekday=6)).strftime('%Y-%m-%d')
        elif r=='year':
            return time.strftime('%Y-12-31')
        return fields.date.context_today()

    date_start = fields.Date(string=u'Date start', required=True, default=_default_date_start, help='Date Start')
    date_stop = fields.Date(string=u'Date stop', required=True, default=_default_date_stop, help='Date Stop')
    employee_ids = fields.Many2many('hr.employee', string=u'Employee',
                                    help='List of Employee will own timesheet sheet')

    @api.multi
    def action_apply(self):
        timesheet_obj = self.env['hr_timesheet_sheet.sheet']
        if not self.date_start or not self.date_stop:
            raise UserError(_("Please Insert correct date !"))
        elif self.date_start >= self.date_stop:
            raise UserError(_("Start date should be older than Stop date !"))
        else:
            # all is correct we can process
            if not self.employee_ids:
                raise UserError(_("You should at least insert 1 employee !"))
            else:
                for employee in self.employee_ids:
                    if not employee.user_id:
                        raise UserError(_("Please, define an user for employee: %s !") % (employee.name,))
                    else:
                        vals = {'employee_id': employee.id,
                                'date_from': self.date_start,
                                'date_to': self.date_stop}
                        timesheet_obj.create(vals)

        return {
            'view_type': 'form',
            'view_mode': 'tree',
            'res_model': 'hr_timesheet_sheet.sheet',
            'type': 'ir.actions.act_window',
            'target': 'current',
            'res_id': False,
        }


#!/usr/bin/env python

"""parse_paper_data.py: reads the data given by the paper on community structure
mentioned in the readme.
"""

import sys
import numpy as np
import matplotlib.pyplot as plt
from scipy.interpolate import interp1d

#-----------------------------------------------------------------------------#
#                                                                             #
#    Global Variables                                                         #
#                                                                             #
#-----------------------------------------------------------------------------#

#data given at stdin, from the paper
DATA = open("bucketing.csv", "r")

#data given at stdin, from random tests
#RANDOM_DATA = open("output_data/random_graph_data" , "r")

#boolean to skip the first line of input file
_FIRST_LINE = True

#Q-values for bucketed clause-to-variable ratio
_RANDOM_VALS = [[] for _ in range(201)]

#-----------------------------------------------------------------------------#
#                                                                             #
#    Build interpolated function from random graph data                       #
#                                                                             #
#-----------------------------------------------------------------------------#

#------------------------------------------------------------------------------#
#                                                                              #
#    Build Plots from Paper Input                                              #
#                                                                              #
#------------------------------------------------------------------------------#

_FIRST_LINE = True

#Q-value
x_axis = []

#running time
y_axis = []

#buckets for averaging running times
VALS = [[] for _ in range(101)]

for line in DATA:
    if _FIRST_LINE:
        _FIRST_LINE = False
    else:
        inst = [i.strip() for i in line.split(",")]
        if inst[4] != "" and inst[49] != "":
            Q_val = float(inst[4])
            x_axis.append(Q_val)
            y_axis.append(float(inst[49]))
            VALS[int(round(Q_val * 100))].append(float(inst[49]))

for i in range(len(VALS)):
    if VALS[i] == []:
        VALS[i] = -100
    else:
        VALS[i] = sum(VALS[i]) / len(VALS[i])

x_axis3 = [float(i) / 100 for i in range(101)]
y_axis3 = VALS

#------------------------------------------------------------------------------#
#                                                                              #
#    Plot Figures                                                              #
#                                                                              #
#------------------------------------------------------------------------------#

#Q-value to running time as in the paper
fig1 = plt.figure()
ax1 = fig1.add_subplot(1, 1, 1)
ax1.set_xlabel("Q-value")
ax1.set_ylabel("running time")
ax1.scatter(x_axis, y_axis)
ax1.axis([0, 1, -200, 1000000])

#Q-value to average running time, as in the paper
fig3 = plt.figure()
ax3 = fig3.add_subplot(1, 1, 1)
ax3.set_xlabel("Q-value")
ax3.set_ylabel("average running time")
ax3.plot(x_axis3, y_axis3, "ro")
ax3.axis([0, 1, -200, 900000])

plt.show()


"""
The file contains tests for books (Adding books, editing, deleting, borrowing and getting
"""
import json

from tests.base_tests import HelloBooksTestCase
from run import jwt

class BooksTestCase(HelloBooksTestCase):
    """This class contains all tests for users"""

    def test_admin_add_book(self):
        """Add book by admin that already exist"""
        add_book = self.add_book(self.add_book_data)
        self.assertEqual(add_book.status_code, 201)
        add_similar_book_isnb = self.add_book(self.add_book_data)
        self.assertEqual(add_similar_book_isnb.status_code, 409)
        add_similar_book_title_author = self.add_book(self.similar_book_data)
        self.assertEqual(add_similar_book_title_author.status_code, 409)

    def test_user_add_book(self):
        """Tests that user cannot make crud operations. Only admin do."""
        user_login = self.login_user(self.user_data)
        user_login_msg = json.loads(user_login.data)
        access_token = user_login_msg['access_token']
        user_add_book = self.client.post('/api/v1/books', data=json.dumps(self.add_book_data),
                                         headers={"Authorization": "Bearer {}".format(access_token)},
                                         content_type='application/json')
        self.assertEqual(user_add_book.status_code, 403)

    # def test_get_all_books(self):
    #     """Test user can get all books"""
    #     self.add_book(self.add_book_data)
    #     get_all_books = self.client.get('/api/v1/books')
    #     self.assertEqual(get_all_books.status_code, 200)

    def test_get_all_books_no_books(self):
        """Tests get books when no books available"""
        get_all_books = self.client.get('/api/v1/books')
        self.assertEqual(get_all_books.status_code, 404)

    def test_get_single_book(self):
        """Test that a user can get a single book"""
        add_book = self.add_book(self.add_book_data)
        book_data = json.loads(add_book.data)
        result = self.client.get('/api/v1/books/{}'.format(book_data['book_added']['book_id']))
        self.assertEqual(result.status_code, 200)

    def test_get_unavailable_book(self):
        """Get a book that is not available"""
        unavailable_book = self.client.get('/api/v1/books/1234')
        self.assertEqual(unavailable_book.status_code, 404)

    def test_admin_can_edit_book(self):
        """Tests that a book can be edited"""
        add_book = self.add_book(self.add_book_data)
        book_data = json.loads(add_book.data)
        login_admin = self.login_admin(self.admin_data)
        login_msg = json.loads(login_admin.data)
        access_token = login_msg['access_token']
        edit_book = self.client.put('/api/v1/books/{}'.format(book_data['book_added']['book_id']),
                                    data=json.dumps(self.edit_book_data), content_type='application/json',
                                    headers={"Authorization": 'Bearer {}'.format(access_token)}
                                    )
        self.assertEqual(edit_book.status_code, 200)

    def test_user_can_edit_book(self):
        """Testing that user can edit a book"""
        admin_add_book = self.add_book(self.add_book_data)
        book_data = json.loads(admin_add_book.data)
        login_user = self.login_user(self.user_data)
        login_user_msg = json.loads(login_user.data)
        access_token = login_user_msg['access_token']
        user_edit_book = self.client.put('/api/v1/books/{}'.format(book_data['book_added']['book_id']),
                                         data=json.dumps(self.edit_book_data), content_type='application/json',
                                         headers={"Authorization": 'Bearer {}'.format(access_token)}
                                         )
        self.assertEqual(user_edit_book.status_code, 403)

    def test_admin_can_delete_book(self):
        add_book = self.add_book(self.add_book_data)
        book_data = json.loads(add_book.data)
        login_admin = self.login_admin(self.admin_data)
        login_msg = json.loads(login_admin.data)
        access_token = login_msg['access_token']
        delete_book = self.client.delete('/api/v1/books/{}'.format(book_data['book_added']['book_id']),
                                         headers={"Authorization": 'Bearer {}'.format(access_token)})
        self.assertEqual(delete_book.status_code, 200)

    def test_user_can_delete_a_book(self):
        """Test user deleting a book"""
        admin_add_book = self.add_book(self.add_book_data)
        book_data = json.loads(admin_add_book.data)
        login_user = self.login_user(self.user_data)
        login_msg = json.loads(login_user.data)
        access_token = login_msg['access_token']
        user_delete_book = self.client.delete('/api/v1/books/{}'.format(book_data['book_added']['book_id']),
                                              headers={"Authorization": 'Bearer {}'.format(access_token)})
        self.assertEqual(user_delete_book.status_code, 403)

    def test_delete_unavailable_book(self):
        """Test deleting book that is not available"""
        login_admin = self.login_admin(self.admin_data)
        login_msg = json.loads(login_admin.data)
        access_token = login_msg['access_token']
        delete_unavailable_book = self.client.delete('/api/v1/books/1234',
                                                     headers={"Authorization": 'Bearer {}'.format(access_token)})
        self.assertEqual(delete_unavailable_book.status_code, 404)

    def test_delete_borrowed_book(self):
        """Trying to delete a book that a user has not yet returned"""
        add_book = self.add_book(self.add_book_data)
        book_data = json.loads(add_book.data)
        login_user = self.login_user(self.user_data)
        login_msg = json.loads(login_user.data)
        access_token = login_msg['access_token']
        self.client.post('/api/v1/users/books/{}'.format(book_data['book_added']['book_id']),
                         headers={"Authorization": "Bearer {}".format(access_token)})
        login_admin = self.login_admin(self.admin_data)
        login_msg = json.loads(login_admin.data)
        access_token = login_msg['access_token']
        delete_book = self.client.delete('/api/v1/books/{}'.format(book_data['book_added']['book_id']),
                                         headers={"Authorization": 'Bearer {}'.format(access_token)})
        self.assertEqual(delete_book.status_code, 403)


#!/usr/bin/env python3
import fileinput

pairs = {}
for a,b in [('i', '1'), ('a', '4'), ('o', '0'), ('e', '3'), ('s', '5'), ('b', '8'), ('g', '9')]:
    pairs[a] = b

def process(l):
    if not l:
        yield ""
        return
    if l[0] not in pairs:
        yield from ((l[0] + x) for x in process(l[1:]))
        return
    t = pairs[l[0]]
    for c in process(l[1:]):
        yield t+c
        yield l[0]+c

if __name__ == '__main__':
    for l in fileinput.input():
        l = l.strip()
        for p in process(l):
            print("n3w_ag3_v1rtu4liz4t1on_" + p)


import numpy as np
from layer import *
from loss import MSELoss, SoftmaxCrossEntropyLoss, CrossEntropyLoss
from model import Model
from utils import *
import matplotlib.pyplot as plt
import pickle

def evaluate(model, data):
    x_data = data['x']
    y_data = data['y']

    batch_size = 100
    size = len(x_data)
    correct = 0
    loss_value = 0
    loss = SoftmaxCrossEntropyLoss('loss')
    for start_idx in range(0, size, batch_size):
        end_idx = min(start_idx + batch_size, size)
        x = np.array(x_data[start_idx: end_idx])
        y = y_data[start_idx: end_idx]

        ans = model.forward(x)
        output = softmax(ans)

        loss_value += len(y) * loss.forward(ans, onehot_encoding(y, 5))
        correct += len(y) * calculate_acc(output, y)

    return loss_value / size, correct / size

def build_model():
    model = Model()
    f1 = DenseLayer('f1', 5, 5, 1)
    f2 = SoftmaxLayer('f2')
    f3 = TanhLayer('f3')
    model.addLayer(f1)
    # model.addLayer(f3)
    return model

def plot(logs):
    x = [i['step'] for i in logs]
    y = [i['train_acc'] for i in logs]
    plt.plot(x, y, label='train_acc')
    y = [i['val_acc'] for i in logs]
    plt.plot(x, y, label='val_acc')
    plt.legend()
    plt.show()
    y = [i['train_loss'] for i in logs]
    plt.plot(x, y, label='train_loss')
    y = [i['val_loss'] for i in logs]
    plt.plot(x, y, label='val_loss')
    plt.legend()
    plt.show()

if __name__ == "__main__":
    np.random.seed(13)

    with open('train.pkl', 'rb') as f:
        train_data = pickle.load(f)

    with open('validation.pkl', 'rb') as f:
        val_data = pickle.load(f)

    with open('data.pkl', 'rb') as f:
        test_data = pickle.load(f)

    x_data = train_data['x']
    y_data = train_data['y']

    model = build_model()
    batch_size = 128
    size = len(x_data)
    global_step = 0

    epoch = 2

    loss = SoftmaxCrossEntropyLoss('loss')

    logs = []
    for i in range(epoch):

        for start_idx in range(0, size, batch_size):
            end_idx = min(start_idx + batch_size, size)
            x = np.array(x_data[start_idx: end_idx])
            label = y_data[start_idx: end_idx]
            y = onehot_encoding(label, 5)

            x = np.array(x)

            val_loss, val_acc = 0, 0
            val_loss, val_acc = evaluate(model, val_data)

            ans = model.forward(x)
            train_acc = calculate_acc(softmax(ans), label)
            train_loss = loss.forward(ans, y)

            log_dict = {'step': global_step, 'train_loss': train_loss, 'train_acc':train_acc, 'val_loss':val_loss, 'val_acc':val_acc}

            logs.append(log_dict)

            msg = 'epoch: %d  steps: %d \n train_loss: %.3f   train_acc: %.3f \n valid_loss: %.3f   valid_acc: %.3f'%(i, global_step, train_loss, train_acc, val_loss, val_acc)
            print(msg)

            grad = loss.backward(ans, y)
            model.backward(grad)
            model.update()

            global_step += len(label)

    plot(logs)


# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from django.core.exceptions import ImproperlyConfigured
from django.core.management.base import NoArgsCommand, CommandError, OutputWrapper
from django.core import serializers
from django.db import router, DEFAULT_DB_ALIAS
from django.utils.datastructures import SortedDict
from django.contrib.auth.models import User

from optparse import make_option
import logging
import sys

#class Command(BaseCommand):
#    option_list = BaseCommand.option_list + (
#        make_option('--format', default='json', dest='format',
#            help='Specifies the output serialization format for fixtures.'),
#        make_option('--indent', default=None, dest='indent', type='int',
#            help='Specifies the indent level to use when pretty-printing output'),
#        make_option('--database', action='store', dest='database',
#            default=DEFAULT_DB_ALIAS, help='Nominates a specific database to dump '
#                'fixtures from. Defaults to the "default" database.'),
#        make_option('-e', '--exclude', dest='exclude',action='append', default=[],
#            help='An appname or appname.ModelName to exclude (use multiple --exclude to exclude multiple apps/models).'),
#        make_option('-n', '--natural', action='store_true', dest='use_natural_keys', default=False,
#            help='Yay CHRIS Use natural keys if they are available.'),
#        make_option('-a', '--all', action='store_true', dest='use_base_manager', default=False,
#            help="Use Django's base manager to dump all models stored in the database, including those that would otherwise be filtered or modified by a custom manager."),
#        make_option('--pks', dest='primary_keys', help="Only dump objects with "
#            "given primary keys. Accepts a comma seperated list of keys. "
#            "This option will only work when you specify one model."),
#    )
#    help = ("Output the contents of the database as a fixture of the given "
#            "format (using each model's default manager unless --all is "
#            "specified).")
#    args = '[appname appname.ModelName ...]'

#    def handle(self, *app_labels, **options):
class Command(NoArgsCommand):
  def handle_noargs(self, *app_labels, **options):
        from django.db.models import get_app, get_apps, get_model

        format = options.get('format')
        indent = options.get('indent')
        using = options.get('database')
        excludes = options.get('exclude')
        show_traceback = options.get('traceback')
        use_natural_keys = options.get('use_natural_keys')
        use_base_manager = options.get('use_base_manager')
        pks = options.get('primary_keys')
        user = options.get('user')
        userid = user.id

        stdout = OutputWrapper(options.get('stdout', sys.stdout))

        if pks:
            primary_keys = pks.split(',')
        else:
            primary_keys = []

        excluded_apps = set()
        excluded_models = set()
        if excludes:
            for exclude in excludes:
                if '.' in exclude:
                    app_label, model_name = exclude.split('.', 1)
                    model_obj = get_model(app_label, model_name)
                    if not model_obj:
                        raise CommandError('Unknown model in excludes: %s' % exclude)
                    excluded_models.add(model_obj)
                else:
                    try:
                        app_obj = get_app(exclude)
                        excluded_apps.add(app_obj)
                    except ImproperlyConfigured:
                        raise CommandError('Unknown app in excludes: %s' % exclude)

        if len(app_labels) == 0:
            if primary_keys:
                raise CommandError("You can only use --pks option with one model")
            app_list = SortedDict((app, None) for app in get_apps() if app not in excluded_apps)
        else:
            if len(app_labels) > 1 and primary_keys:
                raise CommandError("You can only use --pks option with one model")
            app_list = SortedDict()
            for label in app_labels:
                try:
                    app_label, model_label = label.split('.')
                    try:
                        app = get_app(app_label)
                    except ImproperlyConfigured:
                        raise CommandError("Unknown application: %s" % app_label)
                    if app in excluded_apps:
                        continue
                    model = get_model(app_label, model_label)
                    if model is None:
                        raise CommandError("Unknown model: %s.%s" % (app_label, model_label))

                    if app in app_list.keys():
                        if app_list[app] and model not in app_list[app]:
                            app_list[app].append(model)
                    else:
                        app_list[app] = [model]
                except ValueError:
                    if primary_keys:
                        raise CommandError("You can only use --pks option with one model")
                    # This is just an app - no model qualifier
                    app_label = label
                    try:
                        app = get_app(app_label)
                    except ImproperlyConfigured:
                        raise CommandError("Unknown application: %s" % app_label)
                    if app in excluded_apps:
                        continue
                    app_list[app] = None

        # Check that the serialization format exists; this is a shortcut to
        # avoid collating all the objects and _then_ failing.
        if format not in serializers.get_public_serializer_formats():
            try:
                serializers.get_serializer(format)
            except serializers.SerializerDoesNotExist:
                pass

            raise CommandError("Unknown serialization format: %s" % format)

        def get_objects():
            # Collate the objects to be serialized.
            for model in sort_dependencies(app_list.items()):
                if model in excluded_models:
                    continue
                if not model._meta.proxy and router.allow_syncdb(using, model):
                    if use_base_manager:
                        objects = model._base_manager
                    else:
                        objects = model._default_manager

                    queryset = objects.using(using).order_by(model._meta.pk.name)
                    if primary_keys:
                        queryset = queryset.filter(pk__in=primary_keys)
                    queryset = queryset.filter(owner_id=userid)
                    for obj in queryset.iterator():
                        yield obj

        try:
            stdout.ending = None
#            self.stdout.ending = None
            serializers.serialize(format, get_objects(), indent=indent,
                    use_natural_keys=use_natural_keys, stream=stdout)
            stdout
#            serializers.serialize(format, get_objects(), indent=indent,
#                    use_natural_keys=use_natural_keys, stream=self.stdout)
        except Exception as e:
            if show_traceback:
                raise
            raise CommandError("Unable to serialize database: %s" % e)

def sort_dependencies(app_list):
    """Sort a list of app,modellist pairs into a single list of models.

    The single list of models is sorted so that any model with a natural key
    is serialized before a normal model, and any model with a natural key
    dependency has it's dependencies serialized first.
    """
    from django.db.models import get_model, get_models
    # Process the list of models, and get the list of dependencies
    model_dependencies = []
    models = set()
    for app, model_list in app_list:
        if model_list is None:
            model_list = get_models(app)

        for model in model_list:
            models.add(model)
            # Add any explicitly defined dependencies
            if hasattr(model, 'natural_key'):
                deps = getattr(model.natural_key, 'dependencies', [])
                if deps:
                    deps = [get_model(*d.split('.')) for d in deps]
            else:
                deps = []

            # Now add a dependency for any FK or M2M relation with
            # a model that defines a natural key
            for field in model._meta.fields:
                if hasattr(field.rel, 'to'):
                    rel_model = field.rel.to
                    if hasattr(rel_model, 'natural_key') and rel_model != model:
                        deps.append(rel_model)
            for field in model._meta.many_to_many:
                rel_model = field.rel.to
                if hasattr(rel_model, 'natural_key') and rel_model != model:
                    deps.append(rel_model)
            model_dependencies.append((model, deps))

    model_dependencies.reverse()
    # Now sort the models to ensure that dependencies are met. This
    # is done by repeatedly iterating over the input list of models.
    # If all the dependencies of a given model are in the final list,
    # that model is promoted to the end of the final list. This process
    # continues until the input list is empty, or we do a full iteration
    # over the input models without promoting a model to the final list.
    # If we do a full iteration without a promotion, that means there are
    # circular dependencies in the list.
    model_list = []
    while model_dependencies:
        skipped = []
        changed = False
        while model_dependencies:
            model, deps = model_dependencies.pop()

            # If all of the models in the dependency list are either already
            # on the final model list, or not on the original serialization list,
            # then we've found another model with all it's dependencies satisfied.
            found = True
            for candidate in ((d not in models or d in model_list) for d in deps):
                if not candidate:
                    found = False
            if found:
                model_list.append(model)
                changed = True
            else:
                skipped.append((model, deps))
        if not changed:
            raise CommandError("Can't resolve dependencies for %s in serialized app list." %
                ', '.join('%s.%s' % (model._meta.app_label, model._meta.object_name)
                for model, deps in sorted(skipped, key=lambda obj: obj[0].__name__))
            )
        model_dependencies = skipped

    return model_list


#!/usr/bin/python

#from pushover import Pushover
import argparse
from datetime import datetime
from subprocess import check_call, CalledProcessError
from sys import argv

def run(args):
    maxprocs = args.maxprocs
    verify = args.verify
    overload = args.overload_allowed
    message_size = args.message_size
    binding = args.binding
    plot = args.plot

    if verify: vstring = "--verify "
    else: vstring = ""

    if overload: binding += ":overload-allowed"

    if plot: pstring = "--plot "
    else: pstring = ""

    memory = ['sm', 'vader']
    time = datetime.now().strftime("%Y-%m-%d.%H%M%S")

    try:
        check_call("mpicc -g -o bigarrays bigarrays.c -std=c11", shell=True)
    except CalledProcessError as err:
        print("Error in Compiling with code" + str(err.returncode))
        exit(1)

    try:
        check_call("mkdir -p data/" + time, shell=True)
    except CalledProcessError as err:
        print("Error in creating data directory with error code" + str(err.returncode))
        exit(1)

    for nprocs in range(2, maxprocs+1):
        for mem in memory:
            base = "data/" + time + "/np="+ str(nprocs) + "_shmem=" + mem + "log"
            logfile = base + ".out"
            mpirun_logfile = base + ".mpirun"

            run = "mpirun -np  " + str(nprocs) + " --bind-to " + binding + " --mca btl " + mem + ",self bigarrays " + pstring + vstring + "--message_size " + str(message_size) + " > " + logfile

            print(run)
            f = open(mpirun_logfile, 'w')
            f.write(run + "\n")
            f.close()

            try:
                check_call(run, shell=True)
            except CalledProcessError as err:
                print("mpirun failed with code" + str(err.returncode))
                exit(1)

            if plot:
                f = open(logfile, 'r')
                data = []
                for line in f:
                    data += line.split()
                data = [float(i) for i in data]
                f.close()

                data = data[1::2]
                np = len(data)
                avg = sum(data)/len(data)

                f = open("data/" + time + "/" + mem + "-data.txt", 'a')
                f.write(str(np) + " " + str(avg) + "\n")
                f.close()

    if plot:
        try:
            check_call("gnuplot -e \"sm='data/"+time+"/sm-data.txt'\" -e \"vader='data/"+time+"/vader-data.txt'\" -e \"out='data/"+time+"/output-graph.pdf'\" plot.gnu", shell=True)
        except CalledProcessError as err:
            print("plot failed with error code " + str(err.returncode))
            exit(1)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("maxprocs", type=int, help="maximum number of processes to be run")
    parser.add_argument("--verify", action="store_true", help="verify the correctness of the data sent")
    parser.add_argument("--overload-allowed", action="store_true", help="allow overload of cores")
    parser.add_argument("--message_size", default=50000, type=int, help="specify the size of each message sent, defaults to 50000")
    parser.add_argument("--binding", help="set the bind type for mpi runtime. Should be none, core or socket")
    parser.add_argument("--plot", action="store_true", help="use gnuplot to plot the output data to pdf. Specifying this disables debug output")
    args = parser.parse_args()
    run(args)

if __name__ == "__main__":
    main()


from datetime import timedelta
from uuid import UUID

import pytest
from colour import Color
from pytest import raises
from sqlalchemy.util import OrderedSet

from ereuse_devicehub.client import UserClient
from ereuse_devicehub.db import db
from ereuse_devicehub.devicehub import Devicehub
from ereuse_devicehub.resources.device.exceptions import NeedsId
from ereuse_devicehub.resources.device.models import Component, Computer, Desktop, Device, \
    GraphicCard, Laptop, Microtower, Motherboard, NetworkAdapter
from ereuse_devicehub.resources.device.schemas import Device as DeviceS
from ereuse_devicehub.resources.device.sync import MismatchBetweenTags, MismatchBetweenTagsAndHid, \
    Sync
from ereuse_devicehub.resources.event.models import Remove, Test
from ereuse_devicehub.resources.tag.model import Tag
from ereuse_devicehub.resources.user import User
from ereuse_utils.naming import Naming
from teal.db import ResourceNotFound
from tests.conftest import file

@pytest.mark.usefixtures('app_context')
def test_device_model():
    """
    Tests that the correctness of the device model and its relationships.
    """
    pc = Desktop(model='p1mo', manufacturer='p1ma', serial_number='p1s')
    net = NetworkAdapter(model='c1mo', manufacturer='c1ma', serial_number='c1s')
    graphic = GraphicCard(model='c2mo', manufacturer='c2ma', memory=1500)
    pc.components.add(net)
    pc.components.add(graphic)
    db.session.add(pc)
    db.session.commit()
    pc = Desktop.query.one()
    assert pc.serial_number == 'p1s'
    assert pc.components == OrderedSet([net, graphic])
    network_adapter = NetworkAdapter.query.one()
    assert network_adapter.parent == pc

    # Removing a component from pc doesn't delete the component
    pc.components.remove(net)
    db.session.commit()
    pc = Device.query.first()  # this is the same as querying for Desktop directly
    assert pc.components == {graphic}
    network_adapter = NetworkAdapter.query.one()
    assert network_adapter not in pc.components
    assert network_adapter.parent is None

    # Deleting the pc deletes everything
    gcard = GraphicCard.query.one()
    db.session.delete(pc)
    db.session.flush()
    assert pc.id == 1
    assert Desktop.query.first() is None
    db.session.commit()
    assert Desktop.query.first() is None
    assert network_adapter.id == 2
    assert NetworkAdapter.query.first() is not None, 'We removed the network adaptor'
    assert gcard.id == 3, 'We should still hold a reference to a zombie graphic card'
    assert GraphicCard.query.first() is None, 'We should have deleted it –it was inside the pc'

@pytest.mark.usefixtures('app_context')
def test_device_schema():
    """Ensures the user does not upload non-writable or extra fields."""
    device_s = DeviceS()
    device_s.load({'serialNumber': 'foo1', 'model': 'foo', 'manufacturer': 'bar2'})
    device_s.dump(Device(id=1))

@pytest.mark.usefixtures('app_context')
def test_physical_properties():
    c = Motherboard(slots=2,
                    usb=3,
                    serial_number='sn',
                    model='ml',
                    manufacturer='mr',
                    width=2.0,
                    color=Color())
    pc = Computer()
    pc.components.add(c)
    db.session.add(pc)
    db.session.commit()
    assert c.physical_properties == {
        'usb': 3,
        'serial_number': 'sn',
        'pcmcia': None,
        'model': 'ml',
        'slots': 2,
        'serial': None,
        'firewire': None,
        'manufacturer': 'mr',
        'weight': None,
        'height': None,
        'width': 2.0,
        'color': Color(),
        'depth': None
    }

@pytest.mark.usefixtures('app_context')
def test_component_similar_one():
    snapshot = file('pc-components.db')
    d = snapshot['device']
    snapshot['components'][0]['serial_number'] = snapshot['components'][1]['serial_number'] = None
    pc = Computer(**d, components=OrderedSet(Component(**c) for c in snapshot['components']))
    component1, component2 = pc.components  # type: Component
    db.session.add(pc)
    db.session.flush()
    # Let's create a new component named 'A' similar to 1
    componentA = Component(model=component1.model, manufacturer=component1.manufacturer)
    similar_to_a = componentA.similar_one(pc, set())
    assert similar_to_a == component1
    # Component B does not have the same model
    componentB = Component(model='nope', manufacturer=component1.manufacturer)
    with pytest.raises(ResourceNotFound):
        assert componentB.similar_one(pc, set())
    # If we blacklist component A we won't get anything
    with pytest.raises(ResourceNotFound):
        assert componentA.similar_one(pc, blacklist={componentA.id})

@pytest.mark.usefixtures('auth_app_context')
def test_add_remove():
    # Original state:
    # pc has c1 and c2
    # pc2 has c3
    # c4 is not with any pc
    values = file('pc-components.db')
    pc = values['device']
    c1, c2 = (Component(**c) for c in values['components'])
    pc = Computer(**pc, components=OrderedSet([c1, c2]))
    db.session.add(pc)
    c3 = Component(serial_number='nc1')
    pc2 = Computer(serial_number='s2', components=OrderedSet([c3]))
    c4 = Component(serial_number='c4s')
    db.session.add(pc2)
    db.session.add(c4)
    db.session.commit()

    # Test:
    # pc has only c3
    events = Sync.add_remove(device=pc, components={c3, c4})
    db.session.add_all(events)
    db.session.commit()  # We enforce the appliance of order_by
    assert len(events) == 1
    assert isinstance(events[0], Remove)
    assert events[0].device == pc2
    assert events[0].components == OrderedSet([c3])

@pytest.mark.usefixtures('app_context')
def test_sync_run_components_empty():
    """
    Syncs a device that has an empty components list. The system should
    remove all the components from the device.
    """
    s = file('pc-components.db')
    pc = Computer(**s['device'], components=OrderedSet(Component(**c) for c in s['components']))
    db.session.add(pc)
    db.session.commit()

    # Create a new transient non-db synced object
    pc = Computer(**s['device'])
    db_pc, _ = Sync().run(pc, components=OrderedSet())
    assert not db_pc.components
    assert not pc.components

@pytest.mark.usefixtures('app_context')
def test_sync_run_components_none():
    """
    Syncs a device that has a None components. The system should
    keep all the components from the device.
    """
    s = file('pc-components.db')
    pc = Computer(**s['device'], components=OrderedSet(Component(**c) for c in s['components']))
    db.session.add(pc)
    db.session.commit()

    # Create a new transient non-db synced object
    transient_pc = Computer(**s['device'])
    db_pc, _ = Sync().run(transient_pc, components=None)
    assert db_pc.components
    assert db_pc.components == pc.components

@pytest.mark.usefixtures('app_context')
def test_sync_execute_register_computer_new_computer_no_tag():
    """
    Syncs a new computer with HID and without a tag, creating it.
    :return:
    """
    # Case 1: device does not exist on DB
    pc = Computer(**file('pc-components.db')['device'])
    db_pc = Sync().execute_register(pc)
    assert pc.physical_properties == db_pc.physical_properties

@pytest.mark.usefixtures('app_context')
def test_sync_execute_register_computer_existing_no_tag():
    """
    Syncs an existing computer with HID and without a tag.
    """
    pc = Computer(**file('pc-components.db')['device'])
    db.session.add(pc)
    db.session.commit()

    pc = Computer(**file('pc-components.db')['device'])  # Create a new transient non-db object
    # 1: device exists on DB
    db_pc = Sync().execute_register(pc)
    assert pc.physical_properties == db_pc.physical_properties

@pytest.mark.usefixtures('app_context')
def test_sync_execute_register_computer_no_hid_no_tag():
    """
    Syncs a computer without HID and no tag.

    This should fail as we don't have a way to identify it.
    """
    pc = Computer(**file('pc-components.db')['device'])
    # 1: device has no HID
    pc.hid = pc.model = None
    with pytest.raises(NeedsId):
        Sync().execute_register(pc)

@pytest.mark.usefixtures('app_context')
def test_sync_execute_register_computer_tag_not_linked():
    """
    Syncs a new computer with HID and a non-linked tag.

    It is OK if the tag was not linked, it will be linked in this process.
    """
    tag = Tag(id='FOO')
    db.session.add(tag)
    db.session.commit()

    # Create a new transient non-db object
    pc = Computer(**file('pc-components.db')['device'], tags=OrderedSet([Tag(id='FOO')]))
    returned_pc = Sync().execute_register(pc)
    assert returned_pc == pc
    assert tag.device == pc, 'Tag has to be linked'
    assert Computer.query.one() == pc, 'Computer had to be set to db'

@pytest.mark.usefixtures('app_context')
def test_sync_execute_register_no_hid_tag_not_linked(tag_id: str):
    """
    Validates registering a computer without HID and a non-linked tag.

    In this case it is ok still, as the non-linked tag proves that
    the computer was not existing before (otherwise the tag would
    be linked), and thus it creates a new computer.
    """
    tag = Tag(id=tag_id)
    pc = Computer(**file('pc-components.db')['device'], tags=OrderedSet([tag]))
    returned_pc = Sync().execute_register(pc)
    db.session.commit()
    assert returned_pc == pc
    db_tag = next(iter(returned_pc.tags))
    # they are not the same tags though
    # tag is a transient obj and db_tag the one from the db
    # they have the same pk though
    assert tag != db_tag, 'They are not the same tags though'
    assert db_tag.id == tag.id
    assert Computer.query.one() == pc, 'Computer had to be set to db'

@pytest.mark.usefixtures('app_context')
def test_sync_execute_register_tag_does_not_exist():
    """
    Ensures not being able to register if the tag does not exist,
    even if the device has HID or it existed before.

    Tags have to be created before trying to link them through a Snapshot.
    """
    pc = Computer(**file('pc-components.db')['device'], tags=OrderedSet([Tag()]))
    with raises(ResourceNotFound):
        Sync().execute_register(pc)

@pytest.mark.usefixtures('app_context')
def test_sync_execute_register_tag_linked_same_device():
    """
    If the tag is linked to the device, regardless if it has HID,
    the system should match the device through the tag.
    (If it has HID it validates both HID and tag point at the same
    device, this his checked in ).
    """
    orig_pc = Computer(**file('pc-components.db')['device'])
    db.session.add(Tag(id='foo', device=orig_pc))
    db.session.commit()

    pc = Computer(**file('pc-components.db')['device'])  # Create a new transient non-db object
    pc.tags.add(Tag(id='foo'))
    db_pc = Sync().execute_register(pc)
    assert db_pc.id == orig_pc.id
    assert len(db_pc.tags) == 1
    assert next(iter(db_pc.tags)).id == 'foo'

@pytest.mark.usefixtures('app_context')
def test_sync_execute_register_tag_linked_other_device_mismatch_between_tags():
    """
    Checks that sync raises an error if finds that at least two passed-in
    tags are not linked to the same device.
    """
    pc1 = Computer(**file('pc-components.db')['device'])
    db.session.add(Tag(id='foo-1', device=pc1))
    pc2 = Computer(**file('pc-components.db')['device'])
    pc2.serial_number = 'pc2-serial'
    pc2.hid = Naming.hid(pc2.manufacturer, pc2.serial_number, pc2.model)
    db.session.add(Tag(id='foo-2', device=pc2))
    db.session.commit()

    pc1 = Computer(**file('pc-components.db')['device'])  # Create a new transient non-db object
    pc1.tags.add(Tag(id='foo-1'))
    pc1.tags.add(Tag(id='foo-2'))
    with raises(MismatchBetweenTags):
        Sync().execute_register(pc1)

@pytest.mark.usefixtures('app_context')
def test_sync_execute_register_mismatch_between_tags_and_hid():
    """
    Checks that sync raises an error if it finds that the HID does
    not point at the same device as the tag does.

    In this case we set HID -> pc1 but tag -> pc2
    """
    pc1 = Computer(**file('pc-components.db')['device'])
    db.session.add(Tag(id='foo-1', device=pc1))
    pc2 = Computer(**file('pc-components.db')['device'])
    pc2.serial_number = 'pc2-serial'
    pc2.hid = Naming.hid(pc2.manufacturer, pc2.serial_number, pc2.model)
    db.session.add(Tag(id='foo-2', device=pc2))
    db.session.commit()

    pc1 = Computer(**file('pc-components.db')['device'])  # Create a new transient non-db object
    pc1.tags.add(Tag(id='foo-2'))
    with raises(MismatchBetweenTagsAndHid):
        Sync().execute_register(pc1)

def test_get_device(app: Devicehub, user: UserClient):
    """Checks GETting a Desktop with its components."""
    with app.app_context():
        pc = Desktop(model='p1mo', manufacturer='p1ma', serial_number='p1s')
        pc.components = OrderedSet([
            NetworkAdapter(model='c1mo', manufacturer='c1ma', serial_number='c1s'),
            GraphicCard(model='c2mo', manufacturer='c2ma', memory=1500)
        ])
        db.session.add(pc)
        db.session.add(Test(device=pc,
                            elapsed=timedelta(seconds=4),
                            error=False,
                            author=User(email='bar@bar.com')))
        db.session.commit()
    pc, _ = user.get(res=Device, item=1)
    assert len(pc['events']) == 1
    assert pc['events'][0]['type'] == 'Test'
    assert pc['events'][0]['device'] == 1
    assert pc['events'][0]['elapsed'] == 4
    assert not pc['events'][0]['error']
    assert UUID(pc['events'][0]['author'])
    assert 'events_components' not in pc, 'events_components are internal use only'
    assert 'events_one' not in pc, 'they are internal use only'
    assert 'author' not in pc
    assert tuple(c['id'] for c in pc['components']) == (2, 3)
    assert pc['hid'] == 'p1ma-p1s-p1mo'
    assert pc['model'] == 'p1mo'
    assert pc['manufacturer'] == 'p1ma'
    assert pc['serialNumber'] == 'p1s'
    assert pc['type'] == 'Desktop'

def test_get_devices(app: Devicehub, user: UserClient):
    """Checks GETting multiple devices."""
    with app.app_context():
        pc = Desktop(model='p1mo', manufacturer='p1ma', serial_number='p1s')
        pc.components = OrderedSet([
            NetworkAdapter(model='c1mo', manufacturer='c1ma', serial_number='c1s'),
            GraphicCard(model='c2mo', manufacturer='c2ma', memory=1500)
        ])
        pc1 = Microtower(model='p2mo', manufacturer='p2ma', serial_number='p2s')
        pc2 = Laptop(model='p3mo', manufacturer='p3ma', serial_number='p3s')
        db.session.add_all((pc, pc1, pc2))
        db.session.commit()
    devices, _ = user.get(res=Device)
    assert tuple(d['id'] for d in devices) == (1, 2, 3, 4, 5)
    assert tuple(d['type'] for d in devices) == ('Desktop', 'Microtower',
                                                 'Laptop', 'NetworkAdapter', 'GraphicCard')