"""
This file initialises, allows the modification of parameters and runs the evolutionary algorithm
"""
import numpy as np
import matplotlib.pyplot as plt
import random
from celluloid import Camera
from matplotlib import colors
from scipy.stats import sem

from ea.encoding import real_number_encoding
from ea.selection import tournament_selection
from ea.crossover_mutation import choose_pair, one_point_crossover, uniform_crossover, arithmetic_crossover, mutation
from ea.reproduction import generational_rollover
from ea.evaluation import get_xy_phenotype, cost_rosenbrock, cost_rastrigin
from ea.robot_evaluation import evaluate_genotype

import time

output_directory = "animations/ea_benchmark/diyon/"
plot_directory = "plots/"
phenotype_history = []
population_dictionary = {}

population_size = 30

generations = 100

offsprings_per_generation = 25

# Variables for the encoding strategy
genotype_length = 210
genotype_min_range = -10
genotype_max_range = 10
whole_numbers = False

# Variables for the selection strategy
tournament_k = 6

# Variables for the crossover/mutation strategies
mutation_rate = 5

"""
IMPORTANT:

Any time you change one of these strategies/functions, be sure to check if the arguments you pass are
actually those that the strategy/functions require

"""
encoding_strategy = real_number_encoding
phenotype_computer = get_xy_phenotype
cost_function = cost_rosenbrock
selection_strategy = tournament_selection
reproduction_strategy = generational_rollover

def initialise(room):
    """
    This function populates the population dictionary by using the specified encoding strategy.
    Each individual has a key (integer), a genotype array and a fitness.

    The dictionary will be of the form:

    key : value
    int : [[list], int]
    <ID> : [genotype], fitness

    :return: none
    """

    # generation = []
    for individual in range(population_size):

        genotype = encoding_strategy(genotype_length, genotype_min_range, genotype_max_range, integers=whole_numbers)

        population_dictionary[individual] = [genotype, evaluate_genotype(genotype, individual, room)]

def initialise_rooms(rooms):
    """
    This function populates the population dictionary by using the specified encoding strategy.
    Each individual has a key (integer), a genotype array and a fitness.

    The dictionary will be of the form:

    key : value
    int : [[list], int]
    <ID> : [genotype], fitness

    :return: none
    """

    # generation = []
    for individual in range(population_size):

        genotype = encoding_strategy(genotype_length, genotype_min_range, genotype_max_range, integers=whole_numbers)

        fitness = 0
        for room in rooms:
            print("Room: " + str(room[2]))
            fitness += evaluate_genotype(genotype, individual, room)

        population_dictionary[individual] = [genotype, fitness / len(rooms)]

def run_algorithm(room, file):
    """
    Initialises the data, and then runs evaluation, selection, reproduction and mutations for the specified amount
    of generations

    :param room: Room list
    :param file: An open file to write to
    :return: a max fitness array and an average fitness array, both of length equal to number of generations + 1
            (to include "generation 0" after initialisation but before any reproduction has taken place), containing
            max fitness and average fitness of the population at every generation
    """

    max_fitness_history = []
    avg_fitness_history = []

    start = time.time()
    print(f"Running for {generations} generations with {population_size} genotypes\n"
          f"Initialising all genotypes")

    file.write(f"This file contains a run for {generations} generations with {population_size} genotypes\n"
               f"offspring per generation: {offsprings_per_generation}\n"
               f"k: {tournament_k}\n"
               f"mutation rate: {mutation_rate}\n\n")

    initialise(room)

    print(f"Done initialising in {time.time() - start} seconds")

    file.write("Initialised genotypes:\n")
    write_dictionary(population_dictionary, file)
    file.write("############################\n")

    start = time.time()

    # Get the max and the avg fitness at the start
    features = [individual[1] for individual in population_dictionary.values()]
    max_fitness_history.append(max(features))
    avg_fitness_history.append(sum(features) / len(features))

    for i in range(generations):

        print(f"Evaluating generation: {i+1}")
        file.write(f"GENERATION: {i+1}\n")

        # SELECTION
        # For the number of desired offspring per generation, choose the best parent
        # and momentarily copy them over in a dedicated offspring dictionary
        offspring_dictionary = {}

        for j in range(offsprings_per_generation):

            best_parent = selection_strategy(population_dictionary, k=tournament_k)

            # offspring is initialised (genotype is set as empty, fitness is set to 0)
            offspring_dictionary[j] = [[], 0]

            # genotype of offspring is copied over from parent
            for gene in population_dictionary[best_parent][0]:
                offspring_dictionary[j][0].append(gene)

        # CROSS OVER
        crossover_events = int(random.uniform(0.1, 0.4)*offsprings_per_generation)

        for _ in range(crossover_events):

            offspring_pair = choose_pair(offspring_dictionary)
            rng = random.randint(0, 99)

            if rng < 33:
                one_point_crossover(offspring_dictionary, offspring_pair)
            elif rng < 66:
                uniform_crossover(offspring_dictionary, offspring_pair)
            else:
                arithmetic_crossover(offspring_dictionary, offspring_pair, integers=False)

        # MUTATION
        for target_offspring in offspring_dictionary:
            mutation(offspring_dictionary, target_offspring, mutation_rate, integers=False)

        # REPRODUCTION
        # before offsprings are added into the population, but after they have undergone crossover and mutation events
        # (so that their genotype is "finalised"), their fitness and phenotype is computed
        for offspring in offspring_dictionary:

            genotype = offspring_dictionary[offspring][0]

            offspring_dictionary[offspring][1] = evaluate_genotype(genotype, offspring, room)

        reproduction_strategy(population_dictionary, offspring_dictionary)

        # max fitness and average fitness of the population are extracted and appended to the max fitness and
        # average fitness arrays
        features = [individual[1] for individual in population_dictionary.values()]

        max_fitness = max(features)
        avg_fitness = sum(features) / len(features)
        max_fitness_history.append(max_fitness)
        avg_fitness_history.append(avg_fitness)

        end = time.time()
        print(f"Finished generation {i+1} in {end - start} seconds.\n "
              f"Max fitness: {max_fitness}\n "
              f"Avg. fitness: {avg_fitness}")

        start = time.time()

        write_dictionary(population_dictionary, file)
        file.write("############################\n")

    best_genotype = max(population_dictionary.values(), key=lambda x: x[1])
    print(f"The best genotype is:\n"
          f"{best_genotype}")

    file.write(f"The best genotype is: {best_genotype}")

    return max_fitness_history, avg_fitness_history

def write_dictionary(dictionary, file):

    for genotype_id in dictionary:

        line = f"ID: {str(genotype_id)}\n" \
               f"Genotype: {str(dictionary[genotype_id][0])}\n" \
               f"Fitness: {str(dictionary[genotype_id][1])}\n\n"

        file.write(line)
        file.write("------------------------------------\n")

def animate_evolution():
    """
    A function to animate the plotting of graphs

    :return: None
    """
    fig, ax = plt.subplots()
    camera = Camera(fig)

    n = 100
    X = np.linspace(genotype_min_range, genotype_max_range, n)
    Y = np.linspace(genotype_min_range, genotype_max_range, n)
    X, Y = np.meshgrid(X, Y)

    Z = cost_function(X, Y)

    for i in range(len(phenotype_history)):

        generation = phenotype_history[i]

        # Holds all phenotypes of all genotypes
        x, y = [], []
        for phenotype in generation:
            # Get the phenotype
            phenotype_x, phenotype_y = phenotype
            x.append(phenotype_x)
            y.append(phenotype_y)

        # Plot all phenotypes
        ax.plot(x, y, 'bo', c="#000000")
        ax.set_xlim([genotype_min_range, genotype_max_range])
        ax.set_ylim([genotype_min_range, genotype_max_range])
        ax.text(0.4, 1.01, f"Generation {i+1}", transform=ax.transAxes)
        camera.snap()

    im = plt.contourf(X, Y, Z, levels=50, cmap="jet") if cost_function == cost_rastrigin else \
         plt.pcolor(X, Y, Z, norm=colors.LogNorm(vmin=Z.min(), vmax=Z.max()), cmap='jet', shading='auto')

    fig.colorbar(im, orientation='vertical', extend='max')
    ax.set_xlabel("x")
    ax.set_ylabel("y")

    # Animates
    animation = camera.animate(interval=200, repeat=False, repeat_delay=500)
    # animation.save(output_directory +
    #                f"{cost_function.__name__}_pop{population_size}_offs{offsprings_per_generation}_gens{generations}_"
    #                f"tk{tournament_k}_mr{mutation_rate}.gif")
    plt.show()

def test_helper(results, value, tests, room):
    """
    A helper function to make code more concise
    Initialises average max fitness and average average fitness arrays, runs the evolutionary algorithm multiple times
    and fills said arrays across iterations and appends said arrays into the dictionary of results

    :param results: The dictionary of results
    :param value: The value for the parameter being investigated
    :param tests: The number of times the evolutionary algorithm should be run
    :return: None
    """

    avg_max_fitnesses = [0 for _ in range(generations+1)]
    avg_avg_fitnesses = [0 for _ in range(generations+1)]
    results[value] = []

    for itter in range(tests):
        max_fitness_history, avg_fitness_history = run_algorithm(room)
        for i in range(len(max_fitness_history)):
            avg_max_fitnesses[i] += max_fitness_history[i]
            avg_avg_fitnesses[i] += avg_fitness_history[i]

    # the dictionary of results is modified in place
    results[value].append(avg_max_fitnesses)
    results[value].append(avg_avg_fitnesses)

def testing_routine(parameter_set, parameter, room, tests=100, short=False):
    """
    The function that handles running multiple experiments with different values for a given parameter
    to be investigated

    :param parameter_set: A list of values for the parameter to be investigated
    :param parameter: The parameter of the evolutionary algorithm under investigation
    :param tests: The number of times the evolutionary algorithm should be run to test a single value for the parameter
    :param short: A boolean to indicate whether the user wants to run a simple and quick testing routine

    :return: A dictionary of results, the keys are values for a parameter being investigated in a testing routine
            and values are lists of two arrays, the first containing average max fitness at every generation
            of the genetic algorithm and the second containing average average fitness at every generation
    """

    if short:
        global generations
        generations = 10
        tests = 10

    results = {}

    if parameter == "tournament_k":

        for value in parameter_set:

            global tournament_k
            tournament_k = value
            test_helper(results, value, tests, room)
            print("Finished with value " + str(value))

    elif parameter == "offsprings_per_generation":

        for value in parameter_set:

            global offsprings_per_generation
            offsprings_per_generation = value
            test_helper(results, value, tests, room)
            print("Finished with value " + str(value))

    elif parameter == "mutation_rate":

        for value in parameter_set:

            global mutation_rate
            mutation_rate = value
            test_helper(results, value, tests, room)
            print("Finished with value "+str(value))

    return results

def run_algorithm_with_parameters(rooms, gens, k, num_offspring, mr, file):
    """
    This function allows the value of parameters to be changed by passing in values

    Initialises the data, and then runs evaluation, selection, reproduction and mutations for the specified amount
    of generations

    :param file: An open file object
    :param mr: Integer for mutation rate
    :param num_offspring: Integer for number of offspring per generation
    :param k: Integer for tournament k
    :param gens: Integer for number of generations
    :param room: Room list

    :return: a max fitness array and an average fitness array, both of length equal to number of generations + 1
            (to include "generation 0" after initialisation but before any reproduction has taken place), containing
            max fitness and average fitness of the population at every generation
    """

    max_fitness_history = []
    avg_fitness_history = []

    start = time.time()
    print(f"Running for {gens} generations with {population_size} genotypes\n"
          f"Initialising all genotypes")

    file.write(f"This file contains a run for {gens} generations with {population_size} genotypes\n"
               f"offspring per generation: {num_offspring}\n"
               f"k: {k}\n"
               f"mutation rate: {mr}\n\n")

    initialise_rooms(rooms)

    print(f"Done initialising in {time.time() - start} seconds")

    file.write("Initialised genotypes:\n")
    write_dictionary(population_dictionary, file)
    file.write("############################\n")

    start = time.time()

    # Get the max and the avg fitness at the start
    features = [individual[1] for individual in population_dictionary.values()]
    max_fitness_history.append(max(features))
    avg_fitness_history.append(sum(features) / len(features))

    for i in range(gens):

        print(f"Evaluating generation: {i + 1}")
        file.write(f"GENERATION: {i + 1}\n")

        # SELECTION
        # For the number of desired offspring per generation, choose the best parent
        # and momentarily copy them over in a dedicated offspring dictionary
        offspring_dictionary = {}

        for j in range(num_offspring):

            best_parent = selection_strategy(population_dictionary, k=k)

            # offspring is initialised (genotype is set as empty, fitness is set to 0)
            offspring_dictionary[j] = [[], 0]

            # genotype of offspring is copied over from parent
            for gene in population_dictionary[best_parent][0]:
                offspring_dictionary[j][0].append(gene)

        # CROSS OVER
        crossover_events = int(random.uniform(0.1, 0.4)*num_offspring)

        for _ in range(crossover_events):

            offspring_pair = choose_pair(offspring_dictionary)
            rng = random.randint(0, 99)

            if rng < 33:
                one_point_crossover(offspring_dictionary, offspring_pair)
            elif rng < 66:
                uniform_crossover(offspring_dictionary, offspring_pair)
            else:
                arithmetic_crossover(offspring_dictionary, offspring_pair, integers=False)

        # MUTATION
        for target_offspring in offspring_dictionary:
            mutation(offspring_dictionary, target_offspring, mr, integers=False)

        # REPRODUCTION
        # before offsprings are added into the population, but after they have undergone crossover and mutation events
        # (so that their genotype is "finalised"), their fitness and phenotype is computed
        for offspring in offspring_dictionary:

            genotype = offspring_dictionary[offspring][0]
            fitness = 0
            for room in rooms:
                fitness += evaluate_genotype(genotype, offspring, room)
            offspring_dictionary[offspring][1] = fitness / len(rooms)

        reproduction_strategy(population_dictionary, offspring_dictionary)

        # max fitness and average fitness of the population are extracted and appended to the max fitness and
        # average fitness arrays
        features = [individual[1] for individual in population_dictionary.values()]

        max_fitness = max(features)
        avg_fitness = sum(features) / len(features)
        max_fitness_history.append(max_fitness)
        avg_fitness_history.append(avg_fitness)

        end = time.time()
        print(f"Finished generation {i+1} in {end - start} seconds.\n "
              f"Max fitness: {max_fitness}\n "
              f"Avg. fitness: {avg_fitness}")

        write_dictionary(population_dictionary, file)
        file.write("############################\n")

        start = time.time()

    best_genotype = max(population_dictionary.values(), key=lambda x: x[1])
    print(f"The best genotype is:\n"
          f"{best_genotype}")

    file.write(f"The best genotype is: {best_genotype}")
    file.write(f"\n")
    file.write(f"{max_fitness_history}\n")
    file.write(f"{avg_fitness_history}\n")

    return max_fitness_history, avg_fitness_history

def test_algorithm_with_parameters(room, file, plot_name, gens=generations, k=tournament_k, num_offspring=offsprings_per_generation, mr=mutation_rate, tests=100):
    """
    Plots the average max fitness and average fitness from all run tests over a number of generations.

    Plots contain error bars

    :param room: Room list
    :param gens: Integer for number of generations
    :param k: Integer for tournament k
    :param num_offspring: Integer for number of offspring per generation
    :param mr: Integer for mutation rate
    :param tests: Integer for number of tests
    :return: None - saves figure
    """
    # Holds lists of max and avg fitness per run test
    max_fitness = []
    avg_fitness = []

    print(f"Running the algorithm for {tests} tests, each containing {gens} generations.\n"
          f"Parameters:\n"
          f"k = {tournament_k}\n"
          f"offspring per gen = {num_offspring}\n"
          f"mutation rate = {mr}")

    for i in range(tests):

        print(f"TEST {i+1}:")
        file.write(f"TEST {i+1}\n")

        max_fitness_history, avg_fitness_history = run_algorithm_with_parameters(room, gens, k, num_offspring, mr, file)

        max_fitness.append(max_fitness_history)
        avg_fitness.append(avg_fitness_history)

        file.write("##############################################\n")

    # Holds the average max fitness and average average fitness over all tests, per generation
    avg_max_fitness = []
    avg_avg_fitness = []

    # Holds the errors
    max_fitness_error = []
    avg_fitness_error = []

    # For each generation, go over all run tests and compute the average + errors
    for j in range(gens):

        total_max_fitness = []
        total_avg_fitness = []

        for k in range(tests):

            total_max_fitness.append(max_fitness[k][j])
            total_avg_fitness.append(avg_fitness[k][j])

        avg_max_fitness.append(sum(total_max_fitness) / tests)
        avg_avg_fitness.append(sum(total_avg_fitness) / tests)

        max_fitness_error.append(sem(total_max_fitness))
        avg_fitness_error.append(sem(total_avg_fitness))

    generation_list = [gen for gen in range(gens)]

    # Plot
    plt.errorbar(generation_list, avg_max_fitness, yerr=max_fitness_error, c='b', ls='-', fmt='.', label="Max Fitness")
    plt.errorbar(generation_list, avg_avg_fitness, yerr=avg_fitness_error, c='r', ls='-', fmt='.', label="Average Fitness")

    plt.xlabel("Generations")
    plt.ylabel("Fitness")

    plt.xlim(0, gens)
    plt.legend(loc='lower right')

    # Save
    plt.savefig(plot_directory+plot_name)


# File with important common constants for the API scripts
import api_info

KEY = api_info.KEY
BASE_URL = "http://api.wunderground.com/api/"+ KEY +"/hourly10day/q/"
DAILY_BASE_URL = "http://api.wunderground.com/api/"+ KEY +"/forecast10day/q/"

FILENAME = "hourly_forecast.json"
FOLDERNAME = "/home/danielv/Documents/webscraping_2018/data/"
CITIES = ["BERLIN", "HAMBURG", "MUNICH", "COLOGNE", "FRANKFURT"]

#Coordinates
coordinates = { 'BERLIN': (52.52000659999999, 13.404954),
                'MUNICH': (48.1351253,  11.5819805),
                'HAMBURG': (53.5510846, 9.9936819),
                'FRANKFURT': (50.1109221, 8.6821267),
                'COLOGNE': (50.937531, 6.9602786)
            }


import pytest
from os import path as p
from wwfs.board import Board
from wwfs.word import Word
from wwfs.tiles import TileBag

_wwfs = p.dirname(p.abspath(__file__))
test_data = p.join(_wwfs, '..', 'wwfs', 'data')

@pytest.fixture
def mytilebag():
    return TileBag(p.join(test_data, 'tiles.csv'))

@pytest.fixture
def myword():
    word = Word("TASTY", coord=(5, 5), direction=0)
    layout = p.join(test_data, 'board.csv')
    board = Board(layout)
    word.squares = board.get_square_xy(word, word.x, word.y, word.direction)
    for sq in word.squares:
        sq.tile_used = True
    return word

def test_tilebag_instansiate(mytilebag):
    """Can instansiate Word object."""
    assert isinstance(mytilebag, TileBag)
    assert mytilebag.remaining == 104
    assert mytilebag.played == []

def test_update(myword, mytilebag):
    """Remove tiles from tilebag aftern playing word."""
    mytilebag.update(myword)
    assert mytilebag.played == list('TASTY')
    assert mytilebag.remaining == (104 - len(myword))


import tensorflow as tf
from tensorflow.contrib import slim

def conv2d(x, num_ch, k_size=3, s_size=1, scope='conv_2d', DW=True):
    with tf.variable_scope(name_or_scope=scope):
        if DW:
            out = slim.separable_conv2d(x, num_outputs=num_ch, kernel_size=[k_size, k_size], stride=s_size)
        else:
            out = slim.conv2d(x, num_outputs=num_ch, kernel_size=[k_size, k_size], stride=s_size)
        return out

def downsample_2x(x, num_ch, scope='down2x'):
    with tf.variable_scope(name_or_scope=scope):
        out = slim.conv2d(x, num_ch, kernel_size=[3, 3], stride=2)
        return out

def downsample_4x(x, num_ch, scope='down4x'):
    with tf.variable_scope(name_or_scope=scope):
        out = slim.conv2d(x, num_ch, kernel_size=[3, 3], stride=2)
        out = slim.conv2d(out, num_ch, kernel_size=[3, 3], stride=2)
        return out

def downsample_8x(x, num_ch, scope='down8x'):
    with tf.variable_scope(name_or_scope=scope):
        out = slim.conv2d(x, num_ch, kernel_size=[3, 3], stride=2)
        out = slim.conv2d(out, num_ch, kernel_size=[3, 3], stride=2)
        out = slim.conv2d(out, num_ch, kernel_size=[3, 3], stride=2)
        return out

def upsample_2x(x, num_ch, scope='up2x'):
    with tf.variable_scope(name_or_scope=scope):
        out = slim.conv2d_transpose(x, num_ch, kernel_size=[2, 2], stride=2)
        return out

def upsample_4x(x, num_ch, scope='up4x'):
    with tf.variable_scope(name_or_scope=scope):
        out = slim.conv2d_transpose(x, num_ch, kernel_size=[2, 2], stride=2)
        out = slim.conv2d_transpose(out, num_ch, kernel_size=[2, 2], stride=2)
        return out

def upsample_8x(x, num_ch, scope='up8x'):
    with tf.variable_scope(name_or_scope=scope):
        out = slim.conv2d_transpose(x, num_ch, kernel_size=[2, 2], stride=2)
        out = slim.conv2d_transpose(out, num_ch, kernel_size=[2, 2], stride=2)
        out = slim.conv2d_transpose(out, num_ch, kernel_size=[2, 2], stride=2)
        return out

def model(inputs, is_training=False):
    with slim.arg_scope([slim.conv2d, slim.conv2d_transpose, slim.separable_conv2d],
                        normalizer_fn=slim.batch_norm,
                        activation_fn=tf.nn.relu,
                        weights_initializer=slim.xavier_initializer(),
                        biases_initializer=None,
                        weights_regularizer=slim.l2_regularizer(0.0005),
                        padding='SAME'):
        with slim.arg_scope([slim.batch_norm],
                            center=True,
                            scale=True,
                            is_training=is_training):
            path_ch = 6
            path1_ch = path_ch * 1
            path2_ch = path1_ch * 2
            path3_ch = path2_ch * 2
            path4_ch = path3_ch * 2

            net = slim.conv2d(inputs, 16, kernel_size=3, stride=2, scope='Conv')

            path1_1 = downsample_2x(net, path1_ch, scope='path1_1')
            path1_2 = conv2d(path1_1, path1_ch, scope='path1_2')
            path1_3 = conv2d(path1_2, path1_ch, scope='path1_3')
            path1_3_down_2x = downsample_2x(path1_3, path2_ch, 'path1_3_down_2x')

            path1_4 = conv2d(path1_3, path1_ch, scope='path1_4')
            path1_5 = conv2d(path1_4, path1_ch, scope='path1_5')
            path1_6 = conv2d(path1_5, path1_ch, scope='path1_6')
            path1_6_down_2x = downsample_2x(path1_6, path2_ch, 'path1_6_down_2x')
            path1_6_down_4x = downsample_4x(path1_6, path3_ch, 'path1_6_down_4x')

            with tf.variable_scope(name_or_scope='path2_1'):
                path2_1 = tf.identity(path1_3_down_2x)

            path2_2 = conv2d(path2_1, path2_ch, scope='path2_2')
            path2_3 = conv2d(path2_2, path2_ch, scope='path2_3')
            path2_3_down_2x = downsample_2x(path2_3, path3_ch, scope='path2_3_down_2x')
            path2_3_up_2x = upsample_2x(path2_3, path1_ch, scope='path2_3_up_2x')

            with tf.variable_scope(name_or_scope='path1_7'):
                path1_7 = conv2d(path1_6, path1_ch)
                path1_7 = tf.add(path1_7, path2_3_up_2x)

            path1_8 = conv2d(path1_7, path1_ch, scope='path1_8')
            path1_9 = conv2d(path1_8, path1_ch, scope='path1_9')
            path1_9_down_2x = downsample_2x(path1_9, path2_ch, 'path1_9_down_2x')
            path1_9_down_4x = downsample_4x(path1_9, path3_ch, 'path1_9_down_4x')
            path1_9_down_8x = downsample_8x(path1_9, path4_ch, 'path1_9_down_8x')

            with tf.variable_scope(name_or_scope='path2_4'):
                path2_4 = conv2d(path2_3, path2_ch)
                path2_4 = tf.add(path2_4, path1_6_down_2x)

            path2_5 = conv2d(path2_4, path2_ch, scope='path2_5')
            path2_6 = conv2d(path2_5, path2_ch, scope='path2_6')
            path2_6_down_2x = downsample_2x(path2_6, path3_ch, scope='path2_6_down_2x')
            path2_6_down_4x = downsample_4x(path2_6, path4_ch, scope='path2_6_down_4x')
            path2_6_up_2x = upsample_2x(path2_6, path1_ch, scope='path2_6_up_2x')

            with tf.variable_scope(name_or_scope='path3_1'):
                path3_1 = tf.add(path2_3_down_2x, path1_6_down_4x, name='path3_1')

            path3_2 = conv2d(path3_1, path3_ch, scope='path3_2')
            path3_3 = conv2d(path3_2, path3_ch, scope='path3_3')
            path3_3_down_2x = downsample_2x(path3_3, path4_ch, scope='path3_3_down_2x')
            path3_3_up_2x = upsample_2x(path3_3, path2_ch, scope='path3_3_up_2x')
            path3_3_up_4x = upsample_4x(path3_3, path1_ch, scope='path3_3_up_4x')

            with tf.variable_scope(name_or_scope='path1_10'):
                path1_10 = conv2d(path1_9, path1_ch)
                path1_10 = tf.add(path1_10, path2_6_up_2x)
                path1_10 = tf.add(path1_10, path3_3_up_4x)

            path1_11 = conv2d(path1_10, path1_ch, scope='path1_11')
            path1_12 = conv2d(path1_11, path1_ch, scope='path1_12')

            with tf.variable_scope(name_or_scope='path2_7'):
                path2_7 = conv2d(path2_6, path2_ch)
                path2_7 = tf.add(path2_7, path3_3_up_2x)
                path2_7 = tf.add(path2_7, path1_9_down_2x)

            path2_8 = conv2d(path2_7, path2_ch, scope='path2_8')
            path2_9 = conv2d(path2_8, path2_ch, scope='path2_9')

            with tf.variable_scope(name_or_scope='path3_4'):
                path3_4 = conv2d(path3_3, path3_ch)
                path3_4 = tf.add(path3_4, path1_9_down_4x)
                path3_4 = tf.add(path3_4, path2_6_down_2x)

            path3_5 = conv2d(path3_4, path3_ch, scope='path3_5')
            path3_6 = conv2d(path3_5, path3_ch, scope='path3_6')

            with tf.variable_scope(name_or_scope='path4_1'):
                path4_1 = tf.add(path3_3_down_2x, path2_6_down_4x)
                path4_1 = tf.add(path4_1, path1_9_down_8x, name='path4_1')

            path4_2 = conv2d(path4_1, path4_ch, scope='path4_2')
            path4_3 = conv2d(path4_2, path4_ch, scope='path4_3')

            path4_3_up_8x = upsample_8x(path4_3, path1_ch, scope='path4_3_up_8x')
            path3_6_up_4x = upsample_4x(path3_6, path1_ch, scope='path3_6_up_4x')
            path2_9_up_2x = upsample_2x(path2_9, path1_ch, scope='path2_9_up_2x')

            p3 = tf.concat([path1_12, path2_9_up_2x, path3_6_up_4x, path4_3_up_8x], axis=3, name='p3')
            p_out = slim.conv2d(p3, 24, kernel_size=[3, 3], stride=1, scope='p_out')

            map_class = slim.conv2d(p_out, 1, kernel_size=[1, 1], stride=1,
                                    normalizer_fn=None, activation_fn=tf.nn.sigmoid, biases_initializer=tf.zeros_initializer(),
                                    scope='pred_class')
            print(map_class)

            map_scale = slim.conv2d(p_out, 2, kernel_size=[1, 1], stride=1,
                                    normalizer_fn=None, activation_fn=None, biases_initializer=tf.zeros_initializer(),
                                    scope='pred_scale')
            print(map_scale)

            map_offset = slim.conv2d(p_out, 2, kernel_size=[1, 1], stride=1,
                                     normalizer_fn=None, activation_fn=None, biases_initializer=tf.zeros_initializer(),
                                     scope='pred_offset')
            print(map_offset)

            map_landmark = slim.conv2d(p_out, 10, kernel_size=[1, 1], stride=1,
                                       normalizer_fn=None, activation_fn=None, biases_initializer=tf.zeros_initializer(),
                                       scope='pred_landmark')
            print(map_landmark)

    return map_class, map_scale, map_offset, map_landmark


from SymbolTable import symbolTable  # Importing the symbolTable module
from Errors import Error  # Importing the Error module

# The Object class represents a generic object.
class Object:
    def __init__(self, value):
        self.token = value  # Initialize the token attribute with the provided value

    def return_type(self):
        pass

    def cast(self):
        casted_val = self.token  # Start with the original token value
        data_types = {
            'string': Str,  # Mapping the string data type to the Str class
            'integer': Int,  # Mapping the integer data type to the Int class
            'boolean': Bool,  # Mapping the boolean data type to the Bool class
            'variable': Var  # Mapping the variable data type to the Var class
        }
        if self.token.token_type in data_types.keys():
            # If the token's type is in the data_types dictionary, create an instance of the corresponding class
            casted_val = data_types[self.token.token_type](self.token.value)

        return casted_val  # Return the casted value

# The Var class represents a variable object.
class Var(Object):
    def __init__(self, value):
        self.mode = 1 if type(value).__name__ == 'tuple' else 2  # Determine the mode based on the type of the value

        if self.mode == 1:
            # If mode is 1, it means the value is a tuple containing name, value, and scope
            self.name, self.value, self.scope = value
        else:
            self.name = value  # If mode is 2, it means the value is only the variable name

    def return_type(self):
        return '<\'Anwani\'>'

    def evaluate(self):
        result = False  # Initialize the result variable as False

        if self.mode == 1:
            # If mode is 1, it means the variable is being assigned a value
            symbolTable.table['variables'][self.scope][self.name] = self.value  # Assign the value to the variable
            result = True  # Set the result as True, indicating successful assignment
        else:
            result = symbolTable.fetch_variable(self.name)  # Fetch the value of the variable from the symbol table
            if result:
                result = symbolTable.fetch_variable(self.name).evaluate()
                # If the variable is found, evaluate its value recursively
            else:
                Error.throwException('anwani', self.name)
                # If the variable is not found, throw an exception and set the result as False

        return result  # Return the result

# The Int class represents an integer object.
class Int(Object):
    def evaluate(self):
        return int(self.token)  # Return the integer value of the token

# The Str class represents a string object.
class Str(Object):
    def evaluate(self):
        return '{}'.format(self.token)  # Return the string value of the token

# The Bool class represents a boolean object.
class Bool(Object):
    def evaluate(self):
        return bool(self.token.capitalize())  # Return the boolean value of the token

# The code above defines several classes for different types of objects, such as variables, integers, strings, and booleans.
# These classes inherit from the Object class and provide the evaluate method, which returns the evaluated value of the object.
# The Var class handles variable assignments and retrieval, the Int class converts the token to an integer, the Str class converts the token to a string,
# and the Bool class converts the token to a boolean value.
# The Object class also provides the cast method, which allows for type casting of objects.


from easydict import EasyDict as edict

BATCH_SIZE = 1
POINT_FEATURES = ['x', 'y', 'z', 'intensity', 'timestamp']

data_cfg = edict({
    'POINT_CLOUD_RANGE': [-51.2, -51.2, -5.0, 51.2, 51.2, 3.0],
    'VOXEL_SIZE': [0.2, 0.2, 8.0],
    'CLASSES': ['car','truck', 'construction_vehicle', 'bus', 'trailer',
                'barrier', 'motorcycle', 'bicycle', 'pedestrian', 'traffic_cone'],
    'NUM_POINT_FEATURES': len(POINT_FEATURES)
})

model_cfg = edict({
    'NAME': 'CenterPoint',

    'VFE':
        {
            'NAME': 'DynamicPillarVFE',
            'WITH_DISTANCE': False,
            'USE_ABSOLUTE_XYZ': True,
            'USE_NORM': True,
            'NUM_FILTERS': [64, 64],
            'NUM_RAW_POINT_FEATURES': 5
        },

    'MAP_TO_BEV':
        {
            'NAME': 'PointPillarScatter',
            'NUM_BEV_FEATURES': 64,
            'BATCH_SIZE': BATCH_SIZE
        },

    'BACKBONE_2D':
        {
            'NAME': 'BaseBEVBackbone',
            'LAYER_NUMS': [3, 5, 5],
            'LAYER_STRIDES': [2, 2, 2],
            'NUM_FILTERS': [64, 128, 256],
            'UPSAMPLE_STRIDES': [0.5, 1, 2],
            'NUM_UPSAMPLE_FILTERS': [128, 128, 128]
        },

    'DENSE_HEAD':
        {
            'NAME': 'CenterHead',
            'BATCH_SIZE': BATCH_SIZE,
            'CLASS_NAMES_EACH_HEAD': [
                ['car'], # 0
                ['truck', 'construction_vehicle'],  # 1, 2
                ['bus', 'trailer'],  # 3, 4
                ['barrier'],  # 5
                ['motorcycle', 'bicycle'],  # 6, 7
                ['pedestrian', 'traffic_cone'],  # 8, 9
            ],
            'SHARED_CONV_CHANNEL': 64,
            'USE_BIAS_BEFORE_NORM': True,
            'NUM_CONV_IN_ONE_HEAD': 2,
            'FEATURE_MAP_STRIDE': 4,
            'POST_PROCESSING':{
                'SCORE_THRESH': 0.3,
                'NMS_IOU_THRESH': 0.3,
                'NMS_PRE_MAXSIZE': 200,
                'NMS_POST_MAXSIZE': 30,
                'POST_CENTER_LIMIT_RANGE': [-61.2, -61.2, -10.0, 61.2, 61.2, 10.0],
                'MAX_OBJ_PER_SAMPLE': 500,
            }
        }
})


# Monitors the source files for any new insertions new excel files for loops only. Once it finds new excel file database operations.py does the insertion
from os import listdir
from os.path import isfile, join, isdir
import json
import os

currentPath = os.path.dirname(os.path.realpath(__file__))

def newfiledetector(targetdir, targetYear):
    returningList = []

    # CAUTION! the JSON file must be under json format: {}
    with open(currentPath + '/logs/log.json', 'a+') as data_file:
        PreviousFiles = json.load(data_file)

    targetTypes = [targetdir + '\\' + f for f in listdir(targetdir) if isdir(join(targetdir, f))]

    # onetype is the data type: Class, Volume, Speed
    for onetype in targetTypes:

        # Only select Class for now.
        if onetype.split('\\')[-1:][0] != 'Class':
            continue

        # In case of new data types add in
        if onetype not in PreviousFiles:
            PreviousFiles[onetype] = {}

        # targetYear is the specific Year
        yeardir = onetype + '\\' + targetYear

        # In case of new year
        if yeardir not in PreviousFiles[onetype]:
            PreviousFiles[onetype][yeardir] = {}

        StationIDs = [yeardir + '\\' + f for f in listdir(yeardir) if isdir(join(yeardir, f))]

        # onestation is the StationID
        for onestation in StationIDs:

            # In case of new stations add in
            if onestation not in PreviousFiles[onetype][yeardir]:
                PreviousFiles[onetype][yeardir][onestation] = []

            sourcefiles = [onestation + '\\' + f for f in listdir(onestation) if isfile(join(onestation, f))]

            set_PreviousFiles = set(PreviousFiles[onetype][yeardir][onestation])
            set_CurrentFiles = set(sourcefiles)

            PreviousFiles[onetype][yeardir][onestation] = list(set_CurrentFiles | set_PreviousFiles)
            returningList.extend(list(set_CurrentFiles - set_PreviousFiles))

    with open(currentPath + '/logs/log.json', 'w') as data_file:
        json.dump(PreviousFiles, data_file)

    return returningList

    # Commet out
    # import datetime
    # print newfiledetector('C:\\ProgramData\\Peek Traffic\\Viper\\Output1\\Binned Data Reports', datetime.datetime.now().strftime('%Y'))


import math
import numpy as np

def r(gn0, gn1):
    """Rogers Huff R

    Lift from https://github.com/cggh/scikit-allel/blob/961254bd583572eed7f9bd01060e53a8648e620c/allel/opt/stats.pyx
    """
    # initialise variables
    m0 = m1 = v0 = v1 = cov = n = 0

    # iterate over input vectors
    for i in range(len(gn0)):
        x = gn0[i]
        y = gn1[i]
        # consider negative values as missing
        if x >= 0 and y >= 0:
            n += 1
            m0 += x
            m1 += y
            v0 += x ** 2
            v1 += y ** 2
            cov += x * y

    # early out
    if n < 1:
        return np.nan

    # compute mean, variance, covariance
    m0 /= n
    m1 /= n
    v0 /= n
    v1 /= n
    cov /= n
    cov -= m0 * m1
    v0 -= m0 * m0
    v1 -= m1 * m1

    d = math.sqrt(v0 * v1)
    # np.finfo(np.float32).tiny
    if d < 1.1754944e-38:
        return np.nan

    # compute correlation coeficient
    r = cov / d

    return r

def r2(gn0, gn1):
    return r(gn0, gn1) ** 2


from pathlib import Path

from mlf_core.create.domains.mlflow_creator import MlflowCreator
from mlf_core.create.domains.package_creator import PackageCreator
from mlf_core.custom_cli.questionary import mlf_core_questionary_or_dot_mlf_core

def choose_domain(path: Path, domain: str or None, dot_mlf_core: dict = None):  # type: ignore
    """
    Prompts the user for the template domain.
    Creates the .mlf_core file.
    Prompts the user whether or not to create a Github repository

    :param path: The path, the project should be created at
    :param domain: Template domain
    :param dot_mlf_core: Dictionary created from the .mlf_core.yml file. None if no .mlf_core.yml file was used.
    """
    if not domain:
        domain = mlf_core_questionary_or_dot_mlf_core(
            function="select",
            question="Choose the project's domain",
            choices=["mlflow", "package"],
            default="mlflow",
            dot_mlf_core=dot_mlf_core,
            to_get_property="domain",
        )

    switcher = {"mlflow": MlflowCreator, "package": PackageCreator}

    creator_obj = switcher.get(domain.lower())()  # type: ignore
    creator_obj.create_template(path, dot_mlf_core)  # type: ignore


# -*- coding: utf-8 -*-
import numpy
import math
import pandas
import json
import jpholiday
import statsmodels.api as sm
import collections
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from functools import wraps
import subprocess
import talib as ta
import slack
import time

class TechnicalSetting:
    def __init__(self):
        self.average = 5
        self.long_average = 25
        self.volume_average = 5
        self.rci = 9
        self.rci_long = 27
        self.macd_fast = 12
        self.macd_slow = 26
        self.macd_signal = 9
        self.macd_hist = 100
        self.atr = 14
        self.bbands = 25
        self.env_entity_average = 5

class ShortTechnicalSetting(TechnicalSetting):
    def __init__(self):
        super().__init__()
        self.average = 3
        self.long_average = 9
        self.volume_average = 9
        self.rci = 3
        self.rci_long = 9
        self.macd_fast = 4
        self.macd_slow = 10
        self.macd_signal =3
        self.macd_hist = 30
        self.atr = 5
        self.bbands = 9
        self.env_entity_average = 3

def nan_to_num(data, default=0):
    return data if default is None else numpy.nan_to_num(data)

def add_average_stats(data, technical_setting=None):
    setting = TechnicalSetting() if technical_setting is None else technical_setting
    close = numpy.array(data["close"].values, dtype="f8")
    data["average"]     = ta.SMA(close, timeperiod=setting.average)
    data["long_average"]    = ta.SMA(close, timeperiod=setting.long_average)
    data["volume_average"]    = ta.SMA(data["volume"].astype(float).values, timeperiod=setting.volume_average)
    data["ma_divergence"]     = (data["close"] - data["long_average"]) / data["long_average"]
    return data

def add_tec_stats(data, technical_setting=None):
    setting = TechnicalSetting() if technical_setting is None else technical_setting
    data["rci"]                 = data["close"].rolling(setting.rci).apply(rci, raw=True)
    data["rci_long"]            = data["close"].rolling(setting.rci_long).apply(rci, raw=True)

    close = numpy.array(data["close"].values, dtype="f8")
    macd, macdsignal, macdhist = ta.MACD(close, fastperiod=setting.macd_fast, slowperiod=setting.macd_slow, signalperiod=setting.macd_signal)
    data["macd"]           = macd
    data["macdsignal"]     = macdsignal
    data["macdhist"]       = macdhist
    data["macdhist_convert"] = data["macdhist"].rolling(setting.macd_hist).apply(trend_convert, raw=True)

    # average true range
    data["atr"] = ta.ATR(data["high"].astype(float).values, data["low"].astype(float).values, data["close"].astype(float).values, timeperiod=setting.atr)
    return data

def add_band_stats(data, technical_setting=None):
    setting = TechnicalSetting() if technical_setting is None else technical_setting
    close = numpy.array(data["close"].values, dtype="f8")
    upper, _, lower = ta.BBANDS(close, timeperiod=setting.bbands, nbdevup=1, nbdevdn=1, matype=0)
    upper2, _, lower2 = ta.BBANDS(close, timeperiod=setting.bbands, matype=0)

    data["env12"]          = nan_to_num(upper2)
    data["env11"]          = nan_to_num(upper)
    data["env09"]          = nan_to_num(lower)
    data["env08"]          = nan_to_num(lower2)

    data["env_entity"]     =  each(lambda i, x: x["env12"] - x["env08"], data)
    data["env_entity_average"] = ta.SMA(numpy.array(data["env_entity"].values, dtype="f8"), timeperiod=setting.env_entity_average)

    return data

def add_safety_stats(data, technical_setting=None):
    data["low_noize"]         = data["low"].rolling(2).apply(lambda x: x[-2] - x[-1] if x[-2] - x[-1] < 0 else 0, raw=True)
    data["rising_safety"]     = data["low_noize"].rolling(10).apply(rising_safety, raw=True)
    data["rising_safety"]     = data["low"] + data["rising_safety"]
    data["rising_safety"]     = data["rising_safety"].rolling(3).max() # 過去のsafetyより低い場合は高い方に合わせる

    data["high_noize"]        = data["high"].rolling(2).apply(lambda x: x[-2] - x[-1] if x[-2] - x[-1] > 0 else 0, raw=True)
    data["fall_safety"]       = data["high_noize"].rolling(10).apply(fall_safety, raw=True)
    data["fall_safety"]       = data["high"] + data["fall_safety"]
    data["fall_safety"]       = data["fall_safety"].rolling(3).min() # 過去のsafetyより高い場合は低い方に合わせる

    return data

def add_resistance_stats(data, technical_setting=None):
    data["resistance"] = data["high"].rolling(15).max()

    data["resistance_3d"] = data["high"].rolling(3).max()
    data["resistance_5d"] = data["high"].rolling(5).max()
    data["resistance_10d"] = data["high"].rolling(10).max()
    data["resistance_1m"] = data["high"].rolling(20).max()
    data["resistance_3m"] = data["high"].rolling(60).max()
    data["resistance_6m"] = data["high"].rolling(120).max()
    data["resistance_1y"] = data["high"].rolling(240).max()

    data["resistance_entity"] = (data["resistance_10d"] - data["resistance_3d"]) * (~((data["resistance_3d"] == data["resistance_5d"]) | (data["resistance_5d"] == data["resistance_10d"]) | (data["resistance_3d"] == data["resistance_10d"])) * 1)
    data["resistance_entity"] = data["resistance_entity"] / data["close"]

    return data

def add_support_stats(data, technical_setting=None):
    data["support"] = data["low"].rolling(15).min()

    data["support_3d"] = data["low"].rolling(3).min()
    data["support_5d"] = data["low"].rolling(5).min()
    data["support_10d"] = data["low"].rolling(10).min()
    data["support_1m"] = data["low"].rolling(20).min()
    data["support_3m"] = data["low"].rolling(60).min()
    data["support_6m"] = data["low"].rolling(120).min()
    data["support_1y"] = data["low"].rolling(240).min()

    data["support_entity"] = (data["support_3d"] - data["support_10d"]) * (~((data["support_3d"] == data["support_5d"]) | (data["support_5d"] == data["support_10d"]) | (data["support_3d"] == data["support_10d"])) * 1)
    data["support_entity"] = data["support_entity"] / data["close"]

    return data

def add_stages_stats(data, technical_setting=None):
    data["stages_3d"]                   = stages(data, "resistance_3d", "support_3d", "average", with_eq=True)
    data["stages_5d"]                   = stages(data, "resistance_5d", "support_5d", "average", with_eq=True)
    data["stages_10d"]                  = stages(data, "resistance_10d", "support_10d", "average", with_eq=True)
    data["stages_1m"]                   = stages(data, "resistance_1m", "support_1m", "long_average", with_eq=True)
    data["stages_3m"]                   = stages(data, "resistance_3m", "support_3m", "long_average", with_eq=True)
    data["stages_6m"]                   = stages(data, "resistance_6m", "support_6m", "long_average", with_eq=True)
    data["stages_1y"]                   = stages(data, "resistance_1y", "support_1y", "long_average", with_eq=True)

    data["stages"]                      = stages(data)
    data["stages_average"]              = ta.SMA(data["stages"].astype(float).values, timeperiod=10)
    data["stages_sum"]                  = data["stages_3d"] + data["stages_5d"] + data["stages_10d"] + data["stages_1m"] + data["stages_3m"] + data["stages_6m"] + data["stages_1y"]
    data["stages_sum_average"]          = ta.SMA(data["stages_sum"].astype(float).values, timeperiod=5)
    data["stages_sum_average_long"]     = ta.SMA(data["stages_sum"].astype(float).values, timeperiod=20)
    data["macd_stages"]                 = (data["macd"] > 0) * 1
    data["macdhist_stages"]             = (data["macdhist"] > 0) * 1

    return data

def add_cross_stats(data, technical_setting=None):
    # クロス系
    data["average_cross"] = cross(data["average"], data["long_average"])
    data["macd_cross"] = cross(data["macd"], data["macdsignal"])
    data["rci_cross"] = cross(data["rci"], data["rci_long"])

    data["env12_cross"] = cross(data["high"], data["env12"])
    data["env11_cross"] = cross(data["high"], data["env11"])
    data["env09_cross"] = cross(data["low"], data["env09"])
    data["env08_cross"] = cross(data["low"], data["env08"])

    data["rising_safety_cross"] = cross(data["low"], data["rising_safety"])
    data["fall_safety_cross"] = cross(data["high"], data["fall_safety"])

    return data

def add_trend_stats(data, technical_setting=None):
    # 気配を出力する
    data["average_gradient"]            = diff(data["average"])
    data["long_average_gradient"]       = diff(data["long_average"])
    data["volume_gradient"]             = diff(data["volume_average"])
    data["macd_gradient"]               = diff(data["macd"])
    data["macdhist_gradient"]           = diff(data["macdhist"])
    data["rci_gradient"]                = diff(data["rci"])
    data["rci_long_gradient"]           = diff(data["rci_long"])
    data["stages_gradient"]             = diff(data["stages"])
    data["stages_average_gradient"]     = diff(data["stages_average"])
    data["stages_sum_gradient"]         = diff(data["stages_sum"])
    data["stages_sum_average_gradient"] = diff(data["stages_sum_average"])
    data["stages_sum_average_long_gradient"] = diff(data["stages_sum_average_long"])
    data["fall_safety_gradient"]        = diff(data["fall_safety"])
    data["rising_safety_gradient"]      = diff(data["rising_safety"])

    data["average_trend"]               = data["average_gradient"].rolling(5).apply(trend, raw=True)
    data["long_average_trend"]          = data["long_average_gradient"].rolling(5).apply(trend, raw=True)
    data["volume_average_trend"]        = data["volume_gradient"].rolling(5).apply(trend, raw=True)
    data["macd_trend"]                  = data["macd_gradient"].rolling(5).apply(trend, raw=True)
    data["macdhist_trend"]              = data["macdhist_gradient"].rolling(1).apply(trend, raw=True)
    data["rci_trend"]                   = data["rci_gradient"].rolling(5).apply(trend, raw=True)
    data["rci_long_trend"]              = data["rci_long_gradient"].rolling(5).apply(trend, raw=True)
    data["stages_trend"]                = data["stages_gradient"].rolling(5).apply(trend, raw=True)
    data["stages_average_trend"]        = data["stages_average_gradient"].rolling(5).apply(trend, raw=True)
    data["stages_sum_trend"]            = data["stages_sum_gradient"].rolling(5).apply(trend, raw=True)
    data["stages_sum_average_trend"]    = data["stages_sum_average_gradient"].rolling(5).apply(trend, raw=True)
    data["stages_sum_average_long_trend"]    = data["stages_sum_average_long_gradient"].rolling(5).apply(trend, raw=True)
    data["rising_safety_trend"]         = data["rising_safety_gradient"].rolling(5).apply(trend, raw=True)
    data["fall_safety_trend"]           = data["fall_safety_gradient"].rolling(5).apply(trend, raw=True)

    return data

def add_manda_stats(data, technical_setting=None):
    data["pct_change"] = data["close"].pct_change()
    data["stock_split"]             = (data["pct_change"] < -0.45) * 1
    data["reverse_stock_split"]     = (data["pct_change"] > 2.0) * 1
    data["manda"]                   = ((data["stock_split"] == 1) | (data["reverse_stock_split"] == 1)) * 1

    return data

def stop_high(data):
    before = data[-2]
    current = data[-1]
    limit = price_limit(before)
    gradient = current - before
    return limit == gradient

def stop_low(data):
    before = data[-2]
    current = data[-1]
    limit = price_limit(before)
    gradient = current - before
    return -limit == gradient

def add_stats(data, default=0, names=[], technical_setting=None):
    is_t = lambda name : len(names) == 0 or name in names
    setting = TechnicalSetting() if technical_setting is None else technical_setting
    stats = {
        "average": add_average_stats,
        "resistance": add_resistance_stats,
        "support": add_support_stats,
        "tec": add_tec_stats,
        "band": add_band_stats,
        "safety": add_safety_stats,
        "manda": add_manda_stats,
        "stages": add_stages_stats, # resistance / support
        "cross": add_cross_stats, # average / tec / band / safety
        "trend": add_trend_stats, # average / tec / band / safety / stages
    }

    for name in stats.keys():
        try:
            data = stats[name](data, setting) if is_t(name) else data
        except Exception as e:
            import traceback
            traceback.print_exc()

    if default is None:
        return data

    for column in data.columns.tolist():
        data[column] = nan_to_num(data[column], default)

    return data

def each(callback, data):
    return list(map(lambda x: callback(*x), data.iterrows()))

# ろうそく足のパターン
def add_cs_stats(data):
    toint = lambda x: 1 if x else 0

    # 実体
    data["entity"] = (data["open"] - data["close"]).abs()
    entity = numpy.array(data["entity"].values, dtype="f8")
    entity_average = ta.SMA(entity, timeperiod=5)
    data["entity_average"] = entity_average
    # 上ヒゲ・下ヒゲ
    data["upper_shadow"] = each(lambda i, x: x["high"] - max([x["open"], x["close"]]), data)
    data["lower_shadow"] = each(lambda i, x: min([x["open"], x["close"]]) - x["low"], data)

    ## ここから
    # 長い上ヒゲ・下ヒゲ
    data["long_upper_shadow"] = (data["upper_shadow"] > data["entity"]) * 1
    data["long_lower_shadow"] = (data["lower_shadow"] > data["entity"]) * 1
    # 陽線・陰線
    data["yang"]   = (data["open"] < data["close"]) * 1
    data["yin"]    = (data["open"] > data["close"]) * 1
    data["long_yang"]  = ((data["yang"] == 1) & (data["entity"] > data["entity_average"])) * 1
    data["long_yin"]   = ((data["yin"] == 1) & (data["entity"] > data["entity_average"])) * 1

    # 切り上げ
    data["low_roundup"]  = (data["low"].shift(1) < data["low"]) * 1
    data["high_roundup"] = (data["high"].shift(1) < data["high"]) * 1
    # 切り下げ
    data["low_rounddown"]  = (data["low"].shift(1) > data["low"]) * 1
    data["high_rounddown"] = (data["high"].shift(1) > data["low"]) * 1

    # n日以内の高値安値更新
    data["high_update"] = (data["high"].rolling(5).max() == data["high"]) * 1
    data["low_update"] = (data["low"].rolling(5).min() == data["low"]) * 1

    # ギャップ
    data["yang_gap"] = (data["high"].shift(1) < data["low"]) * 1
    data["yin_gap"]  = (data["low"].shift(1) > data["high"]) * 1
    data["gap"]      = data["yang_gap"] + data["yin_gap"]
    # つつみ線
    data["tsutsumi"]      = (data["entity"].shift(1) < data["entity"]) * 1
    data["yang_tsutsumi"] = ((data["tsutsumi"] == 1) & (data["yin"].shift(1) == 1) & (data["yang"] == 1)) * 1
    data["yin_tsutsumi"]  = ((data["tsutsumi"] == 1) & (data["yang"].shift(1) == 1) & (data["yin"] == 1)) * 1
    # はらみ線
    data["harami"]      = (data["entity"].shift(1) > data["entity"]) * 1
    data["yang_harami"] = ((data["harami"] == 1) & (data["yin"].shift(1) == 1) & (data["yang"] == 1)) * 1
    data["yin_harami"]  = ((data["harami"] == 1) & (data["yang"].shift(1) == 1) & (data["yin"] == 1)) * 1
    # 毛抜き
    data["upper_kenuki"] = (data["upper_shadow"].shift(1) < data["upper_shadow"]) * 1
    data["lower_kenuki"] = (data["lower_shadow"].shift(1) < data["lower_shadow"]) * 1
    # 宵の明星
    data["yoi_mojo"] = ((data["yang_gap"].shift(1) == 1) & (data["yin_gap"] == 1)) * 1
    # 明けの明星
    data["ake_mojo"] = ((data["yin_gap"].shift(1) == 1) & (data["yang_gap"] == 1)) * 1
    # 三空
    data["yang_sanku"] = ((data["yang_gap"].rolling(2).min() == 1) & (data["yang"].rolling(3).min() == 1)) * 1
    data["yin_sanku"]  = ((data["yin_gap"].rolling(2).min() == 1) & (data["yin"].rolling(3).min() == 1)) * 1
    # 三兵
    data["yang_sanpei"] = ((data["yang"].rolling(3).min() == 1) & (data["low_roundup"].rolling(2).min() == 1) & (data["long_upper_shadow"] == 1)) * 1
    data["yin_sanpei"]  = ((data["yin"].rolling(3).min() == 1) & (data["high_rounddown"].rolling(2).min() == 1) & (data["long_lower_shadow"] == 1)) * 1

    # ストップ
    data["stop_high"] = (data["close"].rolling(2).apply(stop_high, raw=True)) * 1
    data["stop_low"] = (data["close"].rolling(2).apply(stop_low, raw=True)) * 1

    # スコア
    data["score"] = each(lambda i, x: score(x), data)

    return data

def score(data):
    plus = ["yang_tsutsumi", "yang_harami", "lower_kenuki", "ake_mojo", "yin_sanku", "yin_sanpei"]
    minus = ["yin_tsutsumi", "yin_harami", "upper_kenuki", "yoi_mojo", "yang_sanku", "yang_sanpei"]

    plus_score = sum(list(map(lambda x: data[x], plus)))
    minus_score = sum(list(map(lambda x: data[x], minus)))

    return plus_score - minus_score

def rising_divergence(data, verbose=False):
    patterns = pattern(data, [
        {"key": "macdhist_convert", "callback": lambda x: x == -1}, # 最安値更新
        {"key": "macd_cross", "callback": lambda x: x == 1}, # ヒストグラムがプラスに
        {"key": "macd_cross", "callback": lambda x: x == -1}, # ヒストグラムがマイナスに
        {"key": "macdhist_trend", "callback": lambda x: x == 1}, # ヒストグラムのトレンドがプラスに
    ])

    if verbose:
        print(patterns)

    if len(list(filter(lambda x: x is None, patterns))) == 0:
        first = data["macdhist"].iloc[-patterns[0]]
        second = data["macdhist"].iloc[-patterns[2]:-patterns[3]].min()
        return first < second # 二番底の方が高い

    return False

def fall_divergence(data, verbose=False):
    patterns = pattern(data, [
        {"key": "macdhist_convert", "callback": lambda x: x == 1}, # 最高値更新
        {"key": "macd_cross", "callback": lambda x: x == -1}, # ヒストグラムがマイナス
        {"key": "macd_cross", "callback": lambda x: x == 1}, # ヒストグラムがプラスに
        {"key": "macdhist_trend", "callback": lambda x: x == -1}, # ヒストグラムのトレンドがマイナスに
    ])

    if verbose:
        print(patterns)

    if len(list(filter(lambda x: x is None, patterns))) == 0:
        first = data["macdhist"].iloc[-patterns[0]]
        second = data["macdhist"].iloc[-patterns[2]:-patterns[3]].max()
        return second < first # 二番天井の方が低い

    return False

# 指定したパターンが順番に現れるか
def pattern(data, patterns):
    results = []
    index = len(data)
    for pattern in patterns:
        if index is not None:
            index = find(data[pattern["key"]], index, pattern["callback"])
        results.append(index)
    return results

def count(data, callback):
    return len(list(filter(callback, data.iterrows())))

def exists(data, callback):
    d = data.values
    length = len(list(filter(lambda x:callback(x), d)))
    return length > 0

# 末尾から探索して最初に見つかるインデックスを返す
def find(data, term, callback, tail=True):
    if len(data) < term:
        return None

    for i in range(term):
        if tail:
            index = -(i+1)
        else:
            index = i

        if callback(data.iloc[index]):
            return abs(index)
    return None

# トレンドが変わった
def trend_convert(data):
    result = 0
    # 前日が直近の最大値だった
    if max(data) == data[-1]:
        result = 1

    # 前日が直近の最小値だった
    if min(data) == data[-1]:
        result = -1
    return result

def diff(data):
#    return numpy.gradient(data).tolist()
    return [0] + numpy.diff(data).tolist()

def cross(base, target):

    base_pre = base.shift(1)
    target_pre = target.shift(1)

    gc = ((base_pre <= target_pre) & (base >= target)) * 1
    dc = ((base_pre >= target_pre) & (base <= target)) * -1

    return gc + dc

# ローソク足の位置
#   -2: 支持線切り下げ
#   -1: 移動平均と支持線の間
#    0: 移動平均とクロス
#    1: 移動平均と抵抗線の間
#    2: 抵抗線切り上げ
def stages(data, resistance="resistance", support="support", average="average", with_eq=False):
    cross = lambda data, line: (data["low"] < data[line]) & (data[line] < data["high"])

    default = cross(data, average)
    p1 = data[average] < data["low"] # ローソク足が平均線より上
    m1 = data["high"] < data[average] # ローソク足が平均線より下

    stage_p1 = (~(default) & (p1)) * 1
    stage_m1 = (~(default) & (m1) ) * -1

    if with_eq:
        stage_p2 = ((cross(data, resistance)) | (data[resistance] <= data["high"])) * 2
        stage_m2 = ((cross(data, support)) | (data["low"] <= data[support])) * -2

        stage = numpy.maximum(stage_p1, stage_p2) + numpy.minimum(stage_m1, stage_m2)
    else:
        stage_p2 = ((cross(data, resistance)) | (data[resistance] < data["high"])) * 2
        stage_m2 = ((cross(data, support)) | (data["low"] < data[support])) * -2

        stage = stage_p1 + stage_p2 + stage_m1 + stage_m2

    return stage

def resistance(data, term):
    return data.iloc[-term:].max()

def support(data, term):
    return data.iloc[-term:].min()

# 上昇中の底値圏
def rising_safety(data):
    noize = list(filter(lambda x: x < 0, data))
    average = numpy.average(noize) if len(noize) > 0 else 0
    average = average * 3 # 係数
    return average

# 下落中の天井圏
def fall_safety(data):
    noize = list(filter(lambda x: x > 0, data))
    average = numpy.average(noize) if len(noize) > 0 else 0
    average = average * 3 # 係数
    return average

def trend(data):
    high_noize = list(filter(lambda x: x > 0, data))
    low_noize = list(filter(lambda x: x < 0, data))

    high_average = abs(numpy.average(high_noize)) if len(high_noize) > 0 else 0
    low_average = abs(numpy.average(low_noize)) if len(low_noize) > 0 else 0

    diff = abs(low_average - high_average)

    if low_average < high_average and len(low_noize) < len(high_noize) and high_average / 2 < diff:
        return 1

    if low_average > high_average and len(low_noize) > len(high_noize) and low_average / 2 < diff:
        return -1

    return 0

def strict_trend(data, term):
    d = data.iloc[-term:]
    high_noize = list(filter(lambda x: x > 0, d.values))
    low_noize = list(filter(lambda x: x < 0, d.values))

    high_average = abs(numpy.average(high_noize)) if len(high_noize) > 0 else 0
    low_average = abs(numpy.average(low_noize)) if len(low_noize) > 0 else 0

    if low_average < high_average and len(low_noize) < len(high_noize):
        return 1

    if low_average > high_average and len(low_noize) > len(high_noize):
        return -1

    return 0

# 定常過程データへ変換
def to_stationary(data, term=2):
    ts = sm.tsa.seasonal_decompose(data.values, freq=7, filt=None, two_sided=True)
    d = numpy.log(ts.trend + ts.resid)
    d = diff(d) # 前日比
    d = pandas.Series(d)
    return replace_invalid(convolve(d, term, average, padding=True, default=0))

# 騰落率
def rate(base, data):
    if base == 0:
        return base
    return (data - base) / float(base)

def get(data, index=-1):
    return data.values[index]

# NaN inf -inf を置換
def replace_invalid(data, source=0.0):
    data = numpy.array(data)
    data[(numpy.isnan(data)) | (data==float("inf")) | (data==float("-inf"))] = source
    return data

# 畳み込み
def convolve(data, term, callback, padding=True, default=0):
    convolved = [default for _ in range(term-1)] if padding else []
    if len(data) < term:
      return convolved[0:len(data)]
    for i in range(term, len(data)+1):
      d = data[0:i]
      convolved.append(callback(d, term))
    return convolved

# 前日比の割合
def gradient_rate(data):
    d = [1]
    for i in range(1, len(data)):
        r = rate(data["close"].iloc[i-1], data["close"].iloc[i])
        d.append(r)
    return d

# 単純移動平均線
def average(data, term):
    term_data = data.values[-term:]
    return numpy.convolve(term_data, numpy.ones(term)/float(term), 'valid')[0]

# RCI
def rci(data):
    term = len(data)
    term_data = data[::-1].reshape(-1)
    index = numpy.argsort(term_data)[::-1]
    d = 0
    for i, si in enumerate(index):
      d += ((i+1) - (si+1)) ** 2

    return int(( 1.0 - ( (6 * d) / float(term ** 3 - term) ) ) * 100)

# 標準偏差
def sigma(data, term):
    term_data = data.values[-term:]
    return math.sqrt((term * sum(term_data ** 2) - sum(term_data ** 2)) / (term * (term - 1)))

# ストキャスティクス
def stochastics(data, term):
    term_data = data.values[-term:]
    min_data = min(term_data)
    max_data = max(term_data)
    A = term_data[-1] - min_data
    B = max_data - min_data
    if B == 0:
        return 0.0
    s = A / float(B)
    return s * 100

# GCまたはDC
def gdc(d1, d2):
    if golden_cross(d1, d2):
        return 1
    elif dead_cross(d1, d2):
        return -1
    return 0

# ゴールデンクロス
def golden_cross(d1, d2):
    if d1 is None or d2 is None:
        return False
    if not (len(d1) == len(d2)):
        return False
    if not isinstance(d1, list):
        d1 = d1.values
    if not isinstance(d2, list):
        d2 = d2.values
    for i in range(len(d1)-1):
        if d1[i] < d2[i] and d1[i+1] > d2[i+1]:
            return True
    return False

# デッドクロス
def dead_cross(d1, d2):
    if d1 is None or d2 is None:
        return False
    if not (len(d1) == len(d2)):
        return False
    if not isinstance(d1, list):
        d1 = d1.values
    if not isinstance(d2, list):
        d2 = d2.values

    for i in range(len(d1)-1):
        if d1[i] > d2[i] and d1[i+1] < d2[i+1]:
            return True
    return False

# ドローダウン
def drawdown(data):
    base = numpy.array(data, dtype=float)
    maximums = numpy.maximum.accumulate(base)
    drawdown = maximums - base
    drawdown = numpy.divide(drawdown, maximums, out=numpy.zeros_like(drawdown), where=maximums!=0)
    drawdown = drawdown.tolist()
    return drawdown

# =======================================
# 値幅制限
def price_limit(price):
    index, price_range = price_limit_with_index(price)
    return price_range

def price_limit_with_index(price):
    from rakuten import Rakuten
    return Rakuten().price_limit(price)

# date util =======================================
def timestamp():
    return datetime.now().strftime("%Y-%m-%d %H:%M:%S")

def to_datetime(date, input_format="%Y-%m-%d"):
    return datetime.strptime(date, input_format)

def format(date, input_format="%Y-%m-%d", output_format="%Y-%m-%d"):
    return to_datetime(date, input_format).strftime(output_format)

def to_format(date, output_format="%Y-%m-%d"):
    return date.strftime(output_format)

def to_format_by_term(date, with_time=False):
    if with_time:
        return to_format(date, output_format="%Y-%m-%d %H:%M:%S")
    else:
        return to_format(date)

def relativeterm(term, with_time=False):
    if with_time:
        return relativedelta(days=term)
    else:
        return relativedelta(months=term)

def to_datetime_by_term(date, with_time=False):
    if with_time:
        return to_datetime(date, input_format="%Y-%m-%d %H:%M:%S")
    else:
        return to_datetime(date)

def is_weekday(date):
    return date.weekday() < 5 and jpholiday.is_holiday_name(date.date()) is None

def select_weekday(date, to_before=True, target=None):
    select = date
    if target is None:
        target = lambda x: True
    while(not (is_weekday(select) and target(select))):
        if to_before:
            select = select - relativeterm(1, with_time=True)
        else:
            select = select + relativeterm(1, with_time=True)
    return select

# end_dateは含まない
def daterange(start_date, end_date, step=1):
    for n in range(0, (end_date - start_date).days, step):
        yield start_date + timedelta(n)

def daterange_by_month(start_date, end_date, step=1):
    delta = relativedelta(end_date, start_date)
    months = delta.years * 12 + delta.months
    for n in range(0, months, step):
        yield start_date + relativedelta(months=n)

# combination util =======================================
# 全組み合わせを取得
def combinations(conditions):
    num = combinations_size(conditions)
    cond = [combination(i, conditions) for i in range(num)]
    cond = sorted(cond, key=lambda x: len(x[0]))
    return cond

# 2進数のフラグで取り出すパターンを決める 0 -> a, 1 -> b
def condition(index, conditions):
    a, b = [], []
    for j in range(len(conditions)): # 1ビットずつどちらに割りふるかチェック
        if int(bin(index & pow(2, j)), 0) != 0:
            a.append(conditions[j])
        else:
            b.append(conditions[j])
    return (a, b)

# 生成される組み合わせのサイズ
def combinations_size(conditions):
    return pow(2, len(conditions))*3 - 3

# 指定したindexの組み合わせを取得する
def combination(index, conditions):
    num = combinations_size(conditions)
    exc = {
        0: ([], conditions),
        num-2: (conditions, []),
        num-1: ([], [])
    }

    if index in exc.keys():
        return exc[index]
    r = divmod(index-1, 3)
    cond = condition(r[0]+1, conditions)
    if r[1] == 1:
        return (cond[0], [])
    elif r[1] == 2:
        return ([], cond[1])
    else:
        return cond

# list util =======================================
def split_list(data, condition):
#    if len(data) <= 1:
#        return data
    for i, d in enumerate(data):
        if condition(d):
            return [data[:i]] + split_list(data[i+1:], condition)
    return [data]

def chunked(iterable, n):
    return [iterable[x:x + n] for x in range(0, len(iterable), n)]

# dict util =======================================
def to_jsonizable(dic):
    to_numpy = lambda x: (x[0], numpy.array([x[1]]).tolist()[0]) if "numpy" in str(type(x[1])) else x
    jsonizable = lambda x: (x[0], to_jsonizable(x[1])) if type(x[1]) is dict else x
    return dict(list(map(lambda x: jsonizable(to_numpy(x)), list(dic.items()))))

# process =======================================
def proc_call(params, retry=3):
    print(params)
    for _ in range(retry):
        ret = subprocess.call(params, timeout=300)
        if ret == 0:
            return

def stop_watch(func) :
    @wraps(func)
    def wrapper(*args, **kargs):
        start = time.time()
        result = func(*args,**kargs)
        elapsed_time =  time.time() - start
        print("%s: %s sec." % (func.__name__, round(elapsed_time, 6)))
        return result
    return wrapper


'''
Created on Oct 21, 2010

@author: joeykblack
'''
from google.appengine.api import mail

def sendResults(resultset):
    giftsuser = resultset.giftsuser

    if mail.is_email_valid(giftsuser.email):
        message = mail.EmailMessage()
        message.sender = 'Chuck Coker <chuck.coker@meclabs.com>'
        message.to = giftsuser.firstname + ' ' + giftsuser.lastname + ' <' + giftsuser.email + '>'
        message.subject = 'Your Spiritual Gifts Results'
        message.body = 'You can access your results at ' + resultset.url

        message.send()


# %%
import torch

from torch import tensor
from torch import nn, optim
from torch.nn import functional as F

from . import Linear

class LRNN(nn.Module):

    def __init__(self, N, dim: list, num_rnn_layers=1, activation=nn.LeakyReLU):

        super(LRNN, self).__init__()

        self.N = N
        self.in_dim = dim[0]

        self.lstm = nn.LSTM(
            input_size=dim[0],
            hidden_size=dim[1],
            num_layers=num_rnn_layers,
            batch_first=True
        )

        if len(dim) >= 3:
            self.nextLayer = LRNN(self.N, dim=dim[1:])
        else:
            self.nextLayer = None

    def encode(self, x):

        x = x.view(-1, self.N, self.in_dim)
        y1, (hn, cn) = self.lstm(x)

        if self.nextLayer is not None:
            y1 = self.nextLayer(y1)

        return y1

    def forward(self, x):

        return self.encode(x)

class aRNN(nn.Module):

    def __init__(self, N, in_dim, output_layer_dim, output_layer=None, num_rnn_layers=1):

        super(aRNN, self).__init__()

        self.N = N
        self.in_lstm = in_dim

        self.lstm = nn.LSTM(
            input_size=self.in_lstm,
            hidden_size=output_layer_dim[0],
            num_layers=num_rnn_layers)

        if output_layer is None:
            self.encoder = Linear.get_Linear_Sequential(
                output_layer_dim, nn.LeakyReLU)
        else:
            self.encoder = output_layer

    def encode(self, x):

        batch = x.size()[0]
        x1 = x.view(batch, -1, self.in_lstm)

        x1 = x1.permute(1, 0, 2)
        y1, (hn, cn) = self.lstm(x1)
        y1 = y1.permute(1, 0, 2)

        y2 = self.encoder(y1)

        return y2

    def forward(self, x):

        return self.encode(x)

# %%

class bRNN(nn.Module):

    def __init__(self, N, in_layer_dim, out_dim, in_layer=None, num_rnn_layers=1):

        super(bRNN, self).__init__()

        if in_layer is None:
            self.encoder = Linear.get_Linear_Sequential(
                in_layer_dim, nn.LeakyReLU)
        else:
            self.encoder = in_layer

        self.N = N
        self.in_lstm = in_layer_dim[-1]

        self.lstm = nn.LSTM(
            input_size=self.in_lstm,
            hidden_size=out_dim,
            num_layers=num_rnn_layers)

    def encode(self, x):

        batch = x.size()[0]

        x1 = self.encoder(x)
        x1 = x1.view(batch, self.N, self.in_lstm)

        x1 = x1.permute(1, 0, 2)
        y1, (hn, cn) = self.lstm(x1)
        y1 = y1.permute(1, 0, 2)

        return y1

    def forward(self, x):

        return self.encode(x)


import os
import random
import numpy as np
import torch
from torch.utils.data import Dataset

from utils import read_ply
from datasets.utils import pc_normalize, offread_uniformed

cats = {'airplane': 0, 'bathtub': 1, 'bed': 2, 'bench': 3, 'bookshelf': 4, 'bottle': 5, 'bowl': 6, 'car': 7, 'chair': 8, 'cone': 9, 'cup': 10, 'curtain': 11, 'desk': 12, 'door': 13, 'dresser': 14, 'flower_pot': 15, 'glass_box': 16, 'guitar': 17, 'keyboard': 18, 'lamp': 19, 'laptop': 20, 'mantel': 21, 'monitor': 22, 'night_stand': 23, 'person': 24, 'piano': 25, 'plant': 26, 'radio': 27, 'range_hood': 28, 'sink': 29, 'sofa': 30, 'stairs': 31, 'stool': 32, 'table': 33, 'tent': 34, 'toilet': 35, 'tv_stand': 36, 'vase': 37, 'wardrobe': 38, 'xbox': 39}

class ModelNet40Align(Dataset):
    '''
        points are randomly sampled from .off file, so the results of this dataset may be better or wrose than our claim results
    '''
    def __init__(self, partition='test', few_num=0, num_points=1024):
        assert partition in ('test', 'train')
        super().__init__()
        self.partition = partition
        self.few_num = few_num
        self.num_points = num_points
        self._load_data()
        if self.partition == 'train' and self.few_num > 0:
            self.paths, self.labels = self._few()

    def _load_data(self):
        DATA_DIR = './data/ModelNet40_manually_aligned'
        self.paths = []
        self.labels = []
        for cat in os.listdir(DATA_DIR):
            cat_path = os.path.join(DATA_DIR, cat, self.partition)
            for case in os.listdir(cat_path):
                if case.endswith('.off'):
                    self.paths.append(os.path.join(cat_path, case))
                    self.labels.append(cats[cat])

    def _few(self):
        num_dict = {i: 0 for i in range(40)}
        few_paths = []
        few_labels = []
        random_list = [k for k in range(len(self.labels))]
        random.shuffle(random_list)
        for i in random_list:
            label = self.labels[i]
            if num_dict[label] == self.few_num:
                continue
            few_paths.append(self.paths[i])
            few_labels.append(self.labels[i])
            num_dict[label] += 1
        return few_paths, few_labels

    def __getitem__(self, index):
        point = torch.from_numpy(offread_uniformed(self.paths[index], self.num_points)).to(torch.float32)
        label = self.labels[index]
        point = pc_normalize(point)
        if self.partition == 'train':
            pt_idxs = np.arange(point.shape[0])
            np.random.shuffle(pt_idxs)
            point = point[pt_idxs]
            return point, label
        return point, label

    def __len__(self):
        return len(self.labels)

class ModelNet40Ply(Dataset):
    '''
        we save the random points in our few-shot learning, so the results are confirmed
    '''
    def __init__(self, partition='test', few_num=0, num_points=1024):
        assert partition in ('test', 'train')
        super().__init__()
        self.partition = partition
        self.few_num = few_num
        self.num_points = num_points
        self._load_data()
        if self.partition == 'train' and self.few_num > 0:
            self.paths, self.labels = self._few()

    def _load_data(self):
        DATA_DIR = './data/ModelNet40_Ply'
        self.paths = []
        self.labels = []
        for case in os.listdir(DATA_DIR):
            self.paths.append(os.path.join(DATA_DIR, case))
            self.labels.append(int(case.split('_')[0]))

    def __getitem__(self, index):
        label = self.labels[index]
        point = torch.from_numpy(read_ply(self.paths[index]))
        point = pc_normalize(point)
        return point, label

    def __len__(self):
        return len(self.labels)


# -*- coding: utf-8 -*-

import os
import glob
import shutil

from datatrusts.utils import safe_mmkdir
from datatrusts import __version__ as __version__

def script_relative_path(file_path):
    '''
    Useful for testing with local files. Use a path relative to where the
    test resides and this function will return the absolute path
    of that file. Otherwise it will be relative to script that
    ran the test

    Note this is expensive performance wise so if you are calling this many
    times you may want to call it once and cache the base dir.
    '''
    # from http://bit.ly/2snyC6s

    import inspect
    scriptdir = inspect.stack()[1][1]
    return os.path.join(os.path.dirname(os.path.abspath(scriptdir)), file_path)

def scaffold_directories_and_notebooks(base_dir):
    """Add basic directories for an initial, opinionated data trust."""

    safe_mmkdir(base_dir, exist_ok=True)
    notebook_dir_name = "notebooks"

    open(os.path.join(base_dir, ".gitignore"), 'w').write("misc/")

    for directory in [notebook_dir_name, "expectation", "resource", "misc", "dag", "algorithm"]:
        safe_mmkdir(os.path.join(base_dir, directory), exist_ok=True)

    for misc in ["validations", "credentials", "documentation", "samples"]:
        safe_mmkdir(os.path.join(base_dir, "misc",
                                 misc), exist_ok=True)

    for notebook in glob.glob(script_relative_path("../notebooks/init_notebooks/*.ipynb")):
        notebook_name = os.path.basename(notebook)
        shutil.copyfile(notebook, os.path.join(
            base_dir, notebook_dir_name, notebook_name))

# !!! This injects a version tag into the docs. We should test that those versioned docs exist in RTD.
greeting_1 = """
Responsibly discover more together with a Data Trust.

If you're new to BrightHive Data Trusts, this tutorial is a good place to start:

    <blue>https://docs.brighthive.io</blue>
""".format(__version__.replace(".", "_"))

msg_prompt_lets_begin = """
Let's get started with a new data trust by scaffolding a new datatrust directory:

    datatrust_name
        ├── datatrust.yml
        ├── trustfile
        ├── resource
        │   ├── datapackages
        │   ├── raw
        │   ├── resource.yml
        ├── expectation
        ├── dag
        ├── algorithm
        ├── notebooks
        ├── misc
        │   ├── credentials
        │   ├── documentation
        │   └── examples
        └── .gitignore

OK to proceed?
"""


import numpy as np
import logging
from typing import Dict
from mypy.types import Type
from mypy.nodes import NameExpr, StrExpr, MemberExpr

from . import register
from ..shortcuts import is_dtypetype, int_type, bool_type, float_type
from ..bind_arguments import BoundArgument

log = logging.getLogger(__name__)

@register('numpy._InferDtypeWithDefault')
def InferDtypeWithDefault(typ: Type, funcname: str, bound_args: Dict[str, BoundArgument]):
    matches = [f for f in bound_args.values() if f is not None and is_dtypetype(f.formal_typ)]
    if len(matches) == 0:
        return typ.args[0]
    elif len(matches) == 1:
        return infer_dtype(matches[0])
    raise ValueError()

@register('numpy._InferDtype')
def InferDtype(typ: Type, funcname: str, bound_args: Dict[str, BoundArgument]):

    matches = [f for f in bound_args.values() if f is not None and is_dtypetype(f.formal_typ)]
    assert len(matches) == 1

    dtype = infer_dtype(matches[0])
    return dtype

###############################################################################

def infer_dtype(formal_arg) -> str:
    arg_type = formal_arg.arg_typ
    arg = formal_arg.arg
    if isinstance(arg, NameExpr):
        # np.zeros(1, int)
        dtype_str = arg.name
    elif isinstance(arg, StrExpr):
        # np.zeros(1, 'int')
        dtype_str = arg.value
    elif isinstance(arg, MemberExpr):
         # np.zeros(1, np.int32)
        dtype_str = arg.name
    else:
        log.error('could not infer dtype %s %s', arg_type, arg)
        raise ValueError()

        return 'Any'
    np_kind = np.dtype(dtype_str).kind
    builtin_kind_str = {
        'i': int_type(),
        'b': bool_type(),
        'f': float_type(),
    }[np_kind]
    return builtin_kind_str


import sys
input = sys.stdin.readline
N = int(input())
arr = []

for _ in range(N):
    num = int(input())
    # 0이 아닌 수가 입력되면 더하고
    if num :
        arr.append(num)
    # 0이 입력되면 마지막 수를 지운다
    else :
        arr.pop()

print(sum(arr))


#!/bin/env pypy

"""
Tools for calling short nucleotide variants.

Usage:
  variant_call calculate BAF <genome_fasta> <SNP_position> <tumor> <normal> [--hetz=N:R] [-q N] [-r REGION]

Options:
  -h --help         Show this screen
  -r <region>       Restrict analysis to chromosomal region
  -q --quality=N    Minimum mapping quality score [default: 10]
  --hetz=N:R        Minimum evidence for heterozygous [default: 4:0.25]
"""

from __future__ import print_function
import sys, docopt, re, os
from collections import defaultdict
from pypette import zopen, shell, shell_stdout, info, error

gt_symbols = ['', '0/0', '0/1', '1/1']

def simple_pileup(bam_paths, genome_path, kgenomes_path, min_mapq=10, min_alt_alleles=3,
	region=None):

	helper_dir = os.path.dirname(os.path.realpath(__file__)) + '/compiled'

	options = []
	if region:
		options.append('%s %s' % ('-l' if region.endswith('.bed') else '-r', region))

	# samtools mpileup will automatically ignore alignments flagged as
	# duplicates
	cmd = 'samtools mpileup -d 100000 -A -x -R -sB %s -q0 -l %s -f %s %s | %s/spileup %d %d' % (' '.join(options), kgenomes_path, genome_path,
		' '.join(bam_paths), helper_dir, min_alt_alleles, min_mapq)
	#info('Pre-filtering mutations with the following command:\n%s' % cmd)
	return shell_stdout(cmd)

def call_genotypes(alt, total, options):
	# 0 = unknown, 1 = ref, 2 = hetz, 3 = homz
	gtypes = [0] * len(alt)
	for s in range(len(alt)):
		if total[s] == 0: continue
		if total[s] - alt[s] >= options.min_ref_reads and \
			(total[s] - alt[s]) / total[s] >= options.min_ref_ratio:
			gtypes[s] = 1
		ratio = float(alt[s]) / total[s]
		if alt[s] >= options.min_hetz_reads and ratio >=options.min_hetz_ratio:
			gtypes[s] = 2
		if alt[s] >= options.min_homz_reads and ratio >=options.min_homz_ratio:
			gtypes[s] = 3
	return gtypes

################
# VARIANT CALL #
################

def calculate_BAF(bam_paths, genome_path, kgenomes_path, options):
	#print(bam_paths, genome_path, options.region, options.homz)
	gt_symbols = ['', '0/0', '0/1', '1/1']
	if not os.path.exists(genome_path):
		error('Could not find genome FASTA file %s.' % genome_path)

	if options.region:
		for bam_path in bam_paths:
			if not os.path.exists(bam_path + '.bai'):
				error('No index found for BAM file %s.' % bam_path)

	samples = [os.path.basename(p).replace('.bam', '') for p in bam_paths]

	# print('CHROM\tPOSITION\tREF\tALT\t%s' % '\t'.join(samples))
	print('CHROM\tPOSITION\tREF\tALT\t%s' %samples[0])

	ignore_mapq = [False] * len(samples)
	if options.ignore_mapq:
		for s, sample in enumerate(samples):
			if re.search(options.ignore_mapq, sample) != None:
				ignore_mapq[s] = True
				info('Ignoring mapping quality for sample %s.' % sample)

	for line in simple_pileup(bam_paths, genome_path, kgenomes_path,
		min_mapq=options.min_mapq, min_alt_alleles=(0 if options.keep_all else options.min_hetz_reads),
		region=options.region):

		if type(line) == bytes:
			line = line.decode('utf8')

		tokens = line[:-1].split('\t')
		if len(tokens) < 3: error('Invalid spileup line:\n%s' % line)
		if tokens[2] == 'N': continue
		pileups = [p.split(' ') for p in tokens[3:]]
		#total_reads = np.zeros(len(samples))
		#allele_reads = defaultdict(lambda: np.zeros(len(samples)))

		total_reads = [0] * len(samples)
		allele_reads = defaultdict(lambda: [0] * len(samples))

		for s, pileup in enumerate(pileups):
			if len(pileup) < 3: continue
			for a in range(0, len(pileup), 3):
				count = int(pileup[a+1]) + \
					(int(pileup[a+2]) if ignore_mapq[s] else 0)
				total_reads[s] += count
				if pileup[a] != '.': allele_reads[pileup[a]][s] = count

		# Call genotypes for each allele.
		# for alt, reads in allele_reads.iteritems():
		for alt, reads in allele_reads.items():
			genotypes = call_genotypes(reads, total_reads, options)

			# if not options.keep_all and all(gt < 2 for gt in genotypes): continue
			# if all(gt != 2 for gt in genotypes): continue
			if genotypes[1] != 2: continue

			gtypes = ('%s:%d:%d' % (gt_symbols[g], reads[s], total_reads[s])
				for s, g in enumerate(genotypes))
			# Reformat indels in VCF4 format
			ref = tokens[2]
			if len(alt) >= 2:
				if alt[1] == '+':    # Insertion
					alt = (ref if alt[0] == '.' else alt[0]) + alt[2:]
				elif alt[1] == '-':  # Deletion
					ref += alt[2:]
					alt = (ref[0] if alt[0] == '.' else alt[0])

			#######################
			## Hetrozygous bases ##
			#######################

			gt_list = list(gtypes)
			gt_col = gt_list[1] ## genotype for the normal sample
			genotype = gt_symbols.index(gt_col[:gt_col.find(':')])
			total_read = float(gt_col.split(':')[2])
			if not (genotype == 2 and total_read >= 15): continue

			#########################
			## calculating the BAF ##
			#########################

			read = gt_list[0].split(':')[1:3] ## reads for the tumor sample
			sys.stdout.write('\t'.join([tokens[0], tokens[1], ref, alt.upper()]))
			alt, total = float(read[0]), int(read[1])
			sys.stdout.write('\tNaN' if total == 0 else '\t%.2f' % (alt / total))
			sys.stdout.write('\n')

#######################
# COMMAND LINE PARSER #
#######################

class Options(object):
	def __init__(self, args):
		self.region = args['-r']
		self.min_mapq = int(args['--quality'])
		self.ignore_mapq = None
		self.keep_all = None
		self.min_ref_reads = 8
		self.min_ref_ratio = 0.9
		hetz = args['--hetz'].split(':')
		self.min_hetz_reads = int(hetz[0])
		self.min_hetz_ratio = float(hetz[1])
		self.min_homz_reads = 4
		self.min_homz_ratio = 0.7

if __name__ == '__main__':
	args = docopt.docopt(__doc__)
	if args['calculate'] and args['BAF']:
		options = Options(args)
		bam_files = [args['<tumor>'], args['<normal>']]
		calculate_BAF(bam_files, args['<genome_fasta>'], args['<SNP_position>'], options)


from typing import Dict, List
import pandas as pd
from utils.utils import Utils
import numpy as np
import random
from vertex.doc_vertex import Vertex

class KmeanCluster:

    def __init__(self, document_vertx=None, k: int = 15):
        if document_vertx is None:
            self.document_vertx: Dict[int, Vertex] = self.get_document_vertex()
        self.k: int = k
        self.clusters: List[Cluster] = list()
        self.create_first_cluster()

    def reassgin(self):
        self.cal_new_centroid_for_clusters()
        self.reassign_cluster_documetns()

    def cal_new_centroid_for_clusters(self):
        for cluster in self.clusters:
            new_centroid = dict()
            for cluster_member in cluster.members:
                member_vertex = cluster_member.term_weight_dict
                for key in member_vertex.keys():
                    if key in new_centroid.keys():
                        new_centroid[key] += member_vertex.get(key)
                    else:
                        new_centroid[key] = member_vertex.get(key)
            cluster_member_num = len(cluster.members)
            new_centroid_ = dict()
            for key, value in new_centroid.items():
                new_centroid_[key] = value / cluster_member_num
            cluster.centroid = new_centroid_

    def reassign_cluster_documetns(self):

        centroid_list = [cluster.centroid for cluster in self.clusters]
        new_clusters: List[Cluster] = list()
        for centroid in centroid_list:
            new_clusters.append(Cluster(centroid))
        for document_vertex in self.document_vertx.values():
            new_centroid = self.get_centorid_of_document(document_vertex.term_weight_dict)
            for cluster in new_clusters:
                if cluster.centroid == new_centroid:
                    cluster.members.append(document_vertex)
                    break

        self.clusters = new_clusters

    def create_first_cluster(self) -> None:
        """
        choosing random k centroids and then clustering
        """
        # random_list = [1000, 2000, 3000, 4000, 5000]
        for i in range(self.k):
            random_centroid_doc_id = random.randint(0, len(self.document_vertx))
            centroid = self.document_vertx.get(random_centroid_doc_id).term_weight_dict
            cluster = Cluster(centroid)
            self.clusters.append(cluster)
        for document_vertx in self.document_vertx.values():
            most_similar_centroid = self.get_centorid_of_document(document_vertx.term_weight_dict)
            for cluster in self.clusters:
                if cluster.centroid == most_similar_centroid:
                    cluster.members.append(document_vertx)
                    break

    def get_centorid_of_document(self, document_vertex):
        most_similar_centroid = None
        max_cosin_similarity = 0
        for index, cluster in enumerate(self.clusters):
            centoroid_vector = cluster.centroid
            cosin_simi = self.calculate_cosin_similarity(centoroid_vector, document_vertex)
            if cosin_simi > max_cosin_similarity:
                max_cosin_similarity = cosin_simi
                most_similar_centroid = centoroid_vector
        return most_similar_centroid

    @staticmethod
    def calculate_cosin_similarity(doc_vertex1: Dict, doc_vertex2: Dict):

        intersection = set(doc_vertex1.keys()).intersection(doc_vertex2.keys())
        if len(intersection):
            vect1 = [doc_vertex1.get(key) for key in intersection]
            vec2 = [doc_vertex2.get(key) for key in intersection]
            norm_vect_1 = np.linalg.norm(list(doc_vertex1.values()))
            norm_vect_2 = np.linalg.norm(list(doc_vertex2.values()))
            inner_product = pd.Series(vect1).dot(pd.Series(vec2))
            cosin_similarity = inner_product / (norm_vect_1 * norm_vect_2)
            return cosin_similarity
        return 0

    @staticmethod
    def get_document_vertex():
        # return Utils.load_object_from_file('doc_vertex_idf=1').doc_vertex
        data = Utils.load_object_from_file('knn_doc_vertex')
        return data

class Cluster:
    def __init__(self, centroid: Dict):
        self.centroid: Dict = centroid  # a dictionary as centroid
        self.members: List[Vertex] = list()  # list of doc_vertex belong to this cluster

if __name__ == '__main__':
    a = KmeanCluster(k=5)
    # a = Utils.load_object_from_file('a_kmeans')
    for i in range(5):
        a.reassgin()
    print(a)
    print(a)
    print(a)


import sys, time, logging
import PyIndi
import threading

class IndiClient(PyIndi.BaseClient):

    device = None

    def __init__(self, exposure_time, gain, offset):
        super(IndiClient, self).__init__()
        self.logger = logging.getLogger('PyQtIndi.IndiClient')
        self.logger.info('creating an instance of PyQtIndi.IndiClient')
        self.exposure_time = exposure_time
        self.gain = gain
        self.offset = offset
        self.abort_thread = threading.Thread(target=self.abort_exposure)
    def newDevice(self, d):
        self.logger.info("new device " + d.getDeviceName())
        if d.getDeviceName() == "QHY CCD QHY5LII-M-60b7e":
            self.logger.info("Set new device QHY CCD QHY5LII-M-60b7e!")
            # save reference to the device in member variable
            self.device = d
    def newProperty(self, p):
        self.logger.info("new property "+ p.getName() + " for device "+ p.getDeviceName())
        if self.device is not None and p.getName() == "CONNECTION" and p.getDeviceName() == self.device.getDeviceName():
            self.logger.info("Got property CONNECTION for QHY-5L-II CCD!")
            # connect to device
            self.connectDevice(self.device.getDeviceName())
            # set BLOB mode to BLOB_ALSO
            self.setBLOBMode(1, self.device.getDeviceName(), None)
        if p.getName() == "SDK_VERSION":
            # take first exposure
            self.takeExposure(self.exposure_time,self.gain, self.offset)
            self.abort_thread.start()
    def removeProperty(self, p):
        self.logger.info("remove property "+ p.getName() + " for device "+ p.getDeviceName())
    def newBLOB(self, bp):
        self.logger.info("new BLOB "+ bp.name)
        # get image data
        img = bp.getblobdata()
        # write image data to BytesIO buffer
        import io
        blobfile = io.BytesIO(img)
        # open a file and save buffer to disk
        #with open("frame.fit", "wb") as f:
         #   f.write(blobfile.getvalue())
        # start new exposure
        self.takeExposure(self.exposure_time, self.gain, self.offset)
    def newSwitch(self, svp):
        self.logger.info ("new Switch "+ svp.name + " for device "+ svp.device)
    def newNumber(self, nvp):
        self.logger.info("new Number "+ nvp.name + " for device "+ nvp.device)
    def newText(self, tvp):
        self.logger.info("new Text "+ tvp.name + " for device "+ tvp.device)
    def newLight(self, lvp):
        self.logger.info("new Light "+ lvp.name + " for device "+ lvp.device)
    def newMessage(self, d, m):
        #self.logger.info("new Message "+ d.messageQueue(m))
        pass
    def serverConnected(self):
        print("Server connected ("+self.getHost()+":"+str(self.getPort())+")")
    def serverDisconnected(self, code):
        self.logger.info("Server disconnected (exit code = "+str(code)+","+str(self.getHost())+":"+str(self.getPort())+")")
    def takeExposure(self, exposure_time, gain, offset):
        self.logger.info(">>>>>>>>")
        #get current exposure time
        ccd_exposure = self.device.getNumber("CCD_EXPOSURE")
        ccd_gain = self.device.getNumber("CCD_GAIN")
        ccd_offset = self.device.getNumber("CCD_OFFSET")
        ccd_abort_exposure = self.device.getSwitch("CCD_ABORT_EXPOSURE")
        #for i in ccd_abort_exposure:
            #print("       "+i.name+" = "+str(i.s))
        # set exposure time for captures
        ccd_exposure[0].value = exposure_time
        ccd_gain[0].value = gain
        ccd_offset[0].value = offset
        #STATUS
        for t in ccd_exposure:
            print("       "+t.name+" = "+str(t.value))
        # send new exposure time to server/device
        self.sendNewNumber(ccd_gain)
        self.sendNewNumber(ccd_offset)
        self.sendNewNumber(ccd_exposure)

    def abort_exposure(self):
        # Deactivate streaming
        #ccd_exposure = self.device.getNumber("CCD_EXPOSURE")
        ccd_abort_exposure = self.device.getSwitch("CCD_ABORT_EXPOSURE")
        choice = input("Abort exposure?: ")
        if choice == "yes" or choice =="YES" or choice == "Yes" or choice == "1":
            print("Ending exposure")
            #ccd_exposure[0].value = 0
            #ccd_abort_exposure[0].s = 0
            #self.sendNewNumber(ccd_exposure)
            self.sendNewSwitch(ccd_abort_exposure)
        else:
            print("Exposure continues")
        #for t in ccd_exposure:
         #   print("       "+t.name+" = "+str(t.value))
        for i in ccd_abort_exposure:
            print("       "+i.name+" = "+str(i.s))

str_exposure_time = input("Choose exposure time: ")
str_gain = input("Choose gain: ")
str_offset = input("Choose offset: ")
exposure_time = float(str_exposure_time)
gain = int(str_gain)
offset = int(str_offset)
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)

# instantiate the client
indiclient=IndiClient(exposure_time, gain, offset)
print(f"Tipo: {type(indiclient)}")
# set indi server localhost and port 7624
indiclient.setServer("localhost",7624)
# connect to indi server
print("Connecting to indiserver")
if (not(indiclient.connectServer())):
     print("No indiserver running on "+indiclient.getHost()+":"+str(indiclient.getPort())+" - Try to run")
     print("  indiserver indi_simulator_telescope indi_simulator_ccd")
     sys.exit(1)

# start endless loop, client works asynchron in background
while True:
    time.sleep(1)