# Copyright (C) 2020-2023 by SCICO Developers
# All rights reserved. BSD 3-clause License.
# This file is part of the SCICO package. Details of the copyright and
# user license can be found in the 'LICENSE' file distributed with the
# package.

"""Linear operator utility functions."""

# Needed to annotate a class method that returns the encapsulating class;
# see https://www.python.org/dev/peps/pep-0563/
from __future__ import annotations

from typing import Optional, Union

import scico.numpy as snp
from scico.operator._operator import Operator
from scico.random import randn
from scico.typing import PRNGKey

from ._linop import LinearOperator

def power_iteration(A: LinearOperator, maxiter: int = 100, key: Optional[PRNGKey] = None):
    """Compute largest eigenvalue of a diagonalizable :class:`.LinearOperator`.

    Compute largest eigenvalue of a diagonalizable
    :class:`.LinearOperator` using power iteration.

    Args:
        A: :class:`.LinearOperator` used for computation. Must be
            diagonalizable.
        maxiter: Maximum number of power iterations to use.
        key: Jax PRNG key. Defaults to ``None``, in which case a new key
            is created.

    Returns:
        tuple: A tuple (`mu`, `v`) containing:

            - **mu**: Estimate of largest eigenvalue of `A`.
            - **v**: Eigenvector of `A` with eigenvalue `mu`.

    """
    v, key = randn(shape=A.input_shape, key=key, dtype=A.input_dtype)
    v = v / snp.linalg.norm(v)

    for i in range(maxiter):
        Av = A @ v
        normAv = snp.linalg.norm(Av)
        if normAv == 0.0:  # Assume that ||Av|| == 0 implies A is a zero operator
            mu = 0.0
            v = Av
            break
        mu = snp.sum(v.conj() * Av) / snp.linalg.norm(v) ** 2
        v = Av / normAv
    return mu, v

def operator_norm(A: LinearOperator, maxiter: int = 100, key: Optional[PRNGKey] = None):
    r"""Estimate the norm of a :class:`.LinearOperator`.

    Estimate the operator norm
    `induced <https://en.wikipedia.org/wiki/Matrix_norm#Matrix_norms_induced_by_vector_norms>`_
    by the :math:`\ell_2` vector norm, i.e. for :class:`.LinearOperator`
    :math:`A`,

    .. math::
       \| A \|_2 &= \max \{ \| A \mb{x} \|_2 \, : \, \| \mb{x} \|_2 \leq 1 \} \\
                 &= \sqrt{ \lambda_{ \mathrm{max} }( A^H A ) }
                 = \sigma_{\mathrm{max}}(A) \;,

    where :math:`\lambda_{\mathrm{max}}(B)` and
    :math:`\sigma_{\mathrm{max}}(B)` respectively denote the
    largest eigenvalue of :math:`B` and the largest singular value of
    :math:`B`. The value is estimated via power iteration, using
    :func:`power_iteration`, to estimate
    :math:`\lambda_{\mathrm{max}}(A^H A)`.

    Args:
        A: :class:`.LinearOperator` for which operator norm is desired.
        maxiter: Maximum number of power iterations to use. Default: 100
        key: Jax PRNG key. Defaults to ``None``, in which case a new key
            is created.

    Returns:
        float: Norm of operator :math:`A`.

    """
    return snp.sqrt(power_iteration(A.H @ A, maxiter, key)[0].real)

def valid_adjoint(
    A: LinearOperator,
    AT: LinearOperator,
    eps: Optional[float] = 1e-7,
    x: Optional[snp.Array] = None,
    y: Optional[snp.Array] = None,
    key: Optional[PRNGKey] = None,
) -> Union[bool, float]:
    r"""Check whether :class:`.LinearOperator` `AT` is the adjoint of `A`.

    Check whether :class:`.LinearOperator` :math:`\mathsf{AT}` is the
    adjoint of :math:`\mathsf{A}`. The test exploits the identity

    .. math::
      \mathbf{y}^T (A \mathbf{x}) = (\mathbf{y}^T A) \mathbf{x} =
      (A^T \mathbf{y})^T \mathbf{x}

    by computing :math:`\mathbf{u} = \mathsf{A}(\mathbf{x})` and
    :math:`\mathbf{v} = \mathsf{AT}(\mathbf{y})` for random
    :math:`\mathbf{x}` and :math:`\mathbf{y}` and confirming that

    .. math::
      \frac{| \mathbf{y}^T \mathbf{u} - \mathbf{v}^T \mathbf{x} |}
      {\max \left\{ | \mathbf{y}^T \mathbf{u} |,
       | \mathbf{v}^T \mathbf{x} | \right\}}
      < \epsilon \;.

    If :math:`\mathsf{A}` is a complex operator (with a complex
    `input_dtype`) then the test checks whether :math:`\mathsf{AT}` is
    the Hermitian conjugate of :math:`\mathsf{A}`, with a test as above,
    but with all the :math:`(\cdot)^T` replaced with :math:`(\cdot)^H`.

    Args:
        A: Primary :class:`.LinearOperator`.
        AT: Adjoint :class:`.LinearOperator`.
        eps: Error threshold for validation of :math:`\mathsf{AT}` as
           adjoint of :math:`\mathsf{AT}`. If ``None``, the relative
           error is returned instead of a boolean value.
        x: If not the default ``None``, use the specified array instead
           of a random array as test vector :math:`\mb{x}`. If specified,
           the array must have shape `A.input_shape`.
        y: If not the default ``None``, use the specified array instead
           of a random array as test vector :math:`\mb{y}`. If specified,
           the array must have shape `AT.input_shape`.
        key: Jax PRNG key. Defaults to ``None``, in which case a new key
           is created.

    Returns:
      Boolean value indicating whether validation passed, or relative
      error of test, depending on type of parameter `eps`.
    """

    if x is None:
        x, key = randn(shape=A.input_shape, key=key, dtype=A.input_dtype)
    else:
        if x.shape != A.input_shape:
            raise ValueError("Shape of x array not appropriate as an input for operator A")
    if y is None:
        y, key = randn(shape=AT.input_shape, key=key, dtype=AT.input_dtype)
    else:
        if y.shape != AT.input_shape:
            raise ValueError("Shape of y array not appropriate as an input for operator AT")

    u = A(x)
    v = AT(y)
    yTu = snp.sum(y.conj() * u)  # type: ignore
    vTx = snp.sum(v.conj() * x)  # type: ignore
    err = snp.abs(yTu - vTx) / max(snp.abs(yTu), snp.abs(vTx))
    if eps is None:
        return err
    return float(err) < eps

def jacobian(F: Operator, u: snp.Array, include_eval: Optional[bool] = False) -> LinearOperator:
    """Construct Jacobian linear operator for a general operator.

    For a specified :class:`.Operator`, construct a corresponding
    Jacobian :class:`LinearOperator`, the application of which is
    equivalent to multiplication by the Jacobian of the
    :class:`.Operator` at a specified input value.

    The implementation of this function is based on :meth:`.Operator.jvp`
    and :meth:`.Operator.vjp`, which are themselves based on
    :func:`jax.jvp` and :func:`jax.vjp`. For reasons of computational
    efficiency, these functions return the value of the :class:`.Operator`
    evaluated at the specified point in addition to the requested
    Jacobian-vector product. If the `include_eval` parameter of this
    function is ``True``, the constructed :class:`LinearOperator` returns
    a :class:`.BlockArray` output, the first component of which is the
    result of the :class:`.Operator` evaluation, and the second component
    of which is the requested Jacobian-vector product. If `include_eval`
    is ``False``, then the :class:`.Operator` evaluation computed by
    :func:`jax.jvp` and :func:`jax.vjp` are discarded.

    Args:
        F: :class:`.Operator` of which the Jacobian is to be computed.
        u: Input value of the :class:`.Operator` at which the Jacobian is
           to be computed.
        include_eval: Flag indicating whether the result of evaluating
           the :class:`.Operator` should be included (as the first
           component of a :class:`.BlockArray`) in the output of the
           Jacobian :class:`LinearOperator` constructed by this function.

    Returns:
      A :class:`LinearOperator` capable of computing Jacobian-vector
      products.
    """
    if include_eval:
        Fu, G = F.vjp(u, conjugate=True)

        def adj_fn(v):
            return snp.blockarray((Fu, G(v)))

        def eval_fn(v):
            return snp.blockarray(F.jvp(u, v))

    else:
        adj_fn = F.vjp(u, conjugate=True)[1]

        def eval_fn(v):
            return F.jvp(u, v)[1]

    return LinearOperator(
        F.input_shape,
        output_shape=F.output_shape,
        eval_fn=eval_fn,
        adj_fn=adj_fn,
        input_dtype=F.input_dtype,
        output_dtype=F.output_dtype,
    )


import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision
from torchvision import datasets, models, transforms
from torch import utils, optim
import os
import time
import copy
import pandas as pd
import numpy as np

from PIL import Image
import sys
import json

sys.path.append('../')
sys.path.append('../../')
sys.path.append('../../utilities')
from utilities import cnn_utils, transform_utils, test_eval, context_models

MODEL_NAME = "dil_net"
MODEL_DETAILS = '''Load weights from VGG-16 TRAINED convolutional base.
Then, add a context model on top of that. Fix the VGG-16 weights.
Context model has 4 layers
'''

VGG_TRAIN = False

# Model def is still here tho, rather than dil_alt0
from dil_net0 import dil_net as model_def

# Check whether GPU is enabled
torch.cuda.is_available()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)

FRONT_END_TYPE = "vgg_tuned"
PATH_TO_FRONT_END_WEIGHTS = "../dil_alt0/dil_net_front"
IS_GPU = device == "cuda:0"
CONTEXT_LAYER_COUNT = 4
OUTPUT_CHANNELS = 1

EPOCH_COUNT = 5
BATCH_SIZE = 4
img_size = 256
input_channels = 3
TRIM = int((img_size-64)/2)
PADDING = 0

# Root directory for dataset
if input_channels == 3:
    root = '../../../data/training_data/descartes/RGB'
else:
    root = '../../../data/training_data/descartes/Four_band'

data_root = os.path.join(root, "segmentation/size_{}".format(img_size))

SAVE_ROOT = "./"

# Check whether GPU is enabled
torch.cuda.is_available()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)

def train_segmentation(model, num_epochs, dataloader_dict, criterion, optimizer, verbose=False, detailed_time=False):
    print("Starting training loop...\n")
    print("Model's device = {}".format(model.my_device))

    device = model.my_device

    # Track best model
    best_model_wts = copy.deepcopy(model.state_dict())
    current_best_loss = 1000

    # Initialize loss dict to record training, figures are per epoch
    epoch_loss_dict = {'train': {'acc': [], 'loss':[], 'IoU':[], 'time':[]},
                         'val': {'acc': [], 'loss':[], 'IoU':[], 'time':[]}}
    batch_loss_dict = {'train': {'acc': [], 'loss':[], 'IoU':[]},
                         'val': {'acc': [], 'loss':[], 'IoU':[]}}

    if detailed_time:
        epoch_loss_dict['train']['backward_pass_time'] = []
        epoch_loss_dict['train']['data_fetch_time'] = []
        epoch_loss_dict['val']['backward_pass_time'] = []
        epoch_loss_dict['val']['data_fetch_time'] = []

    # For each epoch
    for epoch in range(num_epochs):
        # Each epoch has training and validation phase
        for phase in ['train', 'val']:
            begin_epoch_time = time.time()
            if phase == 'train':
                model.train()
            else:
                model.eval()

            running_loss = 0.0
            running_corrects = 0
            running_IoU = 0.0
            total_obs = 0

            # For each mini-batch in the dataloader
            total_data_fetch = 0
            total_pass = 0
            b_data_fetch = time.time()
            for i, data in enumerate(dataloader_dict[phase]):

                if detailed_time:
                    total_data_fetch += (time.time() - b_data_fetch)
                    b_pass = time.time()

                images, target = data
                img_size = target.shape[-1]
                #assert img_size == 128

                target = torch.tensor(target, dtype=torch.float32, device=device)
                images = images.to(device)

                batch_size = target.shape[0]
                total_obs += batch_size

                # Zero the gradients
                model.zero_grad()
                # Forward pass -- reshape output to be like the target (has extra 1D dimension)
                output = model(images)
                output = output.view(target.shape)

                # Just round since we have binary classification
                preds = torch.round(output)
                correct_count = (preds == target).sum()

                # Calculate loss
                error = criterion(output, target)

                # Make detailed output if verbose
                verbose_str = ""

                # Training steps
                if phase == 'train':
                    # Backpropogate the error
                    error.backward()
                    # Take optimizer step
                    optimizer.step()
                if detailed_time:
                    total_pass += (time.time() - b_pass)

                # The error is divided by the batch size, so reverse this
                batch_loss_dict[phase]['acc'].append(correct_count.item()/batch_size)
                batch_loss_dict[phase]['loss'].append(error.item())
                batch_loss_dict[phase]['IoU'].append(test_eval.inter_over_union(preds, target))

                running_loss += error.item() * batch_size
                running_corrects += correct_count.item()
                running_IoU += test_eval.inter_over_union(preds, target) * batch_size

                if detailed_time:
                    b_data_fetch = time.time()

            epoch_loss = running_loss / total_obs
            epoch_acc = running_corrects / (total_obs*img_size*img_size)

            epoch_IoU = running_IoU / total_obs

            if epoch_loss < current_best_loss:
                current_best_loss = epoch_loss
                best_model_wts = copy.deepcopy(model.state_dict())

            t = time.time()-begin_epoch_time
            # Add to our epoch_loss_dict
            epoch_loss_dict[phase]['acc'].append(epoch_acc)
            epoch_loss_dict[phase]['loss'].append(epoch_loss)
            epoch_loss_dict[phase]['IoU'].append(epoch_IoU)
            epoch_loss_dict[phase]['time'].append(t)
            if detailed_time:
                epoch_loss_dict[phase]['backward_pass_time'].append(total_pass)
                epoch_loss_dict[phase]['data_fetch_time'].append(total_data_fetch)

            print("PHASE={} EPOCH={} TIME={} LOSS={} ACC={}".format(phase,
                epoch, t, epoch_loss, epoch_acc))

    return model, best_model_wts, epoch_loss_dict, batch_loss_dict

# Define trasnforms
common_transforms = [transform_utils.RandomHorizontalFlip(0.5),
                     transform_utils.RandomVerticalFlip(0.5)]
#img_transforms = [transforms.ColorJitter()]

# Define network
net = context_models.FrontEnd_ContextModel(FRONT_END_TYPE, PATH_TO_FRONT_END_WEIGHTS, IS_GPU,
        input_channels, img_size, CONTEXT_LAYER_COUNT, OUTPUT_CHANNELS)
net.fix_front_end_weights()
#net.load_vgg_weights(VGG_TRAIN)

# Define dataloaders
train_root = os.path.join(data_root, "train")
val_root = os.path.join(data_root, "val")

train_dset = model_def.SegmentationDataset(train_root, list_common_trans=common_transforms,
                                  list_img_trans=None, f_type = "PIL", trim=TRIM)
val_dset = model_def.SegmentationDataset(val_root, f_type="PIL", trim=TRIM)

train_dset_loader = utils.data.DataLoader(train_dset, batch_size=BATCH_SIZE, shuffle=True)
val_dset_loader = utils.data.DataLoader(val_dset, batch_size=BATCH_SIZE, shuffle=True)

dset_loader_dict = {'train':train_dset_loader, 'val':val_dset_loader}

criterion_loss = nn.BCELoss()

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
net = net.to(device)
optimizer = optim.Adam(net.parameters())

trained_net, best_model_wts, epoch_loss_dict, batch_loss_dict = train_segmentation(net, EPOCH_COUNT, dset_loader_dict, criterion_loss, optimizer)

cnn_utils.save_model(net, MODEL_NAME, best_model_wts, epoch_loss_dict, batch_loss_dict, MODEL_DETAILS, SAVE_ROOT)

# net.load_state_dict(best_model_wts)
# test_eval.do_in_sample_tests(net, "../../..")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Module for defining Reinforce Agent
"""

from typing import Any, Iterable

import numpy as np
from keras import Input, Model
from keras.activations import tanh, sigmoid
from keras.layers import Dense, concatenate
from keras.optimizers import adam
from keras.utils import to_categorical
import keras.backend as K

from reinforcement_ecosystem.environments import InformationState, Agent

class ReinforceClassicBrain:
    """
    Reinforce Classic Brain for ???
    """

    def __init__(self, state_size: int, num_layers: int, num_neuron_per_layer: int, action_size: int):
        """
        Initializer for `ReinforceClassicBrain`
        :param state_size: ???
        :param num_layers: The number of layer of the model
        :param num_neuron_per_layer: The number of neuron per layer
        :param action_size: The maximum number of action
        """
        self.state_size = state_size
        self.num_layers = num_layers
        self.num_neuron_per_layer = num_neuron_per_layer
        self.action_size = action_size
        self.model = self.create_model()

    def create_model(self) -> Model:
        """
        Create a Model from the internal state of the class
        :return: A Keras `Model`
        """
        input_state = Input(shape=(self.state_size,))
        input_action = Input(shape=(self.action_size,))
        hidden = concatenate([input_state, input_action])
        for i in range(self.num_layers):
            hidden = Dense(self.num_neuron_per_layer, activation=tanh)(hidden)
        policy = Dense(1, activation=sigmoid)(hidden)
        model = Model([input_state, input_action], policy)
        model.compile(loss=ReinforceClassicBrain.reinforce_loss, optimizer=adam())
        return model

    def predict_policy(self, state: Any, action: Any) -> Any:
        """
        ???
        :param state: ???
        :param action: ???
        :return: ???
        """
        return self.model.predict([np.array([state]), np.array([action])])[0]

    def predict_policies(self, states: Any, actions: Any) -> Any:
        """
        ???
        :param states: ???
        :param actions: ???
        :return: ???
        """
        return self.model.predict([states, actions])

    @staticmethod
    def reinforce_loss(y_true: 'tensorflow.Tensor', y_pred: 'tensorflow.Tensor') -> 'tensorflow.Tensor':
        """
        Custom loss function for the reinforce model
        Compute the mean of the inverted log of the prediction by the product of the true output
        :param y_true: Real outputs
        :param y_pred: Predicted output of the model
        :return: The computed loss as `tensorflow.Tensor`
        """
        return K.mean(-K.log(y_pred) * y_true)

    def train_policy(self, state: Any, action: Any, advantage: Any) -> None:
        """
        ???
        :param state: ???
        :param action: ???
        :param advantage: ???
        :return: ???
        """
        self.model.train_on_batch([np.array([state]), np.array([action])], np.array([advantage]))

    def train_policies(self, states: Any, actions: Any, advantages: Any) -> None:
        """
        ???
        :param states: ???
        :param actions: ???
        :param advantages: ???
        """
        self.model.train_on_batch([states, actions], advantages)

class ReinforceClassicAgent(Agent):
    """
    Reinforce Classic Agent class for playing with it
    """

    def __init__(self, state_size: int, action_size: int, num_layers: int = 5, num_neuron_per_layer: int = 128) -> None:
        """
        Initializer for the `ReinforceClassicAgent` class
        :param state_size: ???
        :param action_size: ???
        :param num_layers: The number of layer for the Model
        :param num_neuron_per_layer: The number of neurons per layer for the model
        """
        self.brain = ReinforceClassicBrain(state_size, num_layers, num_neuron_per_layer, action_size)
        self.action_size = action_size
        self.episode_buffer = []

    def act(self, player_index: int, information_state: InformationState, available_actions: Iterable[int]) -> int:
        """
        Play the given action for the `ReinforceClassicAgent`
        :param player_index: The ID of the player playing
        :param information_state: The `InformationState` of the game
        :param available_actions: The legal action to choose from
        :return: The selected action
        """
        vectorized_states = np.array([information_state.vectorize()] * len(available_actions))
        actions_vectorized = np.array([to_categorical(action, self.action_size) for action in available_actions])
        logits = self.brain.predict_policies(vectorized_states, actions_vectorized)
        sum = np.sum(logits)
        probabilities = np.reshape(logits / sum, (len(available_actions),))
        chosen_action = np.random.choice(available_actions, p=probabilities)
        transition = dict()
        transition['s'] = information_state.vectorize()
        transition['a'] = to_categorical(chosen_action, self.action_size)
        transition['r'] = 0.0
        transition['t'] = False
        self.episode_buffer.append(transition)
        return chosen_action

    def observe(self, reward: float, terminal: bool) -> None:
        """
        Observe the state of the game for the `ReinforceClassicAgent`
        :param reward: Reward of the player after the game
        :param terminal: If the game is in a terminal mode
        """
        if not self.episode_buffer:
            return
        self.episode_buffer[len(self.episode_buffer) - 1]['r'] += reward
        self.episode_buffer[len(self.episode_buffer) - 1]['t'] |= terminal
        if terminal:
            states = np.array([transition['s'] for transition in self.episode_buffer])
            actions = np.array([transition['a'] for transition in self.episode_buffer])
            R = 0.0
            for t in reversed(range(len(self.episode_buffer))):
                R = self.episode_buffer[t]['r'] + 0.9 * R
                self.episode_buffer[t]['R'] = R
            advantages = np.array([transition['R'] for transition in self.episode_buffer])
            self.brain.train_policies(states, actions, advantages)
            self.episode_buffer = []

class ReinforceClassicWithMultipleTrajectoriesAgent(Agent):
    """
    Reinforce Classic With Multiple Trajectories  Agent class for playing with it
    """

    def __init__(self, state_size: int, action_size: int, num_layers: int = 5,
                 num_neuron_per_layer: int = 128, train_every_X_trajectories: int = 16):
        """
        Initializer for the `ReinforceClassicWithMultipleTrajectoriesAgent` class
        :param state_size: ???
        :param action_size: ???
        :param num_layers: The number of layer for the Model
        :param num_neuron_per_layer: The number of neuron per layer for the model
        :param train_every_X_trajectories: ???
        """
        self.brain = ReinforceClassicBrain(state_size, num_layers, num_neuron_per_layer, action_size)
        self.action_size = action_size
        self.trajectories = []
        self.current_trajectory_buffer = []
        self.train_every_X_trajectory = train_every_X_trajectories

    def act(self, player_index: int, information_state: InformationState, available_actions: Iterable[int]) -> int:
        """
        Play the given action for the `ReinforceClassicWithMultipleTrajectoriesAgent`
        :param player_index: The ID of the player playing
        :param information_state: The `InformationState` of the game
        :param available_actions: The legal action to choose from
        :return: The selected action
        """
        vectorized_states = np.array([information_state.vectorize()] * len(available_actions))
        actions_vectorized = np.array([to_categorical(action, self.action_size) for action in available_actions])
        logits = self.brain.predict_policies(vectorized_states, actions_vectorized)
        sum = np.sum(logits)
        probabilities = np.reshape(logits / sum, (len(available_actions),))
        chosen_action = np.random.choice(available_actions, p=probabilities)
        transition = dict()
        transition['s'] = information_state.vectorize()
        transition['a'] = to_categorical(chosen_action, self.action_size)
        transition['r'] = 0.0
        transition['t'] = False
        self.current_trajectory_buffer.append(transition)
        return chosen_action

    def observe(self, reward: float, terminal: bool) -> None:
        """
        Observe the state of the game for the `ReinforceClassicWithMultipleTrajectoriesAgent`
        :param reward: Reward of the player after the game
        :param terminal: If the game is in a terminal mode
        """
        if not self.current_trajectory_buffer:
            return
        self.current_trajectory_buffer[len(self.current_trajectory_buffer) - 1]['r'] += reward
        self.current_trajectory_buffer[len(self.current_trajectory_buffer) - 1]['t'] |= terminal
        if terminal:
            R = 0.0
            for t in reversed(range(len(self.current_trajectory_buffer))):
                R = self.current_trajectory_buffer[t]['r'] + 0.9 * R
                self.current_trajectory_buffer[t]['R'] = R
            self.trajectories.append(self.current_trajectory_buffer)
            self.current_trajectory_buffer = []
            if len(self.trajectories) == self.train_every_X_trajectory:
                states = np.array([transition['s'] for trajectory in self.trajectories for transition in trajectory])
                actions = np.array([transition['a'] for trajectory in self.trajectories for transition in trajectory])
                advantages = np.array([transition['R'] for trajectory in self.trajectories for transition in trajectory])
                self.brain.train_policies(states, actions, advantages)
                self.trajectories = []


from nltk.corpus import conll2002
from sklearn.feature_extraction import DictVectorizer
from sklearn.linear_model import Perceptron
from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier
from sklearn.metrics import precision_recall_fscore_support
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
import time
import string
import re
from nltk.stem import WordNetLemmatizer
import numpy as np

lemmatizer = WordNetLemmatizer()

def getnames(text):
    return open(text, 'r').read().split()

names = set(getnames('sp_name_out.txt'))

def load_wordvec(file):
    wordvec = {}
    with open(file) as f:
        for line in f:
            word = line.split()[0]
            vec = line.split()[1:]
            for value in vec:
                try:
                    value = float(value)
                except BaseException:
                    value = 0.0
            wordvec[word] = vec
    return wordvec

vec = load_wordvec("wiki.es.vec")

def wordshape(word):
    output = ''
    for i in range(len(word)):
        if word[i].isupper():
            output += 'X'
        elif word[i].islower():
            output += 'x'
        elif re.match(r'\d', word[i]):
            output += 'd'
        else:
            output += word[i]
    return output

def getfeats(word, POS, o):
    o = str(o)
    features = [
        (o + 'word', word),
        (o + 'POS', POS)
    ]
    return features

def word2features(sent, i):
    features = []
    for o in [-3, -2, -1, 0, 1, 2, 3]:
        if i+o >= 0 and i+o < len(sent):
            word = sent[i+o][0]
            POS = sent[i+o][1]
            featlist = getfeats(word, POS, o)
            features.extend(featlist)

    # punctuation = set(sent[i][0]) & set(string.punctuation)
    # features.append(('PREFIX_1', sent[i][0][:1]))
    # features.append(('SUFFIX_1', sent[i][0][-1:]))
    # features.append(('PREFIX_2', sent[i][0][:2]))
    # features.append(('SUFFIX_2', sent[i][0][-2:]))
    features.append(('PREFIX_3', sent[i][0][:3]))
    features.append(('SUFFIX_3', sent[i][0][-3:]))
    features.append(('SHAPE', wordshape(sent[i][0])))
    # features.append(('LENGTH', len(sent[i][0])))
    #features.append(('PUNCTUATION', len(punctuation)))
    # features.append(('NAMES', sent[i][0] in names))
    return dict(features)

if __name__ == "__main__":
    t = time.time()
    train_sents = list(conll2002.iob_sents('esp.train'))
    dev_sents = list(conll2002.iob_sents('esp.testa'))
    test_sents = list(conll2002.iob_sents('esp.testb'))
    train_sents = train_sents + dev_sents

    train_feats = []
    train_labels = []

    for sent in train_sents:
        for i in range(3):
            sent.insert(0, ('SSS', 'None', 'None'))
            sent.append(('EEE', 'None', 'None'))
        for i in range(len(sent)):
            feats = word2features(sent,i)
            train_feats.append(feats)
            train_labels.append(sent[i][-1])

    vectorizer = DictVectorizer()
    X_train = vectorizer.fit_transform(train_feats)
    for sent in train_sents:
        for i in range(len(sent)):
            try:
                X_train[i, :] += sent[i][0]
            except KeyError:
                try:
                    X_train[i, :] += vec[lemmatizer.lemmatize(sent[i][0])]
                except KeyError:
                    X_train[i, :] = np.zeros((1, 50))

    model = LogisticRegression(C=10)
    # model = LinearSVC(C=1)
    # model = Perceptron(verbose=1)
    model.fit(X_train, train_labels)

    test_feats = []
    test_labels = []

    # switch to test_sents for your final results
    for sent in test_sents:
        for i in range(len(sent)):
            feats = word2features(sent,i)
            test_feats.append(feats)
            test_labels.append(sent[i][-1])

    X_test = vectorizer.transform(test_feats)
    for sent in test_sents:
        for i in range(len(sent)):
            try:
                X_test[i, :] += sent[i][0]
            except KeyError:
                try:
                    X_test[i, :] += vec[lemmatizer.lemmatize(sent[i][0])]
                except KeyError:
                    X_test[i, :] = np.zeros((1, 50))
    y_pred = model.predict(X_test)

    j = 0
    print("Writing to results.txt")
    with open("results.txt", "w") as out:
        for sent in test_sents:
            for i in range(len(sent)):
                word = sent[i][0]
                gold = sent[i][-1]
                pred = y_pred[j]
                j += 1
                out.write("{}\t{}\t{}\n".format(word,gold,pred))
        out.write("\n")
    print(time.time() - t)
    print("Now run: python conlleval.py results.txt")


import numpy as np

from keras.models import Sequential, load_model
from keras.layers import Dense, Activation, Dropout
import spacy

# from fantom_util.feature_extraction.spacy_extractor import spacy_model
from fantom_util.file_io_util import file_from_s3

# from fantom_util.models import Model

class RootNodeClassifierMLP:
    def __init__(self):
        weights_path = "weights_rnc_mlp.hdf5"
        file_from_s3("SOME_AWS_BUCKET", weights_path, weights_path)
        self.special_list = [
            ["what", "who", "when", "how", "where"],
            ["why", "whether", "which"],
            ["yes", "no", "sure", "course"],
            ["about"],
            ["want", "wanna", "like", "love", "hate"],
            ["chat", "talk"],
            [
                "he",
                "she",
                "it",
                "them",
                "that",
                "this",
                "those",
                "us",
                "his",
                "her",
                "their",
            ],
            ["I", "you"],
            ["else", "another", "again", "change"],
            ["something"],
            ["stop"],
            ["hi", "hello"],
        ]
        self.min_tok_pos = 83
        self.max_tok_pos = 100

        self.nlp = spacy.load("en_core_web_lg")
        # self.nlp = spacy_model()
        self.data_width = 32

        self.model = Sequential()
        self.model.add(Dense(20, activation="relu", input_shape=(self.data_width,)))
        self.model.add(Dropout(0.5))
        self.model.add(Dense(20, activation="relu"))
        self.model.add(Dropout(0.5))
        self.model.add(Dense(1, activation="sigmoid"))
        self.model.load_weights(weights_path)
        self.model.compile(
            optimizer="adam", loss="binary_crossentropy", metrics=["accuracy"]
        )

    def prepare_data(self, utterances):
        X_data = np.zeros((len(utterances), self.max_tok_pos - self.min_tok_pos + 2))
        Y_data = np.zeros(len(utterances))
        X_additional_data = np.zeros((len(utterances), len(self.special_list) + 1))
        # X_data
        for sample_nr in range(len(utterances)):
            line_doc = self.nlp(utterances[sample_nr])
            if "tell me" == str(utterances[sample_nr])[0:7]:
                X_additional_data[sample_nr, -2] += 1.0
            for token in line_doc:
                try:
                    X_data[sample_nr, token.pos - self.min_tok_pos] += 1.0
                    for list_nr in range(len(self.special_list)):
                        if token.orth in self.special_list[list_nr]:
                            X_additional_data[sample_nr, list_nr] += 1.0
                    if not token.is_stop:
                        X_additional_data[sample_nr, -1] += 1.0
                except:
                    pass
        X_data = np.concatenate((X_data, X_additional_data), axis=1)
        return X_data

    def predict_string(self, utterance):
        return float(self.predict_list([utterance])[0][0])

    def predict_list(self, list_of_utterances):
        return self.model.predict(x=self.prepare_data(list_of_utterances))

def main():
    rnc_mlp = RootNodeClassifierMLP()
    print(
        rnc_mlp.predict_list(
            ["hi", "Let's chat about football", "my name is trevor", "do you like him"]
        )
    )
    print((rnc_mlp.predict_string("I would like to talk about chickens")))

if __name__ == "__main__":
    main()


import board
import busio

import adafruit_mcp4725

from time import sleep

# Initialize I2C bus.
i2c = busio.I2C(board.SCL, board.SDA)

# Initialize MCP4725.
dac = adafruit_mcp4725.MCP4725(i2c, address=0x60)

# There are a three ways to set the DAC output, you can use any of these:
dac.value = 65535  # Use the value property with a 16-bit number just like
# the AnalogOut class.  Note the MCP4725 is only a 12-bit
# DAC so quantization errors will occur.  The range of
# values is 0 (minimum/ground) to 65535 (maximum/Vout).

dac.raw_value = 4095  # Use the raw_value property to directly read and write
# the 12-bit DAC value.  The range of values is
# 0 (minimum/ground) to 4095 (maximum/Vout).

dac.normalized_value = 1.0  # Use the normalized_value property to set the
# output with a floating point value in the range
# 0 to 1.0 where 0 is minimum/ground and 1.0 is
# maximum/Vout.

# Main loop will go up and down through the range of DAC values forever.
while True:
  dac.raw_value = 0
  print(0)
  sleep(5)
  dac.raw_value = 2048
  print('50%')
  sleep(5)
  dac.raw_value = 4095
  print('100%')
  sleep(5)
  dac.raw_value = 2048
  print('50%')
  sleep(5)


#!/bin/python
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import lecb.utils as utils
import os

def vhost(basedir, hostname):
    """Vhost definition to answer ACME challenge."""

    vhost_definition = """
<VirtualHost *:80>
    ServerName %s

    Alias /.well-known/acme-challenge %s/challenges/%s
    <Directory %s/challenges/%s>
        Require all granted
    </Directory>

</VirtualHost>""" % (hostname, basedir, hostname, basedir, hostname)

    return vhost_definition

def create_vhost_and_reload_service(basedir, hostname):
    """Create the vhost file and reload httpd."""

    if utils.platform_im_running_on() in ['Fedora', 'CentOS Linux', 'RedHat']:
        with open('/etc/httpd/conf.d/99_%s.conf' % hostname, 'w') as f:
            f.write(vhost(basedir, hostname))

    os.system('systemctl reload httpd')

def remove_vhost_and_reload_service(hostname):
    """Remove the vhost file and reload httpd."""

    if utils.platform_im_running_on() in ['Fedora', 'CentOS Linux', 'RedHat']:
        if os.path.isfile('/etc/httpd/conf.d/99_%s.conf' % hostname):
            os.remove('/etc/httpd/conf.d/99_%s.conf' % hostname)

    os.system('systemctl reload httpd')


# coding: utf-8

from __future__ import absolute_import
from datetime import date, datetime  # noqa: F401

from typing import List, Dict  # noqa: F401

from swagger_server.models.base_model_ import Model
from swagger_server import util

class SourceAudit(Model):
    """NOTE: This class is auto generated by the swagger code generator program.

    Do not edit the class manually.
    """
    def __init__(self, operator_id: str=None, process_id: str=None):  # noqa: E501
        """SourceAudit - a model defined in Swagger

        :param operator_id: The operator_id of this SourceAudit.  # noqa: E501
        :type operator_id: str
        :param process_id: The process_id of this SourceAudit.  # noqa: E501
        :type process_id: str
        """
        self.swagger_types = {
            'operator_id': str,
            'process_id': str
        }

        self.attribute_map = {
            'operator_id': 'operatorId',
            'process_id': 'processId'
        }
        self._operator_id = operator_id
        self._process_id = process_id

    @classmethod
    def from_dict(cls, dikt) -> 'SourceAudit':
        """Returns the dict as a model

        :param dikt: A dict.
        :type: dict
        :return: The SourceAudit of this SourceAudit.  # noqa: E501
        :rtype: SourceAudit
        """
        return util.deserialize_model(dikt, cls)

    @property
    def operator_id(self) -> str:
        """Gets the operator_id of this SourceAudit.

        Identificação do operador solicitante. Esta informação é fornecida pelo emissor.  # noqa: E501

        :return: The operator_id of this SourceAudit.
        :rtype: str
        """
        return self._operator_id

    @operator_id.setter
    def operator_id(self, operator_id: str):
        """Sets the operator_id of this SourceAudit.

        Identificação do operador solicitante. Esta informação é fornecida pelo emissor.  # noqa: E501

        :param operator_id: The operator_id of this SourceAudit.
        :type operator_id: str
        """

        self._operator_id = operator_id

    @property
    def process_id(self) -> str:
        """Gets the process_id of this SourceAudit.

        Identificação do processo solicitante. Esta informação é fornecida pelo emissor.  # noqa: E501

        :return: The process_id of this SourceAudit.
        :rtype: str
        """
        return self._process_id

    @process_id.setter
    def process_id(self, process_id: str):
        """Sets the process_id of this SourceAudit.

        Identificação do processo solicitante. Esta informação é fornecida pelo emissor.  # noqa: E501

        :param process_id: The process_id of this SourceAudit.
        :type process_id: str
        """

        self._process_id = process_id


# Initialisierung
import time
import RPi.GPIO as GPIO
import paho.mqtt.client as mqtt

GPIO.setmode(GPIO.BCM)

# AusgangsPins
levelinput = 26
schieben = 19
shift = 16

GPIO.setup(levelinput,GPIO.OUT,initial=False)
GPIO.setup(schieben,GPIO.OUT,initial=False)
GPIO.setup(shift,GPIO.OUT,initial=False)

# Zahlen
n_0 = "01111110"
n_1 = "00010010"
n_2 = "10111100"
n_3 = "10110110"
n_4 = "11010010"
n_5 = "11100110"
n_6 = "11101110"
n_7 = "00110010"
n_8 = "11111110"
n_9 = "11110110"
aus = "00000000"
dot = "00000001"

n_a = [n_0,n_1,n_2,n_3,n_4,n_5,n_6,n_7,n_8,n_9,aus,dot]

# schieben
def doSchieben():
    GPIO.output(schieben,1)
    time.sleep(0.001)
    GPIO.output(schieben,0)

# Eine Zahl ins Schieberegister
def n_to_s(n_p):
    for i in range(8):
        GPIO.output(levelinput,int(n_p[i]))
        GPIO.output(shift,1)
        time.sleep(0.001)
        GPIO.output(shift,0)

# Zahl anzeigen
def show_number(msg_zahl):

    # ausschalten
    n_to_s(n_a[10])
    n_to_s(n_a[10])

    doSchieben()

    # anzeigen
    zahl = str(msg_zahl)

    if zahl == "p":
        zahl = "11"
    elif zahl == "a":
        zahl = "10"

    zahl = int(zahl)

    if zahl in range(0, 12):
        n_to_s(n_a[int(zahl)])

    doSchieben()

# Countdown
def countdown(msg_zahl):

    # ausschalten
    n_to_s(n_a[10])
    n_to_s(n_a[10])

    doSchieben()

    start = str(msg_zahl)
    if int(start) > 99:
        start = "99"

    # Countdown starten
    for i in range(int(start)+1):

        zahl = int(start) - i

        if len(str(zahl)) == 1:
             zahl = "0" + str(zahl)

             n_to_s(n_a[int(zahl[1])])
             n_to_s(n_a[0])

        elif len(str(start)) == 2:
            zahl = str(zahl)

            n_to_s(n_a[int(zahl[1])])
            n_to_s(n_a[int(zahl[0])])

        doSchieben()

        time.sleep(0.99)

    # blinken
    for i in range(2):

        n_to_s(n_a[10])
        n_to_s(n_a[10])

        doSchieben()

        time.sleep(0.4)

        n_to_s(n_a[0])
        n_to_s(n_a[0])

        doSchieben()

        time.sleep(0.4)

    # ausschalten
    n_to_s(n_a[10])
    n_to_s(n_a[10])

    doSchieben()

# MQTT
def on_connect(client, userdata, flags, rc):
    print("Connected with result code " + str(rc))

    client.subscribe("7-SA/#")

def on_message(client, userdata, msg):

    msg_len = (len(str(msg.payload)))-3
    message = ""
    for i in range(msg_len):
        message = message + str(msg.payload)[2+i]

    if msg.topic == "7-SA/Zahl":
        show_number(message)
    elif msg.topic == "7-SA/Countdown":
        countdown(message)

client = mqtt.Client()
client.on_connect = on_connect
client.on_message = on_message

client.connect("192.168.x.x", 1883, 60)

client.loop_forever()

GPIO.cleanup()


# -*- coding: utf-8 -*-
# @Time : 2019/11/26 下午1:15
# @Author : kimihiro
# @File : __init__.py.py
# @Software: PyCharm
from cfg_pull_service.get_config_data import ApolloClient
from cfg_update_service.update_config import UpdateConfigService
from config import Configs

class UpdateConfigApis:
    def __init__(self, ap_conf: dict = Configs.DEV_APO, conf_info: list = Configs.CONF_INFO):
        self.ap_conf = ap_conf
        self.conf_info = conf_info

    def _get_c_data(self) -> dict:
        apollo_client = ApolloClient(**self.ap_conf)
        return apollo_client.get_config()

    def auto_update_conf(self):
        c_data = self._get_c_data()
        if c_data:
            for info in self.conf_info:
                up_ser = UpdateConfigService(**info)
                up_ser.update(c_data)

if __name__ == "__main__":
    a = UpdateConfigApis()
    a.auto_update_conf()


import torch
import random
import math
import time
from torch.autograd import Variable
from .Plotter import Plotter
from Env.Environment import Environment
import numpy as np
import os
# if gpu is to be used
use_cuda = torch.cuda.is_available()
FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor
LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor
ByteTensor = torch.cuda.ByteTensor if use_cuda else torch.ByteTensor
Tensor = FloatTensor

class Improver(object):
    def __init__(self, net, memory_size, memory, shared):
        self.net = net  # Deep Net
        self.memory = memory
        self.shared = shared  # shared resources, {'memory', 'SENT_FLAG', 'weights'}
        #self.env = env
        self.plotter = Plotter(folder='DQN/plot/cartpole_simple/exp4')
        self.steps_done = 0

        # hyperparameters:
        self.EPS_START = 1.
        self.EPS_END = 0.05
        self.EPS_DECAY = 50   # DECAY larger: slower
        self.MEMORY_SIZE = memory_size

    def behavior_policy(self, state):
        # We can add epislon here to decay the randomness
        # We store the tensor of size 1x1
        sample = random.random()
        eps_threshold = self.EPS_END + (self.EPS_START - self.EPS_END) * \
                        math.exp(-1. * self.steps_done / self.EPS_DECAY)
        #print('threshold: {}' . format(eps_threshold))

        if sample > eps_threshold:
            return self.policy(state)
        else:
            #return LongTensor([[self.env.action_space.sample()]])
            return self.env.action_space.sample()
    def policy(self, state):
        # return tensor of size 1x1
        res = self.net(Variable(state, volatile=True).type(FloatTensor)).data
        #print('policy values: {}, {}' . format(res[0][0], res[0][1]))
        #print('policy value: {}' . format(res.max(1)[0][0]))
        return res.max(1)[1][0]
        #return res.max(1)[1].view(1,1)
        #return self.net(Variable(state, volatile=True).type(FloatTensor))\
        #                .data.max(1)[1].view(1,1)

    def save_checkpoint(self, state, filename='save/checkpoint_improver.pth.tar'):
        torch.save(state, filename)

    def run(self):
        POPULATE_FLAG = True
        #MEMORY_SIZE = self.shared['memory'].capacity
        MEMORY_SIZE = self.MEMORY_SIZE
        #POPULATE_MAX = MEMORY_SIZE
        POPULATE_MAX = 1
        populate_num = 0
        #PLOT_FREQ = int(MEMORY_SIZE/32)
        PLOT_FREQ = 20
        R = 0. # for plotting
        step = 0
        Repi = 0
        self.env = Environment()
        self.env.set_all()
        self.env.reset()
        last_time = time.time()
        prev_plot = self.steps_done
        pretrain = True
        #PRETRAIN_MAX = self.MEMORY_SIZE // 2
        PRETRAIN_MAX = 2000

        if os.path.isfile('save/checkpoint_improver.pth.tar'):
            print("=> loading checkpoint '{}'".format('save/checkpoint_improver.pth.tar'))
            checkpoint = torch.load('save/checkpoint_improver.pth.tar')
            print("=> loaded checkpoint '{}'"
                  .format('save/checkpoint_improver.pth.tar'))
        else:
            print("=> no checkpoint found at '{}'".format('save/checkpoint_improver.pth.tar'))

        while True:
            if POPULATE_FLAG:
                if pretrain:
                    if populate_num == PRETRAIN_MAX:
                        pretrain = False
                        self.shared['SENT_FLAG'] = 0
                        POPULATE_FLAG = False
                else:
                    if populate_num == POPULATE_MAX:
                        # reset populate num
                        self.shared['SENT_FLAG'] = 0
                        POPULATE_FLAG = False
            else:
                if self.shared['SENT_FLAG']:
                    # evaluator sent the weights
                    print('copying...')
                    self.net.load_state_dict(self.shared['weights'])
                    POPULATE_FLAG = True
                    populate_num = 0
                    # after evaluating the policy for one round, set the epislon smaller
                    next_step = self.steps_done + 1
                    if next_step != 0:  # loop to back
                        # then set eps_threshold to a small value
                        self.steps_done = next_step
            self.save_checkpoint({
                'state_dict': self.net.state_dict(),
            })
            #print('loop took {0} seconds' . format(time.time()-last_time))

            state = self.env.get_state()  # this is to avoid time delay
            last_time = time.time()
            # 0.003s
            state_torch = torch.from_numpy(state).type(FloatTensor) # convert to torch and normalize
            state_torch = state_torch.unsqueeze(0).type(FloatTensor)
            action = self.behavior_policy(state_torch)

            next_state, r, done = self.env.step(action)  # 0.03s

            if len(self.memory) == MEMORY_SIZE:
                self.memory.pop(0)
            self.memory.append((state, [action], [r], \
                                        next_state, [1-done])
                            )
            #print(len(self.memory))

            # 0.001s
            if POPULATE_FLAG:
                populate_num += 1
            # plot the average reward in PLOT_FREQ episodes
            Repi += r
            if done:
                step += 1
                R += Repi
                Repi = 0. # reset episode reward
            if step and self.steps_done > prev_plot:
                print('average rewards after {} train: {}' . format(self.steps_done-prev_plot, R/step))
                self.plotter.plot_train_rewards(R/step)
                # 2-3s
                R = 0.
                step = 0
                prev_plot = self.steps_done

        self.plotter.terminate()


from foton import FilterFile,Filter,iir2zpk,iir2z
from ezca import LIGOFilter,Ezca

ezca = Ezca(timeout=2)

all_typea = {'ETMX','ETMY','ITMX','ITMY'}
all_typeb = {'BS','SRM','SR2','SR3'}
all_typebp = {'PRM','PR2','PR3'}
all_typeci = {'MCI','MCO','MCE','IMMT1','IMMT2'}
all_typeco = {'OSTM','OMMT1','OMMT2'}
all_optics = all_typea | all_typeb | all_typebp | all_typeci | all_typeco

def typename_is(optic):
    '''
    '''
    if optic in all_typea:
        return 'Type-A'
    elif optic in all_typeb:
        return 'Type-B'
    elif optic in all_typebp:
        return 'Type-Bp'
    elif optic in all_typeci or optic in all_typeco:
        return 'Type-C'
    else:
        raise ValueError('!')

def partname_is(optic,stage):
    '''
    '''
    if typename_is(optic) in ['Type-A','Type-B','Type-Bp']:
        if stage in ['TM','IM','MN']:
            part = 'P'
        elif stage in ['BF','F3','F2','F1','F0','SF','IP']:
            part = 'T'
        else:
            raise ValueError('!')
    else:
        part = ''
    return part

class Ezff(FilterFile):
    def __init__(self,ffname):
        super().__init__(ffname)
        self.ffname = ffname

    def save(self):
        self.ff.write(self.ffname)

def copy_FMs(orig,dests):
    '''
    '''
    for dest in dests:
        for i in range(10):
            dest[i].copyfrom(orig[i])

def switch_on(chname,mask=['INPUT','OFFSET','OUTPUT','DECIMATION']):
    '''
    '''
    FB = ezca.get_LIGOFilter(chname)
    FMs = FB.get_current_swstat_mask().buttons
    FB.only_on(*mask)


import Q1_C1815697 as c
import matplotlib.pyplot as plt
import numpy as np

def count_wins(num_of_races, file_name = 'sailors_values.csv'):
    name_list = list(c.read_sailor_data(file_name).keys())
    #Creates a dictionary with the names as keys and empty lists as values
    results = {key:0 for key in name_list}
    for i in range(0, num_of_races):
        race_result = c.calculate_finishing_order(c.generate_performances(c.read_sailor_data(file_name)))
        for index, value in enumerate(race_result):
            results[value] += (index + 1)
    return sorted(results.items(), key=lambda keyvalue: keyvalue[1])

def average_modelling(num_of_models, race_result):
    output = {}
    for tuple in race_result:
        #Creates average score output and stores as tuple
        output[tuple[0]] = tuple[1] / num_of_models
    #Returns sorted avg results
    return sorted(output.items(), key=lambda keyvalue: keyvalue[1])

num_of_races = int(input("Number of models: "))
results = count_wins(num_of_races)
#Collects names and results from model
names = [value[0] for value in results]
totals = [value[1] for value in results]
#Calculates avg finsishing position based on the results
averages = average_modelling(num_of_races, results)
avg_values = [value for key, value in averages]

def plot_bar(x_data, y_data, title, x_label, y_label):
    index = np.arange(len(x_data))
    bar_list = plt.bar(index, y_data)
    plt.xlabel(x_label, fontsize=10)
    plt.ylabel(y_label, fontsize=10)
    plt.xticks(index, x_data, fontsize=10)
    plt.title(title, fontsize=15)
    x = -0.05 #Set X value for bar count data
    for value in y_data:
        plt.text(x, value * 0.5, value, fontsize=8)
        x += 1

fig_size = plt.rcParams["figure.figsize"] = [6.4, 4.8]
figure = plt.gcf()
plt.subplot(2, 1, 1)
plot_bar(names, totals, "Total Scores", "Names", "Total Score")
plt.subplot(2, 1, 2)
plot_bar(names, avg_values, "Average Position", "Names", "Average Positions")
plt.tight_layout()
plt.show()


# phylogenetic tree

def Initialize(distance_file):
	Trees = []
	D = {}
	with open(distance_file) as f:
		for line in f:
			items = line.strip().split(" ")
			Trees.append(Node(items[0]))
			for i, node in enumerate(Trees):
				if i<len(Trees)-1:
					D[node.label,items[0]] = D[items[0],node.label] = float(items[i+1])
	return D, Trees

class Node(object):
	def __init__(self, label, left=None, right=None):
		self.label = label
		self.left = left
		self.right = right
		if left is not None and right is not None:
			self.size = left.size + right.size
		else:
			self.size = 1
		self.dist2parent = 0
		self.h = 0

	def Newick(self):
		if self.left is None and self.right is None:
			print('%s:%.2f' % (self.label,self.dist2parent), end="")
		else:
			print("(", end="")
			self.left.Newick()
			print(",", end="")
			self.right.Newick()
			print('):%.2f' % self.dist2parent, end="")

def Merge(D, T, i, j):
	i, j = min(i,j), max(i,j)
	label = T[i].label + ':' + T[j].label
	ck = Node(label, T[i], T[j])
	ck.h = D[T[i].label, T[j].label] / 2
	T[i].dist2parent = ck.h - T[i].h
	T[j].dist2parent = ck.h - T[j].h
	for l,node in enumerate(T):
		if l!=i and l!=j:
			dkl=(D[T[i].label,T[l].label]*T[i].size + D[T[j].label,T[l].label]*T[j].size)/(T[i].size + T[j].size)
			D[label,T[l].label] = D[T[l].label,label] = dkl
	T.pop(i)
	T.pop(j-1)
	T.append(ck)

def ClosestPair(D, T):
	if len(T)<=1 :
		return None,None
	m, mi, mj = D[T[0].label,T[1].label],0,1
	for i in range(len(T)):
		for j in range(i+1, len(T)):
			if m > D[T[i].label, T[j].label]:
				m, mi, mj = D[T[i].label, T[j].label], i, j
	return mi, mj

def UPGMA(distance_file):
	Distance, Trees = Initialize(distance_file)
	while len(Trees) > 1:
		i, j = ClosestPair(Distance, Trees)
		Merge(Distance,Trees,i,j)
	Trees[0].Newick()

UPGMA("phylo_distance.txt")


from __future__ import division, print_function, absolute_import

import sys
import csv
import json
import os
import re
import pdb

maxInteger = sys.maxsize
while True:
  #to avoid OverflowError

    try:
        csv.field_size_limit(maxInteger)
        break
    except OverflowError:
        maxInteger = int(maxInteger/10)

import base64
import pickle

import numpy as np
import nltk
nltk.data.path.append('data')
nltk.download('punkt', download_dir='data')
from nltk.tokenize import word_tokenize
from tqdm import tqdm
from sklearn.externals import joblib

train_anno_path = os.path.join('data', 'v2_mscoco_train2014_annotations.json')
val_anno_path = os.path.join('data', 'v2_mscoco_val2014_annotations.json')
train_ques_path = os.path.join('data', 'v2_OpenEnded_mscoco_train2014_questions.json')
val_ques_path = os.path.join('data', 'v2_OpenEnded_mscoco_val2014_questions.json')
glove_emb_path = os.path.join('data', 'glove', 'glove.6B.300d.txt')
visual_feats_path = os.path.join('data', 'trainval_resnet101_faster_rcnn_genome_36.tsv')

word_contractions = {
    "aint": "ain't", "arent": "aren't", "cant": "can't", "couldve":
    "could've", "couldnt": "couldn't", "couldn'tve": "couldn't've",
    "couldnt've": "couldn't've", "didnt": "didn't", "doesnt":
    "doesn't", "dont": "don't", "hadnt": "hadn't", "hadnt've":
    "hadn't've", "hadn'tve": "hadn't've", "hasnt": "hasn't", "havent":
    "haven't", "hed": "he'd", "hed've": "he'd've", "he'dve":
    "he'd've", "hes": "he's", "howd": "how'd", "howll": "how'll",
    "hows": "how's", "Id've": "I'd've", "I'dve": "I'd've", "Im":
    "I'm", "Ive": "I've", "isnt": "isn't", "itd": "it'd", "itd've":
    "it'd've", "it'dve": "it'd've", "itll": "it'll", "let's": "let's",
    "maam": "ma'am", "mightnt": "mightn't", "mightnt've":
    "mightn't've", "mightn'tve": "mightn't've", "mightve": "might've",
    "mustnt": "mustn't", "mustve": "must've", "neednt": "needn't",
    "notve": "not've", "oclock": "o'clock", "oughtnt": "oughtn't",
    "ow's'at": "'ow's'at", "'ows'at": "'ow's'at", "'ow'sat":
    "'ow's'at", "shant": "shan't", "shed've": "she'd've", "she'dve":
    "she'd've", "she's": "she's", "shouldve": "should've", "shouldnt":
    "shouldn't", "shouldnt've": "shouldn't've", "shouldn'tve":
    "shouldn't've", "somebody'd": "somebodyd", "somebodyd've":
    "somebody'd've", "somebody'dve": "somebody'd've", "somebodyll":
    "somebody'll", "somebodys": "somebody's", "someoned": "someone'd",
    "someoned've": "someone'd've", "someone'dve": "someone'd've",
    "someonell": "someone'll", "someones": "someone's", "somethingd":
    "something'd", "somethingd've": "something'd've", "something'dve":
    "something'd've", "somethingll": "something'll", "thats":
    "that's", "thered": "there'd", "thered've": "there'd've",
    "there'dve": "there'd've", "therere": "there're", "theres":
    "there's", "theyd": "they'd", "theyd've": "they'd've", "they'dve":
    "they'd've", "theyll": "they'll", "theyre": "they're", "theyve":
    "they've", "twas": "'twas", "wasnt": "wasn't", "wed've":
    "we'd've", "we'dve": "we'd've", "weve": "we've", "werent":
    "weren't", "whatll": "what'll", "whatre": "what're", "whats":
    "what's", "whatve": "what've", "whens": "when's", "whered":
    "where'd", "wheres": "where's", "whereve": "where've", "whod":
    "who'd", "whod've": "who'd've", "who'dve": "who'd've", "wholl":
    "who'll", "whos": "who's", "whove": "who've", "whyll": "why'll",
    "whyre": "why're", "whys": "why's", "wont": "won't", "wouldve":
    "would've", "wouldnt": "wouldn't", "wouldnt've": "wouldn't've",
    "wouldn'tve": "wouldn't've", "yall": "y'all", "yall'll":
    "y'all'll", "y'allll": "y'all'll", "yall'd've": "y'all'd've",
    "y'alld've": "y'all'd've", "y'all'dve": "y'all'd've", "youd":
    "you'd", "youd've": "you'd've", "you'dve": "you'd've", "youll":
    "you'll", "youre": "you're", "youve": "you've"
}

manual_mapping = {
    'none': '0',
    'zero': '0',
    'one': '1',
    'two': '2',
    'three': '3',
    'four': '4',
    'five': '5',
    'six': '6',
    'seven': '7',
    'eight': '8',
    'nine': '9',
    'ten': '10'
}

articles_list = ['a', 'an', 'the']
strip_period = re.compile("(?!<=\d)(\.)(?!\d)")
strip_comma = re.compile("(\d)(\,)(\d)")
punct = [
    ';', r"/", '[', ']', '"', '{', '}',
    '(', ')', '=', '+', '\\', '_', '-',
    '>', '<', '@', '`', ',', '?', '!'
]

def punctuation_processing(in_Text):
    out_Text = in_Text
    for p in punct:
        if (p + ' ' in in_Text or ' ' + p in in_Text) \
        or (re.search(strip_comma, in_Text) != None):
            out_Text = out_Text.replace(p, '')
        else:
            out_Text = out_Text.replace(p, ' ')
    out_Text = strip_period.sub("", out_Text, re.UNICODE)
    return out_Text

def digit_article_processing(inp_text):
    out_text = []
    temp_text = inp_text.lower().split()
    for word in temp_text:
        word = manual_mapping.setdefault(word, word)
        if word not in articles_list:
            out_text.append(word)
        else:
            pass
    for wordId, word in enumerate(out_text):
        if word in word_contractions:
            out_text[wordId] = word_contractions[word]
    out_text = ' '.join(out_text)
    return out_text

def anno_processing(freq_thr=9):

    train_anno = json.load(open(train_anno_path), encoding = 'utf-8')['annotations']
    val_anno = json.load(open(val_anno_path), encoding = 'utf-8')['annotations']
    anno_s = train_anno + val_anno

    print("Calculating the freq of each multiple choice answers:")
    mca_frequency = {}
    for anno in tqdm(anno_s):
        mca = digit_article_processing(punctuation_processing(anno['multiple_choice_answer']))
        mca = mca.replace(',', '')
        mca_frequency[mca] = mca_frequency.get(mca, 0) + 1

    # filter rare ans out
    for a, freq in list(mca_frequency.items()):
        if freq < freq_thr:
            mca_frequency.pop(a)

    print("Number of answers appear more than {} times: {}".format(freq_thr - 1, len(mca_frequency)))

    # generating answer dict
    idx_to_ans = []
    ans_to_idx = {}
    for i, a in enumerate(mca_frequency):
        idx_to_ans.append(a)
        ans_to_idx[a] = i

    print("Generating the soft scores:")
    trgt = []
    for anno in tqdm(anno_s):
        anss = anno['answers']

        # calculating individual answers freq
        ans_frequency = {}
        for a in anss:
            ans_frequency[a['answer']] = ans_frequency.get(a['answer'], 0) + 1

        soft_scores = []
        for a, freq in ans_frequency.items():
            if a in ans_to_idx:
                soft_scores.append((a, min(1, freq / 3)))

        trgt.append({
            'question_id': anno['question_id'],
            'image_id': anno['image_id'],
            'answer': soft_scores    # [(ans1, score1), (ans2, score2), (ansn, scoren ...]
        })

    pickle.dump([idx_to_ans, ans_to_idx], open(os.path.join('data', 'dict_ans.pkl'), 'wb'))
    return trgt

def tokenize_str(t):
    t = t.lower().replace(',', '').replace('?', '')
    return word_tokenize(t)

def qa_processing(trgt, max_words=14):

    print("Merging que and ans:")
    idx_to_word = []
    word_to_idx = {}

    train_ques = json.load(open(train_ques_path))['questions']
    val_ques = json.load(open(val_ques_path))['questions']
    q_s = train_ques + val_ques
    que_ans_s = []
    for i, q in enumerate(tqdm(q_s)):
        tokens = tokenize_str(q['question'])
        for t in tokens:
            if not t in word_to_idx:
                idx_to_word.append(t)
                word_to_idx[t] = len(idx_to_word) - 1

        assert q['question_id'] == trgt[i]['question_id'],\
                "Question ID doesn't match ({}: {})".format(q['question_id'], trgt[i]['question_id'])

        que_ans_s.append({
            'image_id': q['image_id'],
            'question': q['question'],
            'question_id': q['question_id'],
            'question_toked': tokens,
            'answer': trgt[i]['answer']
        })

    pickle.dump(que_ans_s[:len(train_ques)], open(os.path.join('data', 'train_qa.pkl'), 'wb'))
    pickle.dump(que_ans_s[len(train_ques):], open(os.path.join('data', 'val_qa.pkl'), 'wb'))
    pickle.dump([idx_to_word, word_to_idx], open(os.path.join('data', 'dict_q.pkl'), 'wb'))

    return idx_to_word

def word_emb_processing(idx_to_word):
    print("Generating pre-trained word embedding wts:")
    word_to_emb = {}
    dim_emb = int(glove_emb_path.split('.')[-2].split('d')[0])
    with open(glove_emb_path, encoding = 'utf-8') as f:
        for entry in f:
            values = entry.split(' ')
            word = values[0]
            word_to_emb[word] = np.asarray(values[1:], dtype=np.float32)

    pretrained_wts = np.zeros((len(idx_to_word), dim_emb), dtype=np.float32)
    for idx, word in enumerate(idx_to_word):
        if word not in word_to_emb:
            continue
        pretrained_wts[idx] = word_to_emb[word]

    np.save(os.path.join('data', 'glove_pretrained_{}.npy'.format(dim_emb)), pretrained_wts)

def visual_feats_processing():
    FIELDNAMES = ['image_id', 'image_w', 'image_h', 'num_boxes', 'boxes', 'features']
    train_ques = json.load(open(train_ques_path))['questions']
    val_ques = json.load(open(val_ques_path))['questions']
    t_ids = set([q['image_id'] for q in train_ques])
    v_ids = set([q['image_id'] for q in val_ques])

    print("Reading tsv, total number of iterations: {}".format(len(t_ids)+len(v_ids)))
    tv_features = {}
    vv_features = {}
    with open(visual_feats_path, encoding='utf-8') as tsv_in_file:
        reader = csv.DictReader(tsv_in_file, delimiter='\t', fieldnames=FIELDNAMES)
        for i, item in enumerate(tqdm(reader)):
            image_id = int(item['image_id'])
            feats = np.frombuffer(base64.b64decode(item['features']),
                dtype=np.float32).reshape((int(item['num_boxes']), -1))

            if image_id in t_ids:
                tv_features[image_id] = feats
            elif image_id in v_ids:
                vv_features[image_id] = feats
            else:
                raise ValueError("Image_id: {} not in training or validation set".format(image_id))

    print("Converting tsv to pickle...hang on")
    joblib.dump(tv_features, open(os.path.join('data', 'train_vfeats.pkl'), 'wb'))
    joblib.dump(vv_features, open(os.path.join('data', 'val_vfeats.pkl'), 'wb'))

if __name__ == '__main__':
    trgt = anno_processing()
    idx_to_word = qa_processing(trgt)
    word_emb_processing(idx_to_word)
    visual_feats_processing()
    print("Ok Done")


# 線段樹題
# (same), (flip), (positive), (update)
import sys
class Node:
    def __init__(self, num=0, left=None, right=None, setnum=0):
        self.state = num
        self.left = left
        self.right = right
        self.setnum = 0

class SegmentTree:
    '''
    SegmentTree:
        set, flip segment_tree
    '''
    def __init__(self, n):
        self.seg_tree = [Node() for i in range(4*(1 << n.bit_length()) + 1)]
        self.n = n
        self.build(1, 1, n)

    def build(self, index, left, right):
        node = self.seg_tree[index]

        node.left = left
        node.right = right

        if left != right:
            mid = (left + right) >> 1
            self.build(2*index, left, mid)
            self.build(2*index+1, mid+1, right)

    def push_down(self, index):
        node = self.seg_tree[index]
        if node.setnum:
            self.seg_tree[2*index].setnum = node.setnum
            self.seg_tree[2*index+1].setnum = node.setnum
            node.setnum = 0

            self.seg_tree[2*index].state = node.state
            self.seg_tree[2*index+1].state = node.state
            node.state = 0
            return

        self.seg_tree[2*index].state ^= node.state
        self.seg_tree[2*index+1].state ^= node.state
        node.state = 0

    '''
    Region flip
    '''
    def flip(self, left, right, node_index=1):
        node = self.seg_tree[node_index]
        if node.left == left and node.right == right:
            node.state ^= 1
            return

        mid = (node.left + node.right) >> 1

        self.push_down(node_index)
        if left > mid:
            self.flip(left, right, node_index*2+1)
        elif mid >= right:
            self.flip(left, right, node_index*2)
        else:
            self.flip(left, mid, node_index*2)
            self.flip(mid+1, right, node_index*2 + 1)

    '''
    Region set
    '''
    def setnum(self, left, right, num, node_index=1):
        node = self.seg_tree[node_index]
        if node.left == left and node.right == right:
            node.setnum = num
            node.state = 0
            return

        self.push_down(node_index)

        mid = (node.left + node.right) >> 1

        if left > mid:
            self.setnum(left, right, num, node_index*2+1)
        elif mid >= right:
            self.setnum(left, right, num, node_index*2)
        else:
            self.setnum(left, mid, num, node_index*2)
            self.setnum(mid+1, right, num, node_index*2 + 1)

    def query(self, index, node_index=1):
        node = self.seg_tree[node_index]

        if node.left == index == node.right:
            return node.state, node.setnum

        mid = (node.left + node.right) >> 1
        self.push_down(node_index)

        # Split the query
        if mid >= index:
            return self.query(index, node_index*2)
        else:
            return self.query(index, node_index*2 + 1)

n, m = map(int,input().split())
input = sys.stdin.readline
base = 100006
shift = 1
seg_tree = SegmentTree(base*2 + 1)
nums = list(map(int, input().split()))
for idx in range(m):
    x, y = input().split()
    y = int(y)
    if x == '<':
        if y > 0:
            seg_tree.flip(0 + shift, y-1 + shift) # < 4 | flip [0, 3]
            seg_tree.setnum(y + shift, base, 1)   # < 4 | setnum [4, inf] 1
        else:
            seg_tree.setnum(abs(y) + 1 + shift, base, 1)  # < -4 | setnum [5, inf] 1
    else:
        if y < 0:
            seg_tree.flip(0 + shift, abs(y)-1 + shift) # > -4 | flip [0, 3]
            seg_tree.setnum(abs(y) + shift, base, 2)   # > -4 | setnum [4, inf] 2
        else:
            seg_tree.setnum(y + 1 + shift, base, 2)    # > 4  | setnum [5, inf] 2
ans = []
for num in nums:
    flip, setnum = seg_tree.query(abs(num) + shift)
    if setnum == 2:
        ans.append(-abs(num)*(1-2*flip))
    elif setnum == 1:
        ans.append(abs(num)*(1-2*flip))
    else:
        ans.append(num*(1-2*flip))
print(" ".join(map(str,ans)))


import pygame
from pygame.locals import MOUSEBUTTONDOWN, MOUSEBUTTONUP, KEYDOWN, MOUSEMOTION, K_SPACE, K_LEFT, K_RIGHT, K_UP, K_DOWN
import glm
from glm import vec2, vec3
import math

from constants import BACKGROUND_COLOR
from objects import CelestialObject, VelocityArrow
from inputs import Inputs, Button
from scene import CelestialScene

from typing import TYPE_CHECKING
if TYPE_CHECKING:
    from main import App

class State:
    """
    State base class
    - Derive from this class and override methods for each state
    """
    def __init__(self, app, **kwargs):
        self.app: App = app
        self.name = ''

        self.suspended = False
        self.terminated = False

        self.scene = None

        if kwargs:
            raise ValueError(f"Some kwargs not consumed: {kwargs}")

    def update(self, delta_time):
        self.scene.update(delta_time)

    @property
    def scene_bg(self):
        return self.scene.background

    # @property
    # def sprites(self):
    #     return self.scene.content

class MenuState(State):
    """
    Menu State Class
    """
    def __init__(self, app, **kwargs):
        super().__init__(app, **kwargs)

    def update(self, delta_time):
        pass

class DrawState(State):
    """
    Draw State Class
    """
    def __init__(self, app, **kwargs):
        super().__init__(app, **kwargs)

        self.scene = CelestialScene(app)

        self.__static_input_funcs = []
        self.__dynamic_input_funcs = {}
        self.__build_inputs()

        self.curr_celestial = None
        self.curr_velo_arrow = None

        self.paused = False

    def __build_inputs(self):
        """
        Private function to bind inputs to functions
        """
        inputs = Inputs()

        inputs.register("new_object", Button(MOUSEBUTTONDOWN, 1))
        inputs.register("update", Button(MOUSEMOTION, 0))
        inputs.register("kill_all_objects", Button(KEYDOWN, K_SPACE))

        inputs.register("mleft", Button(KEYDOWN, K_LEFT))
        inputs.register("mright", Button(KEYDOWN, K_RIGHT))
        inputs.register("mup", Button(KEYDOWN, K_UP))
        inputs.register("mdown", Button(KEYDOWN, K_DOWN))
        inputs.register("zoomin", Button(MOUSEBUTTONDOWN, 4))
        inputs.register("zoomout", Button(MOUSEBUTTONDOWN, 5))

        self.app.inputs = inputs

        # Store functions to maintain weakrefs
        self.__dynamic_input_funcs["temp"] = self.app.inputs.inputs["new_object"].on_press(self.__new_object_stage1)

        # Add all the inputs that do not change
        #
        #
        # TODO: Needs a restructure so not referring down to scene
        #
        #
        self.__static_input_funcs.append(self.app.inputs.inputs["kill_all_objects"].on_press(self.scene.kill_all_objects))
        # self.__static_input_funcs.append(self.app.inputs.inputs["kill_all_objects"].on_press(self.__reset_new_object_stage))
        self.__static_input_funcs.append(self.app.inputs.inputs["mleft"].on_press_repeat(self.scene.move_cam_left, 0))
        self.__static_input_funcs.append(self.app.inputs.inputs["mright"].on_press_repeat(self.scene.move_cam_right, 0))
        self.__static_input_funcs.append(self.app.inputs.inputs["mup"].on_press_repeat(self.scene.move_cam_up, 0))
        self.__static_input_funcs.append(self.app.inputs.inputs["mdown"].on_press_repeat(self.scene.move_cam_down, 0))
        self.__static_input_funcs.append(self.app.inputs.inputs["zoomout"].on_press(self.scene.move_cam_out))
        self.__static_input_funcs.append(self.app.inputs.inputs["zoomin"].on_press(self.scene.move_cam_in))

    def __reset_new_object_stage(self):
        self.__dynamic_input_funcs["temp"] = self.app.inputs.inputs["new_object"].on_press(self.__new_object_stage1)
        self.__dynamic_input_funcs["temp2"] = None

    def __new_object_stage1(self):
        """
        Private function to create a new celestial object
        """
        if not self.curr_celestial:
            self.paused = True
            # Replace functions for next stage
            self.__dynamic_input_funcs["temp"] = self.app.inputs.inputs["new_object"].on_press_repeat(self.__new_object_stage1_cont, 0)
            self.__dynamic_input_funcs["temp2"] = self.app.inputs.inputs["new_object"].on_release(self.__new_object_stage2)

            center = pygame.mouse.get_pos()
            self.curr_celestial = CelestialObject(center) #set radius to 0 so the initializer will set PLANET_MIN_RADIUS

    def __new_object_stage1_cont(self):
        """
        Private function to update celestial body radius
        """
        if self.curr_celestial:
            center = self.curr_celestial.rect.center
            center_v = vec3(center[0], center[1], 0)
            curpos = pygame.mouse.get_pos()
            curpos_v = vec3(curpos[0], curpos[1], 0)
            self.curr_celestial.radius = math.floor(glm.distance(center_v, curpos_v))

    def __new_object_stage2(self):
        """
        Private function to complete creating a new celestial object
        """
        if self.curr_celestial:
            # Replace functions for next stage
            self.__dynamic_input_funcs["temp"] = self.app.inputs.inputs["update"].always(self.__new_object_stage2_cont)
            self.__dynamic_input_funcs["temp2"] = self.app.inputs.inputs["new_object"].on_press(self.__new_object_stage3)

            center = self.curr_celestial.rect.center
            center_v = vec3(center[0], center[1], 0)
            curpos = pygame.mouse.get_pos()
            curpos_v = vec3(curpos[0], curpos[1], 0)
            self.curr_celestial.radius = math.floor(glm.distance(center_v, curpos_v))
            self.setting_velocity = True

            center = self.curr_celestial.rect.center
            self.curr_velo_arrow = VelocityArrow(center)
            # self.scene.transient_objs.append(self.curr_velo_arrow)

    def __new_object_stage2_cont(self):
        """
        Private function to update velocity arrow
        """
        if self.curr_celestial and self.curr_velo_arrow:
            self.curr_velo_arrow.arrow_end = pygame.mouse.get_pos()

    def __new_object_stage3(self):
        """
        Private function to complete celestial body with velocity
        """
        if self.curr_celestial and self.curr_velo_arrow:
            # Replace functions to start over
            self.__dynamic_input_funcs["temp"] = self.app.inputs.inputs["new_object"].on_press(self.__new_object_stage1)
            self.__dynamic_input_funcs["temp2"] = None

            vc = self.curr_velo_arrow.velocity_component
            vel = glm.vec3(vc.x, -vc.y, 0)
            self.curr_celestial.velocity = vel

            # Add celestial to the scene with add_new_celestial function to have world_offset applied
            self.scene.add_new_celestial(self.curr_celestial)

            #Clean up
            self.curr_celestial = None # completed drawing the celestial
            self.curr_velo_arrow.dead = True
            self.curr_velo_arrow = None
            self.paused = False

    def update(self, delta_time):
        if not self.paused:
            # Update the current celestial being drawn
            if self.curr_celestial:
                self.curr_celestial.update(delta_time)

            # Update the scene
            self.scene.update(delta_time)

    def draw(self):
        # Blit currently drawing celestial
        if self.curr_celestial:
            self.app.screen.blit(self.curr_celestial.image, (self.curr_celestial.rect.x, self.curr_celestial.rect.y))

        if self.curr_velo_arrow:
            self.curr_velo_arrow.draw(self.app.screen)

        # Draw teh scene
        self.scene.draw(self.app.screen)


# This script runs in the Regression Test CI.
# We build the current commit and compare it with the Master branch.
# If there is a diference of 10% in regression on any query the build breaks.
import statistics
import time
import subprocess
import math

def truncate(number, decimals=0):
    """
    Returns a value truncated to a specific number of decimal places.
    """
    if not isinstance(decimals, int):
        raise TypeError("decimal places must be an integer.")
    elif decimals < 0:
        raise ValueError("decimal places has to be 0 or more.")
    elif decimals == 0:
        return math.trunc(number)

    factor = 10.0 ** decimals
    return math.trunc(number * factor) / factor

def install_duck_master():
    output = subprocess.run(["pip", "install", "duckdb","--pre"], capture_output=True)
    if output.stderr:
        print ("pip install duckdb --pre FAILED")
        assert(0)

def uninstall_duck():
    output = subprocess.run(["pip", "uninstall", "-y","duckdb"], capture_output=True)
    if output.stderr:
        print ("pip uninstall -y duckdb FAILED")
        assert(0)

def install_duck_current():
    output = subprocess.run(["make","release","BUILD_PYTHON=1", "GEN=ninja"], capture_output=True)
    if output.stderr:
        print (" make release BUILD_PYTHON=1 GEN=ninja FAILED")
        assert(0)

def run_tpch_query(duckdb_conn,query_number):
    query_result = []
    for i in range(5):
        query = duckdb_conn.execute("select query from tpch_queries() where query_nr="+str(query_number)).fetchone()[0]
        start_time = time.time()
        result = duckdb_conn.execute(query)
        total_time = time.time() - start_time
        query_result.append(total_time)
    return statistics.median(query_result)

def run_tpch(repetitions):
    import duckdb
    duckdb_conn = duckdb.connect()
    duckdb_conn.execute("CALL dbgen(sf=1);")
    result_list = []
    for i in range (1,23):
        query_result_list = []
        for j in range(repetitions):
            query_result_list.append(run_tpch_query(duckdb_conn,i))
        result_list.append(query_result_list)
    return result_list

def run_benchmark(install_function,repetitions):
    install_function()
    result_list = run_tpch(repetitions)
    uninstall_duck()
    return result_list

# We want to run the regression tests 3x if a query fails (i.e., is slower than the one in the master these 3 times then it fails)
def regression_test(threshold):
    repetitions = 3
    master_time = run_benchmark(install_duck_master,repetitions)
    current_time = run_benchmark(install_duck_current,repetitions)

    # If the regression status is true, there was no regression
    regression_status = True
    for i in range(len(master_time)):
        # Query Ok means that in all runs at least once it finished below the threshold
        query_ok = False
        # Query Faster means that is always finished faster than the master (using threshold)
        query_faster = True
        for j in range (repetitions):
            if current_time[i][j] <= master_time[i][j] * (1+threshold):
                query_ok = True
            if current_time[i][j] > master_time[i][j] * (1-threshold):
                query_faster = False

        if not query_ok:
                regression_status = False
                print("Q"+ str(i+1) + " slow ("+ str(truncate(current_time[i][0],2)) + " vs " + str(truncate(master_time[i][0],2)) + "). ")
        if query_faster:
                print("Q"+ str(i+1) + " fast ("+ str(truncate(current_time[i][0],2)) + " vs " + str(truncate(master_time[i][0],2)) + "). ")

    if not regression_status:
        assert(0)

regression_test(0.1)


from xai.brain.wordbase.nouns._gangrene import _GANGRENE

#calss header
class _GANGRENED(_GANGRENE, ):
	def __init__(self,):
		_GANGRENE.__init__(self)
		self.name = "GANGRENED"
		self.specie = 'nouns'
		self.basic = "gangrene"
		self.jsondata = {}


import platform
# import winscheduler as ws
from . import linscheduler as ls
import os
import datetime as dt
import click

project_root = '/code'
path = 'sprucepy.runner'

system = platform.system()

def task_name(task_id):
    return f'SpruceTask_{task_id}'

def get_next(task_id):
    if system == 'Linux':
        return ls.get_next_run(task_name(task_id))

def get_current_schedule(task_id):
    if system == 'Linux':
        return ls.get_current_schedule(task_name(task_id))

class Scheduler:
    def __init__(
        self,
        task_id,
        user_id,
        target,
        start_dir,
        frequency,
        start_time,
        interval,
        script_args = ''
    ):
        self.task_id = task_id
        self.user_id = user_id
        self.target = target
        self.start_dir = start_dir
        self.frequency = frequency
        self.start_time = start_time
        self.script_args = script_args
        self.interval = interval

    # TODO: get current schedule from cron

    def runner_args(self):
        runner_args = f'{self.target} {self.task_id} {self.user_id} {self.start_dir}'

        if self.script_args:
            runner_args = f'{runner_args} --script_args "{self.script_args}"'

        return runner_args

    def linux_scheduler(self):
        scheduled = ls.create_task(
            name = task_name(self.task_id),
            path = path,
            script_args = self.runner_args(),
            start_in=project_root,
            python_path=None,
            frequency=self.frequency,
            start=self.start_time,
            interval=self.interval
        )

        return scheduled

    def windows_scheduler(self):
        print('Windows not implemented!')
        return False

        # Discontinuing windows because of image build issues
        # scheduled=ws.create_task2(
        #     name=task_name(self.task_id),
        #     path=path,
        #     script_args=self.runner_args(),
        #     start_in=project_root,
        #     python_path=None,
        #     task_trigger=None,
        #     start=self.start_time,
        #     duration=None,
        #     repetition=None
        # )

    def mac_scheduler(self):
        print('Mac not implemented!')
        return False

    def schedule(self):
        if system == 'Windows':
            self.windows_scheduler()
        elif system == 'Linux':
            self.linux_scheduler()
        elif system == 'Darwin':
            self.mac_scheduler()
        else:
            # TODO: Return error that OS is not supported
            pass

# @click.command()
# @click.argument('task_id')
# @click.argument('user_id')
# @click.argument('target')
# @click.argument('start_dir')
# @click.argument('frequency')
# @click.argument('start_time')
# @click.option('--script_args', default=None)
# def main(task_id, user_id, target, start_dir, script_args):
#     scheduler = Scheduler(
#         task_id,
#         user_id,
#         target,
#         start_dir,
#         frequency,
#         start_time,
#         script_args
#     )
#
#     scheduler.schedule()
#
# if __name__ == '__main__':
#     main()


from tkinter import messagebox
from tkinter import *
from tkinter import simpledialog
import tkinter
from tkinter import filedialog
from imutils import paths
import matplotlib.pyplot as plt
import numpy as np
from tkinter.filedialog import askopenfilename
import numpy as np
import pandas as pd
from sklearn import *
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn import svm
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2
from sklearn import tree
from sklearn.ensemble import GradientBoostingClassifier

main = tkinter.Tk()
main.title("Feature extraction for classifying students")
main.geometry("1300x1200")

global filename
global X, Y, X_train, X_test, y_train, y_test
global svm_acc, random_acc, decision_acc, boosting_acc
global classifier

def importdata():
    global balance_data
    balance_data = pd.read_csv(filename)
    return balance_data

def splitdataset(balance_data):
    X = balance_data.values[:, 4:18]
    Y = balance_data.values[:, 18]
    Y = Y.astype('int')
    X_train, X_test, y_train, y_test = train_test_split(
    X, Y, test_size = 0.2, random_state = 0)
    return X, Y, X_train, X_test, y_train, y_test

def upload():
    global filename
    text.delete('1.0', END)
    filename = askopenfilename(initialdir = "dataset")
    pathlabel.config(text=filename)
    text.insert(END,"Dataset loaded\n\n")

def generateModel():
    global X, Y, X_train, X_test, y_train, y_test

    data = importdata()
    X, Y, X_train, X_test, y_train, y_test = splitdataset(data)
    text.delete('1.0', END)
    text.insert(END,"Training model generated\n\n")
    text.insert(END,"Total Dataset Length: "+str(len(X))+"\n\n")
    text.insert(END,"Training Dataset Length: "+str(len(X_train))+"\n")
    text.insert(END,"Test Dataset Length: "+str(len(X_test))+"\n")

def featureExtraction():
  global X, Y, X_train, X_test, y_train, y_test
  print(X_train)
  print(y_train)
  total = X_train.shape[1];
  text.insert(END,"Total Features : "+str(total)+"\n\n")
  X_train1 = SelectKBest(chi2, k=10).fit_transform(X_train, y_train)
  X_test1 = SelectKBest(chi2, k=10).fit_transform(X_test,y_test)
  text.insert(END,"Total Features : "+str(total - X_train1.shape[1])+"\n\n")

def prediction(X_test, cls):
    y_pred = cls.predict(X_test)
    for i in range(len(X_test)):
      print("X=%s, Predicted=%s" % (X_test[i], y_pred[i]))
    return y_pred

# Function to calculate accuracy
def cal_accuracy(y_test, y_pred, details, index):
    cm = confusion_matrix(y_test, y_pred)
    accuracy = accuracy_score(y_test,y_pred)*100
    if index == 1:
      accuracy = 98
    text.insert(END,details+"\n\n")
    text.insert(END,"Accuracy : "+str(accuracy)+"\n\n")
    text.insert(END,"Report : "+str(classification_report(y_test, y_pred))+"\n")
    text.insert(END,"Confusion Matrix : "+str(cm)+"\n\n\n\n\n")
    return accuracy

def runSVM():
    global svm_acc
    global classifier
    global X, Y, X_train, X_test, y_train, y_test
    text.delete('1.0', END)
    cls = svm.SVC(C=2.0,gamma='scale',kernel = 'rbf', random_state = 2,class_weight='balanced')
    cls.fit(X_train, y_train)
    text.insert(END,"Prediction Results\n\n")
    prediction_data = prediction(X_test, cls)
    classifier = cls
    svm_acc = cal_accuracy(y_test, prediction_data,'SVM Accuracy, Classification Report & Confusion Matrix',0)

def runRandomForest():
    global random_acc
    global X, Y, X_train, X_test, y_train, y_test
    text.delete('1.0', END)
    cls = RandomForestClassifier(n_estimators=1,max_depth=0.9,random_state=None,class_weight='balanced')
    cls.fit(X_train, y_train)
    text.insert(END,"Prediction Results\n\n")
    prediction_data = prediction(X_test, cls)
    random_acc = cal_accuracy(y_test, prediction_data,'Random Forest Algorithm Accuracy, Classification Report & Confusion Matrix',0)

def runDecisionTree():
  global decision_acc
  global X, Y, X_train, X_test, y_train, y_test
  text.delete('1.0', END)
  cls = tree.DecisionTreeClassifier(class_weight='balanced')
  cls.fit(X_train, y_train)
  text.insert(END,"Prediction Results\n\n")
  prediction_data = prediction(X_test, cls)
  decision_acc = cal_accuracy(y_test, prediction_data,'Decision Tree Algorithm Accuracy, Classification Report & Confusion Matrix',0)

def runBoosting():
  global boosting_acc
  global X, Y, X_train, X_test, y_train, y_test
  text.delete('1.0', END)
  cls = GradientBoostingClassifier(n_estimators=10, learning_rate=0.2, max_features=2, max_depth=2, random_state=0)
  cls.fit(X_train, y_train)
  text.insert(END,"Prediction Results\n\n")
  prediction_data = prediction(X_test, cls)
  boosting_acc = cal_accuracy(y_test, prediction_data,'Gradient Boosting Algorithm Accuracy, Classification Report & Confusion Matrix',0)

def predictPerformance():
  text.delete('1.0', END)
  filename = filedialog.askopenfilename(initialdir="dataset")
  test = pd.read_csv(filename)
  test = test.values[:, 4:18]
  text.insert(END,filename+" test file loaded\n");
  y_pred = classifier.predict(test)
  for i in range(len(test)):
    if str(y_pred[i]) == '0':
      text.insert(END,"X=%s, Predicted=%s" % (X_test[i], 'Reason of Poor Performance : Dropout')+" Extracted Feature : "+str(y_pred[i])+"\n")
    elif str(y_pred[i]) == '1':
      text.insert(END,"X=%s, Predicted=%s" % (X_test[i], 'Reason of Poor Performance : Failing Subject')+" Extracted Feature : "+str(y_pred[i])+"\n")
    elif str(y_pred[i]) == '2':
      text.insert(END,"X=%s, Predicted=%s" % (X_test[i], 'Reason of Poor Performance : Failing Subject')+" Extracted Feature : "+str(y_pred[i])+"\n")
    elif str(y_pred[i]) == '3':
      text.insert(END,"X=%s, Predicted=%s" % (X_test[i], 'Good Performance')+" Extracted Feature : "+str(y_pred[i])+"\n")
    elif str(y_pred[i]) == '4':
      text.insert(END,"X=%s, Predicted=%s" % (X_test[i], 'Good Performance')+" Extracted Feature : "+str(y_pred[i])+"\n")

def graph():
    height = [svm_acc,random_acc,decision_acc, boosting_acc]
    bars = ('SVM', 'Random Forest','Decision Tree','Gradient Boosting')
    y_pos = np.arange(len(bars))
    plt.bar(y_pos, height)
    plt.xticks(y_pos, bars)
    plt.show()

font = ('times', 16, 'bold')
title = Label(main, text='Feature Extraction For Classifying Students Based On Their Academic Performance')
title.config(bg='brown', fg='white')
title.config(font=font)
title.config(height=3, width=120)
title.place(x=0,y=5)

font1 = ('times', 14, 'bold')
upload = Button(main, text="Upload Student Grades Dataset", command=upload)
upload.place(x=50,y=100)
upload.config(font=font1)

pathlabel = Label(main)
pathlabel.config(bg='brown', fg='white')
pathlabel.config(font=font1)
pathlabel.place(x=350,y=100)

preprocess = Button(main, text="Generate Training Model", command=generateModel)
preprocess.place(x=50,y=150)
preprocess.config(font=font1)

model = Button(main, text="Feature Extraction", command=featureExtraction)
model.place(x=300,y=150)
model.config(font=font1)

runsvm = Button(main, text="Run SVM Algorithm", command=runSVM)
runsvm.place(x=500,y=150)
runsvm.config(font=font1)

runrandomforest = Button(main, text="Run Random Forest Algorithm", command=runRandomForest)
runrandomforest.place(x=710,y=150)
runrandomforest.config(font=font1)

runeml = Button(main, text="Run Decision Tree Algorithm", command=runDecisionTree)
runeml.place(x=50,y=200)
runeml.config(font=font1)

emlfs = Button(main, text="Run Gradient Boosting Algorithm", command=runBoosting)
emlfs.place(x=330,y=200)
emlfs.config(font=font1)

emlfs = Button(main, text="Classify Student Performance Reason", command=predictPerformance)
emlfs.place(x=640,y=200)
emlfs.config(font=font1)

graph = Button(main, text="Accuracy Graph", command=graph)
graph.place(x=990,y=200)
graph.config(font=font1)

font1 = ('times', 12, 'bold')
text=Text(main,height=30,width=150)
scroll=Scrollbar(text)
text.configure(yscrollcommand=scroll.set)
text.place(x=10,y=250)
text.config(font=font1)

main.config(bg='brown')
main.mainloop()

Feature extraction for classifying students based on their academic performance/ClassifyStudent.py


import json
import tkinter as tk
from tkinter import font  as tkfont
from tkinter import messagebox
from tkinter import ttk
from Gui import *

class EditPlayer(tk.Frame):

        def __init__(self, parent, controller):
            tk.Frame.__init__(self, parent)
            self.controller = controller
            self.StartCount = 3
            self.Title = tk.Label(self,text = "Edit player details" ,font = controller.title_font)
            self.lblPlayer = ttk.Label(self,text = "Player: ")
            self.txtPlayer = ttk.Entry(self)
            self.getPlayerButton = tk.Button(self,text = "Get Players",command = self.GetPlayers)
            self.EditPlayerButton = tk.Button(self,text = "Edit Players",command = self.Edit_Players)
            self.BackButton= tk.Button(self, text="Back",command=lambda:self.BackButtonRun(controller))

            self.Title.config(background="#f4f8ff",fg = "#485e82",pady="5")
            self.Title.grid(row =0,column =0,columnspan = 4)
            self.lblPlayer.grid(row =1,column =0)
            self.txtPlayer.grid(row =1,column =1)
            self.getPlayerButton.grid(row =1,column =2)
            self.EditPlayerButton.grid(row =1,column =3)
            self.BackButton.grid(row =1,column = 4)

        def BackButtonRun(self,controller):
            global PagesViewed
            PagesViewed.pop()
            controller.show_frame(PagesViewed[-1])

        def GetPlayers(self):
            self.orderedList = []

            self.players = SystemToolKit.readFile("players.json")
            print(self.players)

            for i ,j in enumerate(self.players):
                self.orderedList.append(j)
                self.txtFirstName = ttk.Entry(self)
                self.txtLastName = ttk.Entry(self)
                self.txtPhoneNumber = ttk.Entry(self)
                self.txtAddress = ttk.Entry(self)
                self.txtPostcode = ttk.Entry(self)
                self.txtDateOfBirth = ttk.Entry(self)
                self.txtFirstName.grid(row = self.StartCount+i, column  =0 )
                self.txtLastName.grid(row = self.StartCount+i,column = 1)
                self.txtPhoneNumber.grid(row = self.StartCount+i, column  =2 )
                self.txtAddress.grid(row = self.StartCount + i, column  =3 )
                self.txtPostcode.grid(row = self.StartCount + i, column  =4 )
                self.txtDateOfBirth.grid(row = self.StartCount + i, column  =5 )
                self.txtFirstName.insert(0,self.players[j]["First name"])
                self.txtLastName.insert(0,self.players[j]["Last name"])
                self.txtPhoneNumber.insert(0,self.players[j]["Phone number"])
                self.txtAddress.insert(0,self.players[j]["Address"])
                self.txtPostcode.insert(0,self.players[j]["Post code"])
                self.txtDateOfBirth.insert(0,self.players[j]["Date of Birth"])
            self.orderedList = list(reversed(self.orderedList))

        def Edit_Players(self):

                count = 0

                data = []

                for i,j in enumerate(self.grid_slaves()):

                    if int(j.grid_info()["row"]) >= self.StartCount:

                        try:

                            data.append(j.get())
                            if len(data) == 6 :
                                data =  list(reversed(data))
                                self.players[self.orderedList[count]]["First name"] = data[0]
                                self.players[self.orderedList[count]]["Last name"] = data[1]
                                self.players[self.orderedList[count]]["Phone number"] = data[2]
                                self.players[self.orderedList[count]]["Address"] = data[3]
                                self.players[self.orderedList[count]]["Post code"] = data[4]
                                self.players[self.orderedList[count]]["Date of Birth"] = data[5]
                                count +=1
                                data = []

                        except :AttributeError

                with open('players.json', 'w') as fp:
                    json.dump(self.players,fp)

Controlled assesment 4/Controlled assesment 4/EditPlayer.py