import abc
import copy

from bson import json_util, ObjectId
from flask import request
from flask.views import MethodView
from mongoengine.errors import NotUniqueError

from slots_tracker_server.gsheet import write_expense, update_expense
from slots_tracker_server.models import Expense, PayMethods, Categories
from slots_tracker_server.utils import convert_to_object_id, clean_api_object, next_payment_date

class BaseAPI(MethodView):
    __metaclass__ = abc.ABCMeta

    @property
    @abc.abstractmethod
    def api_class(self):
        raise NotImplementedError

    def get(self, obj_id):
        object_id = convert_to_object_id(obj_id)
        instance = self.api_class.objects.get_or_404(id=object_id)
        return [instance.to_json()]

    def post(self, obj_data):
        # TODO: Check that all reference fields are not inactive before creating a new object
        return self.api_class(**obj_data).save()

    def delete(self, obj_id):
        instance = self.api_class.objects.get_or_404(id=obj_id)
        instance.active = False
        instance.save()
        return '', 200

    def put(self, obj_id, obj_data):
        clean_api_object(obj_data)
        object_id = convert_to_object_id(obj_id)

        # Get and updated the object
        instance = self.api_class.objects.get_or_404(id=object_id)
        instance.update(**obj_data)
        return instance.reload()

    @staticmethod
    def get_obj_data():
        return json_util.loads(request.data)

    def objects_id_to_json(self, obj_data):
        for name, document_type in self.api_class.get_all_reference_fields():
            field_id = obj_data.get(name)
            field_data_as_json = document_type.objects.get(id=field_id).to_json()
            obj_data[name] = field_data_as_json

class BasicObjectAPI(BaseAPI):
    __metaclass__ = abc.ABCMeta

    @property
    @abc.abstractmethod
    def api_class(self):
        raise NotImplementedError

    def get(self, obj_id):
        if obj_id:
            obj_data = super(BasicObjectAPI, self).get(obj_id)
        else:
            obj_data = self.api_class.objects(active=True).order_by('-instances').to_json()

        return json_util.dumps(obj_data[0] if obj_id else obj_data)

    def post(self, obj_data=None):
        obj_data = self.get_obj_data()
        try:
            return json_util.dumps(super(BasicObjectAPI, self).post(obj_data).to_json()), 201
        except NotUniqueError:
            return 'Name value must be unique', 400

    def put(self, obj_id, obj_data=None):
        obj_data = self.get_obj_data()
        try:
            new_expense_as_json = super(BasicObjectAPI, self).put(obj_id, obj_data).to_json()
            self.objects_id_to_json(new_expense_as_json)
            return json_util.dumps(new_expense_as_json)
        except NotUniqueError:
            return 'Name value must be unique', 400

class PayMethodsAPI(BasicObjectAPI):
    api_class = PayMethods

class CategoriesAPI(BasicObjectAPI):
    api_class = Categories

class ExpenseAPI(BaseAPI):
    api_class = Expense

    def get(self, obj_id):
        if obj_id:
            obj_data = super(ExpenseAPI, self).get(obj_id)
        else:
            obj_data = self.api_class.objects(active=True).order_by('one_time', '-timestamp').to_json()

        for name, document_type in self.api_class.get_all_reference_fields():
            for item in obj_data:
                item[name] = document_type.objects.get(id=item.get(name)).to_json()

        limit = int(request.args.get('limit', len(obj_data)))
        return json_util.dumps(obj_data[0] if obj_id else obj_data[:limit])

    def post(self, obj_data=None):
        new_expenses_as_json = self.create_multi_expenses()
        return new_expenses_as_json, 201

    def put(self, obj_id, obj_data=None):
        new_expenses_as_json = self.create_multi_expenses(obj_id=obj_id)
        return new_expenses_as_json

    def reference_field_to_object_id(self, obj_data):
        for name, _ in self.api_class.get_all_reference_fields():
            field_data = obj_data.get(name)
            if field_data and not isinstance(field_data, ObjectId):
                field_data_id = field_data.get('_id') if isinstance(field_data, dict) else field_data
                obj_data[name] = convert_to_object_id(field_data_id)

    def create_multi_expenses(self, obj_id=None):
        obj_data = self.get_obj_data()
        self.reference_field_to_object_id(obj_data)

        new_expenses = []
        payments = int(request.args.get('payments', 1))
        payments_data = copy.deepcopy(obj_data)
        payments_data['amount'] = float(payments_data.get('amount')) / payments
        for i in range(payments):
            payments_data['timestamp'] = next_payment_date(obj_data['timestamp'], payment=i)
            new_expenses.append(self.create_expense(payments_data, obj_id))
        return json_util.dumps(new_expenses)

    def create_expense(self, obj_data, obj_id=None):
        if obj_id:
            new_expense = super(ExpenseAPI, self).put(obj_id, obj_data)
        else:
            new_expense = super(ExpenseAPI, self).post(obj_data)
        new_expense_as_json = new_expense.to_json()
        self.objects_id_to_json(new_expense_as_json)

        if obj_id:
            update_expense(new_expense)
        else:
            write_expense(new_expense)

        return new_expense_as_json


import sys
import os
import yaml
from unicorn import *
from unicorn.arm_const import *

class State:
  """CPU State
  """
  ARCH = ['ARM']
  ARCH_REGS = {
    'ARM': {
      'R0': UC_ARM_REG_R0, 'R1': UC_ARM_REG_R1,
      'R2': UC_ARM_REG_R2, 'R3': UC_ARM_REG_R3,
      'R4': UC_ARM_REG_R4, 'R5': UC_ARM_REG_R5,
      'R6': UC_ARM_REG_R6, 'R7': UC_ARM_REG_R7,
      'R8': UC_ARM_REG_R8, 'R9': UC_ARM_REG_R9,
      'R10': UC_ARM_REG_R10, 'R11': UC_ARM_REG_R11,
      'R12': UC_ARM_REG_R12, 'LR': UC_ARM_REG_LR,
      'PC': UC_ARM_REG_PC, 'SP': UC_ARM_REG_SP,
      'SP': UC_ARM_REG_SP
    },
  }

  def __init__(self, arch, base_addr, memsize, mu):
    # Check arch
    if not arch in self.ARCH_REGS:
      raise ValueError("%s is not a valid arch" % (arch, ) )
    self.arch = arch
    self.base_addr = base_addr
    self.memsize = memsize
    self.mu = mu
    # Init register dictionary
    self.regs = self.get_CPU_state()
    self.mem  = self.get_mem_state()

  def get_CPU_state(self):
    """Get CPU state

    Return:
      Dictionary: CPU State
    """
    regs = {}
    for reg in self.ARCH_REGS[self.arch]:
      regs[reg] =  self.mu.reg_read(self.ARCH_REGS[self.arch][reg])
    return regs

  def get_mem_state(self):
    """Set memory state

    Return:
      String: Memory State
    """
    return self.mu.mem_read(self.base_addr, self.memsize)

  def set_reg(self, reg, value):
    """Set register value

    Args:
      reg (String): Register name
      value (Integer): Register value
    """
    if not reg in self.regs:
      raise ValueError("%s register name not valid" % (reg, ))
    self.regs[reg] = value

  def get_reg(self, reg):
    """Get register value

    Args:
      reg (String): Register name
    Return:
      Integer: Register value
    """
    if not reg in self.regs:
      raise ValueError("%s register name not valid" % (reg, ))
    return self.regs[reg]

  def set_regs(self, regs):
    """Set registers value

    Args:
      reg (Dictionary): Dictionary reg: value
    """
    for reg in regs:
      if not reg in self.regs.keys():
        raise ValueError("%s register name not valid" % (reg, ))
      else:
        self.regs[reg] = regs[reg]

  def get_regs(self):
    """Get registers value

    Return:
      Dictionary: Dictionary reg: value
    """
    return self.regs

  def get_mem(self, addr=None, size=None):
    """Get memory

    Return:
      String: Memory
    """
    if not addr:
      addr = self.base_addr
    if not size:
      size = self.memsize
    return self.mem[addr:addr+size]


import os
from abc import ABCMeta, abstractmethod
from configparser import ExtendedInterpolation

from myutils.myconfigparser import myConfigParser
from myutils.logger import get_logger

class Container(object):
    """Defines a container ABC.

    Attributes:
        name: name of the container
        config: configuration
        data: all the data contained
    """
    __metaclass__ = ABCMeta
    logger = get_logger(__name__, __package__+'.log')

    def __init__(self, name, config_file=None, config=None):
        """Defines a new container.

        Parameters:
            name: container name
        """
        assert not (config_file is not None and config is not None)

        self.name = name
        self.data = {}

        if config_file:
            # Load configuration
            self.logger.debug('Loading configuration file: %s', config_file)
            try:
                assert os.path.isfile(config_file)
            except AssertionError:
                self.logger.exception('File %s does not exist', config_file)
                raise IOError
            self.load_config(config_file)
            self.logger.info('Configuration file loaded')
        elif config is not None:
            self.config = config
            self.logger.info('Configuration assigned')
        else:
            self.config = None

    @abstractmethod
    def load_data(self, key, file_name=None):
        """Load *data* from file and save it in *key*"""
        pass

    def __getitem__(self, key):
        if key not in self.data.keys():
            self.load_data(key, None)
        return self.data[key]

    def __setitem__(self, key, value):
        self[key] = value

    def load_data_from_keys(self, keys, file_names):
        """Load all the data in each file in *file_names* and store it in
        *key*

        Parameters:
            keys (iterable): list of keys for each file
            file_names (iterable): list of files to load
        """
        for k,f in zip(keys, file_names):
            self.load_data(k, f)

    def load_config(self, config_file):
        """Load a configuration file.

        Parameters:
            config_file (str): name of the configuration file
        """
        self.config = myConfigParser(interpolation=ExtendedInterpolation())
        self.config.read(config_file)


from django.shortcuts import render, redirect, reverse, get_object_or_404
from django.shortcuts import HttpResponse
from django.views.decorators.http import require_POST
from django.contrib import messages
from django.conf import settings

from services.models import Service
from .models import OrderLineItem, Order
from profiles.forms import UserProfileForm
from profiles.models import UserProfile
from .forms import OrderForm
from cart.contexts import cart_contents

import stripe
import json

@require_POST
def cache_checkout_data(request):
    try:
        pid = request.POST.get('client_secret').split('_secret')[0]
        stripe.api_key = settings.STRIPE_SECRET_KEY
        stripe.PaymentIntent.modify(pid, metadata={
            'cart': json.dumps(request.session.get('cart', {})),
            'save_info': request.POST.get('save_info'),
            'username': request.user,
        })
        return HttpResponse(status=200)
    except Exception as e:
        messages.error(request, 'Sorry, your payment cannot be \
            processed right now. Please try again later.')
        return HttpResponse(content=e, status=400)

def checkout(request):
    stripe_public_key = settings.STRIPE_PUBLIC_KEY
    stripe_secret_key = settings.STRIPE_SECRET_KEY

    if request.method == 'POST':
        order_form = OrderForm(request.POST)
        cart = request.session.get('cart', {})

        form_data = {
            'full_name': request.POST['full_name'],
            'email': request.POST['email'],
            'company_name': request.POST['company_name'],
            'phone_number': request.POST['phone_number'],
            'description': request.POST['description'],
            'country': request.POST['country'],
            'zipcode': request.POST['zipcode'],
            'city': request.POST['city'],
            'street_address1': request.POST['street_address'],
            'street_address2': request.POST['second_address'],
            'county': request.POST['region'],
        }

        order_form = OrderForm(form_data)
        if order_form.is_valid():
            order = order_form.save(commit=False)
            pid = request.POST.get('client_secret').split('_secret')[0]
            order.stripe_pid = pid
            order.original_cart = json.dumps(cart)
            order.save()
            for item_id, item_data in cart.items():
                try:
                    service = Service.objects.get(id=item_id)
                    if isinstance(item_data, int):
                        order_line_item = OrderLineItem(
                           order=order,
                           service=service,
                           quantity=item_data,
                        )
                        order_line_item.save()
                except Service.DoesNotExist:
                    messages.error(request, (
                        "One of the services in your cart wasn't found in our database."
                        "Please contact us for assistance")
                    )
                    order.delete()
                    return redirect(reverse('view_cart'))

            request.session['save_info'] = 'save-info' in request.POST
            return redirect(reverse('checkout_success', args=[order.order_number]))

        else:
            messages.error(request, 'There was an error with your form. \
                Please double check your information.')

    else:
        cart = request.session.get('cart', {})
        if not cart:
            messages.error(request, "Cart is empty")
            return redirect(reverse('services'))

    current_cart = cart_contents(request)
    total = current_cart['grand_total']
    stripe_total = round(total * 100)
    stripe.api_key = stripe_secret_key
    intent = stripe.PaymentIntent.create(
        amount=stripe_total,
        currency=settings.STRIPE_CURRENCY,
    )

    if request.user.is_authenticated:
        try:
            profile = UserProfile.objects.get(user=request.user)
            order_form = OrderForm(initial={
                'full_name': profile.user.get_full_name(),
                'email': profile.user.get_full_name(),
                'company_name': profile.default_company_name,
                'phone_number': profile.default_phone_number,
                'country': profile.default_country,
                'zipcode': profile.default_zipcode,
                'city': profile.default_city,
                'street_address': profile.default_street_address,
                'second_address': profile.default_second_address,
                'region': profile.default_region,
            })
        except UserProfile.DoesNotExist:
            order_form = OrderForm()
    else:
        order_form = OrderForm()

    if not stripe_public_key:
        messages.warning(request, 'Stripe public key is missing. \
                Did you forget the env variables?')

    template = 'checkout/checkout.html'
    context = {
            'order_form': order_form,
            'stripe_public_key': stripe_public_key,
            'client_secret': intent.client_secret,
        }
    return render(request, template, context)

def checkout_success(request, order_number):
    """
    Handle successful checkouts
    """

    save_info = request.session.get('save_info')
    order = get_object_or_404(Order, order_number=order_number)

    if request.user.is_authenticated:
        profile = UserProfile.objects.get(user=request.user)
        # Attach the user's profile to the order
        order.user_profile = profile
        order.save()

        if save_info:
            profile_data = {
                'default_company_name': order.company_name,
                'default_phone_number': order.phone_number,
                'default_country': order.country,
                'default_zipcode': order.zipcode,
                'default_city': order.city,
                'default_street_address': order.street_address,
                'default_second_address': order.second_address,
                'default.region': order.region,
            }
            user_profile_form = UserProfileForm(profile_data, instance=profile)
            if user_profile_form.is_valid():
                user_profile_form.save()

    messages.success(request, f'Order successfully processed! \
        Your order number is {order_number}. A confirmation \
        email will be sent to {order.email}.')

    if 'cart' in request.session:
        del request.session['cart']

    template = 'checkout/checkout_success.html'
    context = {
        'order': order,
    }

    return render(request, template, context)


import logging

import numpy as np
import pandas as pd
from sklearn.preprocessing import OneHotEncoder, StandardScaler

files_by_day = {
    'mon': ['data/cicids2017/Monday-WorkingHours.pcap_ISCX.csv'],
    'tue': ['data/cicids2017/Tuesday-WorkingHours.pcap_ISCX.csv'],
    'wen': ['data/cicids2017/Wednesday-workingHours.pcap_ISCX.csv'],
    'thu': ['data/cicids2017/Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv',
            'data/cicids2017/Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv'],
    'fri': ['data/cicids2017/Friday-WorkingHours-Morning.pcap_ISCX.csv',
            'data/cicids2017/Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv',
            'data/cicids2017/Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv'],
}

# Configuration for generating the split for this dataset
train_files = [files_by_day['fri'][1]]
train_do_clean = True  # Turn any infinite entries into NaN, drop all rows with NaN entries
train_drop_percentage = 0.7  # Drop percentage of rows (after clean)
train_drop_benign_only = False  # If true, only benign traffic rows will be considered for dropping

test_files = [files_by_day['fri'][2]]
test_do_clean = True
test_drop_percentage = 0.8
test_drop_benign_only = True

random_seed = 1337  # Set fixed random seed to make subsampling repeatable

catcols = [' Destination Port',  'Fwd PSH Flags', ' Bwd PSH Flags', ' Fwd URG Flags', ' Bwd URG Flags']

ratecols = [' Down/Up Ratio']

numcols = [' Flow Duration', ' Total Fwd Packets', ' Total Backward Packets',
           'Total Length of Fwd Packets', ' Total Length of Bwd Packets', ' Fwd Packet Length Max',
           ' Fwd Packet Length Min', ' Fwd Packet Length Mean', ' Fwd Packet Length Std', 'Bwd Packet Length Max',
           ' Bwd Packet Length Min', ' Bwd Packet Length Mean', ' Bwd Packet Length Std', 'Flow Bytes/s',
           ' Flow Packets/s', ' Flow IAT Mean', ' Flow IAT Std', ' Flow IAT Max', ' Flow IAT Min', 'Fwd IAT Total',
           ' Fwd IAT Mean', ' Fwd IAT Std', ' Fwd IAT Max', ' Fwd IAT Min', 'Bwd IAT Total', ' Bwd IAT Mean',
           ' Bwd IAT Std', ' Bwd IAT Max', ' Bwd IAT Min', ' Fwd Header Length', ' Bwd Header Length', 'Fwd Packets/s',
           ' Bwd Packets/s', ' Min Packet Length', ' Max Packet Length', ' Packet Length Mean', ' Packet Length Std',
           ' Packet Length Variance', 'FIN Flag Count', ' SYN Flag Count', ' RST Flag Count', ' PSH Flag Count',
           ' ACK Flag Count', ' URG Flag Count', ' CWE Flag Count', ' ECE Flag Count',
           ' Average Packet Size', ' Avg Fwd Segment Size', ' Avg Bwd Segment Size', ' Fwd Header Length.1',
           'Fwd Avg Bytes/Bulk', ' Fwd Avg Packets/Bulk', ' Fwd Avg Bulk Rate', ' Bwd Avg Bytes/Bulk',
           ' Bwd Avg Packets/Bulk', 'Bwd Avg Bulk Rate', 'Subflow Fwd Packets', ' Subflow Fwd Bytes',
           ' Subflow Bwd Packets', ' Subflow Bwd Bytes', 'Init_Win_bytes_forward', ' Init_Win_bytes_backward',
           ' act_data_pkt_fwd', ' min_seg_size_forward', 'Active Mean', ' Active Std', ' Active Max', ' Active Min',
           'Idle Mean', ' Idle Std', ' Idle Max', ' Idle Min']

def fit_encoders(dataset):
    ohe = OneHotEncoder(handle_unknown='ignore')
    ohe.fit(dataset[catcols].values)
    scaler = StandardScaler()
    scaler.fit(dataset[numcols].values)
    return ohe, scaler

def transform(dataset, ohe, scaler):
    a1 = scaler.transform(dataset[numcols])
    a2 = ohe.transform(dataset[catcols]).toarray()
    a3 = dataset[ratecols].values
    labels = (dataset[' Label'].values != 'BENIGN').astype(int)
    return np.append(np.append(a1, a2, axis=1), a3, axis=1), labels

def read_files():
    np.random.seed(random_seed)

    if len(train_files) > 1:
        _train_list = []
        for filename in train_files:
            df = pd.read_csv(filename, index_col=None)
            _train_list.append(df)
        train_dataset = pd.concat(_train_list, axis=0, ignore_index=True)
    else:
        train_dataset = pd.read_csv(train_files[0])

    if train_do_clean:
        train_dataset = train_dataset.replace([np.inf, -np.inf], np.nan)
        train_dataset = train_dataset.dropna()
    benign_count = len(train_dataset[train_dataset[' Label'] == 'BENIGN'])
    logging.debug('Train dataset ...')
    logging.debug('Sample count: '+str(len(train_dataset)))
    logging.debug('Benign count: '+str(benign_count))
    logging.debug('Benign ratio: '+str(benign_count / len(train_dataset)))

    if train_drop_percentage > 0:
        if train_drop_benign_only:
            drop_indices = np.random.choice(train_dataset[train_dataset[' Label'] == 'BENIGN'].index,
                                            int(len(train_dataset) * train_drop_percentage),
                                            replace=False)
        else:
            drop_indices = np.random.choice(train_dataset.index,
                                            int(len(train_dataset) * train_drop_percentage),
                                            replace=False)
        train_dataset = train_dataset.drop(drop_indices)

        benign_count = len(train_dataset[train_dataset[' Label'] == 'BENIGN'])
        logging.debug('After dropping ...')
        logging.debug('Sample count: '+str(len(train_dataset)))
        logging.debug('Benign count: '+str(benign_count))
        logging.debug('Benign ratio: '+str(benign_count / len(train_dataset)))

    if len(test_files) > 1:
        _test_list = []
        for filename in test_files:
            df = pd.read_csv(filename, index_col=None)
            _test_list.append(df)
        test_dataset = pd.concat(_test_list, axis=0, ignore_index=True)
    else:
        test_dataset = pd.read_csv(test_files[0])

    if test_do_clean:
        test_dataset = test_dataset.replace([np.inf, -np.inf], np.nan)
        test_dataset = test_dataset.dropna()
    benign_count = len(test_dataset[test_dataset[' Label'] == 'BENIGN'])
    logging.debug('Test dataset ...')
    logging.debug('Sample count: '+str(len(test_dataset)))
    logging.debug('Benign count: '+str(benign_count))
    logging.debug('Benign ratio: '+str(benign_count / len(test_dataset)))

    if test_drop_percentage > 0:
        if test_drop_benign_only:
            drop_indices = np.random.choice(test_dataset[test_dataset[' Label'] == 'BENIGN'].index,
                                            int(benign_count * test_drop_percentage),
                                            replace=False)
        else:
            drop_indices = np.random.choice(test_dataset.index,
                                            int(benign_count * test_drop_percentage),
                                            replace=False)
        test_dataset = test_dataset.drop(drop_indices)

        benign_count = len(test_dataset[test_dataset[' Label'] == 'BENIGN'])
        logging.debug('After dropping ...')
        logging.debug('Sample count: '+str(len(test_dataset)))
        logging.debug('Benign count: '+str(benign_count))
        logging.debug('Benign ratio: '+str(benign_count / len(test_dataset)))

    np.random.seed(None)  # Set random random seed

    return train_dataset, test_dataset

def get_coherent_split():
    train_dataset, test_dataset = read_files()
    logging.debug(train_dataset.head())
    ohe, scaler = fit_encoders(train_dataset)
    x_train, y_train = transform(train_dataset, ohe, scaler)
    x_test, y_test = transform(test_dataset, ohe, scaler)
    return x_train, y_train, x_test, y_test

def get_random_split():
    train_dataset_coherent, test_dataset_coherent = read_files()

    train_dataset_anomaly_count = len(train_dataset_coherent[train_dataset_coherent[' Label'] != 'BENIGN'])
    train_dataset_benign_count = len(train_dataset_coherent) - train_dataset_anomaly_count
    logging.debug("CICIDS2017 train set anomaly count : " + str(train_dataset_anomaly_count) +
                  ", benign count : " + str(train_dataset_benign_count))

    test_dataset_anomaly_count = len(test_dataset_coherent[test_dataset_coherent[' Label'] != 'BENIGN'])
    test_dataset_benign_count = len(test_dataset_coherent) - test_dataset_anomaly_count
    logging.debug("CICIDS2017 test set anomaly count : " + str(test_dataset_anomaly_count) +
                  ", benign count : " + str(test_dataset_benign_count))

    while True:  # Try to generate splits until valid (should not be necessary with OHE handle_unknown='ignore')
        try:
            # Combine datasets, build new index
            combined_dataset = pd.concat([train_dataset_coherent, test_dataset_coherent], ignore_index=True)
            combined_dataset_benign = combined_dataset[combined_dataset[' Label'] == 'BENIGN']
            combined_dataset_anomaly = combined_dataset[combined_dataset[' Label'] != 'BENIGN']

            # Keep sample count and anomaly/benign ratio from original split
            test_dataset_benign = combined_dataset_benign.sample(n=test_dataset_benign_count)
            train_dataset_benign = combined_dataset_benign.drop(test_dataset_benign.index)
            test_dataset_anomaly = combined_dataset_anomaly.sample(n=test_dataset_anomaly_count)
            train_dataset_anomaly = combined_dataset_anomaly.drop(test_dataset_anomaly.index)
            train_dataset = pd.concat([train_dataset_benign, train_dataset_anomaly])
            test_dataset = pd.concat([test_dataset_benign, test_dataset_anomaly])
            logging.debug("(Shape CICIDS2017 Random Split) Train : " + str(train_dataset.shape) +
                          ", Test : " + str(test_dataset.shape))
            logging.debug("(Shape CICIDS2017 Coherent Split) Train : " + str(train_dataset_coherent.shape) +
                          ", Test : " + str(test_dataset_coherent.shape))
            logging.debug(train_dataset.head())
            ohe, scaler = fit_encoders(train_dataset)
            x_train, y_train = transform(train_dataset, ohe, scaler)
            x_test, y_test = transform(test_dataset, ohe, scaler)
            return x_train, y_train, x_test, y_test
        except ValueError:
            logging.info("Discarding split, retrying...")


from django.db import models

class Pin(models.Model):
	user = models.ForeignKey(PinterestUser, null=True, blank=True)
	board = models.ForeignKey(Board, null=True, blank=True)
	# Always 18 characters, but set to 31 for room to change
	pin_id = models.CharField(max_length=31, unique=True)
	# taken from the image url
	md5 = models.CharField(max_length=32, blank=True, default='')
	description = models.TextField()
	source = models.CharField(max_length=255, blank=True, default='')
	likes = models.IntegerField(default=0, db_index=True)
	repins = models.IntegerField(default=0, db_index=True)
	comments = models.IntegerField(default=0, db_index=True)
	actions = models.IntegerField(default=0, db_index=True)
	price = models.BigIntegerField(default=0)
	currency = models.CharField(max_length=1)
	etag = models.CharField(max_length=63, blank=True)
	pinned = models.DateTimeField(null=True, blank=True)
	created = models.DateTimeField(auto_now_add=True)
	updated = models.DateTimeField(auto_now=True)


animals = ['bear', 'python', 'peacock', 'kangaroo', 'whale', 'platypus']

"""
1. The second animal is at 1 and is a python even though it's not an animal. The animal at 1 is the second animal and is a python. Oh, cause it's a snake. Caught up.

2. The third animal is at 2 and is a peacock. The animal at 2 is the third animal and is a peacock.

3. The first animal is at 0 and is a bear. The animal at 0 is the first animal and is a bear.

4. The animal at 3 is the fourth animal and is a kangaroo. The fourth animal is at 3 and is a kangaroo.

5. The fifth animal is at 4 and is a whale. The animal at 4 is the fifth animal and is a whale.

6. The animal at 2 is the third animal and is a peacock. The third animal is at 2 and is a peacock.

7. The sixth animal is at 5 and is a platypus. The animal at 5 is the sixth animal and is a platypus.

8. The animal at 4 is the fifth animal and is a whale. The fifth animal is at 4 and is a whale.
"""


import mystery_word as mw

game = mw.MysteryWord()

game.word_list = ["bird", "calf", "river", "stream", "kneecap",  "cookbook",
             "language", "sneaker", "algorithm", "integration", "brain"]

def test_easy_words():
    assert game.easy_words() == \
        ["bird", "calf", "river", "stream", "brain"]

def test_medium_words():
    assert game.medium_words() == \
        ["stream", "kneecap", "cookbook", "language", "sneaker"]

def test_hard_words():
    assert game.hard_words() == \
        ["cookbook", "language", "algorithm", "integration"]

def test_random_word():
    """This test is not very good. Testing things that are random is hard, in
    that there's not a predictable choice. The best we can do is make sure
    we have valid output."""
    word = game.random_word()
    assert word in game.word_list

def test_display_word():
    game = mw.MysteryWord()
    game.word = "integration"

    assert game.display_word() == "_ _ _ _ _ _ _ _ _ _ _"

    game.attempt_guess('z')
    assert game.display_word() == "_ _ _ _ _ _ _ _ _ _ _"

    game.attempt_guess('g')
    assert game.display_word() == "_ _ _ _ G _ _ _ _ _ _"

    game.guesses = []
    game.attempt_guess('i')
    assert game.display_word() == "I _ _ _ _ _ _ _ I _ _"

    game.attempt_guess('g')
    assert game.display_word() == "I _ _ _ G _ _ _ I _ _"

    game.guesses.remove('g')
    game.attempt_guess('n')
    assert game.display_word() == "I N _ _ _ _ _ _ I _ N"

    game.attempt_guess('z')
    assert game.display_word() == "I N _ _ _ _ _ _ I _ N"

def test_is_word_complete():
    game = mw.MysteryWord()
    game.word = "river"

    game.guesses = ["r"]
    assert not game.is_word_complete()

    game.guesses = ["r", "e"]
    assert not game.is_word_complete()

    game.guesses = ["r", "e", "z"]
    assert not game.is_word_complete()

    game.guesses = ["r", "e", "v", "i"]
    assert game.is_word_complete()

def test_winning_game():
    game = mw.MysteryWord()
    game.word = 'landuade'
    game.quick_play(letters_to_guess='rstlneaioubcdfghjkmpqvwxyz')
    assert game.check_win() == True
    assert game.num_guesses_left == 1

def test_losing_game():
    game = mw.MysteryWord()
    game.word = 'carbong'
    game.quick_play(letters_to_guess='rstlneaioubcdfghjkmpqvwxyz')
    assert game.check_win() == False
    assert game.num_guesses_left == 0

def test_invalid_guesses():
    game = mw.MysteryWord()
    game.word = 'calf'

    game.attempt_guess('x')
    assert game.num_guesses_left == 7

    game.attempt_guess('c')
    assert game.num_guesses_left == 7

    game.attempt_guess('c')
    assert game.num_guesses_left == 7

    game.attempt_guess('C')
    assert game.num_guesses_left == 7

    game.attempt_guess('.')
    assert game.num_guesses_left == 7

    game.attempt_guess('a')
    assert game.num_guesses_left == 7

    game.attempt_guess('d')
    assert game.num_guesses_left == 6

    game.attempt_guess(' ')
    assert game.num_guesses_left == 6


import matplotlib.pyplot as plt

def draw_graph(results):
    x = [result.number for result in results]
    y = [result.result.gpa for result in results]
    plt.plot(x, y, '-o')
    plt.grid(linestyle='--')
    plt.xlabel('Semester')
    plt.xlim(1, len(results))
    plt.ylabel('GPA')
    plt.title(f'GPA over past/current semesters', loc='center')
    plt.show()


# -*- coding: utf-8 -*-

import sys    # sys.setdefaultencoding is cancelled by site.py
reload(sys)    # to re-enable sys.setdefaultencoding()
sys.setdefaultencoding('utf-8')
def read_article(path_to_article):
    with open(path_to_article) as my_file:
        article = my_file.read()
    return article.decode('utf-8')

article = read_article(sys.argv[1])
#article = read_article('.\\AML\\1\\article')

from pprint import pprint
import json
def read_json(path_to_json):
    with open(path_to_json) as json_file:
        json_of_article = json.load(json_file)
    return json_of_article

json_of_article = read_json(sys.argv[2])
#json_of_article = read_json('2.out')

from collections import OrderedDict
def get_entities(json_of_article):
    entities = {}
    for j in json_of_article:
        if j != {}:
            s = (j['Type'])
            for pair in j['Boundaries']:
                entities[ pair[0]] = list([pair[1], entity_color[s] ])
    entities = OrderedDict(sorted(entities.items(), key=lambda t:t[0]))
    return entities

color1 ='#' + sys.argv[4][2:];
color2 ='#' + sys.argv[5][2:];
color3 ='#' + sys.argv[6][2:];

entity_color = {'Person':color1, 'PopulatedPlace':color2, 'Organisation':color3}

entities = get_entities(json_of_article)
#print(entities)

from yattag import Doc, indent

def make_html(article, entities, path_to_html):
    doc, tag, text = Doc().tagtext()

    with tag('html'):
        with tag('head'):
            with tag('meta'):
                doc.attr(charset='utf-8')
            with tag('title'):
                text('Marking of article')

        with tag('body'):
            with tag('table'):
                doc.attr(border='2')
                with tag('tr'):
                    with tag('th'):
                        doc.attr(style='padding: 10px;font-size:160%')
                        with tag('p'):
                            doc.attr(style='word-spacing:20px;font-style:italic;')
                            with tag('font'):
                                if(color2 != '#000000'):
                                    doc.attr(color=color2)
                                    text(' Organisation ')
                            with tag('font'):
                                if(color3 != '#000000'):
                                    doc.attr(color=color3)
                                    text(' PopulatedPlace ')
                            with tag('font'):
                                if(color1 != '#000000'):
                                    doc.attr(color=color1)
                                    text(' Person ')
            with tag('pre'):
                doc.attr(style='white-space: pre-wrap')
                with tag('p'):
                    doc.attr(style='width:3000px;font-size:160%')
                    k = 0
                    for d in entities.items():
                        #print article[d[0]: d[1][0]]
                        if (article[k: d[0]] == ' '):
                            text(' ')
                        else:
                            text(article[k: d[0]])
                        with tag('font'):
                            if(d[1][1]!='#000000'):
                                doc.attr(color=d[1][1])
                                with tag('b'):
                                    text(article[d[0]: d[1][0]])
                                    k = d[1][0]
                            else:
                                text(article[d[0]: d[1][0]])
                                k = d[1][0]
                    text(article[k:])

    result = ((doc.getvalue()))
    #print(result)
    Html_file = open(path_to_html, 'w+')
    Html_file.write(result)
    Html_file.close()

make_html(article, entities, sys.argv[3])
#make_html(article, entities, 'h.html')
#print (len(article))


# Authors: Tom Lambert (lambertt) and Yuuma Odaka-Falush (odafaluy)

from decimal import Decimal

def print_separator3():
    """Prints a visual separator for Header 3"""
    print("")
    print("- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -")
    print("")

def print_separator2():
    """Prints a visual separator for Header 2"""
    print("")
    print("--------------------------------------------------------------------------")
    print("")

def print_separator1():
    """Prints a visual separator for Header 1"""
    print("")
    print("")
    print("==========================================================================================")
    print("")
    print("")

def read_integer(msg=None, error_msg=None):
    """
    Asks the user for an integer value (int or long)

    :param msg: The message, displayed to the user.
    :param error_msg: The message, displayed to the user, in case he did not entered a valid int or long.
    :return: An int or a long from the user.
    """
    res = raw_input(msg)

    try:
        return int(res)
    except (TypeError, ValueError):
        pass

    try:
        return long(res)
    except (TypeError, ValueError):
        pass

    if error_msg is not None:
        print(error_msg)
    return read_integer(msg=msg, error_msg=error_msg)

def read_integer_interval(msg=None, error_msg=None, minimum=None, maximum=None):
    """
    Asks the user for an integer value (int or long)

    :param msg: The message, displayed to the user.
    :param error_msg: The message, displayed to the user, in case he did not entered a valid int or long.
    :param minimum: The minimum for the returned value. This border is ignored if this parameter is None.
    :param maximum: The maximum for the returned value. This border is ignored if this parameter is None.
    :return: An int or a long from the user.
    """
    res = read_integer(msg=msg, error_msg=error_msg)
    if minimum is not None and res < minimum:
        print("The value cannot be less then {0}!".format(minimum))
        return read_integer_interval(msg=msg, error_msg=error_msg, minimum=minimum, maximum=maximum)
    if maximum is not None and res > maximum:
        print("The value cannot be greater then {0}!".format(maximum))
        return read_integer_interval(msg=msg, error_msg=error_msg, minimum=minimum, maximum=maximum)
    return res

def read_yesno(msg="[Y/n]", error_msg="Unrecognized input!", default_input=None):
    """
    Asks the user to input yes or no.

    :param msg: The message, displayed to the user.
    :param error_msg: The message, displayed to the user, in case he did not entered "y", "yes", "n" or "no".
    :param default_input: The value to return, in case the user entered "". Pass None to ask the user again.
    :return: True if he entered "y" or "yes"; otherwise False.
    """
    res = raw_input(msg)
    if res.lower() in ["y", "yes"]:
        return True
    if res.lower() in ["n", "no"]:
        return False
    if default_input is not None and res == "":
        return default_input

    if error_msg is not None:
        print(error_msg)
    return read_yesno(msg=msg, error_msg=error_msg, default_input=default_input)

def read_integer_list_in_range(msg=None, minimum=None, maximum=None, default_if_empty=None):
    """
    Asks the user to input a list of integers.

    :param msg: The message, displayed to the user.
    :param minimum: The smallest number the list can contain.
    :param maximum: The greatest number the list can contain.
    :param default_if_empty: The value to return if the user typed an empty string.
    :return: An array of integers, typed by the user.
    """
    inp = raw_input(msg)

    if len(inp) == 0:
        return default_if_empty

    result = []

    for s in inp.split(","):

        try:
            value = long(s)
        except (TypeError, ValueError):
            print("{0} is not an integer. Try Again.".format(s))
            return read_integer_list_in_range(msg, minimum, maximum)

        if minimum is not None and value < minimum:
            print("{0} is less than the allowed minimum of {1}. Try Again.".format(value, minimum))
            return read_integer_list_in_range(msg, minimum, maximum)

        if maximum is not None and value > maximum:
            print("{0} is greater than the allowed maximum of {1}. Try Again.".format(value, maximum))
            return read_integer_list_in_range(msg, minimum, maximum)

        result.append(value)

    return result

def read_decimal(msg, error_msg=None):
    """
    Asks the user for an decimal value

    :param msg: The message, displayed to the user.
    :param error_msg: The message, displayed to the user, in case he did not entered a valid Decimal value.
    :return: A Decimal from the user.
    """
    res = raw_input(msg)

    try:
        return Decimal(res)
    except (TypeError, ValueError):
        pass

    if error_msg is not None:
        print(error_msg)
    return read_integer(msg=msg, error_msg=error_msg)

def read_float(msg, error_msg=None):
    """
    Asks the user for an float value

    :param msg: The message, displayed to the user.
    :param error_msg: The message, displayed to the user, in case he did not entered a valid float value.
    :return: A float from the user.
    """
    res = raw_input(msg)

    try:
        return float(res)
    except (TypeError, ValueError):
        pass

    if error_msg is not None:
        print(error_msg)
    return read_integer(msg=msg, error_msg=error_msg)


def main():
    t = Triangulo(1, 3, 5)
    u = Triangulo(3, 4, 5)
    print(t.retangulo())
    print(u.retangulo())

class Triangulo:

    def __init__(self, a, b, c):
        self.a = a
        self.b = b
        self.c = c

    def retangulo(self):
        if ((self.a**2) == (self.b**2) + (self.c**2)) or ((self.b**2) == (self.a**2) + (self.c**2)) or ((self.c**2) == (self.a**2) + (self.b**2)):
            return True
        else:
            return False
main()


_API_MAP = {
    'current': {
        'api_root': 'https://www.mkmapi.eu/ws/v1.1/output.json',
        'api_sandbox_root': 'https://sandbox.mkmapi.eu/ws/v1.1/output.json',
        'api': {

            'account_management': {

                'account': {
                    'url': '/account',
                    'method': 'get',
                    'description': 'Get the Account entity of the authenticated user'
                },

                'vacation': {
                    'url': '/account/vacation/{vacation}',
                    'method': 'put',
                    'description': 'Updates the vacation status of the authenticated user; returns the Account entity'
                },

                'put_language': {
                    'url': '/account/language/{language}',
                    'method': 'put',
                    'description': 'Updates the display language of the authenticated user; returns the Account entity'
                },

                'get_messages_thread': {
                    'url': '/account/messages',
                    'method': 'get',
                    'description': 'Get the message thread overview for the authenticated user'
                },

                'get_messages_thread_user': {
                    'url': '/account/messages/{user}',
                    'method': 'get',
                    'description': 'Get a complete message thread between the authenticated user and another user'
                },

                'send_message': {
                    'url': '/account/messages/{user}',
                    'method': 'post',
                    'description': 'Write a message to another user'
                },

                'delete_thread': {
                    'url': '/account/messages/{user}',
                    'method': 'delete',
                    'description': 'Delete all messages between the authenticated and another user'
                },

                'delete_single_message': {
                    'url': '/account/messages/{user}/{message}',
                    'method': 'delete',
                    'description': 'Delete a single message between the authenticated and another user'
                }
            },

            'market_place': {

                'games': {
                    'url': '/games',
                    'method': 'get',
                    'description': 'Returns all games supported by MKM and you can sell and buy cards for'
                },

                'metaproduct': {
                    'url': '/metaproduct/{metaproduct}',
                    'method': 'get',
                    'description': 'Returns the Metaproduct entity for the metaproduct specified by its ID'
                },

                'metaproducts': {
                    'url': '/metaproducts/{name}/{game}/{language}',
                    'method': 'get',
                    'description': 'Returns a metaproduct specified by its exact name, game and language'
                },

                'product': {
                    'url': '/product/{product}',
                    'method': 'get',
                    'description': 'Returns a product specified by its ID '
                },

                'products': {
                    'url': '/products/{name}/{game}/{language}/{match}',
                    'method': 'get',
                    'description': 'Returns products specified by their name, game and language'
                },

                'products_paginated': {
                  'url': '/products/{name}/{game}/{language}/{match}/{start}',
                  'method': 'get',
                  'description': 'Returns paginated products specified by their name, game and language'
                },

                'expansion': {
                    'url': '/expansion/{game}',
                    'method': 'get',
                    'description': 'Returns Expansion entities for all expansions with single cards for a specific game'
                },

                'expansion_singles': {
                    'url': '/expansion/{game}/{name}',
                    'method': 'get',
                    'description': 'Returns Product entities for all expansions with single cards for a specific game'
                },

                'articles': {
                    'url': '/articles/{product}',
                    'method': 'get',
                    'description': 'Returns Articles entities for available articles specified by their product ID'
                },

                'articles_paginated': {
                    'url': '/articles/{product}/{start}',
                    'method': 'get',
                    'description': 'Returns paginated Article for available articles specified by their product ID'
                },

                'user': {
                    'url': '/user/{user}',
                    'method': 'get',
                    'description': 'Returns the User entity for the user specified by its ID or exact name'
                },

                'articles_user': {
                    'url': '/articles/user/{user}',
                    'method': 'get',
                    'description': 'Returns Article entities for available articles from a specified user ID or name'
                },

                'articles_user_paginated': {
                    'url': '/articles/user/{user}/{start}',
                    'method': 'get',
                    'description': 'Returns paginated Article for available articles from a specified user ID or name'
                }

            },

            'order_management': {

                'get_order': {
                    'url': '/order/{order}',
                    'method': 'get',
                    'description': 'Returns the Order entity specified by its ID'
                },

                'put_order': {
                    'url': '/order/{order}',
                    'method': 'put',
                    'description': 'Changes the state of an order specified by its ID'
                },

                'order_eval': {
                    'url': '/order/{order}/evaluation',
                    'method': 'post',
                    'description': 'Evaluates an order specified by its ID'
                },

                'orders': {
                    'url': '/orders/{actor}/{state}',
                    'method': 'get',
                    'description': 'Returns the complete collection of filtered orders.'
                },

                'orders_paginated': {
                    'url': '/orders/{actor}/{state}/{start}',
                    'method': 'get',
                    'description': 'Returns the paginated complete collection of filtered orders.'
                }
            },

            'shopping_cart': {

                'get_shopping_cart': {
                    'url': '/shoppingcart',
                    'method': 'get',
                    'description': 'Returns the shopping cart entity for the authenticated user'
                },

                'put_shopping_cart': {
                    'url': '/shoppingcart',
                    'method': 'put',
                    'description': 'Adds or removes article(s) to or from the authenticated user`s shopping cart'
                },

                'delete_shopping_cart': {
                    'url': '/shoppingcart',
                    'method': 'delete',
                    'description': 'Empties the authenticated user`s shopping cart'
                },

                'checkout': {
                    'url': '/shoppingcart/checkout',
                    'method': 'put',
                    'description': 'Checks out the authenticated user`s shopping cart and creates the respective orders'
                },

                'shipping_address': {
                    'url': '/shoppingcart/shippingaddress',
                    'method': 'put',
                    'description': 'Changes the user`s shipping address for all reservations in the shopping cart'
                },

                'get_shipping_method': {
                    'url': '/shoppingcart/shippingmethod/{method}',
                    'method': 'get',
                    'description': 'Returns all possible shipping method for the specified reservation'
                },

                'put_shipping_method': {
                    'url': '/shoppingcart/shipping/{method}',
                    'method': 'put',
                    'description': 'Changes the shipping method of a specified reservation'
                }
            },

            'stock_management': {

                'get_stock': {
                    'url': '/stock',
                    'method': 'get',
                    'description': 'Returns the Article entities in the authenticated user`s stock'
                },

                'get_stock_paginated': {
                    'url': '/stock/{start}',
                    'method': 'get',
                    'description': 'Returns the paginated Article entities in the authenticated user`s stock'
                },

                'post_stock': {
                    'url': '/stock',
                    'method': 'post',
                    'description': 'Adds new articles to the user`s stock'
                },

                'put_stock': {
                    'url': '/stock',
                    'method': 'put',
                    'description': 'Changes articles in the user`s stock'
                },

                'delete_stock': {
                    'url': '/stock',
                    'method': 'delete',
                    'description': 'Removes articles from the user`s stock'
                },

                'get_article': {
                    'url': '/stock/article/{article}',
                    'method': 'get',
                    'description': 'Returns a single article in the authenticated user`s stock'
                },

                'get_articles': {
                    'url': '/stock/articles/{name}/{game}',
                    'method': 'get',
                    'description': 'Searches for and returns articles specified by the article`s name and game'
                },

                'increase_stock': {
                    'url': '/stock/increase',
                    'method': 'put',
                    'description': 'Increase quantities for articles in authenticated user`s stock'
                },

                'decrease_stock': {
                    'url': '/stock/decrease',
                    'method': 'put',
                    'description': 'Decrease quantities for articles in authenticated user`s stock'
                }
            },

            'wants_list': {

                'get_wants_list': {
                    'url': '/wantslist',
                    'method': 'get',
                    'description': 'Returns all wants lists for the authenticated user'
                },

                'get_user_wants_list': {
                    'url': '/wantslist/{user}',
                    'method': 'get',
                    'description': 'Returns all wants lists for the specified user'
                },

                'post_wants_list': {
                    'url': '/wantslist/{wants}',
                    'method': 'post',
                    'description': 'Creates a new wants list for the authenticated user'
                },

                'put_wants_list': {
                    'url': '/wantslist/{wants}',
                    'method': 'put',
                    'description': 'Adds, edits, or removes wants to, at, or from a wants list specified by its ID'
                },

                'delete_wants_list': {
                    'url': '/wantslist/{wants}',
                    'method': 'delete',
                    'description': 'Deletes a wants list specified by its ID for the authenticated user'
                }
            }
        }
    }
}


#!/usr/bin/env python

# Dependencies
from lxml import html, etree
import requests
import pandas as pd
import datetime as dt

class rightmove_data(object):
    """The rightmove_data web scraper works by implementing an instance of the class
    on the URL returned by a search on the rightmove website. Go to rightmove.co.uk
    and search for whatever you want, then create an instance of the class on the URL
    returned by the search. The class returns an object which includes various
    methods for extracting data from the search results, the most useful being the
    .get_results() method which returns all results as a pandas DataFrame object.
    """

    def __init__(self, url):

        self.url = url

        try:
            if "searchType=SALE" in self.url:
                self.rent_or_sale = "SALE"
            elif "searchType=RENT" in self.url:
                self.rent_or_sale = "RENT"
        except ValueError:
            print("Not a valid rightmove search URL.")

        self.results_count = self.__results_count()
        self.result_pages_count = self.__result_pages_count()

    def __results_count(self):
        """Returns an integer of the total number of results returned by the search URL."""

        page = requests.get(self.url)
        tree = html.fromstring(page.content)
        xp_result_count = """//span[@class="searchHeader-resultCount"]/text()"""
        return int(tree.xpath(xp_result_count)[0].replace(",", ""))

    def __result_pages_count(self):
        """Returns the number of result pages returned by the search URL.
        There are 24 results on each results page, but note that the
        rightmove website limits results pages to a maximum of 42 pages."""

        page_count = self.results_count // 24

        if self.results_count % 24 > 0:
            page_count += 1

        # Rightmove will return a maximum of 42 results pages, hence:
        if page_count > 42: page_count = 42

        return page_count

    def __get_page_results(self,page_url):
        """This is a hidden method to scrape the data from a single page
        of search results. It is used iteratively by the .get_results()
        method to scrape data from every page returned by the search."""

        # Set the correct xpath for the price.
        if self.rent_or_sale == "RENT":
            xp_prices = """//span[@class="propertyCard-priceValue"]/text()"""
        elif self.rent_or_sale == "SALE":
            xp_prices = """//div[@class="propertyCard-priceValue"]/text()"""

        # Set the xpaths for listing title, property address,
        # listing URL, and agent URL.
        xp_titles = """//div[@class="propertyCard-details"]\
        //a[@class="propertyCard-link"]\
        //h2[@class="propertyCard-title"]/text()"""
        xp_addresses = """//address[@class="propertyCard-address"]//span/text()"""
        xp_weblinks = """//div[@class="propertyCard-details"]\
        //a[@class="propertyCard-link"]/@href"""
        xp_agent_urls = """//div[@class="propertyCard-contactsItem"]\
        //div[@class="propertyCard-branchLogo"]\
        //a[@class="propertyCard-branchLogo-link"]/@href"""

        # Use the requests library to get the whole web page.
        page = requests.get(page_url)

        # Process the html.
        tree = html.fromstring(page.content)

        # Create data lists from Xpaths.
        price_pcm = tree.xpath(xp_prices)
        titles = tree.xpath(xp_titles)
        addresses = tree.xpath(xp_addresses)
        urlbase = "http://www.rightmove.co.uk"
        weblinks = ["{}{}".format(urlbase, tree.xpath(xp_weblinks)[val]) \
                    for val in range(len(tree.xpath(xp_weblinks)))]
        agent_urls = ["{}{}".format(urlbase, tree.xpath(xp_agent_urls)[val]) \
                      for val in range(len(tree.xpath(xp_agent_urls)))]

        # Store the data in a temporary pandas DataFrame.
        data = [price_pcm, titles, addresses, weblinks, agent_urls]
        temp_df = pd.DataFrame(data)
        temp_df = temp_df.transpose()
        temp_df.columns = ["price", "type", "address", "url", "agent_url"]

        # Drop empty rows which come from placeholders in the html.
        temp_df = temp_df[temp_df["address"].notnull()]

        return temp_df

    def get_results(self):
        """Returns a pandas DataFrame with all results returned by the search."""

        # Create DataFrame to store results.
        full_results = pd.DataFrame(columns={"price", "type", "address", "url", "agent_url"})

        # Iterate through pages of results, using the .__get_page_results method to scrape results.
        for page in range(0, self.result_pages_count+1, 1):

            # Create the URL of the specific results page.
            iteration_url = "{}{}{}".format(str(self.url), "&index=", str((page*24)))

            # Create a temporary dataframe of the page results.
            temp_df = self.__get_page_results(iteration_url)

            # Concatenate the temporary dataframe with the full dataframe.
            frames = [full_results, temp_df]
            full_results = pd.concat(frames)

        # Reset the index.
        full_results = full_results.reset_index(drop=True)

        # Convert price column to numeric type.
        full_results.price.replace(regex=True, inplace=True, to_replace=r"\D", value=r"")
        full_results.price = pd.to_numeric(full_results.price)

        # Extract postcodes to a separate column.
        full_results["postcode"] = full_results["address"].str.extract\
        (r"\b([A-Za-z][A-Za-z]?[0-9][0-9]?[A-Za-z]?)\b", expand=True)

        # Extract number of bedrooms from "type" to a separate column.
        full_results["number_bedrooms"] = full_results.type.str.extract(r"\b([\d][\d]?)\b", expand=True)
        full_results.loc[full_results["type"].str.contains("studio", case=False), "number_bedrooms"]=0

        # Clean up annoying white spaces and newlines in "type" column.
        for row in range(len(full_results)):
            type_str = full_results.loc[row, "type"]
            clean_str = type_str.strip("\n").strip()
            full_results.loc[row, "type"] = clean_str

        # Add column with datetime when the search was run (i.e. now).
        now = dt.datetime.today()
        full_results["search_date"] = now

        return full_results


from flask import Blueprint, render_template, request
from flask_login import login_required

from src.presentation.common import flash_warning, flash_error
from src.domain import Patient
from src.repositories import patient_repository, visit_repository

bp = Blueprint('home', __name__, url_prefix='/home')

@bp.route('')
@login_required
def home():
    return render_template('home.html', last_visits=visit_repository.last())

@bp.route('/search', methods=('POST',))
@login_required
def search():
    name = request.form.get('name', None)

    if not name:
        flash_error('Informe o nome do Paciente!')
        return render_template('home.html')

    patient_list = patient_repository.filter(Patient.name.like('%' + str(name) + '%')).all()

    if patient_list:
        patient_list.sort(key=lambda k: k.name)
    else:
        flash_warning('Paciente %s não encontrado!' % name)

    return render_template('home.html', patients=patient_list)


# sms.py
from twilio.rest import Client
from flask import g
from logging import getLogger
from app import get_key

log = getLogger(__name__)

#-------------------------------------------------------------------------------
def compose(body, to, callback=None):
    """Send SMS message. Returns delivery status.

    :to: destination phone number in international format
         '+17801234567'
    """
    keys = get_key('twilio')

    try:
        client = Client(keys['api']['sid'], keys['api']['auth_id'])
        msg = client.messages.create(
            body=body, to=to, from_=keys['sms']['number'], status_callback=None)
    except Exception as e:
        log.exception('Error creating twilio client or sending msg')
        raise

    log.debug(body)
    return msg.status


import sys
import os
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand, CommandError
from workshops.models import Award, Badge, Person

class Command(BaseCommand):
    help = 'Report inconsistencies in PDF certificates.'

    def add_arguments(self, parser):
        parser.add_argument(
            'path', help='Path to root directory of certificates repository',
        )

    def handle(self, *args, **options):
        '''Main entry point.'''

        path_to_root = options['path']

        badges = self.get_badges()
        for (name, badge) in badges:
            db_people = self.get_db_people(badge)
            cert_path = os.path.join(path_to_root, name)
            if not os.path.isdir(cert_path):
                print('No directory {0}'.format(name))
            else:
                file_people = self.get_file_people(cert_path)
                self.report_missing('database but not disk', name, db_people - file_people)
                self.report_missing('disk but not database', name, file_people - db_people)

    def get_badges(self):
        '''Get all available badges as list of lower-case name and badge pairs.'''

        return [(b.name.lower(), b) for b in Badge.objects.all()]

    def get_db_people(self, badge):
        '''Get set of usernames of all people with the given badge.'''

        return set(Award.objects.filter(badge=badge).values_list('person__username', flat=True))

    def get_file_people(self, path):
        '''Get names of all people with the given certificate.'''

        return set([os.path.splitext(e)[0]
                    for e in os.listdir(path)
                    if e.endswith('.pdf')])

    def report_missing(self, title, kind, items):
        '''Report missing items.'''
        if items:
            print('{0} {1}'.format(kind, title))
            for i in sorted(list(items)):
                try:
                    p = Person.objects.get(username=i)
                    print(' {0}: {1}'.format(i, p))
                except Person.DoesNotExist:
                    print(' {0}'.format(i))


from collections import OrderedDict

favourite_languages = {
    'jen': 'python',
    'arthur': 'C++',
    'sylvia': 'java',
    'jesus': 'pascal',
}
friends = ['jesus']
print("jesus's favourite language is: " +
     favourite_languages['jesus'].title())

print(favourite_languages['jesus'])

person_1 = {
    'name': 'maciek',
    'surname': 'gajewski',
    'age': '26',
    'gender': 'male'
}

#printing the dictionary
for name, language in favourite_languages.items():
    print(name.title() +
    "'s favourite language is " +
    language.title())

#printing special message to keys included in a previous list
for name in favourite_languages.keys():
    print(name.title())
    if name in friends:
        print("hello " + name)

#printing message for someone not in a dictionary
if 'someone' not in favourite_languages.keys():
    print("hey someone, print your work!")

#printing values in languages but only one time,
# set function omits doubles
print("the following languages were mentioned: ")
for language in set(favourite_languages.values()):
    print(language.title())


from TFmini.graph.arg_ops import *
from TFmini.graph.state_ops import *
from TFmini.graph.math_ops import *
from TFmini.graph.edge import *
from TFmini.graph.graph import *
from TFmini.kernels.arg_kernels import *
from TFmini.kernels.state_kernels import *
from TFmini.kernels.math_kernels import *

class Node(object):
    def __init__(self, id=0, name="None", op=None, kernel=None, device=None, graph=None):
        self._id = id
        self._name = name
        self._operation = op
        self._kernel = kernel
        self._in_edges = []
        self._out_edges = []
        self._device = device
        self._graph = graph
        return

    @property
    def id(self):
        return self._id

    def __hash__(self):
        return hash(self.id)

    def __eq__(self, other):
        return (self.name, self.id) == (other.name, other.id)

    def __ne__(self, other):
        return not (self == other)

    @property
    def name(self):
        return self._name

    @property
    def in_edges(self):
        return self._in_edges

    @property
    def out_edges(self):
        return self._out_edges

    def addOutEdge(self, e):
        if not (e in self.out_edges):
            self.out_edges.append(e)

    def addInEdge(self, e):
        if not (e in self.in_edges):
            self.in_edges.append(e)

    @property
    def operation(self):
        return self._operation

    @property
    def kernel(self):
        return self._kernel

    @property
    def device(self):
        return self._device

    @property
    def graph(self):
        return self._graph

    @operation.setter
    def operation(self, new_operation):
        self._operation = new_operation

    @staticmethod
    def Placeholder(name="None"):
        g = default_G
        x = PlaceholderOp()
        y = PlaceholderKernel()
        n = Node(id=g.nextId, name=name, op=x, kernel=y, graph=g)
        y.arg_index = n;
        g.addNode(n)
        return n

    @staticmethod
    def Variable(name="None", init_v=0):
        g = default_G
        x = VariableOp()
        y = Var0Kernel(init_v)
        n = Node(id=g.nextId, name=name, op=x, kernel=y, graph=g)
        g.addNode(n)
        return n

    @staticmethod
    def Plus(name="+"):
        g = default_G
        x = PlusOp()
        y = PlusKernel()
        n = Node(id=g.nextId, name=name, op=x, kernel=y, graph=g)
        g.addNode(n)
        return n

    @staticmethod
    def Minus(name="-"):
        g = default_G
        x = MinusOp()
        y = MinusKernel()
        n = Node(id=g.nextId, name=name, op=x, kernel=y, graph=g)
        g.addNode(n)
        return n

    @staticmethod
    def Mult(name="*"):
        g = default_G
        x = MultOp()
        y = MultKernel()
        n = Node(id=g.nextId, name=name, op=x, kernel=y, graph=g)
        g.addNode(n)
        return n

    def __repr__(self):
        return self.name + str(self.id)

    def dump(self):
        os = self.name + str(self.id)
        os = os + " i{"
        for e in self.in_edges:
            os = os + str(e) + " "
        os = os + "} o{"
        for e in self.out_edges:
            os = os + str(e) + " "
        os = os + "}"
        return os

    def __mul__(self, other):
        g = default_G
        m = Node.Mult();
        e1 = Edge(id=g.nextId, src=self, dst=m, dst_input=0)
        g.addEdge(e1)
        e2 = Edge(id=g.nextId, src=other, dst=m, dst_input=1)
        g.addEdge(e2)
        return m

    def __add__(self, other):
        g = default_G
        p = Node.Plus();
        e1 = Edge(id=g.nextId, src=self, src_output=0, dst=p, dst_input=0)
        g.addEdge(e1)
        e2 = Edge(id=g.nextId, src=other, src_output=0, dst=p, dst_input=1)
        g.addEdge(e2)
        return p

    def __sub__(self, other):
        g = default_G
        m = Node.Minus();
        e1 = Edge(id=g.nextId, src=self, dst=m, dst_input=0)
        g.addEdge(e1)
        e2 = Edge(id=g.nextId, src=other, dst=m, dst_input=1)
        g.addEdge(e2)
        return m


import torch
from transformers import BertTokenizer
from transformers.tokenization_utils import BatchEncoding

class YelpTransformer:
    def __init__(self, tokenizer: BertTokenizer):
        super().__init__()
        self.tokenizer = tokenizer

    def transform(self, review: str):
        tokenized_review = self.tokenizer.tokenize(review)
        returned_dict: BatchEncoding = self.tokenizer.encode_plus(
            review,
            add_special_tokens=True,
            max_length=self.tokenizer.model_max_length,
            pad_to_max_length=True,
            is_pretokenized=True,
            return_token_type_ids=True,
            return_attention_mask=True,
        )
        encoded_reviews, token_type_ids, attention_masks = returned_dict.values()

        # tensor に変換
        tokens_tensor = torch.LongTensor(encoded_reviews).to('cpu')
        token_type_ids_tensor = torch.LongTensor(token_type_ids).to('cpu')
        attention_tensor = torch.FloatTensor(attention_masks).to('cpu')

        return tokens_tensor, token_type_ids_tensor, attention_tensor


add_library('pdf')
from datetime import datetime

def setup():
    global recordPDF, strokeColor
    recordPDF = False
    strokeColor = color(0, 10)
    colorMode(HSB, 360, 100, 100, 100);
    smooth()
    noFill()
    background(360)

def draw():
    global recordPDF, strokeColor
    if(mousePressed):
        pushMatrix()
        translate(width/2,height/2)

        circleResolution = int(map(mouseY+100,0,height,2, 10))
        radius = mouseX-width/2 + 0.5
        angle = TWO_PI/circleResolution

        strokeWeight(2)
        stroke(strokeColor)

        beginShape()
        for i in range(circleResolution+1):
            x = 0 + cos(angle*i) * radius
            y = 0 + sin(angle*i) * radius
            vertex(x, y)
        endShape()
        popMatrix()

def keyReleased():
    global recordPDF, strokeColor
    if (key == DELETE or key == BACKSPACE):
        background(255)
    if (key=='s' or key=='S'):
        saveFrame(datetime.now().strftime("%Y%m%d%H%M%S")+".png")

    if key == "1":
        strokeColor = color(0, 10)
    elif key == "2":
        strokeColor = color(192, 100, 64, 10)
    elif key == "3":
        strokeColor = color(52, 100, 71, 10)

    # pdf export
    if (key =='r' or key =='R'):
        if (recordPDF == False):
            beginRecord(PDF, datetime.now().strftime("%Y%m%d%H%M%S")+".pdf")
            println("recording started")
            recordPDF = True
            colorMode(HSB, 360, 100, 100, 100)
            smooth()
            noFill()
            background(360)
    elif (key == 'e' or key =='E'):
        if (recordPDF):
            println("recording stopped")
            endRecord()
            recordPDF = False
            background(c60)


from flask_app.config.mysqlconnection import connectToMySQL
import re	# the regex module
# create a regular expression object that we'll use later
EMAIL_REGEX = re.compile(r'^[a-zA-Z0-9.+_-]+@[a-zA-Z0-9._-]+\.[a-zA-Z]+$')
from flask import flash

class Email:
    # using a class variable to hold my database name
    db = "email_validation"
    def __init__(self,data):
        self.id = data['id']
        self.email = data['email']
        self.created_at = data['created_at']
        self.updatd_at = data['updated_at']

    # class method to save the email object in the data base
    @classmethod
    def save(cls,data):
        query = "INSERT INTO emails (email) VALUES (%(email)s);"
        return connectToMySQL(cls.db).query_db(query,data)

    # class method to get all the emails fromt the database
    @classmethod
    def get_all(cls):
        query= "SELECT * FROM emails;"
        results = connectToMySQL(cls.db).query_db(query)
        emails = []
        for row in results:
            emails.append( cls(row) )
        return emails

    # class method to delete an email from the database based on the PK.
    @classmethod
    def destroy(cls,data):
        query = "DELETE FROM emails WHERE id = %(id)s;"
        return connectToMySQL(cls.db).query_db(query,data)

    @staticmethod
    def is_valid(email):
        is_valid = True
        query = "SELECT * FROM emails WHERE email = %(email)s;"
        results = connectToMySQL(Email.db).query_db(query,email)
        if len(results) >= 1:
            flash("Email already taken.")
            is_valid=False
        if not EMAIL_REGEX.match(email['email']):
            flash("Invalid Email!!!")
            is_valid=False
        return is_valid


#!/usr/bin/env python
# xia2merger.py
#   Copyright (C) 2011 Diamond Light Source, Graeme Winter
#
#   This code is distributed under the BSD license, a copy of which is
#   included in the root directory of this package.
#
# Wrapper to allow code written to merge reflections to be externalised...
#
# 03/MAR/16
# To resolve the naming conflict between this file and the entire xia2 module
# any xia2.* imports in this directory must instead be imported as ..*

from __future__ import absolute_import, division, print_function

from xia2.Toolkit.Merger import merger

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()

    parser.add_argument("hklin", type=str, help="input reflection file")
    parser.add_argument("-n", "--nbins", type=float, help="number of resolution bins")
    parser.add_argument("-r", "--rmerge", type=float, help="limit on Rmerge")
    parser.add_argument(
        "-c", "--completeness", type=float, help="limit on completeness [0,1]"
    )
    parser.add_argument("-i", "--isigma", type=float, help="limit on I/sigma")
    parser.add_argument("-m", "--misigma", type=float, help="limit on Mn(I/sigma)")
    args = parser.parse_args()

    if args.nbins:
        nbins = args.nbins
    else:
        nbins = 100

    m = merger(args.hklin)
    m.calculate_resolution_ranges(nbins=nbins)

    if args.completeness:
        print("COMPLETENESS %f" % m.resolution_completeness(limit=args.completeness))

    if args.rmerge:
        print("RMERGE %f" % m.resolution_rmerge(limit=args.rmerge))

    if args.isigma:
        print("ISIGMA %f" % m.resolution_unmerged_isigma(limit=args.isigma))

    if args.misigma:
        print("MISIGMA %f" % m.resolution_merged_isigma(limit=args.misigma))


# DEPENDENCIES
import keras
from keras.models import Model
from keras.preprocessing import image
from keras.layers import Lambda, Input, Conv2D, MaxPooling2D, UpSampling2D
from keras.layers.core import Layer, Dense
from keras.utils import np_utils
import keras.backend as K
import pandas as pd
import numpy as np
import random
import matplotlib.pyplot as plt
from scipy.ndimage.filters import laplace
import re
import os
from sklearn.model_selection import train_test_split
from keras.losses import binary_crossentropy
#------------------

# HELPER FUNCTIONS -----------

# DATA PROCESSING FUNCTIONS

def create_priors_posts(file):

    # Reads csv with rows as pixels and separates even/odd rows into prior/post datasets

    print('reading image CSV - %s' %(str(file)) )
    df = pd.read_csv(file, header=None)
    df['frame_num'] = pd.to_numeric(df[0].apply(lambda x: re.search(r'\d+', x).group()))
    df = df.sort_values(by='frame_num')
    df = df.drop(['frame_num', 0], axis=1)

    print('splitting images into prior/post sets')
    priors = df.iloc[::2].reset_index(drop=True)
    posts = df.iloc[1::2].reset_index(drop=True)

    if len(priors) > len(posts):
        priors = priors[:-1]

    return priors, posts

def get_train_test_indices(X, y, test_size):

    # Split prior/post dataset based on user-defined split and return frame indices so they can be re-used
    # Only run once!

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, random_state=42, shuffle=True)

    train_indices = list(X_train.index)
    test_indices = list(X_test.index)

    return train_indices, test_indices

def split_training_data(X, y, train_indices, test_indices):

    # Delete train/test index if out of bounds (happens for bidirectional data)

    train_indices = [i for i in train_indices if i <= max(list(X.index))]
    test_indices = [i for i in test_indices if i <= max(list(X.index)) ]

    # Split dataset based on consistent training/test indices

    X_train, y_train = X.iloc[train_indices], y.iloc[train_indices]
    X_test, y_test = X.iloc[test_indices], y.iloc[test_indices]

    return X_train, X_test, y_train, y_test

def create_bidirectional_data(priors, posts):

    # Shuffle data to match t-1, t+1 frames as training data to label image at time t

    bidirectional_train  = priors.copy().shift(-1).reset_index(drop=True)[:-1]
    bidirectional_train = priors.join(bidirectional_train, lsuffix='_t-1', rsuffix='_t+1', how='inner')
    bidirectional_response = posts[:-1]

    return bidirectional_train, bidirectional_response

def prep_conv_ae_training_data(video_dir, image_dimensions):

    # Create priors/posts with numpy dimensions needed for convolutional AE
    # This is a bit hacky but otherwise having difficulty getting shape to be correct

    print('Converting images to arrays...')
    image_files = []

    for i in os.listdir(video_dir):
        frame_number = re.search('frame(\d+)', i).group(1)
        frame_image = image.load_img(video_dir + '/' + i, color_mode='grayscale', target_size=image_dimensions)
        frame_array = image.img_to_array(frame_image) / 255
        image_files.append([frame_number, frame_array])

    print('Aggregating & sorting image arrays...')
    df = pd.DataFrame(image_files, columns=['frame_num', 'image'])
    df['frame_num'] = pd.to_numeric(df['frame_num'])

    # need to make sure images are in order
    df = df.sort_values(by='frame_num').reset_index(drop=True)

    print('Splitting priors and posts...')

    conv_priors = []
    for prior in df.iloc[::2]['image']:
      conv_priors.append(prior)

    conv_posts = []
    for post in df.iloc[1::2]['image']:
      conv_posts.append(post)

    return np.array(conv_priors), np.array(conv_posts)

# AUTOENCODER FUNCTIONS

def create_mlae(input_size, hidden_size1, hidden_size2, hidden_size3, code_size, output_size):

    # Standard NN autoencoder (used for forward, backward, and bi-directional)

    input_layer = Input(shape=(input_size,)) #input
    hidden_1 = Dense(hidden_size1, activation='relu')(input_layer) #encoder
    hidden_2 = Dense(hidden_size2, activation='relu')(hidden_1) #encoder
    hidden_3 = Dense(hidden_size3, activation='relu')(hidden_2) #encoder
    code = Dense(code_size, activation='relu')(hidden_3) #code
    hidden_4 = Dense(hidden_size3, activation='relu')(code) #decoder
    hidden_5 = Dense(hidden_size2, activation='relu')(hidden_4) #decoder
    hidden_6 = Dense(hidden_size1, activation='relu')(hidden_5) #decoder
    output_layer = Dense(output_size, activation='sigmoid')(hidden_6) #output

    mlae = Model(inputs=input_layer, outputs=output_layer)
    mlae.compile(optimizer='adam', loss='mse')

    return mlae

def create_conv_ae(input_shape, filters):

    # Convolutional autoencoder

    input_img = Input(input_shape)

    x = Conv2D(filters, (3, 3), activation='relu', padding='same')(input_img)
    x = MaxPooling2D((2, 2), padding='same')(x)
    x = Conv2D(filters, (3, 3), activation='relu', padding='same')(x)
    encoded = MaxPooling2D((2, 2), padding='same')(x)
    x = Conv2D(filters, (3, 3), activation='relu', padding='same')(encoded)
    x = UpSampling2D((2, 2))(x)
    x = Conv2D(filters, (3, 3), activation='relu', padding='same')(x)
    x = UpSampling2D((2, 2))(x)
    decoded = Conv2D(1, (3, 3), activation='sigmoid', padding='same')(x)

    conv_autoencoder = Model(input_img, decoded)
    conv_autoencoder.compile(optimizer='adadelta', loss='mse')
    return conv_autoencoder

# Variational autoencoder (VAE) sampling helper function

def sampling(args):
    z_mean, z_log_var = args
    batch = K.shape(z_mean)[0]
    dim = K.int_shape(z_mean)[1]
    # by default, random_normal has mean=0 and std=1.0
    epsilon = K.random_normal(shape=(batch, dim))
    return z_mean + K.exp(0.5 * z_log_var) * epsilon

def create_vae(original_dim, intermediate_dim, latent_dim):

    # Variational autoencoder

    input_shape = (original_dim, )

    # build encoder model
    inputs = Input(shape=input_shape, name='encoder_input')
    x = Dense(intermediate_dim, activation='relu')(inputs)
    z_mean = Dense(latent_dim, name='z_mean')(x)
    z_log_var = Dense(latent_dim, name='z_log_var')(x)

    # use reparameterization trick to push the sampling out as input
    # note that "output_shape" isn't necessary with the TensorFlow backend
    z = Lambda(sampling, output_shape=(latent_dim,), name='z')([z_mean, z_log_var])

    # instantiate encoder model
    encoder = Model(inputs, [z_mean, z_log_var, z], name='encoder')

    # build decoder model
    latent_inputs = Input(shape=(latent_dim,), name='z_sampling')
    x = Dense(intermediate_dim, activation='relu')(latent_inputs)
    outputs = Dense(original_dim, activation='sigmoid')(x)

    # instantiate decoder model
    decoder = Model(latent_inputs, outputs, name='decoder')

    # instantiate VAE model
    outputs = decoder(encoder(inputs)[2])
    vae = Model(inputs, outputs, name='vae_mlp')

    # VAE custom loss function (variables not found if outside this function)
    def vae_loss(inputs, outputs):
          reconstruction_loss = binary_crossentropy(inputs, outputs)
          reconstruction_loss *= original_dim
          kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
          kl_loss = K.sum(kl_loss, axis=-1)
          kl_loss *= -0.5
          vae_loss = K.mean(reconstruction_loss + kl_loss)
          return vae_loss

    models = (encoder, decoder)
    vae.compile(optimizer='adam', loss=vae_loss)
    return models, vae

# Helper function to kick off training for any model type

def train_mlae(model, X_train, Y_train, X_test, Y_test, epochs, batch_size):
    hist = model.fit(X_train, Y_train,
                        batch_size=batch_size, epochs=epochs, verbose=1,
                        validation_data=(X_test, Y_test))
    return hist

# RUN ACTUAL MODELS

#FORWARD MLAE MODEL--------------
#basically: given frame t-1, can we predict frame t
print('\n\nGATHERING DATA FOR FORWARD MLAE MODEL')

# Use subset for testing

#priors, posts = create_priors_posts('small_data.csv')

priors, posts = create_priors_posts('data.csv')

train_indices, test_indices = get_train_test_indices(priors, posts, test_size=0.3)

priors_train, priors_test, posts_train, posts_test = split_training_data(priors, posts, train_indices, test_indices)

#create and train model
print('\nINITIALIZING FORWARD MLAE MODEL')
forward_mlae = create_mlae(20000, 1024, 512, 256, 128, 20000) #params: input_size, hidden_size1, hidden_size2, hidden_size3, code_size, output_size
print('\nTRAINING FORWARD MLAE MODEL')
hist_forward_mlae = train_mlae(forward_mlae, priors_train, posts_train, priors_test, posts_test, 1, 64) #params: model, X_train, Y_train, X_test, Y_test, epochs, batch_size
print('\tDone.')
#--------------

#BACKWARD MLAE MODEL--------------
#basically: given frame t+1, can we predict frame t
print('\n\nGATHERING DATA FOR BACKWARD MLAE MODEL')

#we want to swap priors and posts for this so we can keep the rest the same as the Forward MLAE model
temp = priors #standard switcheroo
priors = posts
posts = temp
del temp

priors_train, priors_test, posts_train, posts_test = split_training_data(priors, posts, train_indices, test_indices)

#create and train model
print('\nINITIALIZING BACKWARD MLAE MODEL')
backward_mlae = create_mlae(20000, 1024, 512, 256, 128, 20000) #params: input_size, hidden_size1, hidden_size2, hidden_size3, code_size, output_size
print('\nTRAINING BACKWARD MLAE MODEL')
hist_backward_mlae = train_mlae(backward_mlae, priors_train, posts_train, priors_test, posts_test, 1, 64) #params: model, X_train, Y_train, X_test, Y_test, epochs, batch_size
print('\tDone.')
#--------------

#BIDIRECTIONAL MLAE MODEL--------------
#basically: given frames t-1, t+1, can we predict frame t
print('\n\nGATHERING DATA FOR BIDIRECTIONAL MLAE MODEL')

# Swap priors/posts back to original
temp = priors #standard switcheroo
priors = posts
posts = temp
del temp

bidirectional_train, bidirectional_response = create_bidirectional_data(priors, posts)
bi_priors_train, bi_priors_test, bi_posts_train, bi_posts_test =  split_training_data(bidirectional_train,
                                                                                bidirectional_response,
                                                                                train_indices, test_indices)
#create and train model
print('\nINITIALIZING BIDIRECTIONAL MLAE MODEL')
bidirectional_mlae = create_mlae(20000*2, 1024, 512, 256, 128, 20000) #params: input_size, hidden_size1, hidden_size2, hidden_size3, code_size, output_size
print('\nTRAINING BIDIRECTIONAL MLAE MODEL')
hist_bidirectional_mlae = train_mlae(bidirectional_mlae, bi_priors_train, bi_posts_train, bi_priors_test, bi_posts_test, 1, 64) #params: model, X_train, Y_train, X_test, Y_test, epochs, batch_size
print('\tDone.')
#--------------

#CONVOLUTIONAL AUTOENCODER--------------

print('\n\nGATHERING DATA FOR CONVOLUTIONAL AE MODEL')

# Run custom conv_ae data pipeline
conv_priors, conv_posts = prep_conv_ae_training_data(video_dir='Frames_bambi', image_dimensions=(200, 100))

conv_priors_train, conv_posts_train = conv_priors[(train_indices)], conv_posts[(train_indices)]
conv_priors_test, conv_posts_test = conv_priors[(test_indices)], conv_posts[(test_indices)]

# Free up some memory
del conv_priors
del conv_posts

#create and train model
print('\nINITIALIZING CONVOLUTIONAL AE MODEL')
conv_autoencoder = create_conv_ae(input_shape=(200, 100, 1), filters=32)
print('\nTRAINING CONVOLUTIONAL AE MODEL')
hist_convolutional_ae = train_mlae(conv_autoencoder, conv_priors_train, conv_posts_train, conv_priors_test, conv_posts_test, epochs=1, batch_size=1)
print('\tDone.')
#--------------

#VARIATIONAL AUTOENCODER--------------
print('\n\nGATHERING DATA FOR VARIATIONAL AE MODEL')
priors_train, priors_test, posts_train, posts_test = split_training_data(priors, posts, train_indices, test_indices)

print('\nINITIALIZING VARIATIONAL AE MODEL')
models, vae = create_vae(original_dim=20000, intermediate_dim=512, latent_dim=3)
print('\nTRAINING VARIATIONAL AE MODEL')
hist_vae = train_mlae(vae, priors_train, posts_train, priors_test, posts_test, epochs=1, batch_size=1)
print('\tDone.')

#--------------