import time
import datetime
from bson import ObjectId
from flask import request, jsonify
from eclogue.middleware import jwt_required, login_user
from eclogue.model import db
from eclogue.models.configuration import Configuration
from eclogue.lib.logger import logger
from eclogue.models.book import Book
from eclogue.models.playbook import Playbook

@jwt_required
def list_config():
    query = request.args
    page = int(query.get('page', 1))
    size = int(query.get('pageSize', 50))
    offset = (page - 1) * size
    keyword = query.get('keyword')
    is_admin = login_user.get('is_admin')
    start = query.get('start')
    end = query.get('end')
    maintainer = query.get('maintainer')
    where = {
        'status': {
            '$ne': -1
        }
    }
    if keyword:
        where['name'] = {
            '$regex': keyword
        }

    if not is_admin:
        where['maintainer'] = {
            '$in': [login_user.get('username')]
        }
    elif maintainer:
        where['maintainer'] = {
            '$in': [maintainer]
        }

    date = []
    if start:
        date.append({
            'created_at': {
                '$gte': int(time.mktime(time.strptime(start, '%Y-%m-%d')))
            }
        })

    if end:
        date.append({
            'created_at': {
                '$lte': int(time.mktime(time.strptime(end, '%Y-%m-%d')))
            }
        })

    if date:
        where['$and'] = date

    cursor = Configuration.find(where, skip=offset, limit=size)
    total = cursor.count()
    bucket = []

    def get_registry(record):
        if not record:
            return None

        book_id = record.get('book_id')
        book = Book().find_by_id(book_id) or {}

        return {
            '_id': record['_id'],
            'playbook': record.get('name'),
            'path': record.get('path'),
            'book_name': book.get('name'),
            'book_id': book_id,
        }

    for item in cursor:
        register = db.collection('playbook').find({'register': {'$in': [str(item['_id'])]}})
        item['registry'] = list(map(get_registry, register))
        bucket.append(item)

    return jsonify({
        'message': 'ok',
        'code': 0,
        'data': {
            'list': bucket,
            'total': total,
            'page': page,
            'pageSize': size
        }
    })

@jwt_required
def add_configuration():
    payload = request.get_json()
    if not payload:
        return jsonify({
            'message': 'invalid params',
            'code': 184000,
        }), 400
    name = payload.get('name')
    if not name:
        return jsonify({
            'message': 'invalid name',
            'code': 184001,
        }), 400

    existed = Configuration.find_one({'name': name})
    if existed:
        return jsonify({
            'message': 'name existed',
            'code': 184002,
        }), 400

    description = payload.get('description')
    maintainer = payload.get('maintainer') or []
    variables = payload.get('variables')
    status = payload.get('status', 0) or 0
    data = {
        'name': name,
        'description': description,
        'maintainer': maintainer,
        'variables': variables,
        'status': status,
        'add_by': login_user.get('username'),
        'created_at': int(time.time())
    }

    result = Configuration.insert_one(data)
    data['_id'] = result.inserted_id

    return jsonify({
        'message': 'ok',
        'code': 0,
    })

@jwt_required
def get_configs_by_ids():
    query = request.args
    ids = query.get('ids')
    if not ids:
        return jsonify({
            'message': 'invalid params',
            'code': 184000
        }), 400
    ids = ids.split(',')
    ids = map(lambda i: ObjectId(i), ids)
    where = {
        '_id': {
            '$in': list(ids)
        }
    }
    records = Configuration.find(where)

    return jsonify({
        'message': 'ok',
        'code': 0,
        'data': list(records)
    })

def get_register_config(playbook_id):
    record = Playbook.find_by_id(playbook_id)
    if not record:
        return jsonify({
            'message': 'record not found',
            'code': 104040,
        }), 404

    records = Configuration().find_by_ids(record.get('register'))

    return jsonify({
        'message': 'ok',
        'code': 0,
        'data': records,
    })

def get_config_info(_id):
    record = Configuration().find_by_id(_id)
    if not record:
        return jsonify({
            'message': 'record not found',
            'code': 104040,
        }), 400

    return jsonify({
        'message': 'ok',
        'code': 0,
        'data': record,
    })

@jwt_required
def update_configuration(_id):
    payload = request.get_json()
    if not payload:
        return jsonify({
            'message': 'invalid params',
            'code': 184000,
        }), 400

    name = payload.get('name')
    record = db.collection('configurations').find_one({'_id': ObjectId(_id)})
    if not record:
        return jsonify({
            'message': 'record not found',
            'code': 104040,
        }), 404

    description = payload.get('description')
    maintainer = payload.get('maintainer') or []
    variables = payload.get('variables')
    status = payload.get('status')
    data = {}
    if name:
        data['name'] = name

    if description:
        data['description'] = description

    if variables:
        data['variables'] = variables

    if maintainer:
        data['maintainer'] = maintainer

    if status is not None:
        data['status'] = int(status)

    if len(data.keys()):
        update = {
            '$set': data,
        }
        db.collection('configurations').update_one({'_id': record['_id']}, update=update)
        msg = 'update configuration, name: {}'.format(record.get('name'))
        logger.info(msg, extra={'record': record, 'changed': data})

    return jsonify({
        'message': 'ok',
        'code': 0,
    })

@jwt_required
def delete(_id):
    record = db.collection('configurations').find_one({'_id': ObjectId(_id)})
    if not record:
        return jsonify({
            'message': 'record not found',
            'code': 104040,
        }), 404

    is_register = Playbook.find_one({'register': {'$in': [str(_id)]}})
    if is_register:
        return jsonify({
            'message': 'target record is bind to playbook: %s' % is_register.get('name'),
            'code': 104038
        }), 403

    Configuration.delete_one({'_id': record['_id']})
    msg = 'update configuration, name: {}'.format(record.get('name'))
    logger.info(msg, extra={'record': record, 'force': False})

    return jsonify({
        'message': 'ok',
        'code': 0,
    })


import functools
import operator
import struct
import array
import numpy as np
import glob
import sys
from fann2 import libfann

class IdxDecodeError(ValueError):
    """Raised when an invalid idx file is parsed."""
    pass

def parse_idx(fd):
    DATA_TYPES = {0x08: 'B',  # unsigned byte
                  0x09: 'b',  # signed byte
                  0x0b: 'h',  # short (2 bytes)
                  0x0c: 'i',  # int (4 bytes)
                  0x0d: 'f',  # float (4 bytes)
                  0x0e: 'd'}  # double (8 bytes)

    header = fd.read(4)
    if len(header) != 4:
        raise IdxDecodeError('Invalid IDX file, file empty or does not contain a full header.')

    zeros, data_type, num_dimensions = struct.unpack('>HBB', header)

    if zeros != 0:
        raise IdxDecodeError('Invalid IDX file, file must start with two zero bytes. '
                             'Found 0x%02x' % zeros)

    try:
        data_type = DATA_TYPES[data_type]
    except KeyError:
        raise IdxDecodeError('Unknown data type 0x%02x in IDX file' % data_type)

    dimension_sizes = struct.unpack('>' + 'I' * num_dimensions,
                                    fd.read(4 * num_dimensions))

    data = array.array(data_type, fd.read())
    data.byteswap()  # looks like array.array reads data as little endian

    expected_items = functools.reduce(operator.mul, dimension_sizes)
    if len(data) != expected_items:
        raise IdxDecodeError('IDX file has wrong number of items. '
                             'Expected: %d. Found: %d' % (expected_items, len(data)))

    return np.array(data).reshape(dimension_sizes)

if __name__ == '__main__':
    if len(sys.argv) < 2:
        files = []
        for file in glob.glob("net/*.net"):
            files.append(file)
        files.sort(reverse=True)
        filename = files[0]
    else:
        filename = sys.argv[1]

    print('Opening net: ' + filename)
    ann = libfann.neural_net()
    ann.create_from_file(filename)

    print('Reading labels')
    with open('source/t10k-labels.idx1-ubyte', 'rb') as fd:
        labels = parse_idx(fd)
    print('Reading images')
    with open('source/t10k-images.idx3-ubyte', 'rb') as fd:
        images = parse_idx(fd)

    print('Testing')
    error_rate = 0
    images = iter(images)
    for label in labels:
        image = next(images)
        input = []
        for row in image:
            for value in row:
                input.append((float(value) - 127.0) / 127.0)
        output = int(label)
        result_raw = ann.run(input)
        result = int(result_raw.index(max(result_raw)))
        if output == result:
            pass
        else:
            # print(output)
            # print(result)
            # print(result_raw)
            error_rate = error_rate + 1
    print('Error rate = ' + ('%.2f' % (float(error_rate)/float(len(labels))*100.0)) + '%')


# coding: utf-8

# In[38]:

import numpy as np
import os

GLOVE_DIR = './data/'

embeddings_index = {}
f = open(os.path.join(GLOVE_DIR, 'glove.6B.300d.txt'), encoding='utf8')
for line in f:
    values = line.split()
    word = values[0]
    coefs = np.asarray(values[1:], dtype='float32')
    embeddings_index[word] = coefs
f.close()

# In[323]:

def getDicKey(dic):
    for key in dic:
        getKey = key
        return getKey
def getNodeAttribute(idx, nodes):
    for j in range(len(train_content['nodes'])):
        if train_content['nodes'][j][0] == idx:
            return getDicKey(train_content['nodes'][j][1])
def flatten(flat_list):
    flatten = []
    for i in flat_list:
        for j in i:
            flatten.append(j)
    return flatten
def getEmbeddingVector(w1, w2, w3, w4):
    n1 = n2 = n3 = n4 = np.zeros(shape=(1,300))
    if w1 in embeddings_index:
        n1 = embeddings_index[w1]
    if w2 in embeddings_index:
        n2 = embeddings_index[w2]
    if w3 in embeddings_index:
        n3 = embeddings_index[w3]
    if w4 in embeddings_index:
        n4 = embeddings_index[w4]
    n1 = n1.reshape(1, 300).tolist()
    n2 = n2.reshape(1, 300).tolist()
    n3 = n3.reshape(1, 300).tolist()
    n4 = n4.reshape(1, 300).tolist()
#     print(type(n1), len(n1), type(n2), len(n2), type(n3), len(n3), type(n4), len(n4))
    n = n1 + n2 + n3 + n4
    return flatten(n)
def setValList(val_list, key):
    if key == 'fact': # fact
        val_list.append(0)
    elif key == 'analogy': # analogy
        val_list.append(1)
    else:
        val_list.append(2) # equivalence
    return val_list

# In[326]:

def load_data(DATA_DIR):
    import json
    count = 0
    data_list = []
    val_list = []
    with open(DATA_DIR) as f:
        for line in f:
            train_content = json.loads(line)
            for i in range(len(train_content['edges'])):
                idx_1 = train_content['edges'][i][0]
                idx_2 = train_content['edges'][i][1]
                key = getDicKey(train_content['edges'][i][2])
                val_list = setValList(val_list, key)
                idx_1_token = train_content['tokens'][idx_1[0]].lower()
                idx_2_token = train_content['tokens'][idx_2[0]].lower()
                idx_1_attri = getNodeAttribute(idx_1, train_content['nodes'])
                idx_2_attri = getNodeAttribute(idx_2, train_content['nodes'])
    #             print(idx_1_token, idx_2_token, idx_1_attri, idx_2_attri)
                tmp = getEmbeddingVector(idx_1_token, idx_1_attri, idx_2_token, idx_2_attri)
                data_list.append(tmp)
#     print(np.vstack(data_list).size, np.hstack(val_list).size)
    return np.vstack(data_list), np.hstack(val_list)

# In[328]:

from sklearn.datasets import load_iris
from sklearn import tree
from sklearn.cross_validation import train_test_split
from sklearn import metrics

TRAIN_DIR = './data/train.json'
TEST_DIR = './data/test.json'

# load data
train_X, train_Y = load_data(TRAIN_DIR)
test_X, test_Y = load_data(TEST_DIR)
# print(edges_X)
# print('-----')
# print(edges_Y)
#  = load_data(TEST_DIR)

# split training set and validation set
# train_X, test_X, train_y, test_y = train_test_split(edges_X, edges_Y, test_size = 0.3)

# build classifier
clf = tree.DecisionTreeClassifier()
edges_clf = clf.fit(train_X, train_Y)
# predict
test_y_predicted = edges_clf.predict(test_X)
# print(test_y_predicted)
# ans
# print(test_Y)

accuracy = metrics.accuracy_score(test_Y, test_y_predicted)
print(accuracy)


num=int(input())
goat=[]
for i in range(num):
    string=input()
    alpha=string.count('a')
    if alpha%2!=0:
        string=string[:len(string)-1]
        if string.count('a')==0:
            string=string+'aa'
    goat.append(string)
highest=len(max(goat))
for x in range(num):
    print((int((highest-len(goat[x]))/2))*' '+goat[x])


import json

from future import standard_library
standard_library.install_aliases()
try:
  import oauth2 as oauth
except:
  oauth = None

import cgi
import logging
import sys
from datetime import datetime

from axes.decorators import axes_dispatch
import django.contrib.auth.views
from django.core.exceptions import SuspiciousOperation
from django.contrib.auth import login, get_backends, authenticate
from django.contrib.sessions.models import Session
from django.http import HttpResponseRedirect
from django.urls import reverse

from hadoop.fs.exceptions import WebHdfsException
from notebook.connectors.base import get_api
from useradmin.models import get_profile, UserProfile, User, Group
from useradmin.views import ensure_home_directory, require_change_password

from desktop.auth import forms as auth_forms
from desktop.auth.backend import OIDCBackend
from desktop.auth.forms import ImpersonationAuthenticationForm, OrganizationUserCreationForm, OrganizationAuthenticationForm
from desktop.conf import OAUTH, ENABLE_ORGANIZATIONS, SESSION
from desktop.lib import fsmanager
from desktop.lib.django_util import render, login_notrequired, JsonResponse
from desktop.lib.exceptions_renderable import PopupException
from desktop.log.access import access_log, access_warn, last_access_map
from desktop.views import samlgroup_check, saml_login_headers
from desktop.settings import LOAD_BALANCER_COOKIE
from django.utils.encoding import smart_str

if sys.version_info[0] > 2:
  from urllib.parse import urlencode as urllib_urlencode
  from django.utils.translation import gettext as _
else:
  from urllib import urlencode as urllib_urlencode
  from django.utils.translation import ugettext as _

LOG = logging.getLogger()

def get_current_users():
  """Return dictionary of User objects and
  a dictionary of the user's IP address and last access time"""
  current_users = {}
  for session in Session.objects.all():
    try:
      uid = session.get_decoded().get(django.contrib.auth.SESSION_KEY)
    except SuspiciousOperation:
      # If secret_key changed, this resolution won't work.
      uid = None

    if uid is not None:
      try:
        userobj = User.objects.get(pk=uid)
        current_users[userobj] = last_access_map.get(userobj.username, {})
      except User.DoesNotExist:
        LOG.debug("User with id=%d does not exist" % uid)

  return current_users

def first_login_ever():
  backends = get_backends()
  for backend in backends:
    if hasattr(backend, 'is_first_login_ever') and backend.is_first_login_ever():
      return True
  return False

# We want unique method name to represent HUE-3 vs HUE-4 method call.
# This is required because of reverse('desktop.auth.views.dt_login') below which needs uniqueness to work correctly.
@login_notrequired
def dt_login_old(request, from_modal=False):
  return dt_login(request, from_modal)

@login_notrequired
@axes_dispatch
def dt_login(request, from_modal=False):
  if request.method == 'GET':
    redirect_to = request.GET.get('next', '/')
  else:
    redirect_to = request.POST.get('next', '/')
  is_first_login_ever = first_login_ever()
  backend_names = auth_forms.get_backend_names()
  is_active_directory = auth_forms.is_active_directory()
  is_ldap_option_selected = 'server' not in request.POST or request.POST.get('server') == 'LDAP' or \
      request.POST.get('server') in auth_forms.get_ldap_server_keys()

  if is_active_directory and is_ldap_option_selected:
    UserCreationForm = auth_forms.LdapUserCreationForm
    AuthenticationForm = auth_forms.LdapAuthenticationForm
  else:
    UserCreationForm = auth_forms.UserCreationForm
    if 'ImpersonationBackend' in backend_names:
      AuthenticationForm = ImpersonationAuthenticationForm
    else:
      AuthenticationForm = auth_forms.AuthenticationForm
    if ENABLE_ORGANIZATIONS.get():
      UserCreationForm = OrganizationUserCreationForm
      AuthenticationForm = OrganizationAuthenticationForm

  if request.method == 'POST':
    request.audit = {
      'operation': 'USER_LOGIN',
      'username': request.POST.get('username', request.POST.get('email'))
    }

    # For first login, need to validate user info!
    first_user_form = is_first_login_ever and UserCreationForm(data=request.POST) or None
    first_user = first_user_form and first_user_form.is_valid()

    if first_user or not is_first_login_ever:
      auth_form = AuthenticationForm(request=request, data=request.POST)

      if auth_form.is_valid():
        # Must login by using the AuthenticationForm. It provides 'backend' on the User object.
        user = auth_form.get_user()
        userprofile = get_profile(user)

        login(request, user)
        # If Test cookie exists , it should be deleted
        if request.session.test_cookie_worked():
          request.session.delete_test_cookie()
        if request.fs is None:
          request.fs = fsmanager.get_filesystem(request.fs_ref)
        try:
          ensure_home_directory(request.fs, user)
        except (IOError, WebHdfsException) as e:
          LOG.error('Could not create home directory at login for %s.' % user, exc_info=e)

        if require_change_password(userprofile):
          return HttpResponseRedirect('/hue' + reverse('useradmin:useradmin.views.edit_user', kwargs={'username': user.username}))

        userprofile.first_login = False
        userprofile.last_activity = datetime.now()
        if userprofile.creation_method == UserProfile.CreationMethod.EXTERNAL: # This is to fix a bug in Hue 4.3
          userprofile.creation_method = UserProfile.CreationMethod.EXTERNAL.name
        userprofile.update_data({'auth_backend': user.backend})
        try:
          userprofile.update_data({'X-Forwarded-For': request.META['HTTP_X_FORWARDED_FOR']})
        except KeyError as e:
          LOG.error('HTTP-X-FORWARDED-FOR header not found: %s' % smart_str(e))
        try:
          userprofile.update_data({'X-CSRF-TOKEN': request.META['CSRF_COOKIE']})
        except KeyError as e:
          LOG.error('X-CSRF-TOKEN header not found: %s' % smart_str(e))
        userprofile.save()

        msg = 'Successful login for user: %s' % user.username
        request.audit['operationText'] = msg
        access_warn(request, msg)
        if from_modal or request.GET.get('fromModal', 'false') == 'true':
          return JsonResponse({'auth': True})
        else:
          return HttpResponseRedirect(redirect_to)
      else:
        request.audit['allowed'] = False
        msg = 'Failed login for user: %s' % request.POST.get('username', request.POST.get('email'))
        request.audit['operationText'] = msg
        access_warn(request, msg)
        if from_modal or request.GET.get('fromModal', 'false') == 'true':
          return JsonResponse({'auth': False})
  else:
    first_user_form = None
    auth_form = AuthenticationForm()
    # SAML/OIDC user is already authenticated in djangosaml2.views.login
    if hasattr(request, 'fs') and (
        'KnoxSpnegoDjangoBackend' in backend_names or 'SpnegoDjangoBackend' in backend_names or 'OIDCBackend' in backend_names or
        'SAML2Backend' in backend_names
      ) and request.user.is_authenticated:
      if request.fs is None:
        request.fs = fsmanager.get_filesystem(request.fs_ref)
      try:
        ensure_home_directory(request.fs, request.user)
      except (IOError, WebHdfsException) as e:
        LOG.error('Could not create home directory for %s user %s.' % ('OIDC' if 'OIDCBackend' in backend_names else 'SAML', request.user))
    if request.user.is_authenticated and not from_modal:
      return HttpResponseRedirect(redirect_to)

  if is_active_directory and not is_ldap_option_selected and \
      request.method == 'POST' and request.user.username != request.POST.get('username'):
    # local user login failed, give the right auth_form with 'server' field
    auth_form = auth_forms.LdapAuthenticationForm()

  if not from_modal and SESSION.ENABLE_TEST_COOKIE.get():
    request.session.set_test_cookie()

  if 'SAML2Backend' in backend_names:
    request.session['samlgroup_permitted_flag'] = samlgroup_check(request)
    saml_login_headers(request)

  renderable_path = 'login.mako'
  if from_modal:
    renderable_path = 'login_modal.mako'

  response = render(renderable_path, request, {
    'action': reverse('desktop_auth_views_dt_login'),
    'form': first_user_form or auth_form,
    'next': redirect_to,
    'first_login_ever': is_first_login_ever,
    'login_errors': request.method == 'POST',
    'backend_names': backend_names,
    'active_directory': is_active_directory,
    'user': request.user
  })

  if not request.user.is_authenticated:
    response.delete_cookie(LOAD_BALANCER_COOKIE) # Note: might be re-balanced to another Hue on login.

  return response

def dt_logout(request, next_page=None):
  """Log out the user"""
  username = request.user.get_username()
  request.audit = {
    'username': username,
    'operation': 'USER_LOGOUT',
    'operationText': 'Logged out user: %s' % username
  }

  # Close Impala session on logout
  session_app = "impala"
  if request.user.has_hue_permission(action='access', app=session_app):
    session = {"type": session_app, "sourceMethod": " dt_logout"}
    try:
      get_api(request, session).close_session(session)
    except PopupException as e:
      LOG.warning("Error closing %s session: %s" % (session_app, e.message.encode('utf-8')))
    except Exception as e:
      LOG.warning("Error closing %s session: %s" % (session_app, e))

  backends = get_backends()
  if backends:
    for backend in backends:
      if hasattr(backend, 'logout'):
        try:
          response = backend.logout(request, next_page)
          if response:
            return response
        except Exception as e:
          LOG.warning('Potential error on logout for user: %s with exception: %s' % (username, e))

  if len([backend for backend in backends if hasattr(backend, 'logout')]) == len(backends):
    LOG.warning("Failed to log out from all backends for user: %s" % (username))

  response = django.contrib.auth.views.LogoutView.as_view(next_page=next_page)(request)
  response.delete_cookie(LOAD_BALANCER_COOKIE)
  return response

def profile(request):
  """
  Dumps JSON for user-profile information.
  """
  return render(None, request, _profile_dict(request.user))

def _profile_dict(user):
  return dict(
    username=user.username,
    first_name=user.first_name,
    last_name=user.last_name,
    last_login=str(user.last_login), # datetime object needs to be converted
    email=user.email
  )

# OAuth is based on Twitter as example.

@login_notrequired
def oauth_login(request):
  assert oauth is not None

  consumer = oauth.Consumer(OAUTH.CONSUMER_KEY.get(), OAUTH.CONSUMER_SECRET.get())
  client = oauth.Client(consumer)
  resp, content = client.request(
      OAUTH.REQUEST_TOKEN_URL.get(), "POST", body=urllib_urlencode({
      'oauth_callback': 'http://' + request.get_host() + '/login/oauth_authenticated/'
  }))

  if resp['status'] != '200':
    raise Exception(_("Invalid response from OAuth provider: %s") % resp)

  request.session['request_token'] = dict(cgi.parse_qsl(content))

  url = "%s?oauth_token=%s" % (OAUTH.AUTHENTICATE_URL.get(), request.session['request_token']['oauth_token'])

  return HttpResponseRedirect(url)

@login_notrequired
def oauth_authenticated(request):
  consumer = oauth.Consumer(OAUTH.CONSUMER_KEY.get(), OAUTH.CONSUMER_SECRET.get())
  token = oauth.Token(request.session['request_token']['oauth_token'], request.session['request_token']['oauth_token_secret'])
  client = oauth.Client(consumer, token)

  resp, content = client.request(OAUTH.ACCESS_TOKEN_URL.get(), "GET")
  if resp['status'] != '200':
    raise Exception(_("Invalid response from OAuth provider: %s") % resp)

  access_token = dict(cgi.parse_qsl(content))

  user = authenticate(access_token=access_token)
  login(request, user)

  redirect_to = request.GET.get('next', '/')
  return HttpResponseRedirect(redirect_to)

@login_notrequired
def oidc_failed(request):
  if request.user.is_authenticated:
    return HttpResponseRedirect('/')
  access_warn(request, "401 Unauthorized by oidc")
  return render("oidc_failed.mako", request, dict(uri=request.build_absolute_uri()), status=401)


class Solution(object):
    def removeDuplicates(self, nums):
        """
        :type nums: List[int]
        :rtype: int
        """
        if len(nums) <= 1:
            return len(nums)

        j = 0
        for i in range(len(nums)):
            if i == j:  # i和j位于同一位置时，i+=1
                continue
            if nums[j] != nums[i]: # nums[j] != nums[i]时，j += 1 然后nums[j] = nums[i]
                j += 1
                nums[j] = nums[i]
        return j + 1

if __name__ == '__main__':
    nums = [1,1,2,3,4,5]
    result = Solution().removeDuplicates(nums)
    print(result)


# -*- coding: utf-8 -*-
"""
Created on Mon Dec  7 12:16:20 2015

@author: mittelberger
"""

import os
import numpy as np
from scipy.interpolate import interp1d
from ScanMap.find_positions import Positionfinder

class Positioncollector(object):

    positionsfilepath = '/home/mittelberger2/git/ScanMap/positioncollection.npz'

    def __init__(self, **kwargs):
        self.paths = kwargs.get('paths')
        if os.path.isfile(Positioncollector.positionsfilepath):
            self.positionsfile = np.load(Positioncollector.positionsfilepath)
        else:
            self.positionsfile = None

        if self.positionsfile is None:
            # the structure is always: (y,x) on the first axis, datasets on the second axis (line or column, depending
            # on if it is x- or y-data) and the actual deviations from the ideal positions on the third axis.
            self.firstlines = np.empty((2, 0, 101))
            self.evenlines = np.empty((2, 0, 101))
            self.oddlines = np.empty((2, 0, 101))
            # mapnames contain name and pixelsize of the maps
            self.mapnames = (np.empty(0)).astype(np.dtype([('mapname', 'U256'), ('pixelsize', 'f32'),
                                                           ('size_overview', 'f32'), ('size_frames', 'f32'),
                                                           ('number_frames', '2f32')]))
        else:
            self.firstlines = self.positionsfile['firstlines']
            self.evenlines = self.positionsfile['evenlines']
            self.oddlines = self.positionsfile['oddlines']
            self.mapnames = self.positionsfile['mapnames']

    def get_data(self, folder):
        Finder = Positionfinder(framepath=folder)
        Finder.load_data()
        bad_data = False

        if Finder.optimized_positions is None or Finder.positions is None:
            print('No position data found in ' + folder + '.')
            bad_data = True
        if not Finder.number_frames:
            print('Number of frames not found in ' + folder + '.')
            bad_data = True
        if not Finder.size_overview or not Finder.size_frames:
            print('Size of the overview image and/or the single frames not found in ' + folder + '.')
            bad_data = True
        if Finder.overview is None:
            print('No overview image was found in ' + folder + '.')
            bad_data = True

        if not bad_data:
            return (Finder.optimized_positions.copy(), Finder.positions.copy(),
                    Finder.number_frames, Finder.size_overview, Finder.overview.shape, Finder.size_frames)

    def append_data(self, folder):
        data = self.get_data(folder)
        if data is None:
            print('No data received.')
            return
        else:
            optimized_positions, positions, number_frames, size_overview, shape_overview, size_frames = data
            pixelsize = size_overview/np.amax(shape_overview)

        if folder in self.mapnames['mapname']:
            print('\nReplacing ' + folder + ' because it is already in the saved data.')
            i = np.where(self.mapnames['mapname'] == folder)[0].item()
        else:
            print('\nAppending ' + folder + '.')
            shape = self.evenlines.shape
            i = shape[1]

            resized_evenlines = np.empty((shape[0], i+1, shape[2]), dtype=self.evenlines.dtype)
            resized_evenlines[:, :i, :] = self.evenlines
            self.evenlines = resized_evenlines

            resized_oddlines = np.empty((shape[0], i+1, shape[2]), dtype=self.oddlines.dtype)
            resized_oddlines[:, :i, :] = self.oddlines
            self.oddlines = resized_oddlines

            resized_firstlines = np.empty((shape[0], i+1, shape[2]), dtype=self.firstlines.dtype)
            resized_firstlines[:, :i, :] = self.firstlines
            self.firstlines = resized_firstlines

            resized_mapnames = np.empty(i+1, dtype=self.mapnames.dtype)
            resized_mapnames[:i] = self.mapnames
            self.mapnames = resized_mapnames

#        for i in range(self.evenlines.shape[1]):
#            if np.isnan(self.evenlines[0, i]).all():
        self.mapnames['mapname'][i] = folder
        self.mapnames['pixelsize'][i] = pixelsize
        self.mapnames['number_frames'][i] = np.array(number_frames)
        self.mapnames['size_frames'][i] = size_frames
        self.mapnames['size_overview'][i] = size_overview
        differences = positions - optimized_positions
        # Even lines have odd indices
        evenlines = differences[1::2]
        yevenlines = np.median(evenlines[:,:,0], axis=1)
        xevenlines = np.median(evenlines[:,:,1], axis=0)
        inter_yevenlines = interp1d(np.mgrid[0:100:len(yevenlines)*1j], yevenlines)
        inter_xevenlines = interp1d(np.mgrid[0:100:len(xevenlines)*1j], xevenlines)
        self.evenlines[0, i] = inter_yevenlines(np.arange(101))
        self.evenlines[1, i] = inter_xevenlines(np.arange(101))
        # odd lines have even indices
        oddlines = differences[2::2]
        yoddlines = np.median(oddlines[:,:,0], axis=1)
        xoddlines = np.median(oddlines[:,:,1], axis=0)
        inter_yoddlines = interp1d(np.mgrid[0:100:len(yoddlines)*1j], yoddlines)
        inter_xoddlines = interp1d(np.mgrid[0:100:len(xoddlines)*1j], xoddlines)
        self.oddlines[0, i] = inter_yoddlines(np.arange(101))
        self.oddlines[1, i] = inter_xoddlines(np.arange(101))
        # special treatment for first line
        firstlines = differences[0]
        yfirstlines = np.median(firstlines[:,0])
        xfirstlines = firstlines[:,1]
        inter_xfirstlines = interp1d(np.mgrid[0:100:len(xfirstlines)*1j], xfirstlines)
        self.firstlines[0, i] = np.resize(yfirstlines, (101))
        self.firstlines[1, i] = inter_xfirstlines(np.arange(101))

#        break

    def save(self):
        savedict = {}
        savedict['firstlines'] = self.firstlines
        savedict['evenlines'] = self.evenlines
        savedict['oddlines'] = self.oddlines
        savedict['mapnames'] = self.mapnames

        np.savez(Positioncollector.positionsfilepath, **savedict)

    def main(self, prefix='/3tb/maps_data'):
        for path in self.paths:
            print('\nAppending data of ' + path + '...')
            if prefix:
                path = os.path.join(prefix, path)

            self.append_data(path)

        print('Done Appending.\n\nSaving data...')
        self.save()
        print('Done')

if __name__ == '__main__':
#    paths = ['map_2015_04_15_13_13', 'map_2015_04_16_00_25', 'map_2015_06_30_14_44/all', 'map_2015_08_18_17_07',
#             'map_2015_10_19_17_49', 'map_2015_10_19_20_44', 'map_2015_10_19_22_26', 'map_2015_12_10_12_30',
#             'map_2015_12_10_17_27']
    paths = ['map_2017_05_24_00_07']

    Collector = Positioncollector(paths=paths)
    Collector.main()


import math
import numpy as np
import torch

import mindspore as ms
import mindspore.nn as nn
import mindspore.ops as P
from mindspore import set_seed, Tensor
from mindspore.common import initializer as init

from model import Generator, Discriminator

set_seed(1)
np.random.seed(1)

param = {
'main.0.weight': 'model.0.weight',
'main.1.gamma': 'model.1.weight',
'main.1.beta': 'model.1.bias',
'main.3.weight': 'model.3.weight',
'main.4.gamma': 'model.4.weight',
'main.4.beta': 'model.4.bias',
'main.6.weight': 'model.6.weight',
'main.7.gamma': 'model.7.weight',
'main.7.beta': 'model.7.bias',
'main.9.main.0.weight': 'model.9.conv_block.0.weight',
'main.9.main.1.gamma': 'model.9.conv_block.1.weight',
'main.9.main.1.beta': 'model.9.conv_block.1.bias',
'main.9.main.3.weight': 'model.9.conv_block.3.weight',
'main.9.main.4.gamma': 'model.9.conv_block.4.weight',
'main.9.main.4.beta': 'model.9.conv_block.4.bias',
'main.10.main.0.weight': 'model.10.conv_block.0.weight',
'main.10.main.1.gamma': 'model.10.conv_block.1.weight',
'main.10.main.1.beta': 'model.10.conv_block.1.bias',
'main.10.main.3.weight': 'model.10.conv_block.3.weight',
'main.10.main.4.gamma': 'model.10.conv_block.4.weight',
'main.10.main.4.beta': 'model.10.conv_block.4.bias',
'main.11.main.0.weight': 'model.11.conv_block.0.weight',
'main.11.main.1.gamma': 'model.11.conv_block.1.weight',
'main.11.main.1.beta': 'model.11.conv_block.1.bias',
'main.11.main.3.weight': 'model.11.conv_block.3.weight',
'main.11.main.4.gamma': 'model.11.conv_block.4.weight',
'main.11.main.4.beta': 'model.11.conv_block.4.bias',
'main.12.main.0.weight': 'model.12.conv_block.0.weight',
'main.12.main.1.gamma': 'model.12.conv_block.1.weight',
'main.12.main.1.beta': 'model.12.conv_block.1.bias',
'main.12.main.3.weight': 'model.12.conv_block.3.weight',
'main.12.main.4.gamma': 'model.12.conv_block.4.weight',
'main.12.main.4.beta': 'model.12.conv_block.4.bias',
'main.13.main.0.weight': 'model.13.conv_block.0.weight',
'main.13.main.1.gamma': 'model.13.conv_block.1.weight',
'main.13.main.1.beta': 'model.13.conv_block.1.bias',
'main.13.main.3.weight': 'model.13.conv_block.3.weight',
'main.13.main.4.gamma': 'model.13.conv_block.4.weight',
'main.13.main.4.beta': 'model.13.conv_block.4.bias',
'main.14.main.0.weight': 'model.14.conv_block.0.weight',
'main.14.main.1.gamma': 'model.14.conv_block.1.weight',
'main.14.main.1.beta': 'model.14.conv_block.1.bias',
'main.14.main.3.weight': 'model.14.conv_block.3.weight',
'main.14.main.4.gamma': 'model.14.conv_block.4.weight',
'main.14.main.4.beta': 'model.14.conv_block.4.bias',
'main.15.weight': 'model.15.weight',
'main.16.gamma': 'model.16.weight',
'main.16.beta': 'model.16.bias',
'main.18.weight': 'model.18.weight',
'main.19.gamma': 'model.19.weight',
'main.19.beta': 'model.19.bias',
'main.21.weight': 'model.21.weight',
}
# Print the parameter names and shapes of all parameters in the PyTorch parameter file and return the parameter dictionary.
def pytorch_params(pth_file):
    par_dict = torch.load(pth_file, map_location='cpu')
    pt_params = {}
    for name in par_dict:
        parameter = par_dict[name]
        print(name, parameter.numpy().shape)
        pt_params[name] = parameter.numpy()
    return pt_params

# Print the names and shapes of all parameters in the MindSpore cell and return the parameter dictionary.
def mindspore_params(network):
    ms_params = {}
    for param in network.get_parameters():
        name = param.name
        value = param.data.asnumpy()
        print(name, value.shape)
        ms_params[name] = value
    return ms_params

def param_convert(ms_params, pt_params, ckpt_path):
    # Parameter name mapping dictionary
    new_params_list = []
    for ms_param in ms_params.keys():

            # If the corresponding parameter is found and the shape is the same, add the parameter to the parameter list.
            if param[ms_param] in pt_params and pt_params[param[ms_param]].shape == ms_params[ms_param].shape:
                ms_value = pt_params[param[ms_param]]
                new_params_list.append({"name": ms_param, "data": ms.Tensor(ms_value)})
            else:
                print(ms_param, "not match in pt_params")
    ms.save_checkpoint(new_params_list, ckpt_path)

g_conv_dim=64
c_dim=5
g_repeat_num=6
d_repeat_num=6
d_conv_dim=64
image_size=128
G = Generator(g_conv_dim, c_dim, g_repeat_num)
D = Discriminator(image_size, d_conv_dim, c_dim, d_repeat_num)

pth_path = "generator_49.pth"
pt_param = pytorch_params(pth_path)
print("="*20)
ms_param = mindspore_params(G)

ckpt_path = "torch_model.ckpt"
param_convert(ms_param, pt_param, ckpt_path)


"""
This module contains methods that analyse the recorded
habits and their trackings.
It also contains methods that have a connection to
the database.
"""
import sqlite3
from datetime import datetime
from itertools import groupby
from functools import reduce
from collections import Counter
from operator import itemgetter

from .habit import Habit

class Analytics:
    """Initialize and manipulate SQLite3 database"""

    def __init__(self):
        """
        Connect to the database
        or create it if does not exist.
        """
        try:
            # Connect to database
            self.connection = sqlite3.connect('DB_Habitsbox_app.db')
            # Create a cursor
            self.cursor = self.connection.cursor()
            # Create table habits with six columns if does not exist
            self.cursor.execute("""CREATE TABLE IF NOT EXISTS habits(
                             HabitID INTEGER PRIMARY KEY,
                             Name TEXT UNIQUE,
                             Periodicity TEXT,
                             Motivation TEXT,
                             Description TEXT,
                             Creation_date TEXT
                          )""")
            # Create table trackings with four columns if does not exist
            self.cursor.execute("""CREATE TABLE IF NOT EXISTS trackings(
                             TrackingID INTEGER PRIMARY KEY,
                             Date TEXT,
                             Time TEXT,
                             HabitID INTEGER NOT NULL,
                             FOREIGN KEY (HabitID) REFERENCES habits (HabitID)
                          )""")
        # print error if any error occurs
        except sqlite3.Error as error:
            print('Error while connectiong to SQLite', error)

    def insert_habit(self, name, periodicity, motivation, description):
        """
        Initialize the Habit class in the habit module and
        insert a habit to the DB.

        Parameters
        ----------
        name : str
            Name of the habit
        periodicity : str
            Periodicity of the habit, daily or weekly
        motivation :
            Motivation of the user to perform the habit
        description :
            Description of the habit
        """
        self.habit = Habit(name, periodicity, motivation, description)

        with self.connection:
            self.cursor.execute("""
                            INSERT INTO habits (Name,
                                                Periodicity,
                                                Motivation,
                                                Description,
                                                Creation_date)
                            VALUES (:name,
                                    :periodicity,
                                    :motivation,
                                    :description,
                                    :creation_date)""",
                            {'name': self.habit.name,
                            'periodicity':self.habit.periodicity,
                            'motivation':self.habit.motivation,
                            'description':self.habit.description,
                            'creation_date':self.habit.creation_date})

    def habits_table(self):
        """
        Return all the habits with its information contained in
        the table habits in the DB. ID and name of the habit,
        periodicity, motivation, description, and the day it was recorded.

        Returns
        -------
        list
            a list of tuples with the information of the registered habits

        Example
        -------
        [(1, 'Yoga', 'daily', 'Be more flexible', 'Before lunch', '2021-04-26'),
         (2, 'Run', 'weekly', 'Being healthier', 'At weekends', '2021-04-26')]
        """
        self.cursor.execute("SELECT * FROM habits")
        return self.cursor.fetchall()

    def trackings_table(self):
        """
        Return the date and time when the trackings were
        registered, as well as to which habit-id they correspond.

        Returns
        -------
        list
            a list of tuples with the habit id, date and time

        Example
        -------
        [(1, '2021-02-01', '19:52'), (2, '2021-02-03', '02:42')]
        """
        self.cursor.execute("""SELECT HabitID, Date, Time
                            FROM trackings""")
        return self.cursor.fetchall()

    def habits_trackings_table(self):
        """
        Join the habits table and the trackings table using the
        id of the habit.

        Returns
        -------
        list
            a list of tuples of the join of the habits and trackings table

        Example
        -------
        [(1, 'Yoga', 'daily', 'Be more flexible', 'Mornings', '2021-06-21', '09:06'),
         (3, 'Read', 'daily', 'Read 12 books a year', 'Afternoons', '2021-06-21', '15:26')]
        """
        self.cursor.execute("""SELECT h.HabitID, h.Name, h.Periodicity,
                            h.Motivation, h.Description, t.Date, t.Time
                            FROM habits h
                            JOIN trackings t
                            USING(HabitID)""")
        return self.cursor.fetchall()

    def select_column(self, table, number_column):
        """
        Select a column of the indicated table.

        Parameters
        ----------
        table : list of tuples
            Name of the table
        number_column : int
            Index number, i.e. column

        Returns
        -------
        list
            a list with the selected column
        """
        return list(map(lambda x: x[number_column], table))

    def select_columns(self, table, start=0, stop=1):
        """
        Select several columns of the indicated table.

        Parameters
        ----------
        table : list of tuples
            Name of the table
        start : int, optinal
            Number of the first index (inclusive), i.e. column
            (default is 0)
        stop : int, optional
            Number of the last index (exclusive), i.e. column
            (default is 1)

        Returns
        -------
        list
            a list of tuples with selected columns
        """
        return list(map(lambda x: x[start:stop], table))

    def get_all_ids(self, table):
        """
        First column of a table in a list.

        Parameters
        ----------
        table : list of tuples
            Name of the table

        Returns
        -------
        list
            a list of numbers representing habit ids
        """
        return list(self.select_column(table, 0))

    def unique_ids(self, table):
        """
        Unique ids of a table

        Parameters
        ----------
        table: list of tuples
            Name of the table

        Returns
        -------
        list
            a list of numbers representing unique ids
        """
        return self.unique_data(
            self.get_all_ids(
                table))

    def ids_without_trackings(self, habits_table, habits_trackings_table):
        """
        All habit ids that have no trackings.

        Parameters
        ----------
        habits_table : list of tuples
            a list with the information of the registered habits
        habits_trackings_table : list of tuples
            a list with the information of the habits and
            its trackings

        Returns
        -------
        list
            a list of numbers representing habit ids without
            trackings
        """
        return list(
                set(self.unique_ids(habits_table))-set(self.unique_ids(habits_trackings_table))
                )

    def ids_with_trackings(self, habits_table, ids_without_trackings):
        """
        All habit ids that have trackings.

        Parameters
        ----------
        habits_table : list of tuples
            a list with the information of the registered habits
        ids_without_trackings : list
            a list of habit ids without trackings

        Returns
        -------
        list
            a list of numbers representing habit ids trackings
        """
        return list(set(self.unique_ids(habits_table))-set(ids_without_trackings))

    def select_rows(self, table, number_column, feature):
        """
        Select all rows with the same feature.

        Parameters
        ----------
        table : list of tuples
            Name of the table
        number_column : int
            Index number, column where the criterion
            will be searched
        feature : int, str
            the criteria used for selection

        Returns
        -------
        list
            a list of tuples with all rows that
            satisfy the condition
        """
        return list(filter(lambda x: x[number_column]==feature, table))

    def display_elements(self, my_list, separator='\n'):
        """
        Each item in the list is separated by a separator.

        Parameters
        ----------
        my_list : list
            List of elements to be separated
        separator : str, optional
            (default is '\n')

        Returns
        -------
        str
            a string with elements separated by the separator
        """
        return separator.join(map(str, my_list))

    def unique_data(self, my_list):
        """
        Give unique values from a list of data.

        Parameters
        ----------
        my_list : list
            A list that contains duplicate elements

        Returns
        -------
        list
            a list of unique elements
        """
        return reduce(lambda x, y: x + [y] if y not in x else x, my_list, [])

    def display_unique_elements(self, table, column):
        """
        Unique elements of a column are
        separated by commas in a string.

        Parameters
        ----------
        table : list of tuples
            Name of the table
        column : int
            Index number, i.e. column

        Returns
        -------
        str
            Elements separated by comma
        """
        return self.display_elements(
                self.unique_data(
                    self.select_column(
                        table,
                        column)), ', ')

    def format_to_date(self, column):
        """
        Items are converted to datetime.date.

        Parameters
        ----------
        column : list of tuples
            a list of strings containing the dates to
            be converted

        Returns
        -------
        iterator
            dates in datetime.date
        """
        return map(lambda x: datetime.strptime(x, "%Y-%m-%d").date(), column)

    def format_to_time(self, column):
        """
        Items are converted to datetime.time.

        Parameters
        ----------
        column : list of tuples
            a list of strings containing the times to
            be converted

        Returns
        -------
        iterator
            times in datetime.time
        """
        return map(lambda x: datetime.strptime(x, "%H:%M").time(), column)

    def to_calender_week(self, dates):
        """
        Give the calendar week number for each date.

        Parameters
        ----------
        dates : list
            a list of dates

        Returns
        -------
        iterator
            calendar week numbers
        """
        return map(lambda x: x.isocalendar()[1], dates)

    def zipping_unique_data(self, data):
        """
        A list is sliced into one list without the first element
        and into another list without the last element.
        The first elements of each list are matched in a tuple,
        then the second and so on.

        Parameters
        ----------
        data : list
            a list of dates or numbers

        Returns
        -------
        iterator
            pairs of numbers or dates

        Example
        -------
        [52, 1, 2, 3, 5] has the following result

        [(1, 52), (2, 1), (3, 2), (5, 3)]
        """
        return zip(data[1:], data[:-1])

    def differences(self, pairs):
        """
        Differences between numbers or dates.

        Parameters
        ----------
        pairs : list of tuples
            pairs of numbers or dates

        Returns
        -------
        iterator
            substract of the second element from the first

        Example
        -------
        [(1, 52), (2, 1), (5, 3)]

        results in [-51, 1, 2]
        """
        return map(lambda x: (x[0]-x[1]), pairs)

    def difference_in_days(self, differences):
        """
        Number of days from a datetime.timedelta object.

        Parameters
        ----------
        differences : list
            a list of datetime.timedelta elements

        Returns
        -------
        iterator
            number of days

        Example
        -------
        [timedelta(days=2), timedelta(days=1)]

        results in [2, 1]
        """
        return map(lambda x: x.days, differences)

    def dif_5152_to_1(self, differences):
        """
        Change the number to 1 if the number in the list
        is -51 o -52.

        Parameters
        ----------
        differences : list
            a list of numbers

        Returns
        -------
        iterator
            numbers representing days
        """
        return map(lambda x: 1
                   if x in (-51, -52)
                   else x, differences)

    def grouping_differences(self, differences):
        """
        Same numbers are grouped and designed to a key.
        A new group is formed each time the number(key)
        changes.

        Parameters
        ----------
        differences : list
            a list of numbers

        Returns
        -------
        sub-iterators
            keys and groups from the list

        Example
        -------
        [1, 1, 1, 2, 1, 1, 2, 2] is grouped into

        [[1, [1, 1, 1]], [2, [2]], [1, [1, 1]], [2, [2, 2]]]
        """
        return groupby(differences)

    def streaks(self, grouping_differences):
        """
        Count the number of items contained in the group
        if the key is the number one.

        Parameters
        ----------
        grouping_differences : list
            a list of keys and groups

        Returns
        -------
        list
            a list of numbers

        Example
        -------
        [[1, [1, 1, 1]], [2, [2]], [1, [1, 1]], [2, [2, 2]]]

        results in [3, 2]
        """
        return [sum(group) for key, group in grouping_differences if key == 1]

    def longest_streak(self, streaks):
        """
        Choose the maximum number and add 1 to it.
        If the list is empty, returns the number one.

        Parameters
        ----------
        streaks : list
            a list of numbers

        Returns
        -------
        int
            a number representing the longest streak
        """
        if len(streaks) != 0:
            return max(streaks) + 1
        return 1

    def longest_streak_periodicity(self, habits_trackings_table, periodicity, col_date=5):
        """
        The longest streak of a habit depending on
        the habit's periodicity.

        Parameters
        ----------
        habits_trackings_table : list of tuples
            a list with the information of the habits and
            its trackings
        periodicity : str
            periodicity of the habit, 'weekly' or 'daily'
        col_date : int, optional
            index number i.e. column containing the dates
            (default is number 5)

        Returns
        -------
        int
            a number representing the longest streak of a
            habit
        """
        if len(habits_trackings_table) == 0:
            return []
        if periodicity == 'daily':
            return self.longest_streak(
                self.streaks(
                    self.grouping_differences(
                        self.difference_in_days(
                            self.differences(
                                self.zipping_unique_data(
                                    self.unique_data(
                                        self.format_to_date(
                                            self.select_column(
                                                habits_trackings_table,
                                                col_date
                                                )))))))))
        return self.longest_streak(
            self.streaks(
                self.grouping_differences(
                    self.dif_5152_to_1(
                        self.differences(
                            self.zipping_unique_data(
                                self.unique_data(
                                    self.to_calender_week(
                                        self.format_to_date(
                                            self.select_column(
                                                habits_trackings_table,
                                                col_date
                                                ))))))))))

    def start_habit(self, habits_trackings_table, col_date = 5):
        """
        The earliest date from a column of dates.

        Parameters
        ----------
        habits_trackings_table : list of tuples
            a list with the information of the habits and
            its trackings
        col_date : int, optional
            index number i.e. column containing dates
            (default is number 5)

        Returns
        -------
        datetime.date
            The earliest date
        """
        if len(habits_trackings_table) == 0:
            return []
        return min(
            self.format_to_date(
                self.select_column(
                    habits_trackings_table,
                    col_date
                    )))

    def last_day(self, habits_trackings_table, col_date = 5):
        """
        The most recent date in a column of dates.

        Parameters
        ----------
        habits_trackings_table : list of tuples
            a list with the information of the habits and
            its trackings
        col_date : int, optional
            index number i.e. column containing dates
            (default is number 5)

        Returns
        -------
        datetime.date
            The most recent date
        """
        if len(habits_trackings_table) == 0:
            return []
        return max(
            self.format_to_date(
                self.select_column(
                    habits_trackings_table,
                    col_date
                    )))

    def activity(self, periodicity, trackings, col_date=5):
        """
        Return the number of days or weeks in which the habit
        has been checked off, depending on whether the habit
        is weekly or daily.

        Parameters
        ----------
        periodicity : str
            periodicity of the habit, 'weekly' or 'daily'
        trackings : list of tuples
            a list with the information of the trackings
            registered for a certain habit
        col_date : int, optional
            index number i.e. column containing the dates
            (default is number 5)

        Returns
        -------
        int
            number of days or weeks in which the habit
            has been checked off
        """
        date_column = self.format_to_date(
            self.select_column(
                trackings, col_date
                )
            )

        if periodicity == 'daily':
            return len(self.unique_data(date_column))
        return len(self.unique_data(self.to_calender_week(date_column)))

    def only_hours(self, time):
        """
        Get hours without minutes

        Parameters
        ----------
        time : list
            a list of times

        Returns
        -------
        iterator
            numbers representing the hours
        """
        return map(lambda x: x.hour, time)

    def sort_hours(self, hours):
        """
        Each hour is renamed according to the part of the
        day it corresponds to (Morning, Afternoon, Evening
        or Overnight).

        Parameters
        ----------
        hours : list
            a list of numbers representing the hours

        Returns
        -------
        iterator
            parts of the day in strings

        Example
        -------
        [3, 18, 19, 2] results in

        ['Overnight', 'Evening', 'Evening', 'Overnight']
        """
        return map(lambda x: 'Morning'
                   if 5 <= x < 12
                   else ('Afternoon'
                         if 12 <= x < 18
                         else ('Evening'
                               if 18 <= x < 24
                               else 'Overnight')), hours)

    def count_sorted_hours(self, sort_hours):
        """
        Count the number of times the words of the parts
        of the day appear.

        Parameters
        ----------
        sort_hours : list
            a list of strings representing different parts
            of the day

        Returns
        -------
        counter
             a dictionary whose keys are the parts of the day and
             values are the frequencies with which they appear in
             the list

        Example
        -------
        ['Overnight', 'Evening', 'Evening', 'Overnight', 'Evening']

        results in {'Evening': 3, 'Overnight': 2}
        """
        return Counter(sort_hours)

    def active_time_dict(self, trackings, col_time=6):
        """
        A dictionary whose keys are the parts of the day
        (Morning, Afternoon, Evening and Overnight) in which a
        habit was checked and whose values indicate how often
        the activity is performed in these parts of the day.

        Parameters
        ----------
        trackings : list of tuples
            a list with the information of the trackings
            registered for a certain habit
        col_time : int, optional
            index number i.e. column containing the times
            (default is number 6)

        Returns
        -------
        counter
            a dictionary whose keys are the parts of the day and
            values are the frequencies with which the habit is
            performed
        """
        return self.count_sorted_hours(
            self.sort_hours(
                self.only_hours(
                    self.format_to_time(
                        self.select_column(
                            trackings, col_time)))))

    def max_value(self, active_time_dict):
        """
        The highest value from a dictionary.

        Parameters
        ----------
        active_time_dict : dict
            a dictionary whose keys are the parts of the day
            and values are the frequencies with which the
            habit is performed

        Returns
        -------
        int
            a number which is the highest value
        """
        if len(active_time_dict) == 0:
            return {}
        return max(active_time_dict.values())

    def most_active_time(self, active_time_dict, max_value):
        """
        Get the dictionary key(s) with the maximum value.

        Parameters
        ----------
        active_time_dict : dict
            a dictionary whose keys are the parts of the day
            and values are the frequencies with which the
            habit is performed
        max_value : int
            a number which is the highest value of the
            dictionary

        Returns
        -------
        list
            a list of string(s) representing the part(s) of
            the day when a habit is most frequently checked off

        Example
        -------
        {'Morning':4, 'Evening': 3, 'Overnight': 2}

        results in ['Morning']
        """
        if len(active_time_dict) == 0:
            return {}
        return [k for k, v in active_time_dict.items() if v == max_value]

    def unique_ids_periodicity(self, table, col_periodicity, periodicity):
        """
        Get the ids of the habits with the same periodicity.

        Parameters
        ----------
        table : list of tuples
            name of the table
        col_periodicity : int
            index number i.e. column containing the periodicity
        periodicity : str
            periodicity of the habit, 'weekly' or 'daily'

        Returns
        -------
        list
            a list of numbers representing ids
        """
        return self.unique_ids(
            self.select_rows(table, col_periodicity, periodicity))

    def grouped_habits(self,table, unique_ids):
        """
        Habits grouped by id.

        Parameters
        ----------
        table : list of tuples
            name of the table
        unique_ids : list
            a list of ids of the habits with the same periodicity

        Returns
        -------
        list
            Habits are grouped by id
        """
        return [self.select_rows(table, 0, id_n) for id_n in unique_ids]

    def lists_periodicity(self, table, col_periodicity, periodicity):
        """
        Information and trackings of the habits with the same
        periodicity are grouped together by id.

        Parameters
        ----------
        table : list of tuples
            name of the table
        col_periodicity : int
            index number i.e. column containing the periodicity
        periodicity : str
            periodicity of the habit, 'weekly' or 'daily'

        Returns
        -------
        list
            a list of lists. Each list has information and
            trackings for each habit with the same periodicity

        Example
        -------
        For the daily periodicity in the join of the habits and trackings
        table gives the following
        [[(1, 'Yoga', 'daily', 'Be more flexible', 'Mornings', '2021-07-31', '16:41'),
          (1, 'Yoga', 'daily', 'Be more flexible', 'Mornings', '2021-08-02', '18:44'),
          (1, 'Yoga', 'daily', 'Be more flexible', 'Mornings', '2021-08-03', '13:43')],
         [(3, 'Read', 'daily', 'Read 12 books a year', 'Afternoons', '2021-08-03', '15:44')]]
        """
        return self.grouped_habits(table,
                                   self.unique_ids_periodicity(table,
                                                               col_periodicity,
                                                               periodicity))

    def periodicity_info(self, lists_periodicity, periodicity, col_date=5):
        """
        A list containing information on each habit according
        to its periodicity. ID and name of the habit, date of
        the first and last tracking, part of the day when the
        user is most active to perform this habit, days or weeks
        of activity, and the longest streak.

        Parameters
        ----------
        lists_periodicity : list
            a list of lists of habits with the same periodicity.
            Each list has information of the habit and its trackings
        periodicity : str
            periodicity of the habit, 'weekly' or 'daily'
        col_date : int, optional
            index number i.e. column containing dates
            (default is number 5)

        Returns
        -------
        list
            a list of tuples containing information on each habit

        Example
        -------
        [(1, 'Yoga', '2021-03-18', '2021-04-17', 'Evening', 5, 2),
         (2, 'Reading', '2021-03-19', '2021-04-25', 'Afternoon', 8, 2)]
        """
        habits_info = []

        for habit_info in lists_periodicity:

            active_time_dictionary = self.active_time_dict(
                                habit_info,
                                6)
            max_value_active_time = self.max_value(
                                active_time_dictionary)
            most_active_time = self.most_active_time(
                                active_time_dictionary,
                                max_value_active_time)

            habits_info.append(
                (
                    self.display_unique_elements(habit_info, 0),
                    self.display_unique_elements(habit_info, 1),
                    ''.join(min(self.select_column(habit_info, col_date))),
                    ''.join(max(self.select_column(habit_info, col_date))),
                    self.display_elements(most_active_time, ', '),
                    self.activity(
                        periodicity,
                        habit_info,
                        col_date),
                    self.longest_streak_periodicity(
                        habit_info,
                        periodicity,
                        col_date))
                )

        return habits_info

    def lists_both_periodicities(self, habits_trackings_table):
        """
        Habits are separated according to periodicity.
        Information and trackings of each habit are grouped together.

        Parameters
        ----------
        habits_trackings_table : list of tuples
            a list with the information of the habits and
            its trackings

        Returns
        -------
        list
            list of lists. Information and trackings for each habit
            are grouped together, and habits with the same periodicity
            are grouped are separated.

        Example
        -------
        [[[(1, 'Yoga', 'daily', 'Be more flexible', 'Mornings', '2021-07-31', '16:41'),
           (1, 'Yoga', 'daily', 'Be more flexible', 'Mornings', '2021-08-02', '18:44')],
          [(3, 'Read', 'daily', 'Read 12 books a year', 'Afternoons', '2021-08-03', '15:44')]],
         [[(2, 'Run', 'weekly', 'Be faster', 'At weekends', '2021-07-31', '16:46'),
           (2, 'Run', 'weekly', 'Be faster', 'At weekends', '2021-08-03', '14:36')]]]
        """
        return list(map(self.lists_periodicity, *zip((habits_trackings_table, 2, 'daily'),
                                             (habits_trackings_table, 2, 'weekly'))))

    def both_periodicities_info(self, lists_both_periodicities):
        """
        Habits separated by periodicity with general information
        about each habit. ID and Name of the habit, date of the
        first and last tracking, part of the day when the user
        is most active to perform this habit, days or weeks of
        activity, and the longest streak.

        Parameters
        ----------
        lists_both_periodicities : list
            Habits are separated according to periodicity. Information
            and trackings of each habit are grouped together.

        Returns
        -------
        list
            list of lists with general information about each habit

        Example
        -------
        [[('1', 'Yoga', '2021-07-31', '2021-08-03', 'Afternoon', 3, 2),
          ('3', 'Read', '2021-08-03', '2021-08-03', 'Afternoon', 1, 1)],
         [('2', 'Run', '2021-07-31', '2021-08-03', 'Afternoon', 2, 2)]]
        """
        return list(map(self.periodicity_info, *zip((lists_both_periodicities[0], 'daily'),
                                              (lists_both_periodicities[1], 'weekly'))))

    def info_all_habits(self, both_periodicities_info):
        """
        Information on the habits separated by periodicity
        is merged into a single list.

        Parameters
        ----------
        both_periodicities_info : list
            list of lists with general information about each habit

        Returns
        -------
        list
            a list with general information about each habit

        Example
        -------
        [('1', 'Yoga', '2021-07-31', '2021-08-03', 'Afternoon', 3, 2),
         ('3', 'Read', '2021-08-03', '2021-08-03', 'Afternoon', 1, 1)]
        """
        return [habit for l in both_periodicities_info for habit in l]

    def tracked_habits(self, habits_table, habits_trackings_table):
        """
        Habits with trackings and their information from
        the habits table of the DB.

        Parameters
        ----------
        habits_table : list of tuples
            a list with the information of the registered habits
        habits_trackings_table : list of tuples
            a list with the information of the habits and
            its trackings

        Returns
        -------
        list
            a list of tuples of tracked habit information
        """
        return self.info_all_habits(
            self.grouped_habits(habits_table,
                                self.unique_ids(habits_trackings_table))
            )

    def habits_without_trackings(self, habits_table, ids_without_trackings):
        """
        Untracked habits and their information from the habits table
        of the DB.

        Parameters
        ----------
        habits_table : list of tuples
            a list with the information of the registered habits
        ids_without_trackings : list
            habit ids that have no trackings

        Returns
        -------
        list
            a list of tuples of untracked habits and their information
        """
        return [self.select_rows(habits_table, 0, id_n)[0]
                for id_n in ids_without_trackings]

    def habit_info_longest_streak(self, habits_trackings_table):
        """
        Information of the habit(s) that has(have) the
        maximum streak.

        Parameters
        ----------
        habits_trackings_table : list of tuples
            a list with the information of the habits and
            its trackings

        Returns
        -------
        list
            a list with general information of the habit(s)
            that has(have) the maximum streak.

        Example
        -------
        [('1', 'Yoga', '2021-06-26', '2021-07-23', 'Morning', 17, 9)]
        """
        all_periodicity_habits = self.info_all_habits(
            self.both_periodicities_info(
                self.lists_both_periodicities(
                    habits_trackings_table)))

        if len(all_periodicity_habits) > 1:
            # get the number of the maximum streak
            max_n = max(all_periodicity_habits, key=itemgetter(-1))[-1]
            # get all habits which have the same number of the maximum streak
            return self.select_rows(all_periodicity_habits, -1, max_n)
        return all_periodicity_habits

    def name_habit_longest_streak(self, habit_info_longest_streak):
        """
        Give the name(s) of the habit(s) with the longest streak
        and the streak.

        Parameters
        ----------
        habit_info_longest_streak : list
             a list with general information of the habit(s)
             that has(have) the maximum streak.

        Returns
        -------
        list
            a list with the name and longest streak of
            habit(s) with the longest streak

        Example
        -------
        [('Yoga', 2), ('Read', 2), ('Run', 2)]
        """
        return list(zip(self.select_column(habit_info_longest_streak, 1),
                        self.select_column(habit_info_longest_streak, -1)))

    def lengths(self, habits_info):
        """
        Length of each element

        Parameters
        ----------
        habits_info : list
             a list of tuples with general information
             about each habit

        Returns
        -------
        list
            a list of lists of numbers representing lenghts

        Example
        -------
        [(1, 'Yoga', 'daily', 'Be more flexible', 'Preferably in the morning', '2021-07-20'),
         (3, 'Read', 'daily', 'Read 12 books a year', 'Afternoons', '2021-07-21')]

        Results in
        [[1, 4, 5, 16, 25, 10],
         [1, 4, 5, 20, 10, 10]]
        """
        return [[len(str(x)) for x in row] for row in habits_info]

    def max_lengths(self, lengths, table):
        """
        The longest lengths

        Parameters
        ----------
        lengths : list
             a list of lists of numbers representing lenghts
        table : list of tuples
             name of the table

        Returns
        -------
        list
            a list with the longest lengths

        Example
        -------
        [[1, 4, 5, 16, 25, 10], [1, 4, 5, 20, 10, 10], [1, 3, 6, 9, 11, 10]]

        Results in [1, 4, 6, 20, 25, 10].
        """
        return list(max(map(itemgetter(x), lengths)) for x in range(0, len(table[0])))

    def distance_format(self, max_lengths):
        """
        Create a string formating that represents
        the distance to be maintained between each
        element.

        Parameters
        ----------
        max_lengths : list
             a list with the longest lengths

        Returns
        -------
        str
            a string formating

        Example
        -------
        if max_lengths = [1, 4, 6, 20, 25, 10]

        "%-1s    %-4s    %-6s    %-20s    %-25s    %-10s    "
        """
        return ''.join(map(lambda x: '%%-%ss    ' % x, max_lengths))

    def aligned_columns(self, format_distance, table):
        """
        Elements of a table are separated and
        aligned.

        Parameters
        ----------
        format_distance : str
             a string formating that represents the distance
             to be maintained between each element
        table : list of tuples
             name of the table

        Returns
        -------
        iterator
            strings whose elements are aligned with
            the given distance
        """
        return map(lambda x: format_distance % x, table)

    def line(self, max_lengths):
        """
        Create a line by adding the elements of a list
        of numbers (maximum lengths) and the spaces between
        the elements.

        Parameters
        ----------
        max_lengths : list
             a list with the longest lengths

        Returns
        -------
        str:
            a line with the given length
        """
        return '_' * (sum(max_lengths) + len(max_lengths) * 4)

    def add_colnames(self, col_names, table):
        """
        Add the names of each column.

        Parameters
        ----------
        col_names : str
            contains the names of each column
        table : list of tuples
             name of the table

        Returns
        -------
        list
            a list of tuples
        """
        return [col_names]+table

    def table_line(self, table):
        """
        Adjust the length of the line in relation to
        the data in the table.

        Parameters
        ----------
        table : list of tuples
             name of the table

        Returns
        -------
        str
            a line with the right length
        """
        return self.line(
            self.max_lengths(
                self.lengths(table), table))

    def display_table(self, col_names, table, header):
        """
        Display the data in a table.

        Parameters
        ----------
        col_names : str
            contains the names of each column
        table : list of tuples
            name of the table
        header : list of tuples
            title of the table
        """
        data_colnames = self.add_colnames(col_names, table)

        print(self.table_line(data_colnames))
        print("{}".format(header))
        print(self.table_line(data_colnames))
        print(self.display_elements(
            self.aligned_columns(
                self.distance_format(
                    self.max_lengths(
                        self.lengths(data_colnames),
                        data_colnames)),
                data_colnames)))
        print(self.table_line(data_colnames))

    def remove_habit(self, name):
        """
        The habit is removed from the DB in the habit table along
        with all trackings related to its id.

        Parameters
        ----------
        name : str
            name of the habit
        """
        with self.connection:
            self.cursor.execute("SELECT HabitID FROM habits WHERE Name=:name",
                       {'name': name})
            id_habit = self.cursor.fetchone()[0]
            self.cursor.execute("DELETE from trackings WHERE HabitID = :habitID",
                             {'habitID': id_habit})
            self.cursor.execute("DELETE from habits WHERE HabitID = :habitID",
                             {'habitID': id_habit})

    def insert_day(self, id_habit):
        """
        Register in the trackings table the day and time
        when the user checks a habit off, by choosing
        the id of the habit.

        Parameters
        ----------
        id_habit : int
            id of the habit
        """
        date = datetime.now().strftime("%Y-%m-%d")
        time = datetime.now().strftime("%H:%M")

        with self.connection:
            self.cursor.execute("""INSERT INTO trackings (Date, Time, HabitID)
                                VALUES (:date, :time, :habitID)""",
                            {'date': date, 'time': time, 'habitID': id_habit})

    def close(self):
        """
        Close SQLite3 connection
        """
        self.connection.close()


from rest_framework import serializers
from rest_framework.exceptions import ValidationError

from formbuilder.form_creator.models import Question, FormTemplate, FormSubmission

class FeedBackFormQuestionSerialser(serializers.ModelSerializer):
    class Meta:
        model = Question
        exclude = ('created_by', 'updated_by')

class FormDetailSerialiser(serializers.ModelSerializer):
    questions = serializers.SerializerMethodField()

    class Meta:
        model = FormTemplate
        exclude = ('created_by','updated_by','status','validity_start','vallidity_end')

    def get_questions(self,obj):
        all_qestions = obj.questions.all()
        return FeedBackFormQuestionSerialser(all_qestions,many=True).data

class FormSubmissionSerialiser(serializers.ModelSerializer):
    def __init__(self, *args, **kwargs):
        many = kwargs.pop('many', True)
        super(FormSubmissionSerialiser, self).__init__(many=many, *args, **kwargs)

    class Meta:
        model = FormSubmission
        fields = '__all__'

    def validate(self, attrs):
        question = attrs.get("question")
        if question.required:
            if not attrs.get("answer"):
                raise ValidationError({"answer":"answer_cant_be blank "})
        if question.required and question.question_type in [Question.MULTIPLE_CHOICE,Question.BOOLEAN]:
            if not f"{attrs.get('answer')}" in question.question_options:
                raise ValidationError({"answer": "Question Option does not match"})
        return attrs


from matplotlib import markers
import numpy as np
from datetime import datetime
import alpaca_trade_api as alpaca
from alpaca_trade_api.rest import TimeFrame
import pandas as pd
from secrets import API_KEY_ID, SECRET_KEY
from pandas.core.frame import DataFrame
import mplfinance as mpf
import csv
import os

class Bot:
    api = alpaca.REST(API_KEY_ID, SECRET_KEY)
    sym = ''
    slow_period = 40
    fast_period = 15
    stock_data = pd.DataFrame()
    transactions = pd.DataFrame({
        "price": [],
        "buy": [],
        "balance": [],
        "volume": []
    }, index=[])
    state = 0
    balance_history = pd.DataFrame()
    init_stock_closing = -1

    def __init__(self, sym, initial_balance):
        self.transactions.index.name = 'timestamp'
        self.sym = sym
        self.initial_balance = initial_balance
        self.funds = initial_balance
        self.min_balance = initial_balance
        self.min_funds = initial_balance
        self.holding = {
            'close': 0,
            'volume': 0,
            'timestamp': '',
        }
        now = datetime.now()
        self.init_datetime = now
        self.datadir =  "data/{0}_{1}-{2}-{3}".format(self.sym, now.year, now.month, now.day)

    def get_balance(self):
        return self.funds + (self.holding['close'] * self.holding['volume'])

    def add_to_stockdata(self, data):
        if len(self.stock_data.index) > self.slow_period*4:
            self.stock_data = self.stock_data.tail(self.slow_period*4).append(data)
        else:
            self.stock_data = self.stock_data.append(data)

    def add_to_transactions(self, order):
        if len(self.transactions.index) > self.slow_period*4:
            self.transactions = self.transactions.tail(self.slow_period*4).append(order)
        else:
            self.transactions = self.transactions.append(order)

    def is_holding(self):
        if self.holding['volume'] > 0:
            return True
        return False

    def print_order(self, order):
        order_type = '[BUY]' if order['buy'].values == 1 else '[SELL]'
        datetimeind = pd.DatetimeIndex(order.index)[0]
        order['datetime_pretty'] = '{0}/{1}/{2} {3}:{4}'.format(
            datetimeind.month,
            datetimeind.day,
            datetimeind.year,
            datetimeind.hour,
            datetimeind.minute
        )
        print(order_type + ' {0} - {1}x {2} at price ${3}'.format(order['datetime_pretty'].values[0], order['volume'].values[0], self.sym, order['price'].values[0] // 0.01 / 100))

    def get_transactions_filename(self):
        return '{4}/{0}_transactions_{1}-{2}-{3}.csv'.format(self.sym, self.init_datetime.year, self.init_datetime.month, self.init_datetime.day, self.datadir)

    def save_transaction(self, order):
        file = open(self.get_transactions_filename(), 'a')
        writer = csv.writer(file)
        writer.writerow([
            order.index.values[0],
            order['buy'].values[0],
            order['balance'].values[0],
            order['price'].values[0],
            order['volume'].values[0],
        ])
        file.close()
        return

    def get_all_transactions(self):
        filename = self.get_transactions_filename()
        df = pd.DataFrame()
        try:
            f = open(filename, "r")
        except:
            # no transactions were made
            return pd.DataFrame({
                'buy': [],
                'balance': [],
                'price': [],
                'volume': [],
            }, index=[])
        csvFile = csv.reader(f)
        for line in csvFile:
            row = pd.DataFrame({
                'buy': int(line[1]),
                'balance': float(line[2]),
                'price': float(line[3]),
                'volume': float(line[4]),
            }, index=[line[0]])
            df = df.append(row)
        f.close()
        return df

    def buy(self, shares, data):
        if self.is_holding():
            print('[ERROR] Cannot buy. Already holding {0}'.format(self.sym))
            return

        self.funds = self.funds - (shares * data['close'])
        self.holding['close'] = data['close']
        self.holding['volume'] = shares
        self.holding['timestamp'] = data.name
        buy_order = DataFrame([[1, self.get_balance(), data['close'], shares]], columns=['buy', 'balance', 'price', 'volume'], index=[data.name])
        self.add_to_transactions(buy_order)
        self.save_transaction(buy_order)
        self.print_order(buy_order)

    def sell(self, shares, data):
        if not self.is_holding():
            print("[ERROR] Cannot buy. Not holding any {0}".format(self.sym))

        self.funds = self.funds + (shares * data['close'])
        self.holding['close'] = 0
        self.holding['volume'] = 0
        self.holding['timestamp'] = ''

        sell_order = DataFrame([[0, self.get_balance(), data['close'], shares]], columns=['buy', 'balance', 'price', 'volume'], index=[data.name])
        self.add_to_transactions(sell_order)
        self.save_transaction(sell_order)
        self.print_order(sell_order)

    def tema_strategy(self):
        last_two = self.stock_data.tail(2)
        current = last_two.tail(1)
        previous = last_two.head(1)
        current_price = current['close'].values[0]

        # derivatives = pd.DataFrame()
        # derivatives['dTEMA'] = (self.stock_data['TEMA'].shift(periods=-1) - self.stock_data['TEMA'].shift(periods=1)) / (self.stock_data['TEMA'].shift(periods=-1) + self.stock_data['TEMA'].shift(periods=1))

        # if abs(derivatives.loc[previous.index]['dTEMA'].values[0] <= 10**(-7)) and not self.is_holding():
        #     return 1
        # elif abs(derivatives.loc[previous.index]['dTEMA'].values[0] <= 10**(-7)) and self.is_holding():
        #     return 0
        # return -1

        if current['TEMA'].values > current_price and not self.is_holding():
            return 1
        elif current['TEMA'].values < current_price and self.is_holding():
            return 0
        return -1

    def ewa_strategy(self):
        last_two = self.stock_data.tail(2)
        current = last_two.tail(1)
        previous = last_two.head(1)
        if previous['fast_EWA'].values < previous['slow_EWA'].values and current['fast_EWA'].values > current['slow_EWA'].values:
            return 0
        elif previous['fast_EWA'].values > previous['slow_EWA'].values and current['fast_EWA'].values < current['slow_EWA'].values:
            return 1
        return -1

    def make_decision(self, data):
        if self.init_stock_closing == -1:
            self.init_stock_closing = data['close']
        # Update holding
        if self.is_holding():
            self.holding['close'] = data['close']

        # Append data and calulate EWA
        self.add_to_stockdata(data)
        self.stock_data['slow_EWA'] = self.stock_data['close'].rolling(window=self.slow_period).mean()
        self.stock_data.loc[:self.slow_period, 'slow_EWA'] = np.nan
        self.stock_data['fast_EWA'] = self.stock_data['close'].ewm(span=self.fast_period, adjust=True).mean()
        self.stock_data.loc[:self.fast_period, 'fast_EWA'] = np.nan

        emas = pd.DataFrame()
        emas['EMA'] = self.stock_data['close'].rolling(window=10).mean()
        emas['EMA_1'] = emas['EMA'].rolling(window=10).mean()
        emas['EMA_2'] = emas['EMA_1'].rolling(window=10).mean()
        emas['EMA_3'] = emas['EMA_2'].rolling(window=10).mean()
        self.stock_data['TEMA'] = (3*emas["EMA_1"]) - (3*emas['EMA_2']) + emas['EMA_3']

        # Check it was appended
        current_price = self.stock_data.loc[data.name]['close']
        if not current_price == data['close']:
            print('Could not add data to list')
            return

        tema_res = self.tema_strategy()
        ewa_res = self.ewa_strategy()
        should_buy = -1
        if tema_res == ewa_res:
            should_buy = tema_res

        self.save_stock_record(self.stock_data.tail(1))
        # BUY
        if should_buy == 1 and not self.is_holding():
            if data['close'] >= self.funds:
                return
            shares = self.funds // data['close']
            self.buy(shares, data)
        # SELL
        elif should_buy == 0 and self.is_holding():
            bought_price = self.transactions.tail(1)['price'].values[0]
            shares = self.transactions.tail(1)['volume'].values[0]
            if data['close'] > bought_price*(1 + 0.02) or data['close'] <= bought_price*0.98:
                self.sell(shares, data)

        self.balance_history = self.balance_history.append(pd.DataFrame({
            "balance": self.get_balance(),
        }, index=[data.name]))

        if self.get_balance() < self.min_balance:
            self.min_balance = self.get_balance()
        if self.funds < self.min_funds:
            self.min_funds = self.funds

    def print_account(self):
        print("----- Account Information -----")
        print('Initial balance: ${0}'.format(self.initial_balance))
        print('Current funds: ${0}'.format(round(self.funds, 2)))
        print('Current balance: ${0}'.format(round(self.get_balance(), 2)))
        if self.is_holding():
            print('Total assets: {0}x {1} at ${2} (${3})'.format(self.holding['volume'], self.sym, self.holding['close'], self.holding['volume']*self.holding['close'] // 0.01 / 100))
        diff = (self.get_balance() - self.initial_balance)
        if  diff < 0:
            print('Account is down ${0} (%{1})'.format(diff // 0.01 / 100, (diff / self.initial_balance) // 0.01 / 100))
        elif diff > 0:
            print('Account is up ${0} (%{1})'.format(diff // 0.01 / 100, (diff / self.initial_balance) // 0.01 / 100 ))
        print("Minimum balance (%{0}): ${1}".format(round(self.min_balance / self.initial_balance * 100, 2), round(self.min_balance, 2)))
        print("Minimum funds (%{0}): ${1}".format(round(self.min_funds / self.initial_balance * 100, 2), round(self.min_funds, 2)))

    def run_test(self, start_date, end_date, timeframe):
        print("Getting data from alpaca...")
        data = self.api.get_bars(self.sym, timeframe, start_date, end_date).df
        data = data.filter(['close', 'open', 'volume', 'high', 'low'])
        print("Data received.")

        if not os.path.isdir(self.datadir):
            os.mkdir(self.datadir)
        if os.path.exists(self.get_stock_filename()):
            os.remove(self.get_stock_filename())
        if os.path.exists(self.get_transactions_filename()):
            os.remove(self.get_transactions_filename())

        start = datetime.now()
        for t, row in data.iterrows():
            self.make_decision(row)
        end = datetime.now()
        transactions = self.get_all_transactions()

        print('Elapsed time: {0}s'.format((end-start).total_seconds()))
        print('{0} trades were made'.format(len(transactions.index)))
        print()
        self.print_account()

        try:
            diff = bot.stock_data.tail(1)['close'][0] - bot.init_stock_closing
            diff = diff // 0.01 / 100
            init_volume = bot.transactions.head(1)['volume'][0]
            print("Net gain {0}: ${1} (${2})".format(self.sym, diff, diff*init_volume // 0.01 / 100))
        except:
            print("No transactions were made")

        print("Total decision duration: {0}".format(self.total_decision_time))
        return

    def get_all_stockdata(self):
        filename = self.get_stock_filename()
        df = pd.DataFrame()
        f = open(filename, "r")
        csvFile = csv.reader(f)
        for line in csvFile:
            row = pd.DataFrame({
                'close': float(line[1]),
                'open': float(line[2]),
                'high': float(line[3]),
                'low': float(line[4]),
                'volume': float(line[5]),
                'slow_EWA': float(line[6]),
                'fast_EWA': float(line[7]),
                'TEMA': float(line[8])
            }, index=[line[0]])
            df = df.append(row)
        f.close()
        return df

    def plot(self, candle = False):
        stock_data = self.get_all_stockdata()
        transactions = self.get_all_transactions()
        s = mpf.make_mpf_style(base_mpf_style='yahoo', rc={'font.size': 6})
        balance = mpf.make_addplot(self.balance_history,ylabel='Balance', width=2, panel=0)
        slow = mpf.make_addplot(stock_data['slow_EWA'], panel=1, width=0.5, color='purple')
        fast = mpf.make_addplot(stock_data['fast_EWA'], panel=1, width=0.5, linestyle='--', color='purple')
        close = mpf.make_addplot(stock_data['close'], panel=1, width=1, color='blue')

        tema = mpf.make_addplot(stock_data['TEMA'], panel=1, width=1, color='pink')

        temp = stock_data.filter(['close'])
        temp.index = pd.to_datetime(temp.index).tz_localize('Etc/UCT')
        buy_orders = transactions[transactions['buy'] == 1]
        buy_orders.index = pd.to_datetime(buy_orders.index)

        buys = temp.merge(buy_orders, how='outer', left_index=True, right_index=True)
        bought = mpf.make_addplot(buys['price'], panel=1, color='green', width=5, scatter=True, marker='^')

        sell_orders = transactions[transactions['buy'] == 0]
        sells = temp.merge(sell_orders, how='outer', left_index=True, right_index=True)
        sold = mpf.make_addplot(sells['price'], panel=1, color='red', width=5, scatter=True, marker='v')
        stock_data.index = pd.to_datetime(stock_data.index)

        type = None
        if candle:
            type = 'candle'
        else:
            type = 'line'
        figure, axes = mpf.plot(stock_data, type=type, panel_ratios=(0.3, 1, 0.3),
            volume=True,
            title=self.sym,
            style=s,
            figsize=(10,5),
            returnfig=True,
            addplot=[balance,slow,fast, bought, sold, close, tema],
            main_panel=1,
            volume_panel=2,
            num_panels=3
        )

        for t, b in buys.iterrows():
            axes[1].text(b.name, b.price, "${0}".format(b.price), color='black', zorder=9)

        axes[0].yaxis.set_label_position("left")
        axes[0].yaxis.tick_left()
        axes[4].yaxis.set_label_position("left")
        axes[4].yaxis.tick_left()
        mpf.show()

    def get_stock_filename(self):
        return '{4}/{0}_stockdata_{1}-{2}-{3}.csv'.format(self.sym, self.init_datetime.year, self.init_datetime.month, self.init_datetime.day, self.datadir)

    def save_stock_record(self, data):
        filename = self.get_stock_filename()
        file = open(filename, 'a')
        writer = csv.writer(file)
        writer.writerow([
            data.index.values[0],
            data['close'].values[0],
            data['open'].values[0],
            data['high'].values[0],
            data['low'].values[0],
            data['volume'].values[0],
            data['slow_EWA'].values[0],
            data['fast_EWA'].values[0],
            data['TEMA'].values[0]
        ])
        file.close()
        return

start_date = "2020-01-01"
end_date = "2020-01-30"
timeframe = TimeFrame.Minute

bot = Bot("F", 1000)
bot.run_test(start_date, end_date, timeframe)
stock_data = bot.get_all_stockdata()
#bot.plot()