## original code https://medium.com/towards-data-science/lstm-by-example-using-tensorflow-feb0c1968537
###the adapted code uses word characteristics
from __future__ import print_function

import numpy as np
import tensorflow as tf
from tensorflow.contrib import rnn
import random
import collections
import time
from tensorflow.python import debug as tf_debug

start_time = time.time()
def elapsed(sec):
    if sec<60:
        return str(sec) + " sec"
    elif sec<(60*60):
        return str(sec/60) + " min"
    else:
        return str(sec/(60*60)) + " hr"

#targets log path
logs_path = '/tmp/tensorflow/rnn_words'
writer = tf.summary.FileWriter(logs_path)

#text file containing words for training
training_file = '/home/sean/christina/git/lstm_rnn_word2vec/beiling_the_cat.txt'

def read_data(fname):
    with open(fname) as f:
        content = f.readlines()
    content = [x.strip() for x in content]
    content = [content[i].split() for i in range(len(content))]
    content = np.array(content)
    content = np.reshape(content,[-1,])
    return content

training_data = read_data(training_file)
print("Loaded training data...")

#might be better to do hashing? https://www.tensorflow.org/api_guides/python/string_ops#Conversion
def build_dataset(words):
    count = collections.Counter(words).most_common()
    dictionary = dict()
    for word, _ in count:
        dictionary[word] = len(dictionary)
    reverse_dictionary = dict(zip(dictionary.values(),dictionary.keys()))
    return dictionary, reverse_dictionary

dictionary, reverse_dictionary = build_dataset(training_data)
vocab_size = len(dictionary)

#Parameters
#learning_rate = 0.001
training_iters = 100000
display_step = 50
n_input = 3
characteristics_size = 4
input_size = n_input*characteristics_size

#number of units in RNN cell
n_hidden = 512

x = tf.placeholder("float", [input_size])
y = tf.placeholder("float", [characteristics_size])
y2 = tf.placeholder("int64", [])

weights = {
    'out': tf.Variable(tf.random_uniform([n_hidden, input_size])),
    'dict': tf.Variable(tf.random_uniform([vocab_size,characteristics_size]))
}

biases = {
    'out': tf.Variable(tf.random_uniform([input_size]))
}

def input_list(offset, n_input):
        input_list = tf.Variable([],dtype="float32")
        for i in range(offset,offset+n_input):
            toconcat = tf.slice(weights['dict'],[dictionary[str(training_data[i])],0],[1,characteristics_size])
            toconcat = tf.reshape(toconcat,[-1])
            input_list = tf.concat([input_list,toconcat],0)
        input_list = tf.reshape(input_list,[-1])
        return input_list

def RNN(x, weights, biases):

    #reshape to [1, n_input]
    x = tf.reshape(x, [-1, input_size])
    x = tf.split(x, n_input, 1)

    # print(x.eval())

    #2-layer LSTM, each layer has n_hidden units
    rnn_cell = rnn.MultiRNNCell([rnn.BasicLSTMCell(n_hidden),rnn.BasicLSTMCell(n_hidden)])

    #generate prediction
    outputs, states = rnn.static_rnn(rnn_cell, x, dtype=tf.float32)

    # we only want the last output (due to nature of LSTM)
    y_ = tf.matmul(outputs[-1], weights['out']) + biases['out']
    y_ = y_[:,-characteristics_size:]
    y_ = tf.reshape(y_,[-1])
    return y_

#find closest word with the predicted characteristics
def RNN2(y_):
    y_onehot = tf.subtract(weights['dict'],y_)
    y_onehot = tf.square(y_onehot)
    y_onehot = tf.reduce_sum(y_onehot, 1)
    y_onehot = tf.argmin(y_onehot)

    y_onehot = tf.cast(y_onehot, dtype="int64")

    return y_onehot

#find the characteristics of the predicted word
def pred_characteristics(y_onehot):
    index = tf.cast(y_onehot, dtype="int32")
    y_final = weights['dict'][index, :]
    return y_final

actual_pred = RNN(x, weights, biases)
onehot_pred = RNN2(actual_pred)
pred_characteristics = pred_characteristics(onehot_pred)

cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = pred_characteristics, labels = y))
optimizer = tf.train.AdamOptimizer(learning_rate = .01).minimize(cost)

#model evaluation compares the predicted characteristics and actual characteristics of correct word
# note, the model can predict the correct word and have an accuracy <100% because the predicted characteristics weren't correct
correct_pred = tf.square(tf.subtract(pred_characteristics, y))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

#initialize the variables
init = tf.global_variables_initializer()

#launch the graph
with tf.Session() as session:
    session.run(init)
    #session = tf_debug.LocalCLIDebugWrapperSession(session)
    #session.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
    step = 0
    offset = random.randint(0, n_input+1)
    end_offset = n_input + 1
    acc_total = 0
    loss_total = 0

    writer.add_graph(session.graph)

    while step < training_iters:
        #generate a minibatch. add some randomness on selection process
        if offset > (len(training_data)-end_offset):
           offset = random.randint(0, n_input+1)

        symbols_in_keys = input_list(offset, n_input).eval()

        symbols_out = tf.slice(weights['dict'],[dictionary[str(training_data[offset+n_input])],0],[1,characteristics_size])
        symbols_out = tf.reshape(symbols_out,[-1])
        symbols_out = symbols_out.eval()

        y_hot = offset+n_input

        _, acc, loss, onehot_p, pred, actual_pred2= session.run([optimizer, accuracy, cost, onehot_pred, pred_characteristics, actual_pred], \
                                                feed_dict={x: symbols_in_keys, y: symbols_out, y2: y_hot})
        loss_total += loss
        acc_total += acc
        if (step+1) % display_step == 0:
            print("Iter= " + str(step+1) + ", Current Loss= " + \
                  "{:.6f}".format(loss) + ", Average Accuracy= " + \
                  "{:.2f}%".format(100*acc_total/display_step))
            acc_total = 0
            loss_total = 0
            symbols_in = [training_data[i] for i in range(offset, offset + n_input)]
            symbols_out = training_data[offset + n_input]
            symbols_out_pred = reverse_dictionary[onehot_p]
            print("%s - Actual: [%s] vs Pred: [%s]" % (symbols_in,symbols_out,symbols_out_pred))
            #the below will print the actual and predicted word characteristics
            #print("Actual:[%s] vs Pred:[%s]" % (pred,actual_pred2))
        step += 1
        offset += (n_input+1)
    print("Optimization Finished!")
    print("Elapsed time: ", elapsed(time.time() - start_time))
    print("Run on command line.")
    print("\ttensorboard --logdir=%s" % (logs_path))
    print("Point your web browser to: http://localhost:6006/")
    while True:
        prompt = "%s words: " % n_input
        sentence = input(prompt)
        sentence = sentence.strip()
        words = sentence.split(' ')
        if len(words) != n_input:
            continue
        try:
            symbols_in_keys = [dictionary[str(words[i])] for i in range(len(words))]
            for i in range(32):
                keys = np.reshape(np.array(symbols_in_keys), [-1, n_input, 1])
                onehot_p = session.run(pred, feed_dict={x: keys})
                onehot_pred_index = int(tf.argmax(onehot_p, 1).eval())
                sentence = "%s %s" % (sentence,reverse_dictionary[onehot_pred_index])
                symbols_in_keys = symbols_in_keys[1:]
                symbols_in_keys.append(onehot_pred_index)
            print(sentence)
        except:
            print("Word not in dictionary")


import sys, copy
sys.setrecursionlimit(10**9)

h, w = map(int,input().split())
a = [list(map(int,input().split())) for _ in range(h)]

ans = 0
dx_dy = [[1,0],[0,1]]

def dfs(x,y,route):
    myroute = copy.copy(route)
    if a[x][y] not in set(myroute):
        myroute.append(a[x][y])
        if x==h-1 and y==w-1:
            global ans
            ans += 1

        for dx, dy in dx_dy:
            nx, ny = x+dx, y+dy
            if 0 <= nx < h and 0 <= ny < w:
                dfs(nx, ny, myroute)

dfs(0,0,[])
print(ans)


'''
This script uses SLURM to run in parallel many trials of the same algorithm
on different environments with fixed random seed (seed = trial number).

Command
    python3 run_slurm.py <ALG_NAME> <N_TRIALS> <ENV_LIST>

Example
    python3 run_slurm.py ddpg.ddpg 5 Pendulum-v0 Swimmer-v2

One job per run will be submitted.

Data is still saved as usual in `data-trial`, but instead of the current date and time,
the seed (= trial number) is used. For example, for the above run data will be saved in

data-trial/ddpg/Pendulum-v0/0.dat
data-trial/ddpg/Pendulum-v0/1.dat
...

Additionally, stdout and stderr are flushed to log files. For example

logs-trial/stdout_ddpg.ddpg_Pendulum-v0_0
logs-trial/stderr_ddpg.ddpg_Pendulum-v0_0
...

NOTE: Change the slurm script according to your needs (activate virtual env,
request more memory, more computation time, ...).
'''

import os, errno, sys

logdir = 'logs-trial/' # directory to save log files (where stdout is flushed)
try:
    os.makedirs(logdir)
except OSError as e:
    if e.errno != errno.EEXIST:
        raise

alg_name = sys.argv[1]
n_trials = int(sys.argv[2])
env_list = sys.argv[3:]

for env_name in env_list:
    for trial in range(n_trials):
        run_name = alg_name + '_' + env_name + '_' + str(trial)

        text = """\
#!/bin/bash

# job name
#SBATCH -J """ + run_name + """\

# logfiles
#SBATCH -o """ + logdir + """stdout_""" + run_name + """\
#SBATCH -e """ + logdir + """stderr_""" + run_name + """\

# request computation time hh:mm:ss
#SBATCH -t 24:00:00

# request virtual memory in MB per core
#SBATCH --mem-per-cpu=1000

# nodes for a single job
#SBATCH -n 1

#SBATCH -C avx2
#SBATCH -c 4

# activate virtual env
module load intel python/3.6
source ../.bashrc
python3 -m """ + alg_name + """ """ + env_name + """ """ + str(trial) + """\
    """

        text_file = open('r.sh', "w")
        text_file.write(text)
        text_file.close()

        os.system('sbatch r.sh')
        os.remove('r.sh')


from tkinter import *
from PIL import Image, ImageTk
from tkinter import messagebox

# Cores
white = '#FFFFFF'
gray = '#919191'
font = '#3e4146'

# Configurando janela
app = Tk()
app.title('Calculadora de Salário')
app.geometry('550x330')
app.config(bg=white)
app.resizable(False, False)

# Criando cabeçalho
header = Frame(app, width=551, height=30, bg=white,
               relief='raised', borderwidth=1)
header.place(x=0, y=0)

# Preparando imagens
iconHeader = Image.open('CalculadoraSalario/images/money.png')
iconHeader = iconHeader.resize((28, 28))
iconHeader = ImageTk.PhotoImage(iconHeader)

iconBig = Image.open('calculadoraSalario/images/money.png')
iconBig = iconBig.resize((100, 100))
iconBig = ImageTk.PhotoImage(iconBig)

circle = Image.open('CalculadoraSalario/images/circles.png')
circle = circle.resize((39, 100))
circle = ImageTk.PhotoImage(circle)

# Fazendo header
iconHead = Label(header, image=iconHeader, width=20, height=23,
                 compound=CENTER, anchor=CENTER, bg=white, relief=FLAT)
iconHead.place(x=4, y=0)

nameApp = Label(header, text='Calculadora de salário', width=20,
                height=1, anchor=CENTER, font=('Ivy 12 bold'), bg=white, fg=font)
nameApp.place(x=30, y=3)

# Criando labels e entrys
name = Label(app, text='Escreva seu nome completo: ', width=30,
             height=1, anchor='w', font=('Ivy 9 bold'), bg=white, fg=font)
name.place(x=5, y=50)

nameEntry = Entry(app, width=25, justify='left', relief='solid', borderwidth=1)
nameEntry.place(x=190, y=50)

hoursDay = Label(app, text='Quantas horas você trabalha por dia? ',
                 width=40, height=1, anchor='w', font=('Ivy 9 bold'), bg=white, fg=font)
hoursDay.place(x=5, y=85)

hoursEntry = Entry(app, width=15, justify='center',
                   relief='solid', borderwidth=1)
hoursEntry.place(x=246, y=85)

daysWeek = Label(app, text='Quantos dias você trabalha por semana? ',
                 width=40, height=1, anchor='w', font=('Ivy 9 bold'), bg=white, fg=font)
daysWeek.place(x=5, y=120)

daysEntry = Entry(app, width=15, justify='center',
                  relief='solid', borderwidth=1)
daysEntry.place(x=246, y=120)

yearSalary = Label(app, text='Qual é o seu salário anual ?', width=30,
                   height=1, anchor='w', font=('Ivy 9 bold'), bg=white, fg=font)
yearSalary.place(x=5, y=155)

yearEntry = Entry(app, width=15, justify='center',
                  relief='solid', borderwidth=1)
yearEntry.place(x=246, y=155)

# Criando função
def calc():
    try:
        # Coletando informações dos entrys
        nameGet = nameEntry.get()
        hoursGet = int(hoursEntry.get())
        daysGet = int(daysEntry.get())
        yearGet = float(yearEntry.get())
        hoursWeek = int(hoursGet * daysGet)

        if nameGet == '':
            messagebox.showinfo('Atenção!!!',
                                'Preencha o campo de nome.')
        else:
            nameShow = Label(app, text=nameGet, width=28, height=1, anchor='w', font=(
                'Ivy 10'), relief=RAISED, bg=gray, fg=font)
            nameShow.place(x=5, y=190)

            hoursShow = Label(app, text=f'Horas de trabalho por dia: {hoursGet}', width=28, height=1, anchor='w', font=(
                'Ivy 10'), relief=RAISED, bg=gray, fg=font)
            hoursShow.place(x=5, y=212)

            daysShow = Label(app, text=f'Dias de trabalho por semana: {daysGet}', width=28, height=1, anchor='w', font=(
                'Ivy 10'), relief=RAISED, bg=gray, fg=font)
            daysShow.place(x=5, y=234)

            yearShow = Label(app, text=f"R${yearGet:,.2f}", width=28, height=1, anchor='w', font=(
                'Ivy 10 bold'), relief=RAISED, bg=gray, fg=font)
            yearShow.place(x=5, y=256)

            # Mostrando o salário anual, mensal, diário e por hora
            iconCircles = Label(app, image=circle, width=39, height=100,
                                compound=CENTER, anchor=CENTER, bg=white, relief=FLAT)
            iconCircles.place(x=245, y=195)

            salHour = Label(app, text=f'Salário por hora: R$ {yearGet/(hoursWeek*52):,.2f}', width=25, height=1, anchor='w', font=(
                'Arial 10'), bg=white, fg=font)
            salHour.place(x=290, y=190)

            salDay = Label(app, text=f'Salário Diário: R$ {yearGet/(daysGet*52):,.2f}', width=25, height=1, anchor='w', font=(
                'Arial 10'), bg=white, fg=font)
            salDay.place(x=290, y=212)

            salWeek = Label(app, text=f'Salário Semanal: R$ {yearGet/52:,.2f}', width=25, height=1, anchor='w', font=(
                'Arial 10'), bg=white, fg=font)
            salWeek.place(x=290, y=234)

            salMonth = Label(app, text=f'Salário Mensal: R$ {yearGet/12:,.2f}', width=25, height=1, anchor='w', font=(
                'Arial 10'), bg=white, fg=font)
            salMonth.place(x=290, y=256)

            salYear = Label(app, text=f'Salário Anual: R$ {yearGet:,.2f}', width=25, height=1, anchor='w', font=(
                'Arial 10'), bg=white, fg=font)
            salYear.place(x=290, y=278)
    except:
        messagebox.showerror('Erro!!!',
                              'Insira números')

# Colocando imagem e botão de calcular
img = Label(app, image=iconBig, width=114, height=100,
            compound=CENTER, anchor=CENTER, bg=white, relief=FLAT)
img.place(x=380, y=40)

calc_button = Button(app, command=calc, text='Calcular', width=17, height=1, anchor='center', font=(
    'Ivy 9 bold'), relief='raised', overrelief='sunken', bg=gray, fg=font, borderwidth=2,)
calc_button.place(x=370, y=151)

app.mainloop()


from datetime import datetime

import graphene
import math

from django.contrib.auth.models import Group
from django.db import transaction

from authentication.models import User
from common.methods.create_notification import create_notification
from cs.models import Coupon
from order.models import OrderMaster, ZipCode
from order.models.order_detail import OrderDetail
from order.types.order_master_type import OrderMasterType
from order.types.ordered_product import OrderedProductInputType
from product.models import ProductMaster, Product, Draft, NewDraft

class PlaceOrder(graphene.Mutation):
    class Arguments:
        # ordering_user = OrderingUserInputType()
        user_id = graphene.Int()
        ordered_products = graphene.List(OrderedProductInputType)
        is_pick_up = graphene.Boolean(default_value=False)

    success = graphene.Boolean()
    order_master = graphene.Field(OrderMasterType)

    @classmethod
    @transaction.atomic()
    def mutate(cls, _, info, ordered_products, user_id=None, is_pick_up=False):

        delivery_price_normals = [0]
        delivery_price_divisions = [0]
        delivery_price_individuals = [0]
        if user_id is not None:
            user = User.objects.get(pk=user_id)
        else:
            user = info.context.user

        address = user.addresses.get(default=True)

        if not is_pick_up:
            # product_master가 같은 상품들끼리 묶기
            product_masters = {}
            for ordered_product in ordered_products:
                product_master_id = str(ordered_product.product_master_id)
                if product_master_id in product_masters:
                    product_masters[product_master_id]['quantity'] += ordered_product.quantity
                else:
                    product_masters[product_master_id] = {
                        'quantity': ordered_product.quantity
                    }

            # 도서산간 추가배송비
            extra_delivery = ZipCode.objects.filter(zip_code=address.zip_code).first()

            # 합친 딕셔너리로 배달비 구하기
            for key, value in product_masters.items():
                product_master = ProductMaster.objects.get(pk=int(key))
                delivery_type = product_master.delivery_type
                quantity = value['quantity']
                price_delivery = product_master.price_delivery
                max_quantity_per_box = product_master.max_quantity_per_box
                # 도서산간 지역의 zip code를 가지고 있으면 추가 배송비 지챈
                if extra_delivery:
                    extra_delivery_price = extra_delivery.additional_delivery_price
                    if delivery_type == '일반배송상품':
                        total_price_delivery = price_delivery + extra_delivery_price
                        delivery_price_normals.append(total_price_delivery)
                    elif delivery_type == '분할배송상품':
                        delivery_price_division = math.ceil(quantity / max_quantity_per_box)
                        total_price_delivery = (price_delivery + extra_delivery_price) * delivery_price_division
                        delivery_price_divisions.append(total_price_delivery)
                    else:
                        total_price_delivery = (price_delivery + extra_delivery_price) * quantity
                        delivery_price_individuals.append(total_price_delivery)
                else:
                    if delivery_type == '일반배송상품':
                        delivery_price_normals.append(price_delivery)
                    elif delivery_type == '분할배송상품':
                        delivery_price_division = math.ceil(quantity / max_quantity_per_box)
                        total_price_delivery = delivery_price_division * price_delivery
                        delivery_price_divisions.append(total_price_delivery)
                    else:
                        total_price_delivery = price_delivery * quantity
                        delivery_price_individuals.append(total_price_delivery)

        total_delivery_price = max(delivery_price_normals) + sum(delivery_price_divisions) + sum(delivery_price_individuals)

        # 주어진 파라미터 + 배달비 넣어주기(order master,order detail 생성)

        # order master 생성
        order_master = OrderMaster.objects.create(user=user, receiver=user.gym.name,
                                                  order_number='O{}{}'.format(user.phone[-4:], datetime.now().strftime('%y%m%d%H%M%S')),
                                                  phone=user.phone,
                                                  zip_code=address.zip_code,
                                                  address=address.address,
                                                  is_pick_up=is_pick_up,
                                                  detail_address=address.detail_address,
                                                  price_delivery=total_delivery_price,
                                                  )

        for index, ordered_product in enumerate(ordered_products):
            product = Product.objects.get(pk=ordered_product.product_id)
            print(product)

            draft = None
            price_total = product.product_master.price_gym + product.price_additional
            if ordered_product.draft_id:
                draft = NewDraft.objects.get(pk=ordered_product.draft_id)
                price_total += draft.price_work
                admin = Group.objects.get(name="관리자")
                admin_users = User.objects.filter(groups=admin)
                # for admin_user in admin_users:
                #     Notification.objects.create(user=admin_user, title="주문요청알림 - 작업실배정필요",
                #                                 contents="{}님께서 주문 요청 및 작업실 배정 요청을 하였습니다.. 작업실 배정해주세요"
                #                                 .format(order_master.order_name),
                #                                 notification_type="주문 및 작업실 배정 요청알림")
            else:
                admin = Group.objects.get(name="관리자")
                admin_users = User.objects.filter(groups=admin)
                # for admin_user in admin_users:
                #     Notification.objects.create(user=admin_user, title="주문요청알림",
                #                                 contents="{}님께서 주문 요청을 하였습니다. 주문관리에서 확인 부탁드립니다."
                #                                 .format(order_master.order_name),
                #                                 notification_type="부분입고완료")

            # order detail 생성
            product_master = product.product_master
            order_detail =OrderDetail.objects.create(order_master=order_master,
                                                     state="구매요청",
                                                     order_detail_number=order_master.order_number+str(index),
                                                     product_master=product.product_master,
                                                     product=product,
                                                     quantity=ordered_product.quantity,
                                                     user_request=ordered_product.user_request,
                                                     new_draft=draft,
                                                     price_option=product.price_additional * ordered_product.quantity,
                                                     price_total=price_total * ordered_product.quantity,
                                                     price_products=product.product_master.price_gym * ordered_product.quantity,
                                                     price_work=draft.price_work * ordered_product.quantity if draft is not None else 0,
                                                     price_work_labor=draft.price_work_labor * ordered_product.quantity if draft is not None else 0,
                                                     student_names=ordered_product.student_names,
                                                     price_consumer=product_master.price_consumer,
                                                     price_parent=product_master.price_parent,
                                                     price_gym=product_master.price_gym,
                                                     price_vendor=product_master.price_vendor
                                                     )

            gym = order_detail.order_master.user.gym
            gym.total_purchased_amount += order_detail.price_gym * order_detail.quantity + order_detail.price_work
            gym.save()
        return PlaceOrder(success=True, order_master=order_master)


import re

from virttest import libvirt_version
from virttest import virsh

from provider.migration import base_steps

event_session = None

def run(test, params, env):
    """
    Test live migration with precopy/postcopy.

    :param test: test object
    :param params: Dictionary with the test parameters
    :param env: Dictionary with test environment.
    """
    def setup_postcopy_migration():
        """
        Setup for postcopy migration
        """
        expected_event_src = params.get("expected_event_src")
        if expected_event_src:
            cmd = "event --loop --all"
            global event_session
            event_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC,
                                               auto_close=True)
            event_session.sendline(cmd)
        migration_obj.setup_connection()

    def verify_postcopy_migration():
        """
        Verify for postcopy migration
        """
        expected_event_src = params.get("expected_event_src")
        if expected_event_src:
            global event_session
            src_output = event_session.get_stripped_output()
            test.log.debug("Event output on source: %s", src_output)
            if not re.findall(expected_event_src, src_output):
                test.fail("Unalbe to find event {}".format(expected_event_src))
        migration_obj.verify_default()

    libvirt_version.is_libvirt_feature_supported(params)

    test_case = params.get('test_case', '')
    vm_name = params.get("migrate_main_vm")

    vm = env.get_vm(vm_name)
    migration_obj = base_steps.MigrationBase(test, vm, params)
    setup_test = eval("setup_%s" % test_case) if "setup_%s" % test_case in \
        locals() else migration_obj.setup_connection
    verify_test = eval("verify_%s" % test_case) if "verify_%s" % test_case in \
        locals() else migration_obj.verify_default

    try:
        setup_test()
        migration_obj.run_migration()
        verify_test()
    finally:
        migration_obj.cleanup_connection()


# -*- coding: utf-8 -*-
import os
import sys
__project_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
if __project_path not in sys.path:
    sys.path.append(__project_path)
import mongo_db
from bson.objectid import ObjectId
import datetime
import random
from log_module import get_logger
from werkzeug.contrib.cache import SimpleCache

simple_cache = SimpleCache()
logger = get_logger()

"""
此模块用于根据老师喊单的信号，生成对应的虚拟老师数据
"""

class Deposit(mongo_db.BaseDoc):
    """
    入金/存款(记录),当老师做单的时候,如果资金不足,就会激发入金行为
    """
    _table_name = "deposit"
    type_dict = dict()
    type_dict['_id'] = ObjectId
    type_dict['t_id'] = ObjectId  # 老师id
    type_dict['num'] = float  # 入金金额.
    type_dict['time'] = datetime.datetime

    @classmethod
    def generator_deposit(cls, min_money: float) -> float:
        """
        生成一个加金的数字
        :param min_money: 最低额度.加金金额必须大于此额度.
        :return: 实际的加金额度.
        """
        l = [10000,  15000, 20000, 30000, 50000, 100000]
        r = 0
        for x in l:
            if min_money * 2 <= x:
                r = x
                break
            else:
                pass
        return r

class Teacher(mongo_db.BaseDoc):
    """
    老师,微信项目（Webchat_Server）有一个同名的类，不同的是：
    Message_Server项目下的Teacher类主要负责item_module.Trade.sync_from_signal(生成虚拟老师交易信号)时取老师列表。
    Webchat_Server项目项目下的Teacher类，主要负责老师的管理。（属性，头像的修改）
    两个类属性的字段相同即可.
    Webchat_Server项目项目下的teacher_module函数更丰富一些。
    你会在2个项目下分别看到这两个模块。
    """
    _table_name = "teacher"
    type_dict = dict()
    """真实老师的id取自简道云"""
    type_dict['_id'] = ObjectId
    """真实老师的name可能取自简道云(也可再修改)"""
    type_dict['name'] = str   # 展示的名字，比如青云老师等
    type_dict['real_name'] = str  # 真实姓名，非必须
    type_dict['head_img'] = str  # 头像文件相当与项目根目录的路径
    type_dict['img'] = str  # 半身像文件相当与项目根目录的路径
    type_dict['level'] = str  # 老师等级
    type_dict['motto'] = str  # 座右铭
    type_dict['feature'] = str  # 特性，风格，特点
    type_dict['resume'] = str  # 简历
    type_dict['create_date'] = datetime.datetime
    type_dict['native'] = bool  # 是否是真实的teacher？
    type_dict['from_id'] = ObjectId  # 虚拟老师专有，发源老师id，保持不连，除非修改
    type_dict['direction'] = str  # 虚拟老师专有，跟的方向，有三种 follow/reverse/random
    type_dict['win_ratio'] = float  # 胜率率, 每次close时候计算
    type_dict['case_count'] = int  # 总单数, 每次close时候计算
    type_dict['win_count'] = int  # 盈利的单数, 每次close时候计算
    type_dict['profit_ratio'] = float  # 盈利率, 每次close时候计算
    type_dict['profit_amount'] = float  # 盈利总额.每次close时候计算
    type_dict['deposit'] = float  # 存款,当前本金.每次close时候计算
    type_dict['deposit_amount'] = float  # 历次存款总额,每次close时候计算
    type_dict['lots_range'] = list  # 手数范围.
    # type_dict['show'] = bool  # 是否显示?默认显示

    @classmethod
    def instance(cls, **kwargs):
        if "create_date" not in kwargs:
            kwargs['create_date'] = datetime.datetime.now()
        # if "show" not in kwargs:
        #     kwargs['show'] = True
        if "name" not in kwargs:
            ms = "name必须"
            raise ValueError(ms)
        if "native" not in kwargs:
            ms = "native必须"
            raise ValueError(ms)
        return cls(**kwargs)

    @classmethod
    def init_head_image_list(cls) -> list:
        """
        初始化头像列表
        :return:
        """
        d = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
        p = os.path.join(d, "static", 'images', 'head_image')
        ns = os.listdir(p)
        res = list()
        for name in ns:
            u = os.path.join("/static/images/head_image", name)
            res.append(u)
        key = "t_head_img"
        simple_cache.set(key=key, value=res, timeout=900)
        return res

    @classmethod
    def pop_head_image(cls) -> str:
        """
        尽量不重复的生成老师的头像,如果头像资源用尽,那就重头来.
        :return:
        """
        key = "t_head_img"
        m = simple_cache.get(key)
        if m is None or len(m) == 0:
            m = cls.init_head_image_list()
        else:
            pass
        url = m.pop(random.randint(0, len(m) - 1))
        simple_cache.set(key=key, value=m, timeout=900)
        return url

    @classmethod
    def generator_lots_range(cls) -> list:
        """
        虚拟一个手数范围
        :return:
        """
        begin = random.choice([1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3])
        step = random.choice([2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 7, 7])
        return [begin, begin + step]

    @classmethod
    def generator_init(cls, t_id: (str, ObjectId), t_name: str, create_date: datetime.datetime) -> list:
        """
        根据一个老师的id,名字和创建时间.生成3个虚拟的老师,加上原始的真实老师,一起返回(4个doc)
        :param t_id:
        :param t_name:
        :param create_date:
        :return:
        """
        t_id = ObjectId(t_id) if isinstance(t_id, str) and len(t_id) == 24 else t_id
        """原始的老师的init字典"""
        raw = {
            "_id": t_id,  # 老师id
            "name": t_name,
            "direction": "raw",
            "head_img": cls.pop_head_image(),
            "create_date": create_date,
            "native": True,  # 真实老师
            "lots_range": cls.generator_lots_range(),  # 手数范围,和性格有关
            "deposit": 0.0,     # 当前存款,初始资金,都是0
            "deposit_amount": 0.0,     # 历史存款累计，用于计算盈利率
            "profit_ratio": 0.0,   # 盈利率, 每次close时候计算,profit_amount/deposit得出.百分比数值
            "profit_amount": 0,        # 盈利总额.每次close时候计算,初始0,
            # "from_id": t_id,  # 发源老师id, 主要是名字来源,没有其他方面的意义

        }
        """正向"""
        follow = {
            "_id": ObjectId(),  # 老师id
            "name": "{}_follow".format(t_name),
            "direction": "follow",
            "head_img": cls.pop_head_image(),
            "create_date": create_date,
            "native": False,  # 真实老师
            "lots_range": cls.generator_lots_range(), # 手数范围,和性格有关
            "deposit": 0.0,     # 当前存款,初始资金,都是0
            "deposit_amount": 0.0,  # 历史存款累计，用于计算盈利率
            "profit_ratio": 0.0,   # 盈利率, 每次close时候计算,profit_amount/deposit得出.百分比数值
            "profit_amount": 0,        # 盈利总额.每次close时候计算,初始0,
            "from_id": t_id,  # 发源老师id, 主要是名字来源,没有其他方面的意义
            "direction": "follow"
        }
        """反向"""
        reverse = {
            "_id": ObjectId(),  # 老师id
            "name": "{}_reverse".format(t_name),
            "direction": "reverse",
            "head_img": cls.pop_head_image(),
            "create_date": create_date,
            "native": False,  # 真实老师
            "lots_range": cls.generator_lots_range(),  # 手数范围,和性格有关
            "deposit": 0.0,  # 当前存款,初始资金,都是0
            "deposit_amount": 0.0,  # 历史存款累计，用于计算盈利率
            "profit_ratio": 0.0,  # 盈利率, 每次close时候计算,profit_amount/deposit得出.百分比数值
            "profit_amount": 0,  # 盈利总额.每次close时候计算,初始0,
            "from_id": t_id,  # 发源老师id, 主要是名字来源,没有其他方面的意义
            "direction": "reverse"
        }
        """随机"""
        rand = {
            "_id": ObjectId(),  # 老师id
            "name": "{}_random".format(t_name),
            "direction": "random",
            "head_img": cls.pop_head_image(),
            "create_date": create_date,
            "native": False,  # 真实老师
            "lots_range": cls.generator_lots_range(),  # 手数范围,和性格有关
            "deposit": 0.0,  # 当前存款,初始资金,都是0
            "deposit_amount": 0.0,  # 历史存款累计，用于计算盈利率
            "profit_ratio": 0.0,  # 盈利率, 每次close时候计算,profit_amount/deposit得出.百分比数值
            "profit_amount": 0,  # 盈利总额.每次close时候计算,初始0,
            "from_id": t_id,  # 发源老师id, 主要是名字来源,没有其他方面的意义
            "direction": "random"
        }
        return [raw, follow, reverse, rand]

    @classmethod
    def rebuild(cls) -> None:
        """
        从交易信号中，初始化老师，这个动作会清除所有的老师记录.
        仅在初始化时候使用。
        :return:
        """
        t = datetime.datetime.now()
        t = t - datetime.timedelta(days=90)
        f = {"create_time": {"$gte": t}}
        s = [("create_time", -1)]
        p = ['creator_id', 'creator_name', 'create_time']
        ses = mongo_db.get_conn(table_name="signal_info")
        args = {"filter": f, "sort": s, "projection": p}
        r = ses.find(**args)
        native = dict()
        for x in r:
            _id = ""
            try:
                _id = x['creator_id']
                name = x['creator_name']
            except KeyError as e:
                # print(e)
                # print(x)
                try:
                    name = x['updater_name']
                except KeyError as e:
                    print(e)
                    print(x)
            if (isinstance(_id, str) and len(_id) == 24) or isinstance(_id, ObjectId):
                _id = _id if isinstance(_id, ObjectId) else ObjectId(_id)
                if _id not in native:
                    native[_id] = name
                else:
                    prev_name = native[_id]
                    if prev_name == name:
                        pass
                    else:
                        ms = "新旧老师名字不一致：prev={}, now={}".format(prev_name, name)
                        raise ValueError(ms)
            else:
                ms = "异常的_id：{}".format(_id)
                print(ms)
        print(native)

        res = list()
        now = datetime.datetime.now()
        for k, v in native.items():
            res.extend(cls.generator_init(t_id=k, t_name=v, create_date=now))
        client = mongo_db.get_client()
        t = client[mongo_db.db_name][cls.get_table_name()]
        with client.start_session(causal_consistency=True) as ses:
            with ses.start_transaction():
                t.delete_many(filter=dict(), session=ses)
                t.insert_many(documents=res, session=ses)

    @classmethod
    def direction_map(cls, include_raw: bool = True) -> dict:
        """
        获取以方向为key,老师的doc的list为val的字典.
        用于在生成虚拟信号的时候提供随机特性
        : param include_raw: 是否包含原生的方向.
        :return:
        """
        f = {"create_date": {"$gt": mongo_db.get_datetime_from_str("2018-9-3 0:0:0")}}
        ts = Teacher.find_plus(filter_dict=f, to_dict=True)
        t_dict = dict()  # 方向和老师的字典
        for t in ts:
            d = t.get("direction")
            if include_raw:
                if d is not None:
                    if d in t_dict:
                        t_dict[d].append(t)
                    else:
                        t_dict[d] = [t]
            else:
                if d is not None and d != "raw":
                    if d in t_dict:
                        t_dict[d].append(t)
                    else:
                        t_dict[d] = [t]
        return t_dict

if __name__ == "__main__":
    Teacher.rebuild()
    pass


from geometry import *
from physical import *
from ydata_profiling import ProfileReport
from pathlib import Path
import sweetviz as sv
import os
import seaborn as sns
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.colors import ListedColormap, Normalize

def list2str(lst):
    if(isinstance(lst, list)):
        num_str = ''.join(str(num) for num in lst)
        return num_str
    else:
        return lst

def one_hot_encode(lst):
    num = 0
    for i in range(6):
        if(lst[i]): num += 2**i
    return num

def color_map(data):
    fig = plt.figure()
    ax = sns.heatmap(data, cmap='coolwarm')
    # 设置标题、坐标轴标签、颜色条标签等
    plt.title('Correlation Heatmap')
    plt.xlabel('X Label')
    plt.ylabel('Y Label')
    # plt.colorbar(label='Correlation')
    ax.set_xticklabels(ax.get_xticklabels(), rotation=-45)
    plt.tight_layout()
    fig.savefig("data_process/picture/output.svg")
    plt.show()

def plotegv(all_df):
    eg_df = all_df[['E11','G12']]
    egv_df = all_df[['E11','G12','V12']]
    corr_matrix = eg_df.corr()
    # 创建图表和子图
    fig = plt.figure(figsize=(12, 4))

    # 创建第一个子图
    ax1 = fig.add_subplot(231)
    ax1.scatter(egv_df['E11'], egv_df['G12'], s=3, alpha=0.2)
    ax1.set_xlabel('E11')
    ax1.set_ylabel('G12')

    # 创建第二个子图
    ax2 = fig.add_subplot(232)
    ax2.scatter(egv_df['E11'], egv_df['V12'], s=3, alpha=0.2)
    ax2.set_xlabel('E11')
    ax2.set_ylabel('V12')

    # 创建第三个子图
    ax3 = fig.add_subplot(233)
    ax3.scatter(egv_df['G12'], egv_df['V12'], s=3, alpha=0.2)
    ax3.set_xlabel('G12')
    ax3.set_ylabel('V12')

    # ax4 = fig.add_subplot(234)
    # ax4.scatter(all_df['E11'], all_df['Thickness'], s=3, alpha=0.2)
    # ax4.set_xlabel('E11')
    # ax4.set_ylabel('thickness')

    ax5 = fig.add_subplot(235)
    ax5.scatter(all_df['E11'], all_df['RelDensity'], s=3, alpha=0.2)
    ax5.set_xlabel('E11')
    ax5.set_ylabel('RelDensity')

    ax6 = fig.add_subplot(236)
    ax6.scatter(all_df['V12'], all_df['RelDensity'], s=3, alpha=0.2)
    ax6.set_xlabel('V12')
    ax6.set_ylabel('RelDensity')
    # 调整子图间距
    fig.subplots_adjust(wspace=0.4, hspace=0.4)
    plt.show()

def plotegv3D(all_df):
    egv_df = all_df[['E11', 'G12', 'V12']]

    # 创建三维散点图
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.scatter(egv_df['E11'], egv_df['G12'], egv_df['V12'], s=3, alpha=0.2)

    # 添加坐标轴标签
    ax.set_xlabel('E11')
    ax.set_ylabel('G12')
    ax.set_zlabel('V12')

    # 显示图形
    plt.show()

csv_file_path = 'data_process/all_data_diff_thick.csv'
if os.path.exists(csv_file_path):
    all_df = pd.read_csv(csv_file_path)
    row = all_df.iloc[0]
else:
    geo_df = read_geo_file()
    phy_df = read_phy_files()
    # find the geo data of every phy_data_frame

    geo_dict = {"ID":[], "Type":[], "Fp":[]}
    phy_drop = []
    # 注意，物理属性里面的名字比几何信息的多一个S
    for row in phy_df.itertuples(index=True):
        name = (row.Name).rstrip('S')
        geo_data = geo_df.loc[geo_df['Name'] == name]
        if(geo_data.empty):
            phy_drop.append(row.Index)
            # print(row)
        else:
            geo_dict["ID"].append(geo_data['ID'].values[0])
            geo_dict["Type"].append((geo_data['Type'].values[0]))
            # geo_dict["Type"].append(one_hot_encode(geo_data['Type'].values[0])) #encode to int
            geo_dict["Fp"].append(geo_data['Fp'].values[0])

    new_df = pd.DataFrame(geo_dict)
    phy_df = phy_df.drop(phy_drop, axis=0)
    phy_df = phy_df.reset_index(drop=True)
    all_df = pd.concat([phy_df, new_df], axis=1)
    all_df.to_csv(csv_file_path, index=False)

breakpoint()
plotegv(all_df)
# profile = ProfileReport(all_df, title="Data Report", explorative=True, minimal=True)

# 创建一个数据集比较器
# all_df = all_df.applymap(list2str)

# my_report = sv.compare([all_df, "Original"], [all_df, "New"])
# my_report.show_html(filepath="data_process/big_data_report.html")

# profile = ProfileReport(all_df, title="Data Report", explorative=True)
# profile.to_file(Path("data_process/big_data_report.html"))

'''
num_df = all_df.drop(columns=['Name','ID','Fp','Type'])
corr_matrix = num_df.corr()
color_map(corr_matrix)

profile = ProfileReport(num_df, title="Data Report", explorative=True)
profile.to_file(Path("data_process/big_data_report.html"))
image_ids = profile.get_unique_id()
breakpoint()
'''


import json
import os
import re
import numpy as np
import webview

from pysom.graph import GraphCompileError
from pysom.nodes.calibrate import Calibrate
from pysom.nodes.som import SOM
import traceback
from collections import Counter, defaultdict

# Model training service

class SomViewService:
    def __init__(self, ds):
        self.ds = ds
        self.input_key = None
        self.som = None
        self.cal = None

        self.links = None
        self.nodes = None

        self.palette = np.array([[0.12156863, 0.46666667, 0.70588235],
                                 [0.68235294, 0.78039216, 0.90980392],
                                 [1., 0.49803922, 0.05490196],
                                 [1., 0.73333333, 0.47058824],
                                 [0.17254902, 0.62745098, 0.17254902],
                                 [0.59607843, 0.8745098, 0.54117647],
                                 [0.83921569, 0.15294118, 0.15686275],
                                 [1., 0.59607843, 0.58823529],
                                 [0.58039216, 0.40392157, 0.74117647],
                                 [0.77254902, 0.69019608, 0.83529412],
                                 [0.54901961, 0.3372549, 0.29411765],
                                 [0.76862745, 0.61176471, 0.58039216],
                                 [0.89019608, 0.46666667, 0.76078431],
                                 [0.96862745, 0.71372549, 0.82352941],
                                 [0.49803922, 0.49803922, 0.49803922],
                                 [0.78039216, 0.78039216, 0.78039216],
                                 [0.7372549, 0.74117647, 0.13333333],
                                 [0.85882353, 0.85882353, 0.55294118],
                                 [0.09019608, 0.74509804, 0.81176471],
                                 [0.61960784, 0.85490196, 0.89803922]])
        self.num_colors = len(self.palette)

    def generate_nodes(self):
        def rgb2hex(x):
            return "#{:02x}{:02x}{:02x}".format(
                *np.clip(np.array((x * 255.0), dtype=np.int32), 0, 255))

        def fmt(x):
            return ','.join([f'{i}' for i in x.keys()])

        def mix(w, m):
            return np.sum(m * w[:, np.newaxis], axis=0)

        labelmap = None
        if self.cal is not None:
            labelmap = {k[0] * self.som.size + k[1]: v for k,
                        v in self.cal.get_output(slot=1).items()}
        else:
            return [{'id': i} for i in range(self.som.size ** 2)]

        cmap = np.array([self.palette[i % self.num_colors]
                        for i in range(len(labelmap))])
        logit = self.cal.logit()
        # self.nodes = [{
        #     'id': i,
        #     'l': fmt(labelmap.get(i, Counter())),
        #     'c': rgb2hex(colored.get(i, np.array([0.1, 0.1, 0.1])))
        # } for i in range(self.som.size ** 2)]

        self.nodes = [{
            'id': i,
            'l': fmt(labelmap.get(i, Counter())),
            'c': rgb2hex(mix(logit[i], cmap))
        } for i in range(self.som.size ** 2)]

    def rect_links(self):
        dim = self.som.size
        w = self.som.get_weights()
        links = []
        for y in range(dim):
            for x in range(dim - 1):
                row = y * dim
                l = {'source': row + x, 'target': row + x + 1,
                     'value': np.linalg.norm(w[row + x] - w[row + x + 1]) * 100}
                links.append(l)

        for y in range(dim - 1):
            for x in range(dim):
                row = y * dim
                l = {'source': row + x, 'target': row + x + dim,
                     'value': np.linalg.norm(w[row + x] - w[row + x + dim]) * 100}
                links.append(l)

        return links

    def set_input(self, key):
        self.input_key = key

        data = self.ds.get_object_data(self.input_key)
        if data is None:
            return {'status': False, 'msg': 'Input data does not exist.'}

        if isinstance(data, SOM):
            self.som = data
        elif isinstance(data, Calibrate):
            self.cal = data
            self.som = self.cal.get_input()
        else:
            return {'status': False, 'msg': 'Input data is neither a SOM node or Calibrate node.'}

        self.links = self.rect_links()

        try:
            self.generate_nodes()
        except:
            return {'status': False, 'msg': traceback.format_exc()}

        return {'status': True, 'msg': key}

    def get_som_viz_data(self):
        if self.nodes is None or self.links is None:
            return {'status': False, 'msg': 'Not available'}

        return {'status': True, 'msg': 'Ok', 'obj': {'links': self.links, 'nodes': self.nodes}}


import logging
from math import ceil
from pathlib import Path
from typing import Generator, List

from django.db import models
from django.conf import settings
from django.template import engines
from django.db.models import QuerySet
from django.core.validators import MinLengthValidator
from django.utils.translation import ugettext_lazy as _

from commons.models import UserCreatedDatetimeModel
from staticsites.models import IndexPage, StaticPageBase

logger = logging.getLogger(__name__)

class NewsPage(StaticPageBase):
    index = models.OneToOneField(
        IndexPage,
        on_delete=models.CASCADE,
    )

    def get_total_pages(self, items_per_page: int = 10) -> int:
        return ceil(NewsItem.objects.filter(is_published=True).order_by('-publish_on').count() / items_per_page)

    def get_newsitems_pages(self, items_per_page: int = 10) -> Generator[QuerySet, None, None]:
        qs = NewsItem.objects \
            .filter(
                newspage=self,
                is_published=True,
            )\
            .order_by('-publish_on')

        for page_number in range(self.get_total_pages(items_per_page)):
            offset = items_per_page * page_number
            yield qs[offset: offset + items_per_page]

    def get_latest_n_published(self, n: int = 6) -> QuerySet:
        return NewsItem.objects.filter(is_published=True).order_by('-publish_on')[:n]

    def instantiate(self, root_directory: Path, items_per_page: int = settings.NEWS_ITEMS_PER_PAGE) -> List[dict]:
        """
        Create instantiated HTML and images in given root_directory
        """
        # prepare template
        django_engine = engines['django']
        template = django_engine.from_string(self.template)

        result_page_data = []
        for page_count, page_queryset in enumerate(self.get_newsitems_pages(items_per_page)):
            page_numbered_filename = str(self.filename.format(page_count))
            page_newsitems = list(page_queryset)
            relative_filepath = Path(str(self.relative_path), page_numbered_filename)
            absolute_filepath = root_directory / relative_filepath

            # make directories for target news html file
            logger.info(f'Writing ({relative_filepath}) to {root_directory} ...')
            absolute_filepath.parent.mkdir(parents=True, exist_ok=True)
            with absolute_filepath.open('w', encoding='utf8') as html_out:
                html = template.render(context={
                    self.index.newsitems_template_variablename: page_newsitems
                })
                html_out.write(html)

            # instantiate newsitem.images to root_directory
            for newsitem in page_newsitems:
                image_relative_filepath = Path(str(newsitem.image_relpath), str(newsitem.image.name))
                image_absolute_filepath = root_directory / image_relative_filepath
                image_absolute_filepath.parent.mkdir(parents=True, exist_ok=True)
                logger.info(f'Writing ({image_relative_filepath}) to {root_directory} ...')
                with newsitem.image.open('rb') as image_out, image_absolute_filepath.open('wb') as image_in:
                    image_in.write(image_out.read())

            page_data = {
                'relative_path': relative_filepath,
                'absolute_path': absolute_filepath,
                'filename': page_numbered_filename,
                'page_count': page_count
            }
            result_page_data.append(page_data)
        return result_page_data

    def save(self, *args, **kwargs):
        self.filename = 'news_{}.html'  # TODO: May want multiple pages here
        self.relative_path = 'news'
        self.type = 'news'  # auto-populate 'type' field
        super().save(*args, **kwargs)

class NewsItem(UserCreatedDatetimeModel):
    newspage = models.ForeignKey(
        NewsPage,
        on_delete=models.CASCADE
    )
    publish_on = models.DateTimeField()
    is_published = models.BooleanField(
        default=False
    )
    image = models.ImageField(
        null=True,
        blank=True,
    )
    image_relpath = models.CharField(
        max_length=250,
        default='imgs/news'
    )
    title = models.CharField(
        max_length=150,
        help_text=_('News Item Title')
    )
    text = models.TextField(
        validators=[MinLengthValidator(limit_value=10)]
    )


"""EC2 Tag Auditing

This purpose of this script is to provide a working example
of how to audit tags and enforce compliance through serverless
Lambda functions.

This script focuses on enforcing tagging policies. It checks for
required tags on all EC2 instances, checks for appropriate values,
will set tags on first offense and send reminder, then will destroy
instances if they continue to be noncompliant.
"""
from __future__ import print_function
from datetime import datetime, timedelta
import fnmatch
import json
import boto3

client = boto3.client('ec2')

def handler(event, context):

    ### PARAMETERS ###

    # time resources can be non-compliant in days before termination
    noncompliant_threshold = 7

    # required tags, and allowed values
    # allowed values use fnmatch/regex
    required_tags = {
        'environment': [
            'dev',
            'prod',
            'stage'
        ],
        'owner': ['*@gmail.com']
    }

    ### BEGIN AUDIT ###

    # audit resources
    noncompliant_instances = []
    compliant_instances = []
    for instance in get_instance_details(client):
        if not audit_tags(instance['tags'], required_tags):
            noncompliant_instances.append(instance)
        else:
            compliant_instances.append(instance)

    # get records broken down by tasks: terminate, set expiration, terminate in future
    assessed = parse_compliance_status(noncompliant_instances)

    # tag expiration dates
    if assessed['set_expiration']:
        expiration_date = datetime.now() + timedelta(days=noncompliant_threshold)
        client.create_tags(
            Resources=assessed['set_expiration'],
            Tags=[
                {
                    'Key': 'expiration',
                    'Value': expiration_date.strftime("%Y-%m-%d %H:%M:%S")
                }
            ]
        )

    # terminate instances that aren't compliant
    if assessed['terminate']:
        client.terminate_instances(
            InstanceIds=assessed['terminate']
        )

    print("Compliant: " + json.dumps(compliant_instances))
    print("Noncompliant: " + json.dumps(noncompliant_instances))
    print("Set Expiration Date: " + json.dumps(assessed['set_expiration']))
    print("To Terminate Soon: " + json.dumps(assessed['terminate_soon']))
    print("To Terminate: " + json.dumps(assessed['terminate']))

    #print(noncompliant_instances)
    print("Function Complete")

def get_instance_details(ec2_client):
    """ Get Instance Data

    Retrieve and compile just the data we need from EC2 instances
    """

    # find all instances that are not terminated
    response = ec2_client.describe_instances(
        Filters=[
            {
                'Name': 'instance-state-name',
                'Values': [
                    'running',
                    'shutting-down',
                    'stopped',
                    'stopping',
                    'pending'
                ]
            }
        ]
    )

    # aggregate results into easy format to work with
    results = []
    for instance in response['Reservations'][0]['Instances']:
        results.append({
            'id': instance.get('InstanceId'),
            'tags': instance.get('Tags', {})
        })

    return results

def parse_compliance_status(noncompliant_records):
    """Parse Compliance Status

    Assess non compliant record and determine
    whether they need to be terminated, future terminated,
    or expiration date set
    """
    to_terminate = []
    to_set_expiration = []
    to_terminate_soon = []

    for instance in noncompliant_records:
        if 'tags' in instance:
            tag_keys = [t['Key'] for t in instance['tags']]
            if node_first_offense(tag_keys):
                to_set_expiration.append(instance['id'])
            else:
                for tag in instance['tags']:
                    if tag['Key'] == 'expiration':
                        expiration = tag['Value']
                        break

                if node_past_due(expiration):
                    to_terminate.append(instance['id'])
                else:
                    to_terminate_soon.append(instance['id'])
        else:
            to_set_expiration.append(instance['id'])

    return {
        'terminate': to_terminate,
        'terminate_soon': to_terminate_soon,
        'set_expiration': to_set_expiration
    }

def audit_tags(given_tags, required_tags):
    """Audit All Tags

    Assesses the tags in as much detail as possible to reach a conclusion
    about the current resource.
    """
    # validate the keys of tags first
    if validate_tag_keys([t['Key'] for t in given_tags], required_tags.keys()):
        # validate values of each tag
        for tag in given_tags:
            # if tag exist in required_tags, lets check values
            if required_tags.get(tag['Key']):
                return validate_tag_value(tag['Value'], required_tags.get(tag['Key']))
    else:
        return False

def validate_tag_value(given_value, accepted_values):
    """Audit Tag Values

    Will assess the value passed in and determine if it meets the
    requirements that are contained in the `required_tags` array at
    the top of the script

    given_value: string
        The value of the tag we are auditing

    accepted_values: list
        All of the accepted values that are accepted, wildcards included
    """
    for value in accepted_values:
        if fnmatch.fnmatch(given_value, value):
            return True

    return False

def validate_tag_keys(given_tag_keys, required_tag_keys):
    """Validate Set Tags

    Audits the keys provided by the resource and compares them to
    the required tags.

    given_tag_keys: list

    required_tag_keys: list
    """
    return required_tag_keys == list(set(given_tag_keys).intersection(required_tag_keys))

def node_first_offense(given_tags):
    """Node First Offense

    If expiration date hasn't been previously set, retrun True
    """
    return 'expiration' not in given_tags

def node_past_due(expiration_date):
    """Node Past Due

    If an expiration date is set on a node, is it past due.
    If it is past due return true

    Timestamp example: 2016-12-22 09:44:41
    """
    return datetime.now() > datetime.strptime(expiration_date, "%Y-%m-%d %H:%M:%S")


#/usr/bin/env python
#-*- coding:utf-8 -*-

import logging

from datetime import datetime

import threading

import os

from Common import ReadConfig

class Log:

    def __init__(self):

        global logPath,resultPath,rootPath,reportPath

        rootPath = ReadConfig.rootDir
        resultPath = os.path.join(rootPath,"resultReport")

        logPath = os.path.join(resultPath,str(datetime.now().strftime("%Y年%m月%d日%H时%M分%S秒")))

        reportPath = os.path.join(logPath,"reprot.html")

        logInfoPath = os.path.join(logPath,"out.txt")
        if not os.path.exists(resultPath):
            os.mkdir(resultPath)
        if not os.path.exists(logPath):
            os.mkdir(logPath)
        self.logger = logging.getLogger()

        self.logger.setLevel(logging.INFO)

        handler = logging.FileHandler(logInfoPath)

        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

        handler.setFormatter(formatter)

        self.logger.addHandler(handler)
    def getLogger(self):
        return self.logger
    def getReprotPath(self):
        return reportPath
    def getResultPath(self):
        return resultPath

class MyLog:
    log = None
    mutext = threading.Lock()
    def __init__(self):
        pass
    @staticmethod
    def get_log():
        if MyLog.log is None:
            MyLog.mutext.acquire()
            MyLog.log = Log()
            MyLog.mutext.release()
        return MyLog.log


from main import app
from fastapi.testclient import TestClient

client = TestClient(app)

url = "/properties"

def test_get_properties_success():
    """
    Test if get properties endpoint return success status code
    """
    response = client.get(url)
    assert response.status_code == 200

def test_get_properties_list():
    """
    Test if get properties endpoint return a list
    """
    response = client.get(url)
    assert response.status_code == 200
    assert type(response.json()) == list

def test_get_properties_year_param():
    """
    Test if get properties endpoint is successful
    setting the year param
    """
    year_param = {
        "year": 2000
    }
    response = client.get(url, params=year_param)
    assert response.status_code == 200
    assert type(response.json()) == list

def test_get_properties_city_param():
    """
    Test if get properties endpoint is successful
    setting the city param
    """
    city_param = {
        "city": "bogota"
    }
    response = client.get(url, params=city_param)
    assert response.status_code == 200

def test_get_properties_multiple_params():
    """
    Test if get properties endpoint is successful
    setting the multiple params
    """
    params = {
        "year": 2000,
        "city": "bogota"
    }
    response = client.get(url, params=params)
    assert response.status_code == 200

def test_get_properties_invalid_param():
    """
    Test if get properties endpoint is a bad
    request setting an invalid param
    """
    invalid_param = {
        "state": "mexico"
    }
    response = client.get(url, params=invalid_param)
    assert response.status_code == 400

def test_get_properties_error_detail():
    """
    Test if get properties endpoint
    return the detail error response
    """
    invalid_param = {
        "state": "mexico"
    }
    error_detail = {
        "detail": "The state field does not "
                  "exist in db thus this "
                  "parameter is not valid"
    }
    response = client.get(url, params=invalid_param)
    assert response.status_code == 400
    assert response.json() == error_detail


# #Brute Force
# cost=[10,15,20]

# def minCostClimbingStairs1(cost):
#     n=len(cost)
#     cost.append(0)

#     return minCost1(n,cost)

# def minCost1(n,cost):
#     if n<0:
#         return 0
#     elif n==0:
#         return cost[n]
#     elif n==1:
#         return cost[n]

#     return cost[n]+min(minCost1(n-1,cost),minCost1(n-2,cost))

# print(minCostClimbingStairs1(cost))

# DP
# cost=[10,15,20]

# def minCostClimbingStairs2(cost):
#     minCost=[]
#     minCost.append(cost[0])
#     minCost.append(cost[1])
#     cost.append(0)
#     for i in range(2,len(cost)):
#         minCost.append(0)

#     for i in range(2,len(cost)):
#         minCost[i]=cost[i]+min(minCost[i-1],minCost[i-2])

#     return minCost[len(minCost)-1]

# print(minCostClimbingStairs2(cost))

#Memorization
# cost=[10,15,20]

# def minCostClimbingStairs3(cost):
#     n=len(cost)
#     dp=[None]*len(cost)

#     return min(minCost3(n-1,cost,dp),minCost3(n-2,cost,dp))

# def minCost3(n,cost,dp):
#     if n<0:
#         return 0
#     elif n==0:
#         dp[n]=cost[n]
#         return dp[n]
#     elif n==1:
#         dp[n]=cost[n]
#         return dp[n]
#     if dp[n]!=None:
#         return dp[n]

#     dp[n]=cost[n]+min(minCost3(n-1,cost,dp),minCost3(n-2,cost,dp))
#     return dp[n]

# print(minCostClimbingStairs3(cost))

#DP
cost=[10,15,20]

def minCostClimbingStairs4(cost):
    minCost=[]
    minCost.append(cost[0])
    minCost.append(cost[1])

    for i in range(2,len(cost)):
        minCost.append(cost[i]+min(minCost[0],minCost[1]))
        minCost.pop(0)

    return min(minCost[0],minCost[1])

print(minCostClimbingStairs4(cost))


import os
import logging
import time
import sys

from postgresDbloader import dbLoader

import propertyReader as pr

def write_2_qli_app_name_tab_qvd_name(str):
    pass

def print_hi(name):
    # Use a breakpoint in the code line below to debug your script.
    print(f'Hi, {name}')  # Press Ctrl+F8 to toggle the breakpoint.

    config = pr.getPropInstance('config/', 'NLP.properties')
    __prop_dict = dict(config.items('log'))
    logFile = __prop_dict["logfilepath"] + sys.argv[0].split('\\')[-1] + "_" + os.getlogin() + "_" + time.strftime(
        '%Y%m%d%H%M%S') + '.log'

    log_format = '%(asctime)s - [%(levelname)s] - [%(filename)s->%(funcName)s():%(lineno)d] - %(message)s'
    log_date_format = '[%m/%d/%Y %I:%M:%S %p]'

    logging.basicConfig(
        level=logging.INFO
        # , filename = logFile
        , format=log_format
        , datefmt=log_date_format
    )

    ch = logging.StreamHandler(sys.stdout)
    logging.getLogger().addHandler(ch)

    tabs = {}
    qvd_dict = {}
    new_tabs = {}

    path = "D:\\Users\\georgj3\\Documents\\MRO_QVD"

    sql = open("D:\\Users\\georgj3\\Documents\\sap.sql", "w", encoding="utf8")

    qli_app_name_tab_qvd_name = open("D:\\Users\\georgj3\\Documents\\qli_app_name_tab_qvd_name11.sql", "w",
                                     encoding="utf8")

    lists_path = "D:\\Users\\georgj3\\Documents\\mro_qvdtest.txt"

    """ creating db loader

    """
    ldbLoader = dbLoader(__prop_dict)
    ldbLoader.init_db_conn()

    """ truncate table """
    # ldbLoader.truc_qlik_apps_info()
    # ldbLoader.truc_QVD_tab_columns()

    with open(lists_path, "r", encoding="utf8")  as f:
        filenames = f.readlines()
    for filename in filenames:
        filename = filename.strip() + ".txt"
        # for filename in os.listdir(path):
        # logging.info("  {}  ".format(filename))
        tabs = {}

        with open(os.path.join(path, filename), "r", encoding="utf8")  as f:
            lines = f.readlines()

        logging.info("--GEO--C-- file name {}\n".format(filename))
        sql.write("--GEO--C-- file name {}\n".format(filename))
        for l in lines:

            # print(0, l.split("///$tab"))
            temp = l.split("///$tab")
            for ll in temp:
                # print(ll)
                tabs[ll.split("\\r\\n")[0]] = '\n'.join(ll.split("\\r\\n")[1:])
        """loop thru each tabs
        """
        tmpe_tab = {}

        """
        limit to only one TAB
        """
        new_qlik_tabs = {}
        for key, value in tabs.items():
            # print("KEY = " + key  + " " )
            # tab_list = ['TIDPOLIN ALL REVISIONS', "TID WOOOS"]

            # if key.strip() in tab_list :

            lines_in_tabs = value.split("\n")

            new_lines_intab = []
            for lintab in lines_in_tabs:
                if not lintab.strip().startswith("//") and not lintab.strip() == "":
                    new_lines_intab.append(lintab)
            new_qlik_tabs[key.strip()] = '\r\n'.join([str(elem) for elem in new_lines_intab])

        for key, value in new_qlik_tabs.items():

            inc = 0
            # if ("MDM" in key or "MDM" in value) and "(qvd)" in value:
            # logging.info("Valeue before cleanup  {}".format(value[:30]))
            cleaned_value = pr.cleanup_fld(value)
            logging.info("Value after cleanup  {}".format(cleaned_value[:20]))

            qvds = cleaned_value.split("LIB ")  # Split by LIB
            logging.info("key: {}  has 4 Select FROM clauses   {}".format(key, len(qvds)))

            for qvd in qvds:  ## loop thru each LIB
                inc = inc + 1

                qvd_load = qvd
                # replace the 1st \r\r

                qvd_script_lines = qvd.strip().split("\r\n")
                if len(qvd_script_lines) > 1:

                    """
                    Our pattenr will be LIB CONNECT connectionname;
                    qvdtablename:
                    Load

                    Select .....
                    from
                    """
                    semicolondelimetedline = qvd_load.split(";")
                    for line in semicolondelimetedline:
                        # get QVD NAME
                        if "vQVDTableFileName" in line:
                            qlik_qvd_name = line.split(' vQVDTableFileName ')[-1].replace('=', '').strip().replace("'",
                                                                                '')
                            logging.info("qvd_name = {}".format(qlik_qvd_name))
                        # check for connect first and get connection name
                        if "CONNECT" in line.strip():
                            connection_name = line.strip().split(' ')[-1].strip().replace("'", '')
                            logging.info("Conectionname = {}".format(connection_name))
                        elif "SELECT " in line.upper().strip():  # split by from

                            from_stat = line.upper().split('FROM ')
                            # if "WHERE" in from_stat[1].upper():
                            db_tab_name = from_stat[1].upper().split('WHERE')[0].replace('\\\"', "").strip()

                            logging.info("Select  = {}\n  from tab = {} ".format(from_stat[0][:30], db_tab_name))
                            asd = 100

                            selectcolumns = pr.load_qvd_build_info(from_stat[0].replace("\\t", "\t"))
                            # for loop to fire insert
                            # create table qvd_tab_columns(qlik_app_name varchar(100), qlik_tab_name varchar(200), qlik_conn_name varchar(200), db_tab_name varchar(200), db_col_name varchar(200))
                            logging.info(
                                "qlik_conn_name= {}; qlik_qvd_name = {}; db_tab_name = {}; column count = [{}] {}".format(
                                    connection_name, qlik_qvd_name, db_tab_name, len(selectcolumns), key))
                            for column in selectcolumns:
                                column_cleaned = column
                                for rep in ['\\\"', ',']:
                                    column_cleaned = column_cleaned.replace(rep, "")
                                logging.info(
                                    "qlik_app_name= {}; qlik_tab_name = {} qlik_conn_name= {}; qlik_qvd_name = {}; db_tab_name = {}; db_col_name= {} ".format(
                                        filename.replace(".txt", "")
                                        , key, connection_name, qlik_qvd_name, db_tab_name, column_cleaned[:20])
                                )
                                ldbLoader.load_qvd_tab_columns(filename.replace(".txt", ""), key, connection_name,
                                                               qlik_qvd_name, db_tab_name, column_cleaned)

    sql.close()
    qli_app_name_tab_qvd_name.close()

# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    print_hi('PyCharm')


__author__ = 'azeem'

import unittest
from nose.tools import *
from CitationParser import *
import os

#sample item to be cited
sample = {
"id": "ITEM-5",
"title":"Boundaries of Dissent: Protest and State Power in the Media Age",
"author": [
{
"family": "D'Arcus",
"given": "Bruce",
"static-ordering": False
}
],
"publisher": "Routledge",
"publisher-place": "New York",
"issued": {
"date-parts":[[2006]]
},
"type": "book",
"URL": "website.com"
}

class TestConversion(unittest.TestCase):

    def test_JSON_to_harvard1_plain(self):
        expected_output = open("/home/azeem/PycharmProjects/Citations/fixtures/harvard1_plaintext_output.txt").read()
        bibtex_intermediate = to_CSL(sample,os.path.join(CSL_PATH, 'harvard1.csl'), formatter.plain)
        self.assertEqual(bibtex_intermediate, expected_output)

    def test_JSON_to_harvard1_html(self):
        expected_output = open("/home/azeem/PycharmProjects/Citations/fixtures/harvard1_html_output.txt").read()
        bibtex_intermediate = to_CSL(sample,os.path.join(CSL_PATH, 'harvard1.csl'), formatter.html)
        self.assertEqual(bibtex_intermediate, expected_output)

    def test_JSON_to_bibtex(self):
        expected_output = open("/home/azeem/PycharmProjects/Citations/fixtures/bibtex_intermediate_output.txt").read()
        bibtex_intermediate = to_CSL(sample,os.path.join(CSL_PATH, 'bibtex.csl'), formatter.plain)
        self.assertEqual(bibtex_intermediate, expected_output)

    def test_bibtex_to_endnote(self): #fails due to unicode 'u' at beginning
        expected_output = open("/home/azeem/PycharmProjects/Citations/fixtures/endnote_output.txt").read()
        endnote_final = to_final('xml2end', sample)
        self.assertEqual(endnote_final.strip(), expected_output.strip())

    def test_bibtex_to_ris(self):
        expected_output = open("/home/azeem/PycharmProjects/Citations/fixtures/ris_output.txt").read()
        ris_final = to_final('xml2ris', sample)
        self.assertEqual(ris_final.strip(), expected_output.strip())

#print (sample)

if __name__ == '__main__':
    unittest.main()


import random

options = ["rock", "paper", "scissors"]
user_wins = 0
computer_wins = 0

def score():
    print("User wins: ", user_wins, ", Computer wins: ", computer_wins)

def main():
    global user_wins
    global computer_wins
    while True:
        user_input = input("Type Rock/Paper/Scissors or q to quit: ").lower()
#quit system
        if user_input == "q":
            print("Thank you for playing. ")
            break

        if user_input not in options:
            print("This is not a valid option")
            continue
    #assigning values to rock, paper, scissors. rock = 0, paper = 1, scissors = 2
        random_number = random.randint(0, 2)
        computer_pick = options[random_number]
        user_pick = user_input
        print("Computer picked: ", computer_pick + ".")

        if computer_pick == user_pick:
            print("Draw! Please try play again")
            continue

        if user_input == "rock" and computer_pick == "scissors":
            print("You won!")
            user_wins += 1
            score()
            continue

        elif user_input == "paper" and computer_pick == "rock":
            print("You won!")
            user_wins += 1
            score()
            continue

        elif user_input == "scissors" and computer_pick == "paper":
            print("You won!")
            user_wins += 1
            score()
            continue

        else:
            print("You lost!")
            computer_wins += 1
            score()

main()


from rest_framework.filters import BaseFilterBackend

class LessonParentFilter(BaseFilterBackend):
    def filter_queryset(self, request, queryset, view):
        parent_only = request.query_params.get('parent_only')

        if parent_only and parent_only == '1':
            queryset = queryset.filter(**{
                'parent__isnull': True,
            })

        return queryset


class Rectangle:
    def __init__(self, length, breadth):
        self.__length = length
        self.__breadth = breadth
    def __lt__(self, other):
        if self.__length*self.__breadth < other.__length*other.__breadth:
            print("Area of rectangle 1 is less than area of rectangle 2")
r1 = Rectangle(5,4)
r2 = Rectangle(5,5)
r1 < r2


# -*- coding: UTF-8 -*-

from time import sleep
from dogtail.utils import isA11yEnabled, enableA11y
if not isA11yEnabled():
    enableA11y(True)

from common_steps import App
from dogtail.config import config
import os

def before_all(context):
    """Setup gnome-calculator stuff
    Being executed before all features
    """

    try:
        # Skip dogtail actions to print to stdout
        config.logDebugToStdOut = False
        config.typingDelay = 0.2

        context.app_class = App('gnome-calculator')

    except Exception as e:
        print("Error in before_all: %s" % e.message)

def after_step(context, step):
    try:
        if step.status == 'failed' and hasattr(context, "embed"):
            # Embed screenshot if HTML report is used
            os.system("dbus-send --print-reply --session --type=method_call " +
                      "--dest='org.gnome.Shell.Screenshot' " +
                      "'/org/gnome/Shell/Screenshot' " +
                      "org.gnome.Shell.Screenshot.Screenshot " +
                      "boolean:true boolean:false string:/tmp/screenshot.png")
            context.embed('image/png', open("/tmp/screenshot.png", 'r').read())
    except Exception as e:
        print("Error in after_step: %s" % str(e))

def after_scenario(context, scenario):
    """Teardown for each scenario
    Kill gnome-calculator (in order to make this reliable we send sigkill)
    """
    try:
        # Stop gnome-calculator
        context.app_class.kill()

        # Make some pause after scenario
        sleep(1)
    except Exception as e:
        # Stupid behave simply crashes in case exception has occurred
        print("Error in after_scenario: %s" % e.message)


"""Logging utils for films."""

from app import app
from app.schemas.films import FilmSchema

def log_created_film(film: FilmSchema):
    log_msg = "Created film\n"
    log_msg += "\tID: %s\n"
    log_msg += "\tTitle: %s"

    app.logger.info(log_msg, film.id, film.title)

def log_updated_film(film: FilmSchema):
    log_msg = "Updated film\n"
    log_msg += "\tID: %s\n"
    log_msg += "\tTitle: %s"

    app.logger.info(log_msg, film.id, film.title)

def log_deleted_film(film_id: int):
    log_msg = "Deleted film\n"
    log_msg += "\tID: %s"

    app.logger.info(log_msg, film_id)


"""
Parse TGAM data packet
Following ThinkGear Serial Stream Guide
http://developer.neurosky.com/docs/doku.php?id=thinkgear_communications_protocol
"""

import csv
import os
import shutil
import time
import matplotlib.pyplot as plt

# sets the maximum number of lines in a single csv file
MAX_ROW_PER_FILE = 1000000

SYNC = 0xAA

def parse_packet(byte_list, current_time, generate_graph=False):
  """
  Parse the original packet once
  Hand over to parse_data if packet is valid
    This function takes in a byte list and parse out the first data packet
    if it does not see the data packet starting from first byte,
      it will delete the first byte and return False
  """

  packet = byte_list[:]
  p_length = 0

  try:
    # if first byte not SYNC, delete it from byte_list
    current_byte = packet.pop(0)
    if current_byte != SYNC:
      print("Error: First byte not SYNC")
      print_error_package(packet[:10])
      del byte_list[0]
      # byte_list.clear()
      return False

    # second SYNC
    current_byte = packet.pop(0)
    if current_byte != SYNC:
      print("Error: Second byte not SYNC")
      print_error_package(packet[:10])
      return False

    # packet length
    current_byte = packet.pop(0)
    if current_byte >= SYNC:
      print("Error: PLength byte too large")
      print_error_package(packet[:10])
      return False
    p_length = current_byte
    # print(f"p_length: {p_length}")

    data_packet = packet[:p_length]
    packet = packet[p_length:]

    # calculate checksum
    original_sum = 0
    for byte in data_packet:
      original_sum += byte
    chk_sum = original_sum & 0xFF
    chk_sum = ~chk_sum & 0xFF

    check_sum = packet.pop(0)

  except IndexError:
    if not p_length:
      print("Empty packet")
      return False
    print(f"Error: Index out of bound, p_length = {p_length}")
    print_error_package(packet[:p_length+4])
    return False

  # if checksum alright, parse data_packet
  if check_sum == chk_sum:
    # print("CheckSum is correct, preseed to parse data packet")
    packet_length = len(byte_list) - len(packet)
    del byte_list[:packet_length]
    parse_data(data_packet, current_time, generate_graph)
    return True
  else:
    print("CheckSum is wrong")
    print(f"Original Sum is {hex(original_sum)}")
    print(f"chk_sum is {hex(chk_sum)}")
    print(f"CheckSum byte is {hex(check_sum)}")
    print_error_package(packet[:10])
    return False

EXCODE = 0x55
BATTERY = 0x01
POOR_SIGNAL = 0x02
HEART_RATE = 0x03
ATTENTION = 0x04
MEDITATION = 0x05
RAW_WAVE_8 = 0x06
RAW_MARKER = 0x07

RAW_WAVE = 0x80
EEG_POWER = 0x81
ASIC_EEG_POWER = 0x83
RRINTERVAL = 0x86

index = 0

field_list = ["Index", "Time", "Raw_Wave", "Attention", "Meditation", "Delta", "Theta", "LowAlpha", "HighAlpha", "LowBeta", "HighBeta", "LowGamma", "MidGamma", "Poor_Signal", "Battery"]

# stores data parsed from on packet in a dictionary
graph_data_dict = {}
for field_name in field_list:
  graph_data_dict[field_name] = []

def parse_data(byte_list, current_time, generate_graph=False):
  """Parse data packet"""

  # if not byte_list:
  #   return False

  # fill default value in all fields
  global index
  index += 1
  data_packet = byte_list[:]
  data_dict = {}
  for field_name in field_list:
    data_dict[field_name] = "null"

  data_dict["Time"] = current_time
  data_dict["Index"] = index

  while data_packet:

    current_byte = data_packet.pop(0)
    # count Excode
    excode_level = 0
    while current_byte == EXCODE:
      excode_level += 1
      current_byte = data_packet.pop(0)

    code_byte = current_byte
    if code_byte >= 0x80:
      v_length = data_packet.pop(0)

    if code_byte == BATTERY:
      value = data_packet.pop(0)
      full_battery = 0x7F
      battery_level = int(value / full_battery * 100)
      print(f"Battery Level is {battery_level}%")
      data_dict["Battery"] = battery_level

    elif code_byte == POOR_SIGNAL:
      value = data_packet.pop(0)
      if value == 0:
        print("Good Signal")
      elif value == 200:
        print("WARNING: Off head signal detected")
      else:
        print(f"Signal Quality: {value}/255")
      data_dict["Poor_Signal"] = value

    elif code_byte == HEART_RATE:
      value = data_packet.pop(0)
      print(f"Heart Rate: {value}")

    elif code_byte == ATTENTION:
      value = data_packet.pop(0)
      value_meaning = ""
      if value == 0:
        print("WARNING: Unable to calculate Attention eSense")
      elif 1 <= value < 20:
        value_meaning = "strongly lowered"
      elif 20 <= value < 40:
        value_meaning = "reduced"
      elif 40 <= value < 60:
        value_meaning = "neutral"
      elif 60 <= value < 80:
        value_meaning = "slightly elevated"
      elif 80 <= value <= 100:
        value_meaning = "elevated"
      else:
        print("ERROR: Attention eSense value out of bound (0-100)")
      print(f"Attention eSense value: {value}\t{value_meaning}")
      data_dict["Attention"] = value

    elif code_byte == MEDITATION:
      value = data_packet.pop(0)
      value_meaning = ""
      if value == 0:
        print("WARNING: Unable to calculate Meditation eSense")
      elif 1 <= value < 20:
        value_meaning = "strongly lowered"
      elif 20 <= value < 40:
        value_meaning = "reduced"
      elif 40 <= value < 60:
        value_meaning = "neutral"
      elif 60 <= value < 80:
        value_meaning = "slightly elevated"
      elif 80 <= value <= 100:
        value_meaning = "elevated"
      else:
        print("ERROR: Meditation eSense value out of bound (0-100)")
      print(f"Meditation eSense value: {value}\t{value_meaning}")
      data_dict["Meditation"] = value

    # only available on TGEM, not available on TGAM/TGAM1
    elif code_byte == RAW_WAVE_8:
      value = data_packet.pop(0)
      print(f"8 Bit Raw Wave value: {value}")

    # this value is for debug, always 0
    elif code_byte == RAW_MARKER:
      value = data_packet.pop(0)
      print(f"Raw Maker value: {value}")

    # multi byte values
    elif code_byte == RAW_WAVE:
      value = data_packet[:v_length]
      data_packet = data_packet[v_length:]

      raw = value[0]*256 + value[1]
      if raw >= 32768:
        raw = raw - 65536
      print(f"Raw Wave value: {raw}")
      data_dict["Raw_Wave"] = raw

    # I'm not fucking going to spend time parsing this shit!
    # If you want to use it, do it yourself!
    elif code_byte == EEG_POWER:
      value = data_packet[:v_length]
      data_packet = data_packet[v_length:]
      print(f"EEG Power value: {value}")

    elif code_byte == ASIC_EEG_POWER:
      value = data_packet[:v_length]
      data_packet = data_packet[v_length:]

      def to_int(int_list):
        return int_list[0]*256*256 + int_list[1]*256 + int_list[2]

      delta = value[:3]
      delta_value = to_int(delta)
      print(f"delta value: {delta_value}")
      data_dict["Delta"] = delta_value

      theta = value[3:6]
      theta_value = to_int(theta)
      print(f"theta value: {theta_value}")
      data_dict["Theta"] = theta_value

      low_alpha = value[6:9]
      low_alpha_value = to_int(low_alpha)
      print(f"low_alpha value: {low_alpha_value}")
      data_dict["LowAlpha"] = low_alpha_value

      high_alpha = value[9:12]
      high_alpha_value = to_int(high_alpha)
      print(f"high_alpha value: {high_alpha_value}")
      data_dict["HighAlpha"] = high_alpha_value

      low_beta = value[12:15]
      low_beta_value = to_int(low_beta)
      print(f"low_beta value: {low_beta_value}")
      data_dict["LowBeta"] = low_beta_value

      high_beta = value[15:18]
      high_beta_value = to_int(high_beta)
      print(f"high_beta value: {high_beta_value}")
      data_dict["HighBeta"] = high_beta_value

      low_gamma = value[18:21]
      low_gamma_value = to_int(low_gamma)
      print(f"low_gamma value: {low_gamma_value}")
      data_dict["LowGamma"] = low_gamma_value

      mid_gamma = value[21:24]
      mid_gamma_value = to_int(mid_gamma)
      print(f"mid_gamma value: {mid_gamma_value}")
      data_dict["MidGamma"] = mid_gamma_value

      # print(f"ASIC EEG Power value: {value}")

    elif code_byte == RRINTERVAL:
      value = data_packet[:v_length]
      data_packet = data_packet[v_length:]
      print(f"RRInterval value: {value}")

    else:
      print(f"ERROR: Unexpected code detected: {code_byte}")
      print_error_package(byte_list)

  write_to_csv(data_dict)

  if generate_graph:
    for field,data_list in graph_data_dict.items():
      new_data = data_dict[field]
      if new_data == "null":
        new_data = None
      data_list.append(new_data)

def print_error_package(packet):
  """Helper for printing out package that caused error"""

  print("Error packet: [", end = "")
  try:
    end_byte = packet.pop()

    for byte in packet:
      print(hex(byte), end = ", ")
    print(hex(end_byte), end = "")
    print("]")

  except IndexError:
    print("Error packet: []")

# create a dictionary called parse and store the parse result csv files
directory_name = "parse"
# sets the maximum number of lines in a single csv file
max_row_per_file = 50000
# don't touch this
num_of_file = 0

def write_to_csv(data_dict):
  """Write data to csv file, create new file if reach the max_row_per_file"""
  data_row_num = data_dict["Index"]

  global num_of_file
  filename = os.path.join(directory_name, f"parse_{num_of_file}.csv")

  with open(filename, "a+") as f:
    csv_writer = csv.DictWriter(f, fieldnames=field_list)
    if os.stat(filename).st_size == 0:
      csv_writer.writeheader()

    csv_writer.writerow(data_dict)

  if data_row_num % max_row_per_file == 0:
    num_of_file += 1

def read_from_file(filename):
  """
  Parse a txt file into int list
  Function used when parsing data from a pre-recorded txt file
  """
  hex_list = []
  with open(filename, 'r') as f:
    data = f.read()
    while data:
      word = data[:2]
      data = data[2:]
      # print("str:", word)
      if not word.isspace():
        num = int(word, 16)
        hex_list.append(num)

  return hex_list

def generate_graph(field_x = "Time", field_y = "Raw_Wave"):
  """generate a graph with the data"""
  x_list = graph_data_dict[field_x]
  y_list = graph_data_dict[field_y]

  # Create Figure and Axes instances
  fig,ax = plt.subplots(1)

  # Make your plot, set your axes labels
  ax.plot(x_list, y_list)
  ax.set_xlabel(field_x)
  ax.set_ylabel(field_y)

  # Turn off tick labels
  # ax.set_xticklabels([])
  # ax.set_yticklabels([])
  ax.set_xticks([])
  # ax.set_yticks([])
  plt.savefig('foo.png')

  plt.show()

if __name__ == '__main__':

  # Get directory name
  # Try to remove tree; if failed show an error using try...except on screen
  try:
    shutil.rmtree(directory_name)
  except OSError as e:
    print ("Error: %s - %s." % (e.filename, e.strerror))
  os.makedirs(directory_name)

  # choose to read data from pre-recorded file
  # enter file directory here
  data = read_from_file("example_data/capture3.txt")

  start_time = time.time()

  while len(data) >= 8:
    delta_time = "{:.5f}".format(time.time() - start_time)
    parse_packet(data, delta_time, generate_graph = True)