#sensor fusion test
import time
import board
import numpy as np
import adafruit_fxas21002c
import adafruit_fxos8700
from yaw import *

i2c = busio.I2C(board.SCL, board.SDA)
sensor1 = adafruit_fxos8700.FXOS8700(i2c)
sensor2 = adafruit_fxas21002c.FXAS21002C(i2c)

mag_offset = calibrate_mag()
initial_angle = set_initial(mag_offset)
gyro_offset = calibrate_gyro()
yawAM = []
trueYaw = []
xs = []

while True:

    accelX, accelY, accelZ = sensor1.accelerometer #m/s^2
    magX, magY, magZ = sensor1.magnetometer #gauss
    #Calibrate magnetometer readings
    magX = magX - mag_offset[0]
    magY = magY - mag_offset[1]
    magZ = magZ - mag_offset[2]
    gyroX, gyroY, gyroZ = sensor2.gyroscope #rad/s
    gyroX = gyroX * (180/np.pi)- gyro_offset[0]
    gyroY = gyroY * (180/np.pi)- gyro_offset[1]
    gyroZ = gyroZ * (180/np.pi)- gyro_offset[2]
    xs.append(time.time())

    zeroCalib = 365 - initial_angle[2]
    yawAM.append(yaw_am(accelX,accelY,accelZ,magX,magY,magZ, zeroCalib))

    if len(trueYaw) == 0:
        prev_angle = yawAM[0]
    else:
        prev_angle = trueYaw[-1]

    if len(xs) == 1:
        trueYaw.append(prev_angle)
    else:
        delT = xs[-1] - xs[-2]
        trueYaw.append(0.8*(yaw_gy(prev_angle, delT, gyroZ) % 360) + 0.2*yawAM[-1])

    #Keep the plot from being too long
    xs = xs[-20:]
    yawAM = yawAM[-20:]
    trueYaw = trueYaw[-20:]

    print("Gyro and Magnetometer (degrees): ({0:0.3f})".format(trueYaw[-1]))
    time.sleep(0.5)


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
class Solution:
    def balanceBST(self, root: TreeNode) -> TreeNode:
        # 通用的做法是遍历这棵树得到值数组，然后每次选取中间的元素构建新的平衡二叉树，但就这题而言，很明显是丢掉了原树是一棵二叉搜索树这一信息，但好在通用的做法时间复杂度不高
        def get_nums(node): # 得到值数组
            if node is None:
                return
            get_nums(node.left)
            nums.append(node.val)
            get_nums(node.right)

        def get_avl(l, r):
            if l > r:
                return
            m = l +((r-l)>>1)
            node = TreeNode(val=nums[m])
            node.left = get_avl(l, m-1)
            node.right = get_avl(m+1, r)
            return node

        nums = []
        get_nums(root)
        return get_avl(0, len(nums)-1)


#!/usr/bin/python3
# -*- coding: utf-8 -*-

"""
Modulo che contiene la classe SyntacticTreeList
"""

import os
import sys
sys.path.insert(0, "../lib/")
from ParseTreeDef import ParseTreeDef
from Token import Token
from Grammar import Grammar
from nltk.parse import ChartParser

__author__ = "Davide Muzio"
__version__ = "1.0.0"

class SyntacticTreeList:
    """Class SyntacticTreeList
    rappresenta la lista di ParseTree
    in cui puo' essere tradotto un requisito
    """
    # Attributes:
    __id = 0  # (int)
    __numTree = 0  # (int)

    # Operations

    def __init__(self, id=0):
        self.__id = id

    def createTree(self, cfg, tokenList, id):
        """function createTree
        crea una lista di parseTree
        partendo della lista dei token del requisito
        ritorna il miglior parseTree in sottoforma di stringa
        returns String
        """
        token = []
        # crea la lista di token del requisito
        for t in tokenList:
            token.append(t.getValue())
        chartParser = ChartParser(cfg)
        treeList = []
        i = 1
        #crea i possibili parseTree usando il chartParser
        for t in chartParser.parse(token):
            treeList.append(t)
        treeString = []
        # normalizza i parseTree per analizzarli
        for singleTree in treeList:
            req = []
            temp = ' '.join(str(singleTree).split())
            treeString.append(temp)
        parseTree = []
        # crea la lista di parseTree
        for sigleTree in treeString:
            tree = ParseTreeDef(singleTree, tokenList)
            parseTree.append(tree)
        # seleziona il miglior parseTree
        bestTree = self.__selectBestTree(parseTree)
        return bestTree.getString() # should raise NotImplementedError()

    def __selectBestTree(self, parseTree):
        """function selectBestTree
        seleziona il miglior parseTree
        selezionando quello che si estende
        maggiormente in profondita'

        returns ParseTree
        """
        tree = []
        tree = parseTree
        childrenRoot = []
        # controllo il numero dei figli del root di ogni albero
        for t in tree:
            childrenRoot.append(len(t.getRoot().getChildren()))
        minNumChildren = min(childrenRoot)
        bestTree = []
        # analizzo gli alberi che hanno il minor numero di figli della radice
        for t in tree:
            if len(t.getRoot().getChildren()) == minNumChildren:
                bestTree.append(t)
        if len(bestTree) == 1:
            return bestTree[0]
        else:
            best = None
            maxNum = 0
            maxList = []
            for t in range(0, len(bestTree)):
                maxList.append(0)
            for i in range(0, minNumChildren):
                for t in range(0, len(bestTree)):
                    branch = bestTree[t].getRoot().getChildren()[i]
                    num = branch.numBranch()
                    maxList[t] = maxList[t] + num
            for t in range(0, len(bestTree)):
                if maxList[t] > maxNum:
                    best = t
            return bestTree[best]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Sep 15 11:42:23 2020

@author: noahlefrancois
"""

import numpy as np
from scipy.interpolate import CubicSpline

#Define the true function with high resolution
x_true_C = np.linspace(-np.pi/2, np.pi/2, 1000)
cos_true = np.cos(x_true_C)

x_true_L = np.linspace(-1, 1, 1000)
lorentz_true = 1/(1+x_true_L**2)

#Define the 'data' with small np
n=4
m=5
xp_C = np.linspace(-np.pi/2, np.pi/2, n+m-1)
f_C = np.cos(xp_C)

xp_L = np.linspace(-1, 1, n+m-1)
f_L = 1/(1+xp_L**2)

#Fit a polynomial function, evaluate it & get the error compared to f_true
def poly_interp(xp, f_np, n, m, x_true, f_true):
    poly = np.polyfit(xp, f_np, n+m-1)
    f_poly = np.polyval(poly, x_true)
    err_poly = sum(np.abs(f_true-f_poly))/len(f_true)

    return poly, err_poly

#Fit a cubic spline, evaluate it & get the error compared to f_true
def cubicSpline_interp(xp, f_np, x_true, f_true):
    cs = CubicSpline(xp, f_np)
    f_cs = cs(x_true)
    err_cs = sum(np.abs(f_true-f_cs))/len(f_true)

    return cs, err_cs

#Define a rational function interpolation
def rat_eval(p,q,x):
    top=0
    for i in range(len(p)):
        top=top+p[i]*x**i
    bot=1
    for i in range(len(q)):
        bot=bot+q[i]*x**(i+1)
    return top/bot

def rat_fit(x,y,n,m):
    assert(len(x)==n+m-1)
    assert(len(y)==len(x))
    mat=np.zeros([n+m-1,n+m-1])
    for i in range(n):
        mat[:,i]=x**i
    for i in range(1,m):
        mat[:,i-1+n]=-y*x**i
    pars=np.dot(np.linalg.inv(mat),y)
    p=pars[:n]
    q=pars[n:]
    return p,q

#Fit the rational function, evaluate it & get the error compared to f_true
def rat_interp(xp, f_np, n, m, x_true, f_true):
    p, q=rat_fit(xp, f_np, n, m)
    f_rat = rat_eval(p, q, x_true)
    err_rat = sum(np.abs(f_true-f_rat))/len(f_true)

    return p, q, err_rat

#Test the interpolation functions
[poly_C, err_poly_C] = poly_interp(xp_C, f_C, n, m, x_true_C, cos_true)
[cs_C, err_cs_C] = cubicSpline_interp(xp_C, f_C, x_true_C, cos_true)
[p_C, q_C, err_rat_C] = rat_interp(xp_C, f_C, n, m, x_true_C, cos_true)

[poly_L, err_poly_L] = poly_interp(xp_L, f_L, n, m, x_true_L, lorentz_true)
[cs_L, err_cs_L] = cubicSpline_interp(xp_L, f_L, x_true_L, lorentz_true)
[p_L, q_L, err_rat_L] = rat_interp(xp_L, f_L, 4, 5, x_true_L, lorentz_true)

"""
For n=1, m=8, the error of the lorentz rational fit is 1e-15, but for higher
order the error rapidly gets much worse:
    (n=2, m=7, err = 0.6, n=3, m=6, err = 1.5, n=4, m=5, err = 1.2)
This makes sense because the lorentzian function is a rational function
of order 1/x^2, so n=1 provides a good fit.

Switching from inv to pinv results in much better results at high order:
    (n=2, m=7, err = 1e-16, n=3, m=6, err = 1e-16, n=4, m=5, err = 1e-15)
as well as 1e-16 at n=1.

The lorentzian function is an even function, meaning that only even order terms
should be significant. With n=4 m=5, this was observed to be the case using
pinv (other terms were ~1e-14), but using inv all 4 terms of both p and q were order 1
which leads to the observed large increase in error when using inv.
"""


import sentry_sdk
from fastapi import Depends, FastAPI, HTTPException, UploadFile
from fastapi.staticfiles import StaticFiles
from sqlalchemy.orm import Session

from app import config, crud, models, schemas
from app.database import engine
from app.notify import send_slack_message
from app.utils import get_db

sentry_sdk.init(
    dsn=config.SENTRY_DSN,
    traces_sample_rate=1.0,
)

models.Base.metadata.create_all(bind=engine)

api_app = FastAPI(title="API")
app = FastAPI(title="Main App")
app.mount("/api", api_app)
app.mount("/", StaticFiles(directory="app/frontend", html=True), name="frontend")

@api_app.post("/reports/", response_model=schemas.TipReport)
async def create_tip_report(
    tip_report: schemas.TipReport, db: Session = Depends(get_db)
):
    report = crud.create_tip_report(db=db, tip_report=tip_report)
    send_slack_message(f"Tip report with id {report.id} created")
    return report

@api_app.post("/reports/{tip_report_id}/attach/")
async def attach_image(
    tip_report_id: int, image: UploadFile, db: Session = Depends(get_db)
):
    tip_report = crud.get_tip_report(db=db, tip_report_id=tip_report_id)
    if tip_report is None:
        raise HTTPException(status_code=404, detail="Tip report not found")
    crud.attach_image(db=db, tip_report=tip_report, image=image)
    return {"tip_report_id": tip_report_id, "filename": image.filename}


import unittest
from unittest.mock import patch

import os
import json
import tkinter
import _tkinter

from simplabel import ImageClassifier

class TestLabelingTool(unittest.TestCase):

    def setUp(self):

        self.test_folder = 'tests/test_images'
        self.label_file = os.path.join(self.test_folder, '.labels.json')

        self.cleanup_files()

        # Create label file
        labels = ["Label1", "Label2"]
        with open(self.label_file, 'w') as f:
            json.dump(labels, f)

        self.root=tkinter.Tk()
        self.pump_events()
        self.classifier = ImageClassifier(self.root, directory=self.test_folder, username="testuser")

    def tearDown(self):
        if self.classifier.gotLock:
            self.classifier.lock.release()
        if self.root:
            self.root.destroy()
            self.pump_events()

        self.cleanup_files()

    def pump_events(self):
        while self.root.dooneevent(_tkinter.ALL_EVENTS | _tkinter.DONT_WAIT):
            pass

    def cleanup_files(self):
        # Delete any saved files
        savefiles = [file for file in os.listdir(self.test_folder) if file.startswith("labeled_") and file.endswith(".json")]
        if savefiles:
            for file in savefiles:
                os.remove(os.path.join(self.test_folder, file))

        # Delete all lock files
        lockfiles = [file for file in os.listdir(self.test_folder) if file.endswith("_lock.txt")]
        if lockfiles:
            for file in lockfiles:
                os.remove(os.path.join(self.test_folder, file))

        # Delete label file
        if os.path.exists(self.label_file):
            os.remove(self.label_file)

    def test_next_image_button(self):
        prevValue = self.classifier.counter
        self.classifier.nextButton.invoke()
        self.assertEqual(self.classifier.counter, prevValue + 1)

    def test_previous_image_button(self):
        while self.classifier.counter == 0:
            self.classifier.next_image()
        prevValue = self.classifier.counter
        self.classifier.prevButton.invoke()
        self.assertEqual(self.classifier.counter, prevValue - 1)

    def test_goto_last_image_button(self):
        self.classifier.lastButton.invoke()
        self.assertEqual(self.classifier.counter, len(self.classifier.image_list) - 1)

    def test_label_one_image(self):
        '''Checks that clicking on a label button advances to the next image and has labeled the previous image'''
        prevValue = self.classifier.counter
        imgName = self.classifier.image_list[prevValue]
        self.classifier.catButton[0].invoke()
        if prevValue < len(self.classifier.image_list) - 1:
            self.assertEqual(self.classifier.counter, prevValue + 1)
        else:
            self.assertEqual(self.classifier.counter, prevValue)
        self.assertEqual(self.classifier.labeled[imgName], "Label1")

    def test_labeled_image_has_colored_button(self):
        '''Checks that a labeled image has a colored label button with the user's color'''
        self.classifier.catButton[1].invoke()
        self.classifier.firstButton.invoke()

        # Check that the Label2 button has the color of the user
        self.assertEqual(self.classifier.catButton[1].config()['highlightbackground'][-1], self.classifier.userColor)
        self.assertEqual(self.classifier.catButton[1].config()['background'][-1], self.classifier.userColor)

        # Check that the Label1 button has the original color
        self.assertEqual(self.classifier.catButton[0].config()['highlightbackground'][-1], self.classifier.buttonOrigColor)
        self.assertEqual(self.classifier.catButton[0].config()['background'][-1], self.classifier.buttonBgOrigColor)

    def test_save_labels(self):
        '''Checks that the labels are correctly saved to file'''
        imgName = self.classifier.image_list[self.classifier.counter]
        self.classifier.catButton[1].invoke()
        self.classifier.saveButton.invoke()

        with open(os.path.join(self.test_folder, "labeled_testuser.json"), 'r') as savefile:
            savedict = json.load(savefile)

        self.assertEqual(savedict[imgName], self.classifier.labeled[imgName])

    def test_goto_last_unlabeled_image(self):
        self.classifier.catButton[0].invoke()
        self.classifier.firstButton.invoke()
        self.assertEqual(self.classifier.counter, 0)
        self.classifier.nextUnlabeledButton.invoke()
        self.assertEqual(self.classifier.counter, 1)

    @patch('simplabel.simpledialog.askstring')
    def test_add_label(self, mock_dialog):
        '''Check that adding a label through the GUI adds it to the list, creates a new button and saves it to file'''

        # Mock the dialog that requests the name of the label
        mock_dialog.configure_mock(return_value="Label3")

        # Add a label
        self.classifier.addCatButton.invoke()

        # Check that the label was added to the list
        self.assertEqual(self.classifier.categories[2], "Label3")

        # Check that a button was created for it
        self.assertEqual(len(self.classifier.catButton), 3)

        # Check that the label was saved to file
        with open(self.label_file, 'r') as f:
            savedlabels = json.load(f)

        self.assertIn("Label3", savedlabels)

    @patch('simplabel.simpledialog.askstring')
    def test_add_already_existing_label(self, mock_dialog):
        '''Check that adding an already existing label through the GUI does not add it to the list'''

        # Mock the dialog that requests the name of the label
        mock_dialog.configure_mock(return_value="Label1")

        # Add a label
        self.classifier.addCatButton.invoke()

        # Check that the label was not added to the list
        self.assertEqual(len(self.classifier.categories), 2)

        # Check that a button was not created for it
        self.assertEqual(len(self.classifier.catButton), 2)

        # Check that the label was saved to file
        with open(self.label_file, 'r') as f:
            savedlabels = json.load(f)

        self.assertEqual(len(savedlabels),2)

    @patch('simplabel.simpledialog.askstring')
    def test_add_10_labels(self, mock_dialog):
        '''Check that adding 10 labels works as intended (3 rows of labels)'''

        # Add labels
        mock_dialog.configure_mock(return_value="Label3")
        self.classifier.addCatButton.invoke()
        mock_dialog.configure_mock(return_value="Label4")
        self.classifier.addCatButton.invoke()
        mock_dialog.configure_mock(return_value="Label5")
        self.classifier.addCatButton.invoke()
        mock_dialog.configure_mock(return_value="Label6")
        self.classifier.addCatButton.invoke()
        mock_dialog.configure_mock(return_value="Label7")
        self.classifier.addCatButton.invoke()
        mock_dialog.configure_mock(return_value="Label8")
        self.classifier.addCatButton.invoke()
        mock_dialog.configure_mock(return_value="Label9")
        self.classifier.addCatButton.invoke()
        mock_dialog.configure_mock(return_value="Label10")
        self.classifier.addCatButton.invoke()

        # Check that the labels were added to the list
        self.assertEqual(len(self.classifier.categories), 10)

        # Check that a button werecreated
        self.assertEqual(len(self.classifier.catButton), 10)

        # Check that there are 3 rows
        self.assertEqual(len(self.classifier.labelFrameList), 3)

        # Check that the labels were saved to file
        with open(self.label_file, 'r') as f:
            savedlabels = json.load(f)

        self.assertEqual(len(savedlabels),10)

if __name__ == '__main__':
    unittest.main()


import json
from dateutil.parser import parse
from datetime import datetime

target_day = datetime(2013, 1, 8, 12)

count = 0
with open('../data/2013_filtered_by_date_{}.geo.json'.format(target_day.strftime('%Y_%m_%d_%H')), 'w') as outfile:
    with open('../data/ship_lines_2013.json', 'r') as infile:
        for line in infile:
            if count == 0:
                outfile.write('{"type": "FeatureCollection", "features": [\n')
            if count > 1:
                outfile.write(',\n')
            data = {}
            try:
                data = json.loads(line)
            except ValueError:
                continue
            start = parse(data['properties']['trackStartTime'])
            end = parse(data['properties']['trackEndTime'])
            if start < target_day and end > target_day:
                outfile.write(line.strip())
    outfile.write(']}')


from flask import Flask, render_template, request, jsonify
from backend.ethapi import init_sov_24_hours,update_sov_24_hours,total_sov,percent_mined,latest_avg,gasAndNoOfTransactions,get_number_of_holders
from decimal import Decimal

app = Flask(__name__)

def getValues():
    total = total_sov()
    latest = latest_avg()
    holders = get_number_of_holders()

    uPercent = Decimal(percent_mined()) * Decimal(100)
    percent = round(uPercent, 2)

    x = gasAndNoOfTransactions(init_sov_24_hours())
    gasAvg = x[0]
    noOfTransactions = x[1]
    sovMined = x[2]

    valueList = [total, latest, percent, gasAvg, noOfTransactions, sovMined, holders]
    return valueList

@app.route("/")
def home():
    values = getValues()

    return render_template("dashboard.html", total = values[0], latest=values[1],
            percent=values[2],gasAvg=values[3], noOfTransactions = values[4], sovMined=values[5],
            holders = values[6])

@app.route('/update', methods=['POST'])
def update_values():
    func = getValues()

    return jsonify(
        {"total" : func[0],
        "latest" : func[1],
        "percent":func[2],
        "gasAvg":func[3],
        "noOfTransactions":func[4],
        "sovMined" : func[5]
        }
    )

if __name__ == "__main__":
    app.run(debug=True)


import cv2 as cv
import numpy as np

SZ=20
bin_n = 16 # Number of bins
affine_flags = cv.WARP_INVERSE_MAP|cv.INTER_LINEAR

# Antes de proceder enderezamos los dígitos
def deskew(img):
    m = cv.moments(img)
    if abs(m['mu02']) < 1e-2:
        return img.copy()
    skew = m['mu11']/m['mu02']
    M = np.float32([[1, skew, -0.5*SZ*skew], [0, 1, 0]])
    img = cv.warpAffine(img,M,(SZ, SZ),flags=affine_flags)
    return img

# Elegimos como singularidades los gradientes en el histograma de colores (HOG)
def hog(img):
    gx = cv.Sobel(img, cv.CV_32F, 1, 0)
    gy = cv.Sobel(img, cv.CV_32F, 0, 1)
    mag, ang = cv.cartToPolar(gx, gy)
    bins = np.int32(bin_n*ang/(2*np.pi))    # quantizing binvalues in (0...16)
    bin_cells = bins[:10,:10], bins[10:,:10], bins[:10,10:], bins[10:,10:]
    mag_cells = mag[:10,:10], mag[10:,:10], mag[:10,10:], mag[10:,10:]
    hists = [np.bincount(b.ravel(), m.ravel(), bin_n) for b, m in zip(bin_cells, mag_cells)]
    hist = np.hstack(hists)     # hist is a 64 bit vector
    return hist

img = cv.imread('digits.png',0)
cells = [np.hsplit(row,100) for row in np.vsplit(img,50)]

# Usar como datos de partida la mitad de los dígitos
train_cells = [ i[:50] for i in cells ]
test_cells = [ i[50:] for i in cells]
deskewed = [list(map(deskew,row)) for row in train_cells]
hogdata = [list(map(hog,row)) for row in deskewed]

trainData = np.float32(hogdata).reshape(-1,64)
responses = np.repeat(np.arange(10),250)[:,np.newaxis]

# Iniciar el adiestrador con vectores auxiliares
svm = cv.ml.SVM_create()
svm.setKernel(cv.ml.SVM_LINEAR)
svm.setType(cv.ml.SVM_C_SVC)
svm.setC(2.67)
svm.setGamma(5.383)
svm.train(trainData, cv.ml.ROW_SAMPLE, responses)
svm.save('svm_data.dat')

# Verificar la precisión de la clasificación
deskewed = [list(map(deskew,row)) for row in test_cells]
hogdata = [list(map(hog,row)) for row in deskewed]
testData = np.float32(hogdata).reshape(-1,bin_n*4)
result = svm.predict(testData)[1]
mask = result==responses
correct = np.count_nonzero(mask)
print(correct*100.0/result.size)


# hashing with resizing

class HashData:
    def __init__(self,id,*values):
        self.id = id
        self.values = values

    def __str__(self):
        return self.id + str(self.values)

    def __repr__(self):
        return self.id + str(self.values)

class HashTable:
    def __init__(self,size):
        self.size = size
        self.buckets = [[] for e in range(size)]

    def generate_hash(self,k):
        s = sum([ord(e) for e in k])
        hash_index = s % self.size
        return hash_index

    def resize(self):
        self.size = self.size * 2
        self.new_bucket = [ [] for e in range(self.size)]

        for i, bucket in enumerate(self.buckets):
            if bucket:
                index = self.generate_hash(bucket[0].id)
                self.new_bucket[index] = bucket
        print(f"old bucket {self.buckets} new bucket {self.new_bucket}")
        self.buckets = self.new_bucket

    def insert_data(self,k):
        if -1 == self.retrieve_data(k.id):
            self.buckets[self.generate_hash(k.id)].append(k)
        op = sum([1 for x in self.buckets if x])
        if op > self.size * 0.6:
            # trigger resize
            print("triggered resize")
            self.resize()

    def remove_data(self,k):
        if -1 != self.retrieve_data(k.id):
            print("removing ",k)
            bucket = self.buckets[self.generate_hash(k.id)]
            for i, e in enumerate(bucket):
                if k.id == e.id:
                    print("removing inside loop",k)
                    bucket.pop(i)

    def update_data(self,k):
        self.remove_data(k)
        self.buckets[self.generate_hash(k.id)].append(k)

    def retrieve_data(self,k):
        bucket = self.buckets[self.generate_hash(k)]
        for e in bucket:
            if k == e.id:
                return e
        return -1

if __name__ == "__main__":
    o = HashTable(12)
    hd = HashData("prem","email","phone")
    o.insert_data(hd)
    # o.insert_data("prem")
    hd = HashData("prem","fax","bra","penty")
    o.insert_data(HashData("prem","fax","bra","penty"))
    ash = HashData("seema","ashim")
    o.insert_data(ash)
    ash = HashData("seema45","ashim")
    o.insert_data(ash)
    print(o.buckets)
    # o.insert_data("seema")
    # o.remove_data(ash)
    o.update_data(hd)
    print(o.buckets)
    print(o.retrieve_data("prem"))
    print(o.retrieve_data("aseem1"))
    # o.resize()
    ash = HashData("zebra","ashim")
    o.insert_data(ash)
    print(o.buckets)
    o.insert_data(HashData("prem1","fax","bra","penty"))
    o.insert_data(HashData("prem2","fax","bra","penty"))
    o.insert_data(HashData("prem3","fax","bra","penty"))
    o.insert_data(HashData("prem4","fax","bra","penty"))
    o.insert_data(HashData("prem5","fax","bra","penty"))
    o.insert_data(HashData("prem6","fax","bra","penty"))
    o.insert_data(HashData("prem7","fax","bra","penty"))
    o.insert_data(HashData("prem8","fax","bra","penty"))


import os
import getpass
import hashlib
import re
import time
import datetime
import logging
import six

import boto
from boto.emr.step import HiveStep
from boto.emr.step import InstallHiveStep
from boto.emr.connection import EmrConnection
from apiarist.s3 import copy_s3_file, is_dir, upload_file_to_s3
from apiarist.script import generate_hive_script_file, get_script_file_location

logger = logging.getLogger(__name__)

class EMRRunner():

    def __init__(self, job_name=None, input_path=None, hive_query=None,
                 output_dir=None, scratch_uri=None, log_path=None,
                 ami_version=None, hive_version=None, num_instances=None,
                 master_instance_type=None, slave_instance_type=None,
                 iam_instance_profile=None, iam_service_role=None,
                 aws_access_key_id=None, aws_secret_access_key=None,
                 visible_to_all_users=None,
                 s3_sync_wait_time=5, check_emr_status_every=30,
                 label=None, owner=None, temp_dir=None):

        self.job_name = job_name
        self.job_id = self._generate_job_id()
        self.start_time = time.time()

        # AWS credentials can come from arguments or environment
        # set AWS credentials if supplied (override ENV)
        if aws_access_key_id:
            os.environ['AWS_ACCESS_KEY_ID'] = aws_access_key_id
            self.aws_access_key_id = aws_access_key_id
        else:
            logger.debug("Getting AWS access key from ENV")
            self.aws_access_key_id = os.environ['AWS_ACCESS_KEY_ID']

        if aws_secret_access_key:
            os.environ['AWS_SECRET_ACCESS_KEY'] = aws_secret_access_key
            self.aws_secret_access_key = aws_secret_access_key
        else:
            logger.debug("Getting AWS secret key from ENV")
            self.aws_secret_access_key = os.environ['AWS_SECRET_ACCESS_KEY']

        # Set visibility of job flow to all users if visible_to_all_users is None
        if visible_to_all_users is None:
            self.visible_to_all_users = True
        else:
            self.visible_to_all_users = visible_to_all_users

        self.s3_sync_wait_time = s3_sync_wait_time
        self.check_emr_status_every = check_emr_status_every

        # I/O for job data
        self.input_path = input_path
        self.output_dir = output_dir

        # is the input multiple files in a 'directory'?
        try:
            self.input_is_dir = is_dir(input_path)
        except TypeError:
            self.input_is_dir = False

        # the Hive script object
        self.hive_query = hive_query

        #  EMR options
        self.master_instance_type = master_instance_type
        self.slave_instance_type = slave_instance_type
        self.ami_version = ami_version
        self.hive_version = hive_version
        self.num_instances = num_instances
        self.iam_instance_profile = iam_instance_profile
        self.iam_service_role = iam_service_role
        self.label = label
        self.owner = owner

        # give this job a unique name
        self.job_key = self._make_unique_job_key(
            label=self.label, owner=self.owner)

        # S3 'scratch' directory
        if scratch_uri:
            self.base_path = scratch_uri
            os.environ['S3_SCRATCH_URI'] = scratch_uri
        else:
            self.base_path = os.environ['S3_SCRATCH_URI']

        # allow alternate logging path
        self.log_path = log_path or self.base_path + 'logs/'
        # other temp files live in a jobID bucket
        self.job_files = self.base_path + self.job_id + '/'
        self.data_path = self.job_files + 'data'
        if self.input_is_dir:
            self.data_path += '/'
        self.table_path = self.job_files + 'tables/'
        self.script_path = self.job_files + 'script.hql'
        self.output_path = self.output_dir or self.job_files + 'output/'

        # a local temp dir is used to write the script
        self.local_script_file = get_script_file_location(self.job_id,
                                                          temp_dir)

        logger.info("JobID {0}, started at {1}".format(self.job_id,
                                                       self.start_time))

    def _generate_hive_script(self, data_source):
        """Write the HQL to a local (temp) file
        """
        hq = self.hive_query.emr_hive_script(data_source, self.output_path,
                                             self.table_path)
        generate_hive_script_file(hq, self.local_script_file)

    def _generate_job_id(self):
        """Create a unique job run identifier
        """
        run_id = self.job_name + str(time.time())
        digest = hashlib.md5(six.b(run_id)).hexdigest()
        return 'hj-' + digest

    def _make_unique_job_key(self, label=None, owner=None):
        """Come up with a useful unique ID for this job
        """
        # use the name of the job as the label if one wasn't explicitly
        # specified
        if not label:
            label = self.job_name

        if not owner:
            try:
                owner = getpass.getuser()
            except:
                owner = 'no_user'

        now = datetime.datetime.utcnow()
        return '%s.%s.%s.%06d' % (
            label, owner,
            now.strftime('%Y%m%d.%H%M%S'), now.microsecond)

    def _generate_and_upload_hive_script(self):
        self._generate_hive_script(self.data_path)
        upload_file_to_s3(self.local_script_file, self.script_path)

    #  hooks for the with statement ###

    def __enter__(self):
        """Don't do anything special at start of with block"""
        return self

    def __exit__(self, type, value, traceback):
        """Call self.cleanup() at end of with block."""
        self.cleanup()

    def run(self):
        """Run the Hive job on EMR cluster
        """
        #  copy the data source to a new object
        #  (Hive deletes/moves the original)
        copy_s3_file(self.input_path, self.data_path)

        # and create the hive script
        self._generate_and_upload_hive_script()

        logger.info("Waiting {} seconds for S3 eventual consistency".format(
                    self.s3_sync_wait_time))
        time.sleep(self.s3_sync_wait_time)

        # TODO more options like setting aws region
        conn = EmrConnection(self.aws_access_key_id,
                             self.aws_secret_access_key)

        setup_step = InstallHiveStep(self.hive_version)
        run_step = HiveStep(self.job_name, self.script_path)

        cluster_id = conn.run_jobflow(
            self.job_key,
            self.log_path,
            action_on_failure='CANCEL_AND_WAIT',
            master_instance_type=self.master_instance_type,
            slave_instance_type=self.slave_instance_type,
            ami_version=self.ami_version,
            num_instances=self.num_instances,
            job_flow_role=self.iam_instance_profile,
            service_role=self.iam_service_role,
            visible_to_all_users=self.visible_to_all_users)

        conn.add_jobflow_steps(cluster_id, [setup_step, run_step])

        logger.info("Job started on cluster {0}".format(cluster_id))

        self._wait_for_job_to_complete(conn, cluster_id)

        logger.info("Output file is in: {0}".format(self.output_path))

    def cleanup(self):
        # TODO _ remove scratch dirs?
        logger.info("cleaning up ... ")

    # wait for job and log status
    # this method extracted from mrjob.job
    def _wait_for_job_to_complete(self, conn, cluster_id):
        """
        Wait for the job to complete, and raise an exception if
        the job failed.

        Also grab log URI from the job status (since we may not know it)
        """
        success = False
        chk_status_freq = self.check_emr_status_every
        # opts = {'check_emr_status_every': 30}
        # s3_logs = self.log_path
        emr_job_start = self.start_time

        while True:
            # don't antagonize EMR's throttling
            logger.debug('Waiting {0} seconds'.format(chk_status_freq))
            time.sleep(chk_status_freq)

            cluster = conn.describe_cluster(cluster_id)

            job_state = cluster.status.state
            reason = getattr(
                getattr(cluster.status, 'statechangereason', None),
                'message', '')

            # find all steps belonging to us, and get their state
            step_states = []
            running_step_name = ''
            total_step_time = 0.0
            step_nums = []  # step numbers belonging to us. 1-indexed

            steps = conn.list_steps(cluster.id).steps or []
            for i, step in enumerate(steps):

                # ignore steps belonging to other jobs
                if not step.name.startswith(self.job_name):
                    continue

                step_nums.append(i + 1)

                step_states.append(step.status.state)
                if step.status.state == 'RUNNING':
                    running_step_name = step.name

                if hasattr(step, 'startdatetime') and \
                   hasattr(step, 'enddatetime'):

                    start_time = iso8601_to_timestamp(step.startdatetime)
                    end_time = iso8601_to_timestamp(step.enddatetime)
                    total_step_time += end_time - start_time

            if not step_states:
                raise AssertionError("Can't find our steps in the job flow!")

            # if all our steps have completed, we're done!
            if all(state == 'COMPLETED' for state in step_states):
                success = True
                break

            # if any step fails, give up
            if any(state in ('FAILED', 'CANCELLED') for state in step_states):
                break

            # (the other step states are PENDING and RUNNING)

            # keep track of how long we've been waiting
            running_time = time.time() - emr_job_start

            # logging message values
            loginfo = (int(running_time), job_state, reason, running_step_name)

            # otherwise, we can print a status message
            if running_step_name:
                logger.info("Job launched {0} ago. "
                            " Status {1}: {2} ({3})".
                            format(*loginfo))

            # other states include STARTING and SHUTTING_DOWN
            elif reason:
                logger.info("Job launched {0} ago. "
                            "Status {1}: {2}".format(*loginfo))
            else:
                logger.info("Job launched {0} ago. "
                            "Status {1}".format(*loginfo))

        if success:
            logger.info("Job completed on cluster {}.".format(cluster.id))
            logger.info("Running time was {0} "
                        "(excludes time spent waiting for the EC2 instances)".
                        format(total_step_time))
        else:
            msg = "Job on cluster {0} failed with status {1}: {2}".format(
                  cluster.id, job_state, reason)
            logger.info(msg)

            raise Exception(msg)

#  AWS Date-time parsing

# sometimes AWS gives us seconds as a decimal, which we can't parse
# with boto.utils.ISO8601
SUBSECOND_RE = re.compile('\.[0-9]+')

# Thu, 29 Mar 2012 04:55:44 GMT
RFC1123 = '%a, %d %b %Y %H:%M:%S %Z'

def iso8601_to_timestamp(iso8601_time):
    iso8601_time = SUBSECOND_RE.sub('', iso8601_time)
    try:
        return time.mktime(time.strptime(iso8601_time, boto.utils.ISO8601))
    except ValueError:
        return time.mktime(time.strptime(iso8601_time, RFC1123))

def iso8601_to_datetime(iso8601_time):
    iso8601_time = SUBSECOND_RE.sub('', iso8601_time)
    try:
        return datetime.strptime(iso8601_time, boto.utils.ISO8601)
    except ValueError:
        return datetime.strptime(iso8601_time, RFC1123)


from src.data import users, channels, next_channel_id, dreams_stats

from src.error import InputError, AccessError

from src.helpers import check_token, user_from_token, update_channel_stats, update_user_channel_stats

def channels_list_v2(token):
    '''
    <Provide a list of all channels (and their associated details) that the authorised user is part of>

    Arguments:
        <token>        (<string>)     - <User session token>

    Exceptions:
        AccessError - Occurs when the token is not a vaild token

    Return Value:
        Returns <{ channels }>      on <return the list of these channels that user is apart of>

    '''

    global users, channels
    check_token(token)
    auth_user_id = user_from_token(token)['u_id']

    channelList = []

    # search all the channel
    for channel in channels:
        # if the auth user is a menber of this channel, add this channel to the channellist
        if auth_user_id in channel['all_members']:
            channelList.append({ 'channel_id' : channel['channel_id'], 'name': channel['name']})

    # if this user is not in any channel, just return the empty channellist list
    return { 'channels' : channelList }

def channels_listall_v2(token):
    """Summary
        Provide a list of all channels (and their associated details)
    Args:
        token (string): A user session token

    Returns:
        Dictionary: Contains the list of all channels, and their associated details

    Raises:
        AccessError: When an invalid token is given
    """
    check_token(token)

    # Collect channel information for each channel
    output = []
    for channel in channels:
        channel_details = { 'channel_id' : channel['channel_id'], 'name' : channel['name'] }
        output.append(channel_details)

    return { 'channels' : output }

def channels_create_v2(token, name, is_public):
    '''
    <this function will create a channel with the auth_user_id being the creater of the channel, it will give the name of the channel and then set it to public or private>

    Arguments:
        <token>        (<string>)     - <User session token>
        <name>         (<string>)     - <this is the name of the new channel which will be created in this function>
        <is_public>    (<boolean>)    - <this parameter will discrible if this channel is a public channel or a private channel>

    Exceptions:
        InputError  - Occurs when name is more than 20 characters long
        AccessError - Occurs when the token is not a vaild token
    Return Value:
        Returns <{ channel_id }>       on <when channel is successfully created, return the id of this new channel>

    '''

    global users, channels, next_channel_id

    check_token(token)
    auth_user = user_from_token(token)
    auth_user_id = auth_user['u_id']

    # Name is more than 20 characters long
    if len(name) > 20:
        raise InputError(f"The name entered: {name} is more than 20 characters long")

    channel_id = next_channel_id['id']

    next_channel_id['id'] += 1

    # create a new channel
    new_channel = {
        'channel_id' : channel_id,
        'name' : name,
        'owner_members' : [auth_user_id],
        'all_members': [auth_user_id],
        'is_public' : is_public,
        'messages' : [],
        'standup' : {
                      'is_active' : False,
                      'creator' :0,
                      'messages' : [],
                      'time_finish' : 0
                    }
    }

    # add the new channel to the channels list
    channels.append(new_channel)

    update_channel_stats() # Record newly increased number of channels in Dreams stats

    auth_user['channels'].append(channel_id) # Add the channel's id to the user's list of channels
    update_user_channel_stats(auth_user_id)

    return { 'channel_id' : new_channel['channel_id'] }


import functools
import pickle
import heapq
import math
import numpy as np
import src.generator
import src.reader

read = False

def setseed():
    np.random.seed(0)
    random.seed(0)

if read:
    namestate = "../huge-data/debates/topic_state_500"
    namedocs = "../huge-data/debates/mc-20170814-stemmed.txt"

    data = src.reader.TextData(namestate, namedocs)
    data.counts
    data.sorts
    data.sums
    data.save("data/data.p")

with open("data/data.p", "rb") as f:
    print("reading", flush=True)
    data = pickle.load(f)
    print("read", flush=True)

g = src.generator.Generator(data)
v = src.generator.Visualizer(data, "../docs/")

g.save("data/", "../docs/json/")
v.front("front/")
v.wordcloud("cloud/")


#!/usr/bin python3
# -*- coding: utf-8 -*-

import re
from os import path
from codecs import open
from setuptools import setup
from pkg_resources import parse_version

VERSIONFILE = "vafnet/__version__.py"
verstrline = open(VERSIONFILE, "rt").read()
VSRE = r"^__version__ = ['\"]([^'\"]*)['\"]"
mo = re.search(VSRE, verstrline, re.M)
if mo:
    __version__ = mo.group(1)
else:
    raise RuntimeError("Unable to find version string in %s." % (VERSIONFILE,))

here = path.abspath(path.dirname(__file__))

with open(path.join(here, 'README.md'), encoding='utf-8') as f:
    long_description = f.read()

setup(
    name="vafnet",
    version=__version__,
    license="Apache License 2.0",
    author="Max W. Y. Lam",
    author_email="maxingaussian@gmail.com",
    description="Variational Activation Functions for Deep Learning",
    long_description=long_description,
    keywords="machine-learning deep-learning neural-networks gaussian-process",
    url="https://github.com/MaxInGaussian/VAFnet",
    download_url='https://github.com/MaxInGaussian/VAFnet/tarball/'+__version__,
    classifiers=[
        'License :: OSI Approved :: Apache Software License'
        'Natural Language :: English',
        'Operating System :: MacOS :: MacOS X',
        'Operating System :: Microsoft :: Windows',
        'Operating System :: POSIX :: Linux',
        'Intended Audience :: Developers',
        'Programming Language :: Python :: 3.5',
        'Topic :: Scientific/Engineering :: Artificial Intelligence'
    ],
    packages=["vafnet"],
    package_dir={'vafnet': 'vafnet'},
    py_modules=['vafnet.__init__'],
    install_requires=['numpy>=1.7',
                      'six>=1.10.0'],
    extras_require={
        'edward': ['edward>=1.3.4'],
        'tensorflow': ['tensorflow>=1.2.0rc0'],
        'tensorflow with gpu': ['tensorflow-gpu>=1.2.0rc0'],
        'notebooks': ['jupyter>=1.0.0'],
        'visualization': ['matplotlib>=1.3']},
    tests_require=['pytest', 'pytest-pep8'],
)


from django.shortcuts import render
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import generics
from .serializer import *
from .email import *
from rest_framework import views

class Send_otp(APIView):
    def post(self, request):
        try:
            data = request.data
            serializer = UserSerializer(data = data)
            usr = client.objects.filter(email = data['email'])
            if usr:
                send_otp(request.data['email'])
                return Response({'status':200,'message':'OTP Send Successfull','data':request.data['email']})
            if serializer.is_valid():
                serializer.save()
                send_otp(serializer.data['email'])
                return Response({'status':200,'message':'OTP created succesfull check email','data':serializer.data })
            return Response({'status':400,'message':'Something went Wrong','data':serializer.errors})
        except Exception as e:
            print(e)

class verify_otp(APIView):
    def post(self, request):
        try:
            data = request.data
            serializer = verifyaccount(data = data)
            if serializer.is_valid():
                email = serializer.data['email']
                otp = serializer.data['otp']
                user = client.objects.filter(email = email)
                if not user.exists():
                    return Response({
                    'status':400,
                    'message':'Something went Wrong',
                    'data':'invalid email'
                    })
                if user[0].otp != otp:
                    return Response({
                    'status':400,
                    'message':'Something went Wrong',
                    'data':'wrong otp'
                    })
                user.update(is_verified = True)
                return Response({
                    'status':200,
                    'message':'account verified',
                    'data':{}
                })
            return Response({
                'status':400,
                'message':'Something went Wrong',
                'data':serializer.errors
            })
        except Exception as e:
            print(e)


from django.db.models import Q
from django.shortcuts import render, get_object_or_404
from . import models
import markdown
import pygments
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger

def make_paginator(objects, page, num=10):
    paginator = Paginator(objects, num)
    try:
        object_list = paginator.page(page)
    except PageNotAnInteger:
        object_list = paginator.page(1)
    except EmptyPage:
        object_list = paginator.page(paginator.num_pages)
    return object_list, paginator

def pagination_data(paginator, page):
    if paginator.num_pages == 1:
        return {}
    left = []
    right = []
    left_has_more = False
    right_has_more = False
    first = False
    last = False
    try:
        page_number = int(page)
    except ValueError:
        page_number = 1
    except:
        page_number = 1
    total_pages = paginator.num_pages
    page_range = paginator.page_range
    if page_number == 1:
        right = page_range[page_number:page_number+4]
        if right[-1] < total_pages - 1:
            right_has_more = True
        if right[-1] < total_pages:
            last = True

    elif page_number == total_pages:
        left = page_range[(page_number - 3) if (page_number - 3) > 0 else 0:page_number - 1]
        if left[0] > 2:
            left_has_more = True
        if left[0] > 1:
            first = True

    else:
        left = page_range[(page_number-3) if (page_number - 3) > 0 else 0:page_number-1]
        right = page_range[page_number:page_number+2]

        if right[-1] < total_pages - 1:
            right_has_more = True
        if right[-1] < total_pages:
            last = True
        if left[0] > 2:
            left_has_more = True
        if left[0] > 1:
            first = True
    data = {
        'left': left,
        'right':right,
        'left_has_more':left_has_more,
        'right_has_more':right_has_more,
        'first':first,
        'last':last,
    }
    return data

def index(request):
    entries = models.Entry.objects.all()
    page = request.GET.get('page', 1)
    entry_list, paginator = make_paginator(entries, page)
    page_data = pagination_data(paginator, page)
    return render(request, 'blog/index.html', locals())

def detail(request, blog_id):
    # entry = models.Entry.objects.get(id=blog_id)
    # 如果id存在就返回正确页面，否则返回404页面
    entry = get_object_or_404(models.Entry, id=blog_id)
    md = markdown.Markdown(extensions=[
        'markdown.extensions.extra',
        'markdown.extensions.codehilite',
        'markdown.extensions.toc',
    ])
    entry.body = md.convert(entry.body)
    entry.toc =md.toc
    entry.increase_visiting()
    return render(request, 'blog/detail.html', locals())

def category(request, category_id):
    categy = get_object_or_404(models.Category, id=category_id)
    # categy = models.Category.objects.get(id=category_id)
    entries = models.Entry.objects.filter(category=categy)
    page = request.GET.get('page', 1)
    entry_list, paginator = make_paginator(entries, page)
    page_data = pagination_data(paginator, page)
    return render(request, 'blog/index.html', locals())

def tags(request, tag_id):
    tag = get_object_or_404(models.Tag, id=tag_id)
    # tag =models.Tag.objects.get(id=tag_id)
    if tag.name == '全部':
        entries = models.Entry.objects.all()
    else:
        entries = models.Entry.objects.filter(tags=tag)
    page = request.GET.get('page', 1)
    entry_list, paginator = make_paginator(entries, page)
    page_data = pagination_data(paginator, page)
    return render(request, 'blog/index.html', locals())

def search(request):
    keyword = request.GET.get('keyword',None)
    if not keyword:
        error_msg = "请输入关键字"
        return render(request, 'blog/index.html', locals())

    entries = models.Entry.objects.filter(Q(title__icontains=keyword) | Q(body__icontains=keyword))
    page = request.GET.get('page', 1)
    entry_list, paginator = make_paginator(entries, page)
    page_data = pagination_data(paginator, page)
    return render(request, 'blog/index.html', locals())

def archives(request, year, month):
    entries = models.Entry.objects.filter(created_time__year=year, created_time__month=month)
    page = request.GET.get('page', 1)
    entry_list, paginator = make_paginator(entries, page)
    page_data = pagination_data(paginator, page)
    return render(request, 'blog/index.html', locals())

def permission_denied(request, exception, template_name='blog/403.html'):
    return render(request, template_name, locals())

def page_not_found(request, exception, template_name='blog/404.html'):
    return render(request, template_name, locals())

def server_error(exception):
    return render("blog/500.html", status=500)


import random
from elasticsearch import Elasticsearch
from datetime import datetime
import twitterbot

from operator import itemgetter #for sorting

client = Elasticsearch()
'''
import requests

def stat_indices() :
    resp_indices = requests.get('http://localhost:9200/_cat/indices')
    return(resp_indices.content)

def stat_wordlist() :
    resp_wordlist = requests.get('http://localhost:9200/wordlist/_search?pretty')
    return(resp_wordlist.json())

def stat_quotelist() :
    resp_wordlist = requests.get('http://localhost:9200/quotelist/_search?pretty')
    return(resp_wordlist.json())

def stat_photolist() :
    resp_wordlist = requests.get('http://localhost:9200/photolist/_search?pretty')
    return(resp_wordlist.json())
'''

def get_a_random_word() :
    index_name = "wordlist"
    docs_count = client.count(index=index_name)['count']
    print("docs_count = %d" % docs_count)
    random_doc_id = random.randint(1,docs_count)
    print("generate random doc_id = %d" % random_doc_id)
    doc = client.get(index=index_name, id=str(random_doc_id))
    return(doc["_source"])

def get_a_quote_and_author_by_id(id) :
    index_name = "quotelist"
    doc = client.get(index=index_name, id=str(id))
    return(doc["_source"]["Quote"], doc["_source"]["Author"])

def get_a_random_quote() :
    index_name = "quotelist"
    docs_count = client.count(index=index_name)['count']
    print("docs_count = %d" % docs_count)
    random_doc_id = random.randint(1,docs_count)
    print("generate random doc_id = %d" % random_doc_id)
    doc = client.get(index=index_name, id=str(random_doc_id))
    return(doc["_source"]["Quote"] + " —" + doc["_source"]["Author"])

def get_a_random_quote_and_author() :
    index_name = "quotelist"
    docs_count = client.count(index=index_name)['count']
    random_doc_id = random.randint(1,docs_count)
    doc = client.get(index=index_name, id=str(random_doc_id))
    return(doc["_source"]["Quote"], doc["_source"]["Author"], doc["_source"]["TweetID"])

def get_all_quotes_by_category(category):
    index_name = "quotelist"
    results=client.search(index=index_name, body=\
    {
        "query": {
            "match": {
                "Category": {
                    "query": category
                }
            }
        }
    })
    hit_count = len(results["hits"]["hits"])
    quotes = results["hits"]["hits"]
    return quotes, hit_count

def get_a_random_quote_by_category(category):

    quotes, count = get_all_quotes_by_category(category)
    i = random.randint(1,count) - 1
    return(quotes[i]["_source"]["Quote"], quotes[i]["_source"]["Author"])

#test
#get_all_quotes_by_category("Action")
#get_a_random_quote_by_category("Action")

def get_a_random_fact() :
    index_name = "factlist"
    docs_count = client.count(index=index_name)['count']
    print("docs_count = %d" % docs_count)
    random_doc_id = random.randint(1,docs_count)
    print("generate random doc_id = %d" % random_doc_id)
    doc = client.get(index=index_name, id=str(random_doc_id))
    #print("generate random fact = %s" % doc["_source"]["Fact"])
    return(doc["_source"]["Fact"], doc["_source"]["Image File Name"])
#test
#get_a_random_fact()

def get_a_random_love_quote() :
    index_name = "lovequotelist"
    docs_count = client.count(index=index_name)['count']
    print("docs_count = %d" % docs_count)
    random_doc_id = random.randint(1,docs_count)
    print("generate random doc_id = %d" % random_doc_id)
    doc = client.get(index=index_name, id=str(random_doc_id))
    print(doc)
    return(doc["_source"]["Quote"])

def get_a_random_photo() :
    index_name = "photolist"
    docs_count = client.count(index=index_name)['count']
    print("docs_count = %d" % docs_count)
    random_doc_id = random.randint(1,docs_count)
    print("generate random doc_id = %d" % random_doc_id)
    doc = client.get(index=index_name, id=str(random_doc_id))
    print(doc)
    return(doc["_source"]["Photo File Name"], doc["_source"]["Media ID"])

def get_all_photos() :
    index_name = "photolist"
    photolist = []
    results=client.search(index=index_name,body={"size":999,"query":{"match_all":{}}})
    allphotos = results["hits"]["hits"]
    for num, doc in enumerate(allphotos):
        photo_dict = {
            "Photo File Name": doc["_source"]["Photo File Name"],
            "Media ID": doc["_source"]["Media ID"]
        }
        photolist.append(photo_dict)
        random.shuffle(photolist)
    return(photolist)

def get_a_random_book_tweet_id() :
    index_name = "booklist"
    results=client.search(index=index_name, body={"size":999,"query": { "exists": { "field": "TweetID"} } })
    ''' #No longer need to retweet external tweets
    results=client.search(index=index_name, body=\
        {
            "size":999,
            "query": {
                "bool": {
                    "should": [{
                        "exists": {
                            "field": "TweetID"
                        }
                    },
                    {
                        "exists": {
                            "field": "External TweetID"
                        }
                    }
                    ]
                }
            }
        }
    )
    '''
    hit_count = len(results["hits"]["hits"])
    print("%d hits" % hit_count)
    random_doc_index = random.randint(1,hit_count)
    print(random_doc_index)
    tweet_id = results["hits"]["hits"][random_doc_index-1]["_source"]["TweetID"]
    ext_tweet_id = results["hits"]["hits"][random_doc_index-1]["_source"]["External TweetID"]
    random_tweet_id = tweet_id if tweet_id != None else ext_tweet_id
    print(random_tweet_id)
    return(random_tweet_id)

def get_a_random_book() :
    index_name = "booklist"
    docs_count = client.count(index=index_name)['count']
    print("docs_count = %d" % docs_count)
    random_doc_id = random.randint(1,docs_count)
    print("generate random doc_id = %d" % random_doc_id)
    doc = client.get(index=index_name, id=str(random_doc_id))
    print(doc)
    bookinfo = "Title: " + doc["_source"]["Book Title"] + ".  Author: " + doc["_source"]["Author"] + ".  Year Published: " + doc["_source"]["Year Published"]
    bookimage = doc["_source"]["Image File Name"]
    print(bookinfo)
    return(bookinfo, bookimage)

def get_a_random_book_with_detail() :
    index_name = "booklist"
    docs_count = client.count(index=index_name)['count']
    random_doc_id = random.randint(1,docs_count)
    doc = client.get(index=index_name, id=str(random_doc_id))
    return(doc["_source"]["Book Title"], doc["_source"]["Author"], doc["_source"]["Year Published"], doc["_source"]["Image File Name"], doc["_source"]["TweetID"], doc["_source"]["Rating"])

def get_total_book_count() :
    index_name = "booklist"
    docs_count = client.count(index=index_name)['count']
    print("docs_count = %d" % docs_count)
    return(docs_count)

def get_all_book() :
    results=client.search(index="booklist",body={"size":999,"query":{"match_all":{}}})
    hit_count = len(results["hits"]["hits"])
    #print("%d hits" % hit_count)
    allbooks = results["hits"]["hits"]
    #print(allbooks)
    return(allbooks, hit_count)

def get_all_book_sorted_by_likes() :
    index_name = "booklist"
    results=client.search(index=index_name, body=\
    {
        "size":999,
        "query":{"match_all":{}},
        #"sort" : [ { "Likes" : "desc" } ] #ES does not allow sorting texts, only numbers
    })
    hit_count = len(results["hits"]["hits"])
    allbooks = results["hits"]["hits"]
    # Sorting the array by Likes
    items = [{}] * hit_count
    oneitem = {}
    for num, doc in enumerate(allbooks):
        oneitem = doc["_source"]
        items[num] = oneitem.copy()
    items.sort(key=itemgetter('Likes'),reverse=True)
    return(items, hit_count)

def get_all_bookchat() :
    index_name = "bookchat"
    results=client.search(index=index_name, body=\
    {
        "size":999,
        "query":{"match_all":{}}
    })
    hit_count = len(results["hits"]["hits"])
    allbooks = results["hits"]["hits"]
    return(allbooks, hit_count)

def get_book_statistics():
    index_name = "booklist"
    #total_count = client.count(index=index_name)['count']
    allbooks, total_count = get_all_book()

    results=client.count(index=index_name, body=\
    {
        "query": {
            "match": {
                "Genre": {
                    "query": "Fiction"
                }
            }
        }
    })
    fiction_count = results["count"]
    nonfiction_count = total_count - fiction_count

    #query "Audio Book" return invalid result as ES/query does not search exact string
    #query "Audio" return correct result
    #For same reason, query "Book" will not return physical book count
    results=client.count(index=index_name, body=\
    {
        "query": {
            "match": {
                "Book Type": {
                    "query": "Audio"
                }
            }
        }
    })
    audio_book_count = results["count"]
    results=client.count(index=index_name, body=\
    {
        "query": {
            "match": {
                "Book Type": {
                    "query": "EBook"
                }
            }
        }
    })
    ebook_count = results["count"]
    phy_book_count = total_count - audio_book_count - ebook_count

    ''' aggregration on text field not possible
    results=client.search(index=index_name, body=\
    {
        "aggs": {
            "avg_page_count": { "avg": { "field": "Page Count" } }
        }
    })
    avg_page_count = results["aggregations"]["avg_page_count"]
    '''

    min_page_count = 500
    max_page_count = 0
    total_page_count = 0
    count = 0
    books_read_2018 = 0
    books_read_2019 = 0
    books_read_2020 = 0
    books_read_2021 = 0
    books_read_2022 = 0
    books_read_2023 = 0
    books_published_2000s = 0
    books_published_1900s = 0
    books_published_1800s = 0
    oldest_book = 2021
    shortest_book_image = ""
    oldest_book_image = ""
    longest_book_image = ""
    books_rated_5 = 0
    books_rated_4 = 0
    books_rated_3 = 0
    for doc in allbooks:
        if( doc["_source"]["Page Count"] != None ) :
            page_count = int(doc["_source"]["Page Count"])
            count += 1
            total_page_count = total_page_count + page_count
            if( page_count < min_page_count ):
                min_page_count = page_count
                shortest_book_image = doc["_source"]["Image File Name"]
            if( page_count > max_page_count ):
                max_page_count = page_count
                longest_book_image = doc["_source"]["Image File Name"]
        if( doc["_source"]["Date Finished"] != None ) :
            try:
                year = datetime.strptime(doc["_source"]["Date Finished"], '%m/%d/%Y').year
            except:
                try :
                    year = datetime.strptime(doc["_source"]["Date Finished"], '%Y').year
                except:
                    print("datetime parse exception %s" % doc["_source"]["Date Finished"])
            if( year <= 2018 ) :
                books_read_2018 += 1
            elif( year == 2019 ) :
                books_read_2019 += 1
            elif( year == 2020 ) :
                books_read_2020 += 1
            elif( year == 2021 ) :
                books_read_2021 += 1
            elif( year == 2022 ) :
                books_read_2022 += 1
            else :
                books_read_2023 += 1

        if( doc["_source"]["Year Published"] != None ) :
            year = int(doc["_source"]["Year Published"])
            if( year >= 2000 ) :
                books_published_2000s += 1
            elif( year >= 1900 ) :
                books_published_1900s += 1
            else :
                books_published_1800s += 1
            if( year < oldest_book ) :
                oldest_book = year
                oldest_book_image = doc["_source"]["Image File Name"]

        if( doc["_source"]["Rating"] != None ) :
            if( len(doc["_source"]["Rating"]) == 5 ) :
                books_rated_5 += 1
            elif( len(doc["_source"]["Rating"]) == 4 ) :
                books_rated_4 += 1
            elif( len(doc["_source"]["Rating"]) == 3 ) :
                books_rated_3 += 1

    avg_page_count = int(total_page_count / count)
    print("min_page_count: %d" % min_page_count)
    print("max_page_count: %d" % max_page_count)
    print("total_page_count: %d" % total_page_count)
    print("avg_page_count: %d" % avg_page_count)
    print("books rated 5: %d" % books_rated_5)
    print("books rated 4: %d" % books_rated_4)
    print("books rated 3: %d" % books_rated_3)

    book_stats = \
    {
        "total count": total_count,
        "fiction count": fiction_count,
        "nonfiction count": nonfiction_count,
        "physical book count": phy_book_count,
        "ebook count": ebook_count,
        "audio book count": audio_book_count,
        "min page count": min_page_count,
        "max page count": max_page_count,
        "total page count": total_page_count,
        "total word count": total_page_count*299, # 250-300 words per page
        "average page count": avg_page_count,
        "books read in 2018 and prior": books_read_2018,
        "books read in 2019": books_read_2019,
        "books read in 2020": books_read_2020,
        "books read in 2021": books_read_2021,
        "books read in 2022": books_read_2022,
        "books read in 2023": books_read_2023,
        "books published in 2000s": books_published_2000s,
        "books published in 1900s": books_published_1900s,
        "books published in 1800s and prior": books_published_1800s,
        "oldest book in year": oldest_book,
        "oldest book image": oldest_book_image,
        "shortest book image": shortest_book_image,
        "longest book image": longest_book_image,
        "books rated 5": books_rated_5,
        "books rated 4": books_rated_4,
        "books rated 3": books_rated_3
    }
    return book_stats

# code no longer used
def generate_likes_in_booklist():
    index_name = "booklist"
    results=client.search(index=index_name,body={"size":999,"query":{"match_all":{}}})
    hit_count = len(results["hits"]["hits"])
    allbooks = results["hits"]["hits"]
    for doc in allbooks:
        #if(doc["_source"]["TweetID"]):
        #    like_count = twitterbot.get_likes(doc["_source"]["TweetID"])
        #else:
        #    like_count = 0
        like_count = int(doc["_source"]["Likes"])
        client.update(index=index_name,doc_type='_doc',
            id=doc["_source"]["id"],
            #body={"doc": {"Likes": like_count} }
            body={"doc": {"Likes_": like_count} }  #Likes_ is a number not string
        )
    return
#generate_likes_in_booklist()

def get_all_twitter_books():
    index_name = "twitterbooklist"
    results=client.search(index=index_name, body=\
    {
        "size":999,
        "query":{"match_all":{}},
        "sort" : [ { "Likes" : "desc" } ]
    })
    hit_count = len(results["hits"]["hits"])
    alldocs = results["hits"]["hits"]
    return alldocs, hit_count

def get_all_twitter_books_from_booklist():
    index_name = "booklist"
    results=client.search(index=index_name, body=\
    {
        "size":999,
        "query": {
            "exists": {
                "field": "TweetID"
            }
        }
    })
    hit_count = len(results["hits"]["hits"])
    alldocs = results["hits"]["hits"]
    return alldocs, hit_count
'''
def add_a_twitter_book(title, tweet_id, likes, nonfiction,id) :
    index_name = "twitterbooklist"
    client.index(index=index_name, doc_type='post', body= \
    {
        'Book Title': title,
        'TweetID': tweet_id,
        'Likes': likes,
        'Nonfiction': nonfiction,
        'ID': id
    })
'''
def delete_twitter_book_index():
    index_name = "twitterbooklist"
    client.indices.delete(index=index_name, ignore=[400, 404])

def create_twitter_book_index():
    index_name = "twitterbooklist"
    client.indices.create(index=index_name)

def get_all_love_quotes() :
    results=client.search(index="lovequotelist",body={"size":999,"query":{"match_all":{}}})
    hit_count = len(results["hits"]["hits"])
    print("%d hits" % hit_count)
    allrecords = results["hits"]["hits"]
    #print(allrecords)
    return(allrecords, hit_count)

def get_all_words() :
    results=client.search(index="wordlist",body={"size":999,"query":{"match_all":{}}})
    hit_count = len(results["hits"]["hits"])
    print("%d hits" % hit_count)
    allrecords = results["hits"]["hits"]
    #print(allrecords)
    return(allrecords, hit_count)
'''
def get_all_quotes() :
    results=client.search(index="quotelist",body={"size":999,"query":{"match_all":{}}})
    print("%d hits" % hit_count)
    allrecords = results["hits"]["hits"]
    #print(allrecords)
    return(allrecords, hit_count)
'''
def get_all_quotes() :
    '''  # aggregation needs to enable fielddata for Category text field,
         # which cause performance degradation.
         # By default text field fielddata is disabled
    results=client.search(index="quotelist",
        body= {
          "size": 0, #No need to return any documents
            "aggs": {
                "get_categories": {
                    "terms": {
                        "field": "Category",
                        "size": 20  #up to 20 categories
                    }
                }
            }
        }
    )
    '''
    results=client.search(index="quotelist",body={"size":999,"query":{"match_all":{}}})
    hit_count = len(results["hits"]["hits"])
    print("%d hits" % hit_count)
    allrecords = results["hits"]["hits"]
    category_list = []
    for num, doc in enumerate(allrecords):
        if(doc["_source"]["Category"] not in category_list):
            category_list.append(doc["_source"]["Category"])
    #print(category_list)
    #print(allrecords)
    return(allrecords, hit_count, category_list)

#get_all_quotes()

def get_subscriber_count():
    index_name = "subscriptionlist"
    count = client.count(index=index_name)['count']
    return count

def get_all_subscribers():
    index_name = "subscriptionlist"
    results=client.search(index=index_name,body={"size":999,"query":{"match_all":{}}})
    hit_count = len(results["hits"]["hits"])
    #print("%d hits" % hit_count)
    allrecords = results["hits"]["hits"]
    #print(allrecords)
    return(allrecords, hit_count)

def get_all_subscribers_test():
    index_name = "subscriptionlist_test"
    results=client.search(index=index_name,body={"size":999,"query":{"match_all":{}}})
    hit_count = len(results["hits"]["hits"])
    print("%d hits" % hit_count)
    allrecords = results["hits"]["hits"]
    #print(allrecords)
    return(allrecords, hit_count)

def add_a_subscription(email, interest) :
    client.index(index='ingestsubscriptionlist', doc_type='post', body= \
    {
        'Email': email,
        'Interest': interest
    })

def remove_a_subscription(email, interest) :
    client.index(index='unsubscriptionlist', doc_type='post', body= \
    {
        'Email': email,
        'Interest': interest
    })

def add_a_love_quote(quote, author) :
    client.index(index='lovequotelist_new', doc_type='post', body= \
    {
        'Quote': quote,
        'Author': author
    })

def add_a_comment(comment, author) :
    client.index(index='commentlist', doc_type='post', body= \
    {
        'Comment': comment,
        'Author': author
    })

def global_copy(id, text) :
    client.index(index='globalcopypaste', id=id, body= \
    {
        'Text': text
    })

def global_paste(id) :
    try:
        doc = client.get(index='globalcopypaste', id=id)
        print(doc)
        return doc["_source"]["Text"]
    except:
        return ""

def file_upload(filename) :
    client.index(index='globalfiletransfer', id="1", body= \
    {
        'File Name': filename
    })

def file_download() :
    try:
        doc = client.get(index='globalfiletransfer', id="1")
        print(doc)
        return doc["_source"]["File Name"]
    except:
        return ""

def apittracking_get_callcount(api_name) :
    index_name = "apitracking"
    doc = client.get(index=index_name, id=api_name)
    return doc["_source"]["callcount"]

#api_name is document id
def apittracking_increment_callcount(api_name) :
    index_name = "apitracking"

    if client.exists(index=index_name, id=api_name) == False :
        print("es apittracking_increment_callcount, index or id does not exist, create the first entry")
        count = 0
        client.index(index=index_name, id=api_name, body= \
        {
            'api_name': api_name,
            'callcount' : count
        })

    doc = client.get(index=index_name, id=api_name)
    count = doc["_source"]["callcount"]
    count += 1

    client.update(index=index_name,doc_type='_doc',
            id=api_name,
            body={"doc": {"callcount": count} } )

    return count

def save_journal(date, title, text, type) :
    index_name = "journal"
    if client.indices.exists(index_name) == False:
        print("es save_journal, index does not exist, create the first entry")
        id = 1 #first entry
    else:
        id = client.count(index=index_name)['count']
        print("save_journal, docs_count = %d" % id)
        id = id+1
    print("id= %s" % str(id))
    client.index(index='journal', id=str(id), body= \
    {
        'Date': date,
        'Title' : title,
        'Text': text,
        'Type': type
    })
    return

def edit_journal(id, title, text, type) :
    index_name = "journal"
    print("edit_journal, id=%s" % id)

    doc = client.get(index='journal', id=id)

    client.index(index='journal', id=str(id), body= \
    {
        'Date': doc["_source"]["Date"],
        'Title' : title,
        'Text': text,
        'Type': type
    })
    return

def get_all_journals() :
    index_name = "journal"
    results=client.search(index=index_name,body=\
    {   "size":999,
        "query":{"match_all":{}},
        "sort" : [ { "Date" : "desc" } ]
    })
    hit_count = len(results["hits"]["hits"])
    alljournals = results["hits"]["hits"]
    print(alljournals)
    return(alljournals, hit_count)

def get_all_blogs():
    index_name = "journal"
    results=client.search(index=index_name, body=\
    {
        "query": {
            "match": {
            "Type": {
                "query": "Blog"
            }
            }
        },
        "sort" : [ { "Date" : "desc" } ]
    })
    hit_count = len(results["hits"]["hits"])
    allblogs = results["hits"]["hits"]
    return allblogs, hit_count
#test
#for num, doc in enumerate(results["hits"]["hits"]):
#    print("index = %d" % num)
#    print(doc)

#get_a_random_book_tweet_id()
#get_a_random_quote_and_author()
#get_all_journals()
#get_book_statistics()
#get_all_twitter_books()


import pyrealsense2 as rs
import cv2

import numpy as np
import pandas as pd
import os
import time

import dlib
import face_recognition as facerec
import tensorflow as tf

#=============== Reference Link ===============
#
# Face Recognition
#http://www.daydev.com/developer/s6-programming-language/python/face-recognition-real-time-python-opencv.html
#
# Intel Realsense SDK
#https://github.com/IntelRealSense/librealsense
#
# Tensorflow: Binary classifier
#https://towardsdatascience.com/performing-classification-in-tensorflow-95368fde289c
#https://medium.com/analytics-vidhya/save-and-load-a-tensorflow-estimator-model-for-predictions-233b798620a9
#
#=============================================

# Detect data during VIDEO
data_locations = []
data_encodings = []
data_names = []
frameProcess = True
# Database for recognition
face_names = []
face_encodings = []

# Attribute for frame
colorF = (255, 0, 0)
colorL = (255, 255, 255)
colorT = (0, 0, 255)
font = cv2.FONT_HERSHEY_SIMPLEX

name = 'UNKNOWN'

# Import all face from ./Profile
# Name your picture as your name
# Such as "PRUETIKORN.jpg"
entries = os.listdir('Profile/')
for pic in entries:
    if '.jpg' in pic or '.png' in pic:
        face_names.append(pic.split('_')[0])
        database_image = facerec.load_image_file("Profile/"+pic)
        database_encoding = facerec.face_encodings(database_image)[0]
        face_encodings.append(database_encoding)

# dataFrame from depth image
depth_points = {
    "Chin": [],
    "Nose": [],
    "RightEye1": [],
    "RightEye2": [],
    "LeftEye1": [],
    "LeftEye2": [],
    "MouthRight": [],
    "MouthTop": [],
    "MouthLeft": [],
    "MouthBottom": [],
}
input_fn = pd.DataFrame(data = depth_points, dtype = np.float64)

# Collecting data from depth image
depth_ps = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
data_ps = [9, 34, 37, 40, 43, 46, 49, 52, 55, 58]

# Load the model
path_model = './Model_dir/model'
new_model = tf.saved_model.load(path_model)

# Load the detector
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")

# Configure depth and color streams
pipeline = rs.pipeline()
config = rs.config()

# Get device product line for setting a supporting resolution
pipeline_wrapper = rs.pipeline_wrapper(pipeline)
pipeline_profile = config.resolve(pipeline_wrapper)
device = pipeline_profile.get_device()
device_product_line = str(device.get_info(rs.camera_info.product_line))

config.enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 30)
config.enable_stream(rs.stream.color, 640, 480, rs.format.bgr8, 30)

#predict the output from the trained model
def predict(dfeval, importedModel):
    colNames = dfeval.columns
    dtypes = dfeval.dtypes
    predictions = []
    for row in dfeval.iterrows():
        example = tf.train.Example()
        for i in range(len(colNames)):
            dtype = dtypes[i]
            colName = colNames[i]
            value = row[1][colName]
            if dtype == "object":
                value = bytes(value, "utf-8")
                example.features.feature[colName].bytes_list.value.extend(
                    [value])
            elif dtype == "float":
                example.features.feature[colName].float_list.value.extend(
                    [value])
            elif dtype == "int":
                example.features.feature[colName].int64_list.value.extend(
                    [value])

        predictions.append(
           importedModel.signatures["predict"](
           examples=tf.constant([example.SerializeToString()])))
    return predictions

#optional features
max_time = 0
record = cv2.VideoWriter('presenter.avi', cv2.VideoWriter_fourcc(*'MJPG'), 10, (640, 480))

# Start streaming
pipeline.start(config)

try:
    while True:
        time_s = time.time()

        # Wait for a coherent pair of frames: depth and color
        frames = pipeline.wait_for_frames()
        depth_frame = frames.get_depth_frame()
        color_frame = frames.get_color_frame()
        if not depth_frame or not color_frame:
            continue

        # Convert images to numpy arrays
        depth_image = np.asanyarray(depth_frame.get_data())
        color_image = np.asanyarray(color_frame.get_data())

        resizing = cv2.resize(color_image, (0, 0), fx=0.25, fy=0.25)
        result = resizing[:, :, ::-1]
        gray = cv2.cvtColor(color_image, cv2.COLOR_BGR2GRAY)

        #Check that there is a face or not
        if frameProcess:
            data_l = detector(gray)
            data_n = []
            newPred = []
            for f in data_l:
                landmarks = predictor(gray, f)
                for n in data_ps:
                     x = landmarks.part(n).x
                     y = landmarks.part(n).y
                     try:
                         depth_ps[data_ps.index(n)] = depth_image[y, x]
                     except IndexError:
                         pass
            input_fn.loc[0] = depth_ps
            face_pred = predict(input_fn, new_model)
            for pred in face_pred:
                newPred.append(np.argmax(pred["probabilities"]))

            data_l = facerec.face_locations(result)
            data_e = facerec.face_encodings(result, data_l)
            for enc in data_e:
                recog = facerec.compare_faces(face_encodings, enc)
                name = 'UNKNOWN'
                if True in recog:
                    match_index = recog.index(True)
                    name = face_names[match_index]
                    data_n.append(name)

        frameProcess = not frameProcess

        for (t, r, b, l), n, p in zip(data_l, data_n, newPred):
            if p == 1:
                t *= 4
                r *= 4
                b *= 4
                l *= 4
                cv2.rectangle(color_image, (l, t), (r, b), colorF, 2)
                cv2.rectangle(color_image, (l, b-35), (r, b), colorF, cv2.FILLED)
                cv2.putText(color_image, n, (l+6, b-6), font, 1.0, colorL, 1)

        # Show images
        record.write(color_image)
        cv2.namedWindow('RealSense Face Recognition', cv2.WINDOW_AUTOSIZE)
        cv2.imshow('RealSense Face Recognition', color_image)
        key = cv2.waitKey(1)

        process_time = time.time() - time_s
        print('\033[1;35;40m Time process: ' + str(process_time) + ' ms')
        if max_time < process_time:
            max_time = process_time

        if key & 0xFF == ord('q') or key == 27:
            cv2.destroyAllWindows()
            break

finally:
    print("\033[1;36;40m-----------------------------------------------------------------")
    print('Maximum time: ' + str(max_time) + ' ms')

    # Stop streaming
    pipeline.stop()


import torch.nn as nn
import torch.nn.functional as F
import torch

class UNetDown(nn.Module):
    def __init__(self, in_size, out_size, normalize=True, dropout=0.0):
        super(UNetDown, self).__init__()
        layers = [nn.Conv2d(in_size, out_size, kernel_size=4, stride=2, padding=1)]
        if normalize:
            layers.append(nn.InstanceNorm2d(out_size))
        layers.append(nn.LeakyReLU(0.2))
        if dropout:
            layers.append(nn.Dropout(dropout))
        self.model = nn.Sequential(*layers)

    def forward(self, x):
        return self.model(x)

class UNetUp(nn.Module):
    def __init__(self, in_size, out_size, dropout=0.0):
        super(UNetUp, self).__init__()
        layers = [nn.ConvTranspose2d(in_size, out_size, kernel_size=4, stride=2, padding=1),
                nn.InstanceNorm2d(out_size),
                nn.ReLU()]
        if dropout:
            layers.append(nn.Dropout(dropout))
        self.model = nn.Sequential(*layers)

    def forward(self, x, skip_input):
        x = self.model(x)
        x = torch.cat((x, skip_input), 1)

        return x

class GeneratorUNet(nn.Module):
    def __init__(self, in_channels=3, out_channels=3):
        super(GeneratorUNet, self).__init__()

        self.down1 = UNetDown(in_channels, 64, normalize=False)
        self.down2 = UNetDown(64, 128)
        self.down3 = UNetDown(128, 256, dropout=0.5)
        self.down4 = UNetDown(256, 512, dropout=0.5)
        self.down5 = UNetDown(512, 512, dropout=0.5)
        self.down6 = UNetDown(512, 512, dropout=0.5, normalize=False)

        self.up1 = UNetUp(512, 512, dropout=0.5)
        self.up2 = UNetUp(1024, 512, dropout=0.5)
        self.up3 = UNetUp(1024, 256, dropout=0.5)
        self.up4 = UNetUp(512, 128, dropout=0.5)
        self.up5 = UNetUp(256, 64)

        self.final = nn.Sequential(
            nn.Upsample(scale_factor=2),
            nn.ZeroPad2d((1, 0, 1, 0)),
            nn.ZeroPad2d((1, 0, 1, 0)),
            nn.ZeroPad2d((1, 0, 1, 0)),
            nn.Conv2d(128, out_channels=out_channels, kernel_size=4, stride=1),
            nn.Tanh()
        )

    def forward(self, x):
        d1 = self.down1(x)
        d2 = self.down2(d1)
        d3 = self.down3(d2)
        d4 = self.down4(d3)
        d5 = self.down5(d4)
        d6 = self.down6(d5)
        u1 = self.up1(d6, d5)
        u2 = self.up2(u1, d4)
        u3 = self.up3(u2, d3)
        u4 = self.up4(u3, d2)
        u5 = self.up5(u4, d1)

        return self.final(u5)

class Discriminator(nn.Module):
    def __init__(self, in_channels=3):
        super(Discriminator, self).__init__()

        def discriminator_block(in_channels, out_channels, normalization=True):
            layers = [nn.Conv2d(in_channels, out_channels, 4, stride=2, padding=1)]
            if(normalization):
                layers.append(nn.InstanceNorm2d(out_channels))
            layers.append(nn.LeakyReLU(0.2, inplace=True))

            return layers

        self.model = nn.Sequential(
            *discriminator_block(in_channels, 64, normalization=False),
            *discriminator_block(64, 128),
            *discriminator_block(128, 256),
            nn.ZeroPad2d((1, 0, 1, 0)),
            nn.Conv2d(256, 1, 4, padding=1),
        )

        self.lin = nn.Linear(64, 1)

    def forward(self, x):
        return (self.lin(self.model(x).view(-1, 64)))


import os
import random
import shutil
import argparse

import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)

def count_images(directory, image_extensions):
    image_count = 0
    for root, _, files in os.walk(directory):
        for file in files:
            if any(file.lower().endswith(ext) for ext in image_extensions):
                image_count += 1
    return image_count

def split_dataset(source_dir, train_dir, test_dir, split_ratio=0.8):
    image_extensions = ['.jpg', '.jpeg', '.png', '.JPG', '.JPEG', '.PNG']

    for root, _, files in os.walk(source_dir):
        for file in files:
            if any(file.lower().endswith(ext) for ext in image_extensions):
                base_filename, file_extension = os.path.splitext(file)
                source_image_path = os.path.join(root, file)
                source_xml_path = os.path.join(root, base_filename + ".xml")

                # Determine whether to put the file in the train or test set
                if random.uniform(0, 1) < split_ratio:
                    destination_dir = train_dir
                else:
                    destination_dir = test_dir

                # Recreate the subfolder structure in the destination directory
                relative_path = os.path.relpath(root, source_dir)
                destination_subdir = os.path.join(destination_dir, relative_path)
                os.makedirs(destination_subdir, exist_ok=True)

                # Copy the image and its corresponding XML file
                destination_image_path = os.path.join(destination_subdir, file)
                destination_xml_path = os.path.join(destination_subdir, base_filename + ".xml")
                shutil.copy(source_image_path, destination_image_path)
                shutil.copy(source_xml_path, destination_xml_path)

    train_image_count = count_images(train_dir, image_extensions)
    test_image_count = count_images(test_dir, image_extensions)

    print("--------------------------DATASET SPLIT COMPLETED------------------------------")
    print(f'Train Dataset Directory: {train_dir}')
    print(f'Train Dataset Count: {train_image_count} Images')
    print(f'Test Dataset Directory: {test_dir}')
    print(f'Test Dataset Count: {test_image_count} Images')
    print("-------------------------------------------------------------------------------")

def main():

    # source_directory = "/Users/ofotech_fitri/Documents/tf_object_detection/script/script_temp/dataset/temp"
    # train_directory = "/Users/ofotech_fitri/Documents/tf_object_detection/script/script_temp/dataset/train"
    # test_directory = "/Users/ofotech_fitri/Documents/tf_object_detection/script/script_temp/dataset/test"
    # split_ratio = 0.8  # 80% for training, 20% for testing

        # Initiate argument parser
    parser = argparse.ArgumentParser(description="Partition dataset of images into training and testing sets",
                                     formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument(
        '-i', '--imageDir',
        dest = "imageDir",
        help='Path to the folder where the image dataset is stored. If not specified, the CWD will be used.',
        type=str,
        default=os.getcwd()
    )
    parser.add_argument(
        '-o', '--trainDir',
        dest = "trainDir",
        help='Path to the output folder where the train and test dirs should be created. '
             'Defaults to the same directory as IMAGEDIR.',
        type=str,
        default=None
    )
    parser.add_argument(
        '-t', '--testDir',
        dest = "testDir",
        help='Path to the output folder where the train and test dirs should be created. '
             'Defaults to the same directory as IMAGEDIR.',
        type=str,
        default=None
    )
    parser.add_argument(
        '-r', '--ratio',
        dest = "ratio",
        help='The ratio of the number of test images over the total number of images. The default is 0.1.',
        default=0.1,
        type=float)

    args = parser.parse_args()

    os.makedirs(args.trainDir, exist_ok=True)
    os.makedirs(args.testDir, exist_ok=True)

    # split_dataset(source_directory, train_directory, test_directory, split_ratio)
    split_dataset(args.imageDir, args.trainDir, args.testDir, args.ratio)

if __name__ == '__main__':
    main()


l=[12,22,33,44,65]
print (l)

# loop to traverse each element in the l
# and, remove elements
# which are EVEN (divisible by 2)
for i  in l:
	if(i%2 == 0):
	    l.remove(i)

# print l after removing EVEN elements
print (l)


class GhgEmissionIntensity(dict):
    def __init__(self, emission):
        super().__init__(
            code=emission['code'],
            name=emission['name'],
            value=float(emission['value']) if emission['value'] is not None else None,
            flag=emission['flag']
        )


import matplotlib.pyplot as plt
from numpy import arange

def plot_loss_acc(losses, accuracies, epochs):

    train_loss, val_loss = losses
    train_acc, val_acc = accuracies
    epochs_range = range(1, epochs +1)
    plt.figure(1)
    plt.plot(epochs_range, train_loss, 'r-', label='train')
    plt.plot(epochs_range, val_loss, 'b-', label='val')
    plt.xlabel('Epoch')
    plt.ylabel('Loss value')
    plt.title('Loss')
    plt.savefig('loss.jpg')

    plt.figure(2)
    plt.plot(epochs_range, train_acc, 'r-', label='train')
    plt.plot(epochs_range, val_acc, 'b-', label='val')
    plt.xlabel('Epoch')
    plt.ylabel('Accuracy')
    plt.title('Training/Validation Accuracy')
    plt.savefig('accuracy.jpg')


class PIDCurrentRegulator:
    def __init__(self, Kp, Ki, Kd, Ki_min, Ki_max):
        # Define the PID gains
        self.Kp = Kp
        self.Ki = Ki
        self.Kd = Kd

        self.Ki_min = Ki_min
        self.Ki_max = Ki_max

        # Define the variables
        self._last_error = 0
        self._integral = 0

    def regulate_current(self, setpoint_current_A, measured_current_A):
        # Calculate the error
        error = setpoint_current_A - measured_current_A

        # Calculate the integral
        self._integral = self._integral + error

        if self._integral > self.Ki_max:
            self._integral = self.Ki_max
        elif self._integral < self.Ki_min:
            self._integral = self.Ki_min

        # Calculate the derivative
        derivative = error - self._last_error

        # Calculate the output voltage
        output_voltage = self.Kp * error + self.Ki * self._integral + self.Kd * derivative

        # Save the last error
        self._last_error = error

        return output_voltage


# each level of the cache
# size (bytes), latency (cycles), block size (bytes),
# set associativity, write policy

# storage array
# [ [valid (0/1), dirty (0/1), data] ], data, data, ...]
# store newest data at the end, LRU is the front element and we will pop off

from Block import Block
from math import log2
import random

class LevelCache:
    def __init__(self, size, latency, blockSize, setAssociativity, writePolicy, level):
        self.size = size # total capacity in bytes
        self.latency = latency # access latency in cycles
        self.blockSize = blockSize # block size in bytes
        self.setAssociativity = setAssociativity # set associativity in ways
        self.sets = (self.size // self.blockSize) // self.setAssociativity
        self.writePolicy = writePolicy # 0 = write-back, 1 = write-through
        self.level = level
        self.lastLevel = True
        self.tagBits = 32 - int(log2(self.blockSize)) - int(log2(self.sets))

        self.contents = [] # [ [ [block 1], [block 2], ... ] , [ [block 1], [block 2], ... ] ]
                           # block 1: [ valid , dirty , tag , data ]

        self.fillEmptyContents()

        self.hits = 0
        self.misses = 0

    def fillEmptyContents(self):
        layers = self.sets
        for i in range(layers):
            setArr = []
            for j in range(self.setAssociativity):
                block = Block(0, 0, 0, self.blockSize)
                block.data = [0 for i in range(self.blockSize)]
                setArr.append(block)

            self.contents.append(setArr)

    def write(self, address, dirty, data, arrivalTime):
        # set index and tag
        setIndex = (address // self.blockSize) % self.sets
        tag = address >> (32 - self.tagBits)

        # create new block to write
        newBlock = Block(1, dirty, tag, self.blockSize)
        newBlock.data = data
        newBlock.timeAccessed = arrivalTime

        for i, block in enumerate(self.contents[setIndex]):
            # the first empty block we come across, fill
            if not block.valid:
                self.contents[setIndex][i] = newBlock
                return

        # if none were empty, find the LRU and place there
        LRUIndex = self.LRU(setIndex)
        self.contents[setIndex][LRUIndex] = block

    def writeAccess(self, address):
        setIndex = (address // self.blockSize) % self.sets
        tag = address >> (32 - self.tagBits)

        for i, block in enumerate(self.contents[setIndex]):
            # if the block is valid and the tag matches, return true
            if block.valid and block.tag == tag:
                self.hits += 1
                print(self.level, 'hit write')
                return True

        # if the address cannot be found, miss, return false
        self.misses += 1
        return False

    def writeUp(self, address, dirty, newBlock, arrivalTime):
        # block address
        setIndex = (address // self.blockSize) % self.sets
        tag = address >> (32 - self.tagBits)

        newBlock.tag = tag

        for i, block in enumerate(self.contents[setIndex]):
            # the first empty block we come across, fill
            if block.timeAccessed == arrivalTime:
                self.contents[setIndex][i] = newBlock
                return

    def read(self, address, arrivalTime):
        # which block in a set is accessed
        blockOffset = address % self.blockSize

        # block address
        setIndex = (address // self.blockSize) % self.sets

        tag = address >> (32 - self.tagBits)

        # for each block in the set, search
        for block in self.contents[setIndex]:
            # if you find your tag and it's valid, hit
            if block.tag == tag and block.valid:
                self.hits += 1
                block.timeAccessed = arrivalTime
                return [True, block, self.level]

        # if miss, then write into first empty block
        # increment miss
        self.misses += 1

        # create new block from memory
        newBlock = Block(1, 0, tag, self.blockSize)
        newBlock.data = [random.randint(0, 255) for i in range(self.blockSize)]
        newBlock.timeAccessed = arrivalTime

        for i, block in enumerate(self.contents[setIndex]):
            # the first empty block we come across, fill
            if not block.valid:
                self.contents[setIndex][i] = newBlock
                return [False, newBlock, self.level]

        # if none were empty, find the LRU and place there
        LRUIndex = self.LRU(setIndex)
        self.contents[setIndex][LRUIndex] = newBlock

        return [False, newBlock, self.level]

    # function to return the LRU block
    def LRU(self, setIndex):
        LRUTime, index = self.contents[setIndex][0].timeAccessed, 0
        for i in range(len(self.contents[setIndex])):
            if self.contents[setIndex][i].timeAccessed < LRUTime:
                LRUTime = self.contents[setIndex][i].timeAccessed
                index = i

        return index

    # function to calculate hit rate
    def hitRate(self):
        try:
            hitRate = self.hits / (self.hits + self.misses)
        except:
            hitRate = 0

        return int(hitRate*100)

    # function to calculate miss rate
    def missRate(self):
        try:
            missRate = self.misses / (self.hits + self.misses)
        except:
            missRate = 0

        return int(missRate*100)

    # function to print cache contents
    def printContents(self):
        setNum = 0
        for setObj in self.contents:
           print('Set #', setNum, end=': ')
           setNum += 1
           for i in range(len(setObj)):
               print('Block #' + str(i) + ': ' + str(setObj[i].data) + ' (' + str(setObj[i].tag), str(setObj[i].timeAccessed) + ')', end=' ')

           print('')


import requests
from bs4 import BeautifulSoup

url ="https://www.mygov.in/corona-data/covid19-statewise-status"
req = requests.get(url)

bs = BeautifulSoup(req.text,"html.parser")

data= bs.find_all("div",class_="field-item odd")

maharashtra = data[7].text
Confirmed  =42 # index = 42
Discharged = 73 # index = 73
Deaths = 83 # index = 83

confirmed = int(maharashtra[int(Confirmed):46])
discharged = int(maharashtra[int(Discharged):76])
deaths = int(maharashtra[int(Deaths):])

print("===MAHARASHTRA REPORT COVID-19===")
print("Total number of confermed cases:",confirmed)
print("Total dischargged:",discharged)
print("Total deaths:",deaths)


from aiogram import types
import re

from aiogram.types import InlineKeyboardMarkup

from data.config import group_likes
from data.text import text
from filters import IsGroup
from loader import dp
from aiogram.utils.deep_linking import get_start_link
from utils.check_like import get_owner, check_like
from utils.db_api.mongo import post_db

instagram_regex = r'Dx([1-9]|[1-4]\d|50) (?:(?:https?|ftp):\/\/)?(?:www\.)?(?:instagram\.com|instagr\.am)\/([A-Za-z0-9-_]+)\/'

@dp.message_handler(IsGroup())
async def get_link(message: types.Message):
    if re.search(instagram_regex, message.text):
        ### Wait text
        await message.reply(text['checking_list'].format(message.from_user.get_mention(message.from_user.full_name)))

        ### Get instagram link, shortcode and owner
        instagram_link = message.text.split(' ')[-1]
        shortcode = instagram_link.split('/')[4]
        owner_id = "@{}".format(await get_owner(shortcode))

        if not await check_like(shortcode):
            await message.delete()
            await message.answer(
                text['warning_text'].format(message.from_user.get_mention(message.from_user.full_name)))
            return

        temp = post_db.insert_one({'link': instagram_link})
        print(temp.inserted_id)
        ### Create Button with link
        deeplink = await get_start_link(group_likes[message.chat.id])
        keyboard = InlineKeyboardMarkup(inline_keyboard=[
            [(types.InlineKeyboardButton(text['instagram_button'], url=instagram_link)),
             (types.InlineKeyboardButton(text['instagram_link_list'], url=deeplink))],
        ], row_width=2)

        ### Message delete
        await message.delete()

        ### Send message
        await message.answer(text['instagram_text'].format(message.from_user.get_mention(message.from_user.full_name),
                                                           message.text.split(" ")[0],
                                                           owner_id,
                                                           instagram_link), reply_markup=keyboard)
        return

    await message.delete()
    await message.answer(text['warning_text'].format(message.from_user.get_mention(message.from_user.full_name)))


# Definition for singly-linked list.
# class ListNode:
#     def __init__(self, x):
#         self.val = x
#         self.next = None

class Solution:
    # @param two ListNodes
    # @return a ListNode
    def mergeTwoLists(self, l1, l2):

        current = head = ListNode(0)

        while l1 != None or l2 != None:

            temp = None
            if l1 == None: temp = l2.val
            elif l2 == None: temp = l1.val
            else: temp = min(l1.val, l2.val)

            current.next = ListNode(temp)
            current = current.next

            if l1 != None and l1.val == temp: l1 = l1.next
            elif l2 != None and l2.val == temp: l2 = l2.next

        return head.next