# 返回值的关键字是return，存在函数中

# def buyIceCream():
#
#     return '冰激凌'
#
# # 使用一个变量来接受函数的返回值
# food = buyIceCream()
#
# print(food)

# 案例练习
# 定义一个函数 然后让函数计算两个数值的和  并且返回这个计算之后的结果

def sum(a,b):
    c = a + b
    return c

a = sum(123,456)

print(a)

尚硅谷Python学科爬虫教程/代码/045_尚硅谷_爬虫_函数_函数的返回值.py

045_尚硅谷_爬虫_函数_函数的返回值.py


from datetime import datetime
from weakref import WeakValueDictionary

from sqlalchemy import inspect
from sqlalchemy.orm import aliased

from app import db

class MetaModel(db.Model.__class__):
    # Define MetaClass for the BaseModel
    def __init__(self, *args):
        super().__init__(*args)
        self.aliases = WeakValueDictionary()

    def __getitem__(self, key):
        try:
            alias = self.aliases[key]
        except KeyError:
            alias = aliased(self)
            self.aliases[key] = alias
        return alias

class Model:
    print_filter = ()
    to_json_filter = ()

    def __repr__(self):
        # Defining a basway to print models
        return "%s(%s)" % (
            self.__class__.__name__,
            {
                column: value
                for column, value in self._to_dict().items()
                if column not in self.print_filter
            },
        )

    @property
    def json(self):
        return {
            column: value
            if not isinstance(value, datetime)
            else value.strftime("%Y-%m-%d")
            for column, value in self._to_dict().items()
            if column not in self.to_json_filter
        }

    def _to_dict(self):
        # private to_json, allowing overrides without impacting __repr__
        # and being able to add a filter to a lists.
        return {
            column.key: getattr(self, column.key)
            for column in inspect(self.__class__).attrs
        }

    def save(self):
        db.session.add(self)
        db.session.commit()

    def delete(self):
        db.session.delete(self)
        db.session.commit()


from django.urls import path

from orders.views import CreateOrderView, OrdersDetailsView, ShippingAdressView, CartView

urlpatterns = [
    path('create_order', CreateOrderView.as_view(), name="create_order"),
    path('order_details/<int:pk>/', OrdersDetailsView.as_view(), name="order_details"),

    path('shipping_address', ShippingAdressView.as_view(), name="shipping_address"),
    path('cart', CartView.as_view(), name="cart"),

]


#!/usr/bin/env python
# encoding: utf-8
""" Solution to Project Euler problem 1: Multiples of 3 and 5

    If we list all the natural numbers below 10 that are multiples of 3 or 5, we
    get 3, 5, 6 and 9. The sum of these multiples is 23.

    Find the sum of all the multiples of 3 or 5 below 1000.

"""

def multiples(N):
    """ Main function

    """
    numbers = [i for i in range(N)]
    result = 0
    for n in numbers:
        if n % 3 == 0 or n % 5 == 0:
            result += n
    print(result)

if __name__ == '__main__':
    multiples(1000)


pkgname = "nano"
_mver = "5"
pkgver = f"{_mver}.9"
pkgrel = 0
build_style = "gnu_configure"
configure_args = ["--enable-utf8"]
makedepends = ["ncurses-devel", "file-devel", "linux-headers"]
pkgdesc = "GNU nano text editor"
maintainer = "q66 <q66@chimera-linux.org>"
license = "GPL-3.0-or-later"
url = "https://www.nano-editor.org"
source = f"https://www.nano-editor.org/dist/v{_mver}/nano-{pkgver}.tar.xz"
sha256 = "757db8cda4bb2873599e47783af463e3b547a627b0cabb30ea7bf71fb4c24937"

def post_install(self):
    self.install_file("syntax/nanorc.nanorc", "usr/share/examples/nano")


import sys
import os
import ROOT
import pandas
import Systematics
filedir = os.path.dirname(os.path.realpath(__file__))
pyrootdir = os.path.dirname(filedir)

sys.path.append(pyrootdir)
import util.tools.plotClasses as plotClasses

# samples
# input path
path  = "/nfs/dust/cms/user/vdlinden/legacyTTH/ntuples/nobtags/2018/"

path_4FS_ttbb_SL  = path+"/TTbb_4f*SemiLeptonic*/*nominal*.root"
path_4FS_ttbb_DL  = path+"/TTbb_4f*2l2nu*/*nominal*.root"
path_4FS_ttbb_FH  = path+"/TTbb_4f*Hadronic*/*nominal*.root"

path_5FS_ttbar_SL = path+"/TTToSemiLeptonic*/*nominal*.root"
path_5FS_ttbar_DL = path+"/TTTo2L2Nu*/*nominal*.root"
path_5FS_ttbar_FH = path+"/TTToHadronic*/*nominal*.root"

path_ttH_bb       = path+"/ttHTobb*/*nominal*.root"
path_ttH_nonbb    = path+"/ttHToNonbb*/*nominal*.root"

path_ttZ_qq       = path+"/TTZToQQ*/*nominal*.root"
path_ttZ_ll       = path+"/TTZToLL*/*nominal*.root"
# SELECTIONS
# need to veto muon events in electron dataset to avoid double counting and vice versa
sel_singleel="((N_LooseMuons==0 && N_TightElectrons==1))"
sel_singlemu="(N_LooseElectrons==0 && N_TightMuons==1)"
# jet tag base selection

sel_jettag = "(N_Jets>=4)*(Evt_MET_Pt>20.)"

# ======= #
# WEIGHTS #
# ======= #
defaultWeight = sel_jettag+"*Weight_GEN_nom*Weight_pu69p2"

# lepton scale factors
electronSFs = "((N_TightElectrons==1)&&(Electron_IdentificationSF[0]>0.)&&(Electron_ReconstructionSF[0]>0.))*Electron_IdentificationSF[0]*Electron_ReconstructionSF[0]"
muonSFs     = "((N_TightMuons==1)&&(Muon_IdentificationSF[0]>0.)&&(Muon_IsolationSF[0]>0.))*Muon_IdentificationSF[0]*Muon_IsolationSF[0]"
electronTrigger = "("+sel_singleel+"&&(internalEleTriggerWeight>0.))*internalEleTriggerWeight"
muonTrigger = "("+sel_singlemu+"&&(Weight_MuonTriggerSF>0.))*Weight_MuonTriggerSF"

defaultWeight += "*"+"("+electronSFs+"+"+muonSFs+")"+"*"+"("+electronTrigger+"+"+muonTrigger+")"
# dictionary of expressions to replace in systematics csv

weightReplacements = {
    # default weight
    "FINALWEIGHTNJET":  defaultWeight+"*internalCSVweight*sf__NJet__btag_NOMINAL",
    "FINALWEIGHTHT":    defaultWeight+"*internalCSVweight*sf__HT_vs_NJet__btag_NOMINAL",
    "DEFAULTWEIGHT":    defaultWeight+"*internalCSVweight",
    }

# Lumi weight
lumi = '59.7'
sel_ttb  =  "(GenEvt_I_TTPlusCC==0&&GenEvt_I_TTPlusBB==1)"
sel_tt2b = "(GenEvt_I_TTPlusCC==0&&GenEvt_I_TTPlusBB==2)"
sel_ttbb = "(GenEvt_I_TTPlusCC==0&&GenEvt_I_TTPlusBB==3)"
sel_tthf = "(GenEvt_I_TTPlusCC==0&&GenEvt_I_TTPlusBB>=1)"
sel_ttcc = "(GenEvt_I_TTPlusCC==1)"
sel_ttlf = "(GenEvt_I_TTPlusCC==0&&GenEvt_I_TTPlusBB==0)"

# nominal weight
nominalweight="NomWeight:=("+defaultWeight+")"

sampleDict=plotClasses.SampleDictionary()
sampleDict.doPrintout()
doReadTrees=True

# data samples
samplesDataControlPlots=[
]

samples  = [
     plotClasses.Sample('t#bar{t}+H(bb)',ROOT.kBlue+1,
            path_ttH_bb,
            lumi,
            'ttH_bb',
            samDict=sampleDict, readTrees=doReadTrees, typ = "signal"),

     plotClasses.Sample('t#bar{t}+H(nonbb)',ROOT.kBlue+1,
            path_ttH_nonbb,
            lumi,
            'ttH_nonbb',
            samDict=sampleDict, readTrees=doReadTrees, typ = "signal"),

     plotClasses.Sample('t#bar{t}+b#bar{b} (4FS,SL)',ROOT.kRed+3,
             path_4FS_ttbb_SL,
             lumi+"*"+sel_tthf,
             'ttbb_SL',
             samDict=sampleDict, readTrees=doReadTrees),

     plotClasses.Sample('t#bar{t}+b#bar{b} (4FS,DL)',ROOT.kRed+3,
             path_4FS_ttbb_DL,
             lumi+"*"+sel_tthf,
             'ttbb_DL',
             samDict=sampleDict, readTrees=doReadTrees),

     plotClasses.Sample('t#bar{t}+b#bar{b} (4FS,FH)',ROOT.kRed+3,
             path_4FS_ttbb_FH,
             lumi+"*"+sel_tthf,
             'ttbb_FH',
             samDict=sampleDict, readTrees=doReadTrees),

     plotClasses.Sample('t#bar{t}+b#bar{b} (5FS,SL)',ROOT.kRed+3,
             path_5FS_ttbar_SL,
             lumi+"*"+sel_tthf,
             'ttbb_5FS_SL',
             samDict=sampleDict, readTrees=doReadTrees),

     plotClasses.Sample('t#bar{t}+b#bar{b} (5FS,DL)',ROOT.kRed+3,
             path_5FS_ttbar_DL,
             lumi+"*"+sel_tthf,
             'ttbb_5FS_DL',
             samDict=sampleDict, readTrees=doReadTrees),

     plotClasses.Sample('t#bar{t}+b#bar{b} (5FS,FH)',ROOT.kRed+3,
             path_5FS_ttbar_FH,
             lumi+"*"+sel_tthf,
             'ttbb_5FS_FH',
             samDict=sampleDict, readTrees=doReadTrees),

     #plotClasses.Sample('t#bar{t}+b (4FS,SL)',ROOT.kRed+3,
     #        path_4FS_ttbb_SL,
     #        lumi+"*"+sel_ttb,
     #        'ttb_4FS_SL',
     #        samDict=sampleDict, readTrees=doReadTrees),

     #plotClasses.Sample('t#bar{t}+b (4FS,DL)',ROOT.kRed+3,
     #        path_4FS_ttbb_DL,
     #        lumi+"*"+sel_ttb,
     #        'ttb_4FS_DL',
     #        samDict=sampleDict, readTrees=doReadTrees),

     #plotClasses.Sample('t#bar{t}+b (4FS,FH)',ROOT.kRed+3,
     #        path_4FS_ttbb_FH,
     #        lumi+"*"+sel_ttb,
     #        'ttb_4FS_FH',
     #        samDict=sampleDict, readTrees=doReadTrees),

     #plotClasses.Sample('t#bar{t}+2b (4FS,SL)',ROOT.kRed+3,
     #        path_4FS_ttbb_SL,
     #        lumi+"*"+sel_tt2b,
     #        'tt2b_4FS_SL',
     #        samDict=sampleDict, readTrees=doReadTrees),

     #plotClasses.Sample('t#bar{t}+2b (4FS,DL)',ROOT.kRed+3,
     #        path_4FS_ttbb_DL,
     #        lumi+"*"+sel_tt2b,
     #        'tt2b_4FS_DL',
     #        samDict=sampleDict, readTrees=doReadTrees),

     #plotClasses.Sample('t#bar{t}+2b (4FS,FH)',ROOT.kRed+3,
     #        path_4FS_ttbb_FH,
     #        lumi+"*"+sel_tt2b,
     #        'tt2b_4FS_FH',
     #        samDict=sampleDict, readTrees=doReadTrees),

     #plotClasses.Sample('t#bar{t}+b#bar{b} (5FS)',ROOT.kRed+3,
     #        path_5FS_ttbar,
     #        lumi+"*"+sel_ttbb,
     #        'ttbb_5FS',
     #        samDict=sampleDict, readTrees=doReadTrees),

     #plotClasses.Sample('t#bar{t}+b (5FS)',ROOT.kRed-2,
     #        path_5FS_ttbar,
     #        lumi+"*"+sel_ttb,
     #        'ttb_5FS',
     #        samDict=sampleDict, readTrees=doReadTrees),

     #plotClasses.Sample('t#bar{t}+2b (5FS)',ROOT.kRed+2,
     #        path_5FS_ttbar,
     #        lumi+"*"+sel_tt2b,
     #        'tt2b_5FS',
     #        samDict=sampleDict, readTrees=doReadTrees),

    plotClasses.Sample('t#bar{t}+lf (SL)',ROOT.kRed-7,
            path_5FS_ttbar_SL,
            lumi+"*"+sel_ttlf,
            'ttlf_SL',
            samDict=sampleDict, readTrees=doReadTrees),

    plotClasses.Sample('t#bar{t}+lf (DL)',ROOT.kRed-7,
            path_5FS_ttbar_DL,
            lumi+"*"+sel_ttlf,
            'ttlf_DL',
            samDict=sampleDict, readTrees=doReadTrees),

    plotClasses.Sample('t#bar{t}+lf (FH)',ROOT.kRed-7,
            path_5FS_ttbar_FH,
            lumi+"*"+sel_ttlf,
            'ttlf_FH',
            samDict=sampleDict, readTrees=doReadTrees),

    plotClasses.Sample('t#bar{t}+c#bar{c} (SL)',ROOT.kRed+1,
            path_5FS_ttbar_SL,
            lumi+"*"+sel_ttcc,
            'ttcc_SL',
            samDict=sampleDict, readTrees=doReadTrees),

    plotClasses.Sample('t#bar{t}+c#bar{c} (DL)',ROOT.kRed+1,
            path_5FS_ttbar_DL,
            lumi+"*"+sel_ttcc,
            'ttcc_DL',
            samDict=sampleDict, readTrees=doReadTrees),

    plotClasses.Sample('t#bar{t}+c#bar{c} (FH)',ROOT.kRed+1,
            path_5FS_ttbar_FH,
            lumi+"*"+sel_ttcc,
            'ttcc_FH',
            samDict=sampleDict, readTrees=doReadTrees),

    plotClasses.Sample('t#bar{t}+Z(qq)',ROOT.kOrange+7,
            path_ttZ_qq,
            lumi,
            'ttZqq',
            samDict=sampleDict, readTrees=doReadTrees),

    plotClasses.Sample('t#bar{t}+Z(ll)',ROOT.kOrange,
            path_ttZ_ll,
            lumi,
            'ttZll',
            samDict=sampleDict, readTrees=doReadTrees),
    ]

processes = []
for sample in samples:
    processes.append(sample.nick)
list_of_processes   = processes
datacard_processes  = processes

plottingsamples = [
     ]

# sort subset of processes in plots. descending order
sortedProcesses = []


import cv2
import numpy as np
from PIL import Image
import os, sys
path = "/home/hashrin/project/segmentation/word_input/"
dirs = (os.listdir( path ))
dirs=sorted(dirs,key=lambda x: int(os.path.splitext(x)[0]))
def seg():
    count=1
    for item in dirs:
        if os.path.isfile(path+item):
            image = cv2.imread(path+item)
            f, e = os.path.splitext(path+item)
            #print(image.shape[1])
            #grayscale
            gray = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)
            # cv2.imshow('gray',gray)
            # cv2.waitKey(0)

            #binary
            ret,thresh = cv2.threshold(gray,127,255,cv2.THRESH_BINARY)
            # cv2.imshow('second',thresh)
            # cv2.waitKey(0)

            # #dilation
            # kernel = np.ones((5,5), np.uint8)
            # img_dilation = cv2.dilate(thresh, kernel, iterations=1)
            # cv2.imshow('dilated',img_dilation)
            # cv2.waitKey(0)

            # #erosion
            # kernel = np.ones((5,5), np.uint8)
            # img_eroded = cv2.erode(img_dilation, kernel, iterations=1)
            # cv2.imshow('eroded',img_eroded)
            # cv2.waitKey(0)

            #closing
            kernel = np.ones((image.shape[0]//6,image.shape[1]//55), np.uint8)
            closing=cv2.morphologyEx(thresh,cv2.MORPH_CLOSE,kernel)
            # cv2.imshow('closed',closing)
            # cv2.waitKey(0)

            #find contours
            im2,ctrs, hier = cv2.findContours(closing.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
            #sort contours
            sorted_ctrs = sorted(ctrs, key=lambda ctr: cv2.boundingRect(ctr)[0])

            for i, ctr in enumerate(sorted_ctrs):
                # Get bounding box
                x, y, w, h = cv2.boundingRect(ctr)

                # Getting ROI
                roi = image[y:y+h, x:x+w]

                # show ROI
                #cv2.imshow('segment no:'+str(i),roi)
                if(roi.shape[1]>image.shape[1]//50):
                    # if(image.shape[1]>1000):
                    #     cv2.namedWindow('segment no:'+str(i),cv2.WINDOW_NORMAL)
                    #     cv2.resizeWindow('segment no:'+str(i),(x//2,y//2))
                    # cv2.imshow('segment no:'+str(i),roi)
                    cv2.imwrite("/home/hashrin/project/segmentation/word_output/"+"00"+str(count)+".jpg",roi)
                    count=count+1
                    #cv2.rectangle(image,(x,y),( x + w, y + h ),(90,0,255),2)
                    # cv2.waitKey(0)

            # cv2.imshow('marked areas',image)
            # cv2.waitKey(0)
seg()


from collections import defaultdict
from math import sqrt
import operator
class PriorityQueue:
    def __init__(self):
        self.data=[]
        self.pqlocator={}
        self.maxi=0
    def enqueue(self,key,value):
        self.data.append((key,value))
        #print("Sunu ekliyom : ",value," Suraya ekliyom : ",self.maxi)
        self.pqlocator[value]=self.maxi
        self.maxi+=1
    def find_min(self):
        minkv=min(self.data, key=operator.itemgetter(0))[1]
        return minkv
    def dequeue(self):
        minv=float('inf')
        for i in range(len(self.data)):
            if(self.data[i][0]<=minv):
                minv=self.data[i][0]
                mink=i
                minci=self.data[i][1]
        del(self.data[mink])
        #print("Mink",mink)
        self.maxi-=1
        for i in range(mink,self.maxi):
            self.pqlocator[self.data[i][1]]-=1

        return minv,minci
    def __len__(self):
        return len(self.data)
    def update(self,location,nk,nv):
        locationr=self.pqlocator[location]
        #print("Pq Locator : ",self.pqlocator)
        #print("Location : ",location,"Locationr : ",locationr,"Self Data : ",self.data)
        self.data[locationr]=(nk,nv)
    def changePqLocator(self,index):
        del(self.pqlocator[index])

def dijkstra(graph,weights,start,end):
    d={}
    cloud={}
    pqlocator={}
    pq=PriorityQueue()
    for elem in graph.keys():
        if(elem==start):
            d[elem]=0
        else:
            d[elem]=float('inf')
        pq.enqueue(d[elem],elem)
    #print("PQ Data: ",pq.data)
    while pq.find_min()!=end:
        key,u=pq.dequeue()
        cloud[u]=key
        pq.changePqLocator(u)
        for vertex in graph[u]:
            if vertex not in cloud:
                weight=weights[u,vertex]
                if d[u]+weight<d[vertex]:
                    d[vertex]=d[u]+weight
                    pq.update(vertex,d[vertex],vertex)
    key,u=pq.dequeue()
    cloud[u]=key
    return cloud[end]

N,E=map(int,input().strip().split())
graph=defaultdict(list)
coordinates={}
weights={}
for i in range(E):
    a,b = map(int,input().strip().split())
    graph[a].append(b)
    graph[b].append(a)
for i in range(N):
    a,x,y,z=map(int,input().strip().split())
    coordinates[a]=[x,y,z]
p1=int(input().strip())
p2=int(input().strip())
hedef=int(input().strip())
merkez=int(input().strip())
for i in range(E):
    for elem in graph[i]:
        weights[(elem,i)]=sqrt(((coordinates[elem][0]-coordinates[i][0])**2)+((coordinates[elem][1]-coordinates[i][1])**2)+((coordinates[elem][2]-coordinates[i][2])**2))
a1=dijkstra(graph,weights,merkez,p1)+dijkstra(graph,weights,p1,p2)+dijkstra(graph,weights,p2,hedef)
a2=dijkstra(graph,weights,merkez,p2)+dijkstra(graph,weights,p2,p1)+dijkstra(graph,weights,p1,hedef)
print(min(a1,a2))


"""Extract data from CSV and JSON files.

The `load_neos` function extracts NEO data from a CSV file, formatted as
described in the project instructions, into a collection of `NearEarthObject`s.

The `load_approaches` function extracts close approach data from a JSON file,
formatted as described in the project instructions, into a collection of
`CloseApproach` objects.

The main module calls these functions with the arguments provided at the
command line, and uses the resulting collections to build an `NEODatabase`.

You'll edit this file in Task 2.
"""
import csv
import json
from models import NearEarthObject, CloseApproach

def load_neos(neo_csv_path='data/neos.csv'):
    """Read near-Earth object information from a CSV file.

    :param neo_csv_path: A path to a CSV file containing data about near-Earth
    objects.
    :return: A collection of `NearEarthObject`s.
    """
    neos = []

    with open(neo_csv_path, 'r') as infile:

        reader = csv.DictReader(infile)
        for line in reader:
            des, name, pha, diameter = (line['pdes'], line['name'],
                                        line['pha'], line['diameter'])
            neo = NearEarthObject(designation=des,
                                  name=name,
                                  diameter=diameter,
                                  hazardous=pha)
            neos.append(neo)
        return neos

def load_approaches(cad_json_path='data/cad.json'):
    """Read close approach data from a JSON file.

    :param neo_csv_path: A path to a JSON file containing data about close
    approaches.
    :return: A collection of `CloseApproach`es.
    """
    approaches = []

    with open(cad_json_path, 'r') as infile:
        contents = json.load(infile)
        close_approaches = contents['data']
        for ca in close_approaches:
            ca_dict = dict(zip(contents['fields'], ca))
            des, cd, dist, v_rel = (ca_dict['des'], ca_dict['cd'],
                                    ca_dict['dist'], ca_dict['v_rel'])
            ca_object = CloseApproach(designation=des,
                                      time=cd,
                                      distance=dist,
                                      velocity=v_rel)
            approaches.append(ca_object)
        return approaches


import random
import array
import numpy

end = 1
executions = 0
#newfile for debug information
file_pointer = open("debug.txt","w+")

while end == 1:
    f = open('data.txt', 'r')
    file_pointer.write("Execution #%d\n" % (executions))
    #print("From beginning") #debug for unlimited executions
    #reading from a file
    for line in f:
        numbs = f.readlines()

    #Now to convert the list to a proper array that we can work with
    stri = ''.join(numbs) #converting to string to easily remove comma
    stri = stri.replace("\n",",") #changing newline to comma for next step
    stri = stri.split(",")
    numb_array = numpy.asarray(stri) #converting to numerical array
    numb_array = numb_array.astype(numpy.float32)#converting everything to float so it works
    w1 = random.uniform(-1,1)  #random in range of -1 to 1 (can be double)
    w2 = random.uniform(-1,1)
    b = random.uniform(-1,1)
    eta = 0.5 # starting eta value

    i = 0

    #splitting the array for perceptron
    x1 = numb_array[::3]
    x2 = numb_array[1::3]
    d = numb_array[2::3]
    e = [0.0] * 12

    while i < len(x1):
        ans = w1*x1[i]+w2*x2[i]+b
        #print(ans) #debug
        ans = round(ans,2)
        if ans > 0:
            y = 1
        else:
            y = -1
        e[i] = d[i] - y
        i = i+1

    e_total = 0
    i = 0
    q = 0
    r = 0
    #Calculating the total error
    #while i < len(e):
        #e_total = numpy.abs(e_total) + numpy.abs(e[i])
        #i = i + 1
    e_total = sum(e)
    calculations = 0
    #perceptron training algorithm
    while e_total != 0:
        if q == len(x1): #each execution uses new x1 and x2 values, so after limit we reset to 0
            q = 0
        calculations = calculations + 1
        i = 0
        #initial parameter update for current example
        w1 = w1 + eta*e_total*x1[q]
        w2 = w2 + eta*e_total*x2[q]
        b = b + eta * e_total
        #now we test the updated w1, w2 and b
        while i < len(x1):
            ans = w1*x1[i]+w2*x2[i]+b
            ans = round(ans,2)
            if ans > 0:
                y = 1
            else:
                y = -1
            e[i] = d[i] - y
            i = i + 1
        #to make every e value abs
        #while r < len(e):
            #e[r] = abs(e[r])
            #r = r + 1
            #if r == len(e):
                #r = 0
                #e_total = sum(e)
                #break
        e_total = sum(e)
        q = q + 1
        file_pointer.write("%d\n" % (e_total))
        #print(e_total)
        if calculations > 3000:
            print("Learning algorithm couldn't find correct answer in 1500 calculations. e_total last value was: ",e_total)
            file_pointer.close()
            break
    if calculations != 3001:
        print("It took", calculations ,"calculations of learning algorithm for no errors to be found.")
        executions = executions + 1
        file_pointer.write("Amount of calculations it took during this cycle %d\n" % (calculations))
    elif calculations > 3000:
        print("Amount of executions is equal to",executions)
        break


import sys, time, xlsxwriter, os, openpyxl, keyboard
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from fake_useragent import UserAgent
from selenium.webdriver.common.action_chains import ActionChains
from os.path import join, dirname
from datetime import datetime
from openpyxl import load_workbook
from threading import Thread
from os.path import join, dirname, realpath
import http.client
# from dotenv import load_dotenv

class ZomatoThread(Thread):

    def __init__(self, locality_name, cur_row, url):
        Thread.__init__(self)
        self.locality_name = locality_name
        self.cur_row = cur_row
        self.url = url

    def run(self):
        global details, num_threads
        time_limit = 60
        start_time = datetime.now()
        ua = UserAgent()
        userAgent = ua.random
        userAgent = userAgent.split(" ")
        userAgent[0] = "Mozilla/5.0"
        userAgent = " ".join(userAgent)
        print("userAgent = " + userAgent)
        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument('user-agent={0}'.format(userAgent))
        chrome_options.add_argument("--headless")
        # chrome_options.add_argument("window-size=1280,800")

        chrome_options.add_argument('--log-level=0')
        path = join(dirname(__file__), 'webdriver', 'chromedriver.exe')
        driver = webdriver.Chrome (executable_path = path, options = chrome_options )
        # driver.maximize_window()
        driver.get(self.url)

        while True:
            try:
                rest_name = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "//main/div/section[3]/section/section[1]/div/h1"))).text
                break
            except:
                if (datetime.now() - start_time).total_seconds() > time_limit:
                    details[self.cur_row] = {"rest_name": "", "rating": "", "commeters": "", "cuisine": "", "cost_alcohol": "", "cost": "", "address": "", "opening_hours": "", "company": ""}
                    num_threads -= 1
                    conn = http.client.HTTPConnection(driver.service.service_url.split("//")[1])
                    conn.request("GET", "/shutdown")
                    conn.close()
                    del driver
                    return
                time.sleep(0.1)
                pass

        company = driver.find_element_by_xpath("//main/div/section[3]/section/section[1]/section[1]/a").text
        rating = ''
        commeters = ''
        try:
            rating = driver.find_element_by_xpath("//main/div/section[3]/section/section[2]/section/div[1]/p").text
            commeters = driver.find_element_by_xpath("//main/div/section[3]/section/section[2]/section/div[2]/p").text
            commeters = driver.split(" ")[0]
        except:
            pass
        cuisine = ", ".join([e.text for e in driver.find_elements_by_xpath("//h3[contains(text(), 'Cuisines')]/following-sibling::section/a")])
        cost = ''
        cost_alcohol = ''
        try:
            cost_1 = driver.find_element_by_xpath("//h3[contains(text(), 'Average Cost')]/following-sibling::p[1]").text
            if cost_1.find("with alcohol") > -1:
                cost_alcohol = cost_1
            else:
                cost = cost_1
        except:
            pass

        try:
            cost_2 = driver.find_element_by_xpath("//h3[contains(text(), 'Average Cost')]/following-sibling::p[3]").text
            if cost_2.find("₺") > -1:
                if cost_2.find("with alcohol") > -1:
                    cost_alcohol = cost_2
                else:
                    cost = cost_2
        except:
            pass

        address = driver.find_element_by_xpath("//h5[contains(text(), 'Direction')]/following-sibling::p").text
        opening_hours = ''
        try:
            opening_hours = driver.find_element_by_xpath("//span[contains(text(), 'Close')]/following-sibling::span[1]").text
        except:
            try:
                opening_hours = driver.find_element_by_xpath("//span[contains(text(), 'Open')]/following-sibling::span[1]").text
            except:
                pass
        # xlsfile_name = "xls\\zomato\\" + self.locality_name + ".xlsx"
        # wb = load_workbook(xlsfile_name)
        # ws = wb.active
        # ws.cell(row=self.cur_row, column=2).value = rest_name
        # ws.cell(row=self.cur_row, column=3).value = rating
        # ws.cell(row=self.cur_row, column=4).value = commeters
        # ws.cell(row=self.cur_row, column=5).value = cuisine
        # ws.cell(row=self.cur_row, column=6).value = cost_alcohol
        # ws.cell(row=self.cur_row, column=7).value = cost
        # ws.cell(row=self.cur_row, column=8).value = address
        # ws.cell(row=self.cur_row, column=10).value = opening_hours
        details[self.cur_row] = {"rest_name": rest_name, "rating": rating, "commeters": commeters, "cuisine": cuisine, "cost_alcohol": cost_alcohol, "cost": cost, "address": address, "opening_hours": opening_hours, "company": company}
        # wb.save(xlsfile_name)
        # wb.close()
        # print("wrote at row " + str(self.cur_row), "  :: address=", address, ", opening_hours=", opening_hours)
        conn = http.client.HTTPConnection(driver.service.service_url.split("//")[1])
        conn.request("GET", "/shutdown")
        conn.close()
        del driver
        num_threads -= 1

def get_urls(locality_index):
    cur_row = 1
    start_time = datetime.now()
    ua = UserAgent()
    userAgent = ua.random
    userAgent = userAgent.split(" ")
    # userAgent[0] = "Mozilla/5.0"
    userAgent = " ".join(userAgent)
    print("userAgent = " + userAgent)
    chrome_options = webdriver.ChromeOptions()
    chrome_options.add_argument('user-agent={0}'.format(userAgent))
    chrome_options.add_argument("--headless")
    # chrome_options.add_argument("window-size=1280,800")

    chrome_options.add_argument('--log-level=0')
    path = join(dirname(__file__), 'webdriver', 'chromedriver.exe')
    driver = webdriver.Chrome (executable_path = path, options = chrome_options )
    # driver.maximize_window()

    #Remove navigator.webdriver Flag using JavaScript
    # driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
    driver.get("https://www.zomato.com/istanbul")

    while True:
        try:
            # popular_localities = driver.find_elements_by_xpath("//div[@class='title']/following-sibling::div/div/div")
            popular_localities = WebDriverWait(driver, 20).until(EC.presence_of_all_elements_located((By.XPATH, "//div[@class='title']/following-sibling::div/div/div")))
            print("get Popular localities: ", popular_localities)
            break
        except:
            time.sleep(0.1)
            pass
    popular_localities_url = []
    popular_localities_name = []
    for locality in popular_localities:
        popular_localities_url.append(locality.find_element_by_xpath("./a").get_attribute("href"))
        locality_name = locality.find_element_by_xpath("./a/div/div/h5").text
        locality_name= locality_name[:locality_name.find(" (")]
        popular_localities_name.append(locality_name)
    print("get Popular localities URL")
    print("popular_localities_url = ", popular_localities_url)

    # for locality_name, locality_url in zip(popular_localities_name, popular_localities_url):

    locality_name = popular_localities_name[locality_index]
    locality_url = popular_localities_url[locality_index]
    xlsfile_name = "xls\\zomato\\" + locality_name + ".xlsx"
    workbook = xlsxwriter.Workbook(xlsfile_name)
    worksheet = workbook.add_worksheet()
    col_headers = ['No', 'Restaurant', 'Rating', 'Number of Comments', 'Cuisine', 'Cost with alcohol', 'Cost without alcohol', 'Address', 'Address as coordinate', 'Opening hours', 'Locality', 'URL', 'Image URL']

    for c, i in zip(col_headers, range(len(col_headers))):
        worksheet.write(0, i, c)

    print("time = ",(datetime.now() - start_time).total_seconds())
    print("locality_name = ", locality_name)
    # rest_data = {}
    rest_urls = []
    img_urls = []
    count = 0

    driver.get(locality_url) # + "?sort=cd&category=2")
    print("go to new page '" + locality_url + "'")
    # while True:
    #     try:
    #         categories = driver.find_elements_by_xpath("(//section[@role='tablist'])[1]/div")
    #         print("get categories")
    #         break
    #     except:
    #         time.sleep(0.1)
    #         pass
    # is_dining_out = True
    # time.sleep(0.5)
    # print("len of categories = ", len(categories))
    # for category in categories:
    #     if not is_dining_out:
    #         pass
    restaurants = driver.find_elements_by_xpath("//div[@class='jumbo-tracker']")
    restaurants_count = len(restaurants)
    print(restaurants_count)
    restaurants_pre_count = 0

    while restaurants_pre_count != restaurants_count:
        driver.execute_script("window.scrollTo(0,document.body.scrollHeight)")
        ok = False
        restaurants_pre_count = restaurants_count
        restaurants_pre_count_2 = restaurants_count
        for i in range(5):
            time.sleep(1)
            restaurants = driver.find_elements_by_xpath("//div[@class='jumbo-tracker']")
            restaurants_count = len(restaurants)
            if restaurants_pre_count_2 != restaurants_count:
                restaurants_pre_count_2 = restaurants_count
                ok = True
            elif ok:
                break
        print("pre_count = " + str(restaurants_pre_count) + "  count = " + str(restaurants_count))

    try:
        end_of_results = driver.find_element_by_xpath("//h3[contains(text(), 'End of search results')]")
        print("####  End of search results  ####")
    except:
        pass

    cur_time = datetime.now()

    for restaurant in restaurants:
        # //div[@class='jumbo-tracker']/div/a/div/img
        rest_url = restaurant.find_element_by_xpath("./div/a[1]").get_attribute("href")
        img_url = restaurant.find_element_by_xpath("./div/a[1]/div[1]/img").get_attribute("src")

        if not rest_url in rest_urls:
            print("rest_url = ", rest_url)
            rest_urls.append(rest_url)
            img_urls.append(img_url)
            worksheet.write(cur_row, 0, cur_row)
            worksheet.write(cur_row, 10, locality_name)
            worksheet.write(cur_row, 11, rest_url)
            # worksheet.write(cur_row, 12, img_url)
            cur_row += 1

    workbook.close()

def get_details(locality_name, start_row, end_row, time_interval):
    global details, num_threads
    breaked = False
    xlsfile_name = "xls\\zomato\\" + locality_name + ".xlsx"
    wb = load_workbook(xlsfile_name)
    ws = wb.active
    total_rows = 2
    rest_urls = []
    while True:
        if ws.cell(row=total_rows, column=1).value == None: break
        url = ws.cell(row=total_rows, column=12).value
        rest_urls.append(url)
        total_rows += 1
    total_rows -= 1

    if end_row == -1:
        end_row = total_rows
    else:
        end_row += 1

    for i in range(start_row + 1, end_row + 1):
        t = ZomatoThread(locality_name, i, rest_urls[i - 2])
        t.start()
        num_threads += 1

        print("completed = ", len(details), " number of threads = ", num_threads, " total = ", end_row - start_row)
        time.sleep(time_interval)
        # for i in range(5):
        #     try:
        #         if keyboard.is_pressed(' '):  # if key 'q' is pressed
        #             print('You Pressed A Key!')
        #             breaked = True
        #             break
        #     except:
        #         pass
        #     time.sleep(time_interval / 5)
        # if breaked: break

    while True:
        if breaked or len(details) == end_row - start_row:
            for row in details:
                ws.cell(row=row, column=2).value = details[row]['rest_name']
                ws.cell(row=row, column=3).value = details[row]['rating']
                ws.cell(row=row, column=4).value = details[row]['commeters']
                ws.cell(row=row, column=5).value = details[row]['cuisine']
                ws.cell(row=row, column=6).value = details[row]['cost_alcohol']
                ws.cell(row=row, column=7).value = details[row]['cost']
                ws.cell(row=row, column=8).value = details[row]['address']
                ws.cell(row=row, column=9).value = details[row]['company']
                ws.cell(row=row, column=10).value = details[row]['opening_hours']
            wb.save(xlsfile_name)
            wb.close()
            break

        print("completed = ", len(details), " number of threads = ", num_threads, " total = ", end_row - start_row)
        for i in range(5):
            try:
                if keyboard.is_pressed(' '):  # if key 'q' is pressed
                    print('You Pressed A Key!')
                    breaked = True
                    break
            except:
                pass
            time.sleep(time_interval / 5)

def excel_merge():
    xlsfile_name = "xls\\zomato\\total.xlsx"
    # wb = load_workbook(xlsfile_name)
    wb = openpyxl.Workbook()
    ws = wb.active
    cur_row = 1

    directory = join(dirname(realpath(__file__)), "xls", "zomato")
    print("=##", directory, "##")
    urls = []
    for filename in os.listdir(directory):
        if filename.endswith(".xlsx"):
            wb_2 = load_workbook(join(directory, filename))
            ws_2 = wb_2.active
            if cur_row == 1:
                for c in range(1, 13):
                    ws.cell(row=1, column=c).value = ws_2.cell(row=1, column=c).value
                cur_row = 2
            cur_row_2 = 2
            while True:
                if ws_2.cell(row=cur_row_2, column=1).value == None: break
                url = ws_2.cell(row=cur_row_2, column=12).value
                if not url in urls:
                    urls.append(url)

                    ws.cell(row=cur_row, column=1).value = cur_row - 1
                    for c in range(2, 13):
                        ws.cell(row=cur_row, column=c).value = ws_2.cell(row=cur_row_2, column=c).value
                    print(cur_row)
                    cur_row = cur_row + 1

                cur_row_2 += 1
    wb.save(xlsfile_name)
    wb.close()

    # total_rows = 2
    # rest_urls = []
    # while True:
    #     if ws.cell(row=total_rows, column=1).value == None: break
    #     url = ws.cell(row=total_rows, column=12).value
    #     rest_urls.append(url)
    #     total_rows += 1
    # total_rows -= 1

    # for i in range(start_row + 1, total_rows + 1):
    #     t = ZomatoThread(locality_name, i, rest_urls[i - 2])
    #     t.start()
    #     time.sleep(time_interval)

    #     print(len(details))

    # while True:
    #     print(len(details))
    #     if len(details == tatal_rows - start_row):
    #         for row in details:
    #             ws.cell(row=row, column=2).value = details[row].rest_name
    #             ws.cell(row=row, column=3).value = details[row].rating
    #             ws.cell(row=row, column=4).value = details[row].commeters
    #             ws.cell(row=row, column=5).value = details[row].cuisine
    #             ws.cell(row=row, column=6).value = details[row].cost_alcohol
    #             ws.cell(row=row, column=7).value = details[row].cost
    #             ws.cell(row=row, column=8).value = details[row].address
    #             ws.cell(row=row, column=9).value = details[row].company
    #             ws.cell(row=row, column=10).value = details[row].opening_hours

    #     time.sleep(time_interval)

        # rest_name = restaurant.find_element_by_xpath("./div/a[2]/p[1]").text
        # print("Consuming Time: 2", (datetime.now() - start_time).total_seconds())
        # if rest_name in rest_data: continue
        # rest_rating = ''
        # rest_comments = ''
        # try:
        #     rest_rating = restaurant.find_element_by_xpath("./div/a[2]/div/section/div[1]/p").text
        #     rest_comments = restaurant.find_element_by_xpath("./div/a[2]/div/section/div[2]/p").text
        #     rest_comments = rest_comments.split(" ")[0][1:]
        # except:
        #     pass
        # print("Consuming Time: 3", (datetime.now() - start_time).total_seconds())

        # rest_cuisine = restaurant.find_element_by_xpath("./div/a[2]/p[2]").text
        # print("Consuming Time: 4", (datetime.now() - start_time).total_seconds())
        # rest_budget = restaurant.find_element_by_xpath("./div/a[2]/p[4]/span").text
        # print("Consuming Time: 5", (datetime.now() - start_time).total_seconds())
        # rest_data[rest_name] = {'rating': rest_rating, 'comments': rest_comments, 'cuisine': rest_cuisine, 'budget': rest_budget}
        # count += 1
        # print(count, rest_name, rest_rating, rest_comments, rest_cuisine, rest_budget)

        # is_dining_out = False

    # for rest_url, img_url in zip(rest_urls, img_urls):

details = {}
num_threads = 0
file_open_flag = False

if sys.argv[1] == "url":
    get_urls(int(sys.argv[2]))
elif sys.argv[1] == "detail":
    get_details(sys.argv[2], int(sys.argv[3]), int(sys.argv[4]), int(sys.argv[5]) )
elif sys.argv[1] == "merge":
    excel_merge()

# End of search results h3


import torch
import numpy as np
import torch.nn as nn
from torch.utils.data import DataLoader
# pylint: disable=E1101, W0612

class Network(nn.Module):

    def __init__(self, num_features =512, num_layers = 2):
        super(Network, self).__init__()
        self.gru = nn.GRU(321, num_features, num_layers = num_layers, bidirectional = True, batch_first = True)
        self.fc = nn.Linear(num_features*2, 12)

    def forward(self, x):
        x = torch.transpose(x,1,2) #batch_size x sequence_length x features
        x, _ = self.gru(x)
        x = self.fc(x[:, -1, :])
        return x

def accuracy(model, device, dataset, filename, batchsize=2):
    total, correct = 0, 0
    model.eval()
    dataloader = DataLoader(dataset, batch_size = batchsize, drop_last = False)

    with torch.no_grad():
        for i_batch, batch in enumerate(dataloader):
            outputs = model(batch['spec'].to(device))
            _, predicted = torch.max(outputs.data, 1)
            total += batchsize
            correct += (predicted == batch['label'].to(device)).sum().item()

    with open(filename, 'a') as f:
        f.write(str(100 * correct / float(total))+'\n')
    model.train()
    return(100 * correct / float(total))

def class_accuracy(model, device, dataset, filename, batchsize=2):
    labels = ['yes','no','up','down','left','right','on','off','stop','go','unknown','silence']
    class_correct = list(0. for i in range(12))
    class_total = list(0. for i in range(12))
    model.eval()
    dataloader = DataLoader(dataset, batch_size = batchsize, drop_last = False)
    with torch.no_grad():
        for i_batch, batch in enumerate(dataloader):
            outputs = model(batch['spec'].to(device))
            _, predicted = torch.max(outputs.data, 1)
            c = (predicted == batch['label'].to(device)).squeeze()

            for i in range(batchsize):
                label = batch['label'][i]
                class_correct[label] += c[i].item()
                class_total[label] += 1
    with open(filename, 'w') as myFile:
        for i in range(12):
            myFile.write('Accuracy of %5s : %2d %%' % (
            labels[i], 100 * class_correct[i] / class_total[i])+'\n')
    model.train()


animal_dict = {
    "lion": 58,
    "cheater": 110,
    "zebla": 60,
    "tonakai": 80,
}

li = sorted(
    animal_dict.items(),
    key=lambda x : x[1],
    reverse=True)
for name,speed in li:
    print(name, speed)


import os

from scipy.constants import c as clight, e as qe
import numpy as np

from PyHEADTAIL.particles.slicing import UniformBinSlicer

def gen_matched_multibunch_beam(machine, n_macroparticles_per_bunch, filling_pattern,
        b_spac_s, bunch_intensity, epsn_x, epsn_y, sigma_z, non_linear_long_matching, min_inten_slice4EC):

    bucket_length_m = machine.circumference/(machine.longitudinal_map.harmonics[0])
    b_spac_m =  b_spac_s*machine.beta*clight
    b_spac_buckets = np.round(b_spac_m/bucket_length_m)

    if np.abs(b_spac_buckets*bucket_length_m-b_spac_m)/b_spac_m > 0.03:
        raise ValueError('Bunch spacing is not a multiple of the bucket length!')

    if non_linear_long_matching:
        generate_bunch = machine.generate_6D_Gaussian_bunch_matched
    else:
        generate_bunch = machine.generate_6D_Gaussian_bunch

    list_genbunches = []
    for i_slot, inten_slot in enumerate(filling_pattern):
        if inten_slot>0:
            print(('Generating bunch at slot %d/%d'%(i_slot, len(filling_pattern))))
            bunch = generate_bunch(n_macroparticles_per_bunch, inten_slot*bunch_intensity, epsn_x, epsn_y, sigma_z=sigma_z)
            bunch.z -= b_spac_buckets*bucket_length_m*i_slot
            list_genbunches.append(bunch)

    beam = sum(list_genbunches)

    bucket = machine.longitudinal_map.get_bucket(gamma=machine.gamma, mass=machine.mass, charge=machine.charge)
    # z_beam_center = bucket.z_ufp_separatrix + bucket_length - self.circumference/2.

    # Here the center of the bucket
    bucket.z_sfp

    # I want to re-separate the bunches
    buncher = UniformBinSlicer(n_slices = 0, z_sample_points = np.arange(bucket.z_sfp-len(filling_pattern)*bucket_length_m*b_spac_buckets,
                                            bucket.z_sfp+bucket_length_m, bucket_length_m*b_spac_buckets))
    buncher_slice_set = beam.get_slices(buncher, statistics=True)
    list_bunches = beam.extract_slices(buncher, include_non_sliced='never')
    # The bunch at position 0 is the tail

    # If last bunch is empty remove it
    if (list_bunches[0].intensity<min_inten_slice4EC):
        list_bunches = list_bunches[1:]

    # Add further information to bunches
    for i_bb, bb in enumerate(list_bunches[::-1]): # I want bunch 0 at the head of the train
        slice4EC = bb.intensity>min_inten_slice4EC
        bb.slice_info['slice_4_EC'] = slice4EC
        bb.slice_info['interact_with_EC'] = slice4EC
        bb.slice_info['N_bunches_tot_beam'] = len(list_bunches)
        bb.slice_info['i_bunch'] = i_bb
        bb.slice_info['i_turn'] = 0

    return list_bunches

def load_multibunch_beam(dirname, reset_i_turns=True):
    import PyPARIS.myfilemanager as mfm
    import PyPARIS.communication_helpers as ch

    print(('Loading the beam from %s'%dirname))

    bzero = ch.buffer_2_beam(mfm.dict_of_arrays_and_scalar_from_h5(
            dirname+'/bunch0.h5')['bunchbuffer'])
    N_bunches_tot_beam = bzero.slice_info['N_bunches_tot_beam']
    list_bunches = [bzero]
    for ibun in range(1, N_bunches_tot_beam):
        list_bunches.append(ch.buffer_2_beam(
            mfm.dict_of_arrays_and_scalar_from_h5(
            dirname+'/bunch%d.h5'%ibun)['bunchbuffer']))

    list_bunches = list_bunches[::-1] # We want the last bunch to be in pos 0
    if reset_i_turns:
        for bb in list_bunches:
            bb.slice_info['i_turn'] = 0

    return list_bunches

def save_bunch_to_folder(bunch, dirname):
    import PyPARIS.myfilemanager as mfm
    import PyPARIS.communication_helpers as ch
    if not os.path.exists(dirname):
        os.makedirs(dirname)
    buf = ch.beam_2_buffer(bunch)
    bpath = dirname+'/bunch%d.h5'%bunch.slice_info['i_bunch']
    print(('Saving: ' + bpath))
    mfm.dict_to_h5(dict_save={'bunchbuffer':buf},
                   filename=bpath)


file = open('f','r')
total = 0
for line in file:
    line = line.split("x")
    length = int(line[0])
    width = int(line[1])
    height = int(line[2])
    sides = [(length,width), (width,height), (length,height)]
    newl = []
    for x in sides:
        newl.append(2*x[0] + 2*x[1])
    smallest= min(newl)
    total += smallest + length * width * height
    # sides = [2*length*width, 2*length*height, 2*width*height]
    # smallest = min(sides)
    # total += (sum(sides) + .5*smallest)

print("The elves will need %d square feet of ribbon" % total)


import pandas as pd
import numpy as np
import copy, time
import warnings
warnings.filterwarnings('ignore')
from IPython.display import display
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import LabelEncoder

# 檢查 DataFrame 空缺值的狀態
def na_check(df_data):
    data_na = (df_data.isnull().sum() / len(df_data)) * 100
    data_na = data_na.drop(data_na[data_na == 0].index).sort_values(ascending=False)
    missing_data = pd.DataFrame({'Missing Ratio' :data_na})
    display(missing_data.head(10))

########## load data start
data_path = '../csv/3rd-ml100marathon-midterm/'
df_train = pd.read_csv(data_path + 'train_data.csv')
df_test = pd.read_csv(data_path + 'test_features.csv')

########## load data start end

########## 前處理理 (Processing) start
#薪水
var_salary = ['salary']

#獎金
var_bonus = ['bonus']

#業務報銷
var_business_expenses_group = ['long_term_incentive','deferred_income','loan_advances'
        ,'deferral_payments','other','expenses']

#董事费
var_director_fees = ['director_fees']

#總支出
var_total_payments = ['total_payments']

#股權
var_10_11_12 = ['exercised_stock_options','restricted_stock','restricted_stock_deferred']

#股權 total
var_total_stock_value = ['total_stock_value']

#email回覆
var_email_count_group = ['to_messages','from_poi_to_this_person','from_messages','from_this_person_to_poi','shared_receipt_with_poi']

#train_Y = df_train['poi']
train_Y = df_train['poi'].map(lambda x:1 if x==True else 0)

ids_train = df_train['name']
ids = df_test['name']
df_train = df_train.drop(['name', 'poi'] , axis=1)
df_test = df_test.drop(['name'] , axis=1)
df = pd.concat([df_train,df_test])
var_df_columns = df.columns
var_df_columns

var_df_dtypes = df.dtypes

df_cp = df.copy()

df_cp = df_cp.drop(['email_address'] , axis=1)

## 股權計算 start
#df_cp = df_cp.drop(var_10_11_12 , axis=1)
df_cp[var_10_11_12] = df[var_10_11_12].fillna(0)

df_cp[var_total_stock_value] = df[var_total_stock_value].fillna(0)

## 股權計算 end

## 公司支出計算 strat
df_cp[var_salary] = df_cp[var_salary].fillna(df_cp[var_salary].median())
df_cp[var_bonus] = df_cp[var_bonus].fillna(df_cp[var_bonus].median())
df_cp[var_director_fees] = df_cp[var_director_fees].fillna(0)

df_cp.dtypes
df_cp['total_payments_new']=np.zeros([146])

df_cp['total_payments_new']= df_cp['total_payments_new'] + df_cp['salary']
df_cp['total_payments_new'] = df_cp['total_payments_new'] + df_cp['bonus']

for col in var_business_expenses_group:
    df_cp[col] = df_cp[col].fillna(0)
    df_cp['total_payments_new'] = df_cp['total_payments_new'] + df_cp[col]

df_cp['total_payments_new'] = df_cp['total_payments_new'] + df_cp['director_fees']

df_cp = df_cp.drop('total_payments' , axis=1)

## 公司支出計算 end

## email 關係計算 start
for col_other in var_email_count_group:
    df_cp[col_other] = df_cp[col_other].fillna(df_cp[col_other].mean())

## email 關係計算 end

na_check(df_cp)

########## 前處理理 (Processing) end

# 將資料最大最小化
df_cp = MinMaxScaler().fit_transform(df_cp)

# 將前述轉換完畢資料 df , 重新切成 train_X, test_X
train_num = train_Y.shape[0]
train_X = df_cp[:train_num]
test_X = df_cp[train_num:]

# 使用三種模型 : 邏輯斯迴歸 / 梯度提升機 / 隨機森林, 參數使用 Random Search 尋找
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
lr = LogisticRegression(tol=0.001, penalty='l2', fit_intercept=True, C=1.0)
gdbt = GradientBoostingClassifier(tol=100, subsample=0.75, n_estimators=250, max_features=19,
                                  max_depth=3, learning_rate=0.03)
rf = RandomForestClassifier(n_estimators=100, min_samples_split=2, min_samples_leaf=1,
                            max_features='sqrt', max_depth=6, bootstrap=True)

# 線性迴歸預測檔 (結果有部分隨機, 請以 Kaggle 計算的得分為準, 以下模型同理)
lr.fit(train_X, train_Y)
lr_pred = lr.predict_proba(test_X)[:,1]
sub = pd.DataFrame({'name': ids, 'poi': lr_pred})

#sub['Survived'] = sub['Survived'].map(lambda x:1 if x>0.5 else 0)
sub.to_csv('test1023_lr.csv', index=False)

# 梯度提升機預測檔
gdbt.fit(train_X, train_Y)
gdbt_pred = gdbt.predict_proba(test_X)[:,1]
sub = pd.DataFrame({'name': ids, 'poi': gdbt_pred})
#sub['Survived'] = sub['Survived'].map(lambda x:1 if x>0.5 else 0)
sub.to_csv('test_gdbt1023--.csv', index=False)

# 隨機森林預測檔
rf.fit(train_X, train_Y)
rf_pred = rf.predict_proba(test_X)[:,1]
sub = pd.DataFrame({'name': ids, 'poi': rf_pred})
#sub['Survived'] = sub['Survived'].map(lambda x:1 if x>0.5 else 0)
sub.to_csv('test1023_rf.csv', index=False)

# 混合泛化預測檔
"""
Your Code Here
"""
blending_pred = lr_pred*0.3  + gdbt_pred*0.4 + rf_pred*0.3
sub = pd.DataFrame({'name': ids, 'poi': blending_pred})

sub.to_csv('test1023_blending.csv', index=False)


from . import spider_task
from flask import (current_app, make_response, redirect, render_template, jsonify,
                   request, url_for, flash, session)
from .forms import SpiderTask
from ..models import FictionListAll, db, SpiderTask as st
from .thread_pool_task import add_spider_task_pool
import json
from datetime import datetime

@spider_task.route('/spider_task')
def index():
    form = SpiderTask().get_all()

    return render_template('spider_task/spider_task.html', form=form)

@spider_task.route('/get_spider_task_all', methods=['post', 'get'])
def get_spider_task_all():
    data = st().query.all()
    print(json.loads(str(data)))
    return jsonify(json.loads(str(data)))

@spider_task.route('/add_spider_task', methods=['post'])
def add_spider_task():
    data = request.form
    print(data)
    task = st().query.filter_by(task_name=data.get('task_name')).first()
    if task is None:
        task = st(task_name=data.get('task_name'),
                  task_type=data.get('task_type'),
                  task_status=data.get('task_status'),
                  creation_date=str(datetime.now().date()))
        db.session.add(task)
        db.session.commit()
    return 'ok'

@spider_task.route('/run_task', methods=['post'])
def run_task():
    print(request.args, request.form, request.get_data(as_text=True))
    datas = json.loads(request.get_data(as_text=True))
    for data in datas:
        res = st().query.filter_by(task_id=data['task_id']).first()
        add_spider_task_pool(task_id=data['task_id'], task=res.task_type, para=res.task_name)
        # print(res)
        db.session.commit()
    res = {'status': 'SUCCESS'}
    return jsonify(res)


# POKECORD AUTOBOT
# BY PABLO M. DUVAL - abusiv0 (GitHub)

import discord
import sys
from discord.ext import commands
import json
from xyz import xyz
import asyncio
import random
import os
from getpass import getpass
import getpass
from os import system
from keep_alive import keep_alive
from Crypto.Cipher import AES
from Crypto.Random import get_random_bytes
from colorama import init

system("title "+'Pokecord AutoBot v1.0.1')

init(autoreset=True)
width = os.get_terminal_size().columns
print('')
print('')
print('\u001b[36;1m' + 'Welcome to Pokecord AutoBot'.center(width))
print('')
print('\u001b[36;1m' + 'Released date: 30/06/2019'.center(width))
print('')
print('\u001b[36;1m' + 'If u like it, you can donate. (link on github)'.center(width))
print('')
print('')
print('')

with open('settings.json') as s:
  settings = json.load(s)

client = commands.Bot(command_prefix=settings['prefix'])
token = settings['token']
status = settings['status']
totaltrade = settings['totaltrade']
spamv = settings ['spamv']
spamch = settings ['spamch']
pokeprefix = settings ['pokeprefix']

@client.event
async def on_ready():
  print(f'{client.user.name} Online')
  await client.change_presence(activity=discord.Activity(type=discord.ActivityType.playing, name='v1.0.1'+' '+ status))

@client.event
async def background_task():
    await client.wait_until_ready()
    counter = 0
    channel = client.get_channel(spamch)
    while not client.is_closed():
        counter += 1
        await channel.send(counter)
        await asyncio.sleep(spamv)

client.loop.create_task(background_task())

@client.event

async def on_message(message):
  if message.author.id == 365975655608745985:
    with open('settings.json') as s:
      settings = json.load(s)

    if message.channel.id in settings['allowed_channels_ids']:
      if message.embeds:
        if 'Guess' in message.embeds[0].description:
          url = message.embeds[0].image.url
          name = xyz(url).start()
          if 'alolan' in name.lower():
              name = 'Alolan ' + name[6:]
          timer = settings['timer']

          await asyncio.sleep(timer)

          await message.channel.send(pokeprefix+'catch '+name)

  await client.process_commands(message)

@client.command(pass_context=True)
async def say(ctx, msg):
    await ctx.message.delete()
    await ctx.send(msg)

@client.command()
async def commands(ctx):
  with open('settings.json') as s:
    settings = json.load(s)

  if ctx.channel.id in settings['allowed_channels_ids']:
    if ctx.author.id in settings['allowed_users_ids']:
      await ctx.send('```Commands:\nalolan\nshiny\nlegendary\nmythical\npokedex\ntrade\nlist\nclaim\ndetailed\niv\nname\nnumber\nlevel\ndexnumber\nconfirm```')

@client.command()
async def list(ctx):
  with open('settings.json') as s:
    settings = json.load(s)

  if ctx.channel.id in settings['allowed_channels_ids']:
    if ctx.author.id in settings['allowed_users_ids']:
      await ctx.send(pokeprefix+'pokemon')

@client.command()
async def alolan(ctx):
  with open('settings.json') as s:
    settings = json.load(s)

  if ctx.channel.id in settings['allowed_channels_ids']:
    if ctx.author.id in settings['allowed_users_ids']:
      await ctx.send(pokeprefix+'pokemon --alolan')

@client.command()
async def pokedex(ctx):
  with open('settings.json') as s:
    settings = json.load(s)

  if ctx.channel.id in settings['allowed_channels_ids']:
    if ctx.author.id in settings['allowed_users_ids']:
      await ctx.send(pokeprefix+'pokedex')

@client.command()
async def shiny(ctx):
  with open('settings.json') as s:
    settings = json.load(s)

  if ctx.channel.id in settings['allowed_channels_ids']:
    if ctx.author.id in settings['allowed_users_ids']:
      await ctx.send(pokeprefix+'pokemon --shiny')

@client.command()
async def legendary(ctx):
  with open('settings.json') as s:
    settings = json.load(s)

  if ctx.channel.id in settings['allowed_channels_ids']:
    if ctx.author.id in settings['allowed_users_ids']:
      await ctx.send(pokeprefix+'pokemon --legendary')

@client.command()
async def mythical(ctx):
  with open('settings.json') as s:
    settings = json.load(s)

  if ctx.channel.id in settings['allowed_channels_ids']:
    if ctx.author.id in settings['allowed_users_ids']:
      await ctx.send(pokeprefix+'pokemon --mythical')

@client.command()
async def bal(ctx):
  with open('settings.json') as s:
    settings = json.load(s)

  if ctx.channel.id in settings['allowed_channels_ids']:
    if ctx.author.id in settings['allowed_users_ids']:
      await ctx.send(pokeprefix+'bal')

@client.command()
async def detailed(ctx):
  with open('settings.json') as s:
    settings = json.load(s)

  if ctx.channel.id in settings['allowed_channels_ids']:
    if ctx.author.id in settings['allowed_users_ids']:
      await ctx.send(pokeprefix+'detailed')

@client.command()
async def iv(ctx):
  with open('settings.json') as s:
    settings = json.load(s)

  if ctx.channel.id in settings['allowed_channels_ids']:
    if ctx.author.id in settings['allowed_users_ids']:
      await ctx.send(pokeprefix+'order iv')

@client.command()
async def level(ctx):
  with open('settings.json') as s:
    settings = json.load(s)

  if ctx.channel.id in settings['allowed_channels_ids']:
    if ctx.author.id in settings['allowed_users_ids']:
      await ctx.send(pokeprefix+'order level')

@client.command()
async def name(ctx):
  with open('settings.json') as s:
    settings = json.load(s)

  if ctx.channel.id in settings['allowed_channels_ids']:
    if ctx.author.id in settings['allowed_users_ids']:
      await ctx.send(pokeprefix+'order alphabetical')

@client.command()
async def number(ctx):
  with open('settings.json') as s:
    settings = json.load(s)

  if ctx.channel.id in settings['allowed_channels_ids']:
    if ctx.author.id in settings['allowed_users_ids']:
      await ctx.send(pokeprefix+'order number')

@client.command()
async def dexnumber(ctx):
  with open('settings.json') as s:
    settings = json.load(s)

  if ctx.channel.id in settings['allowed_channels_ids']:
    if ctx.author.id in settings['allowed_users_ids']:
      await ctx.send(pokeprefix+'pokedex')

@client.command()
async def claim(ctx):
  with open('settings.json') as s:
    settings = json.load(s)

  if ctx.channel.id in settings['allowed_channels_ids']:
    if ctx.author.id in settings['allowed_users_ids']:
      await ctx.send(pokeprefix+'pokedex claim all')

@client.command()
async def confirm(ctx):
  with open('settings.json') as s:
    settings = json.load(s)

  if ctx.channel.id in settings['allowed_channels_ids']:
    if ctx.author.id in settings['allowed_users_ids']:
      await ctx.send(pokeprefix+'confirm')

@client.command()
async def trade(ctx):
  with open('settings.json') as s:
    settings = json.load(s)

  if ctx.channel.id in settings['allowed_channels_ids']:
    if ctx.author.id in settings['allowed_users_ids']:
      await ctx.send(pokeprefix+'trade '+ctx.author.mention)

      def check(m):
        return m.content == pokeprefix+'accept' and m.channel == ctx.channel and m.author == ctx.author

      msg = await client.wait_for('message',check = check)
      if msg:
        await ctx.send(pokeprefix+'pokemon')
        await asyncio.sleep(.3)
        await ctx.send(totaltrade)

        def check(m):
          return m.channel == ctx.channel and m.author.id == ctx.author.id

        msg2 = await client.wait_for('message',check = check)
        if msg2:
          try:
            poks = int(msg2.content)
            temp = ''

            if poks <= 300:
              for x in range(1,poks+1):
                temp += str(x)
                temp += ' '

            await ctx.send(pokeprefix+'p add '+temp)
            await asyncio.sleep(4)
            await ctx.send(pokeprefix+'confirm')
            await asyncio.sleep(1)
            await ctx.send('Say .confirm to confirm trade.')

          except:
            await ctx.send('https://github.com/abusiv0/pokecord-autobot --> Pokecord AutoBot FREE Version')

client.run(token,bot=False)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import json

from django.contrib.auth.models import User

from rest_framework import status

from dibiup_common.tests.base_tests import BaseTest

from dibiup_project.models import (
    Project,
    ProjectUser,
    ProjectUserArchive
)

class TestProjectArchive(BaseTest):

    def init_test(self):
        users = self.create_user(2)
        user01_json = json.loads(users[0])
        user01_id = user01_json.get('id', None)
        user01 = User.objects.get(id=user01_id)

        user02_json = json.loads(users[1])
        user02_id = user02_json.get('id', None)
        user02 = User.objects.get(id=user02_id)

        projects = self.create_project(user01_id, 1)
        project01_json = json.loads(projects[0])
        project01_id = project01_json.get('id', None)
        project01 = Project.objects.get(id=project01_id)

        projects = self.create_project(user02_id, 1)
        project02_json = json.loads(projects[0])
        project02_id = project02_json.get('id', None)
        project02 = Project.objects.get(id=project02_id)

        return user01, user02, project01, project02

    def test_func01_set_project_archive_by_creator(self):
        print("\n\n")
        print(">>>> test_func01_set_project_archive")

        user01, user02, project01, project02 = self.init_test()

        # set project archive by creator
        url = "/users/{0}/projects/{1}/archives/".format(user01.id, project01.id)
        data = {}
        resp = self.client.post(url, data, format="json")
        assert resp.status_code == status.HTTP_201_CREATED

        project_archive = ProjectUserArchive.objects.filter(user=user01, project=project01)
        assert project_archive.count() == 1

        project = Project.objects.get(id=project01.id)
        assert project.is_admin_archive == True
        print(">> set project archive by creator PASS")

    def test_func02_release_project_archive_by_creator(self):
        print("\n\n")
        print(">>>> test_func02_release_project_archive_by_creator")

        user01, user02, project01, project02 = self.init_test()

        # set project archive by creator
        url = "/users/{0}/projects/{1}/archives/".format(user01.id, project01.id)
        data = {}
        resp = self.client.post(url, data, format="json")
        assert resp.status_code == status.HTTP_201_CREATED

        project_archive = ProjectUserArchive.objects.filter(user=user01, project=project01)
        assert project_archive.count() == 1

        project = Project.objects.get(id=project01.id)
        assert project.is_admin_archive == True
        print(">> set project archive by creator PASS")

        # list users archive
        url = "/users/{0}/archives/".format(user01.id)
        resp = self.client.get(url)
        assert resp.status_code == status.HTTP_200_OK
        resp_json = json.loads(resp.content)
        assert resp_json.get('count') == 1

        # release users archive
        url = "/users/{0}/unarchive/".format(user01.id)
        data = {
            'project_ids': str(project01.id)
        }
        resp = self.client.patch(url, data, format="json")
        assert resp.status_code == status.HTTP_200_OK
        project_archive = ProjectUserArchive.objects.filter(user=user01, project=project01)
        assert project_archive.count() == 0
        project = Project.objects.get(id=project01.id)
        assert project.is_admin_archive == False
        print(">> release project archive by creator")

    def test_func03_set_project_archive_by_normal_user(self):
        print("\n\n")
        print(">>>> test_func03_set_project_archive_by_normal_user")

        user01, user02, project01, project02 = self.init_test()

        # join user02 to project01
        url = "/users/{0}/projects/{1}/projectusers/".format(user02.id, project01.id)
        data = {}
        resp = self.client.post(url, data, format="json")
        assert resp.status_code == status.HTTP_201_CREATED
        project_user = ProjectUser.objects.filter(user=user02, project=project01)
        assert project_user.count() == 1
        print(">> Join project01 PASS")

        # set project archive
        url = "/users/{0}/projects/{1}/archives/".format(user02.id, project01.id)
        data = {}
        resp = self.client.post(url, data, format="json")
        assert resp.status_code == status.HTTP_201_CREATED
        project_archive = ProjectUserArchive.objects.filter(user=user02, project=project01)
        assert project_archive.count() == 1
        project = Project.objects.get(id=project01.id)
        assert project.is_admin_archive == False
        print (">> set archive by normal user PASS")

        # Release project archive
        url = "/users/{0}/unarchive/".format(user02.id)
        data = {
            "project_ids": str(project01.id)
        }
        resp = self.client.patch(url, data, format="json")
        assert resp.status_code == status.HTTP_200_OK
        project_archive = ProjectUserArchive.objects.filter(user=user02, project=project01)
        assert project_archive.count() == 0
        project = Project.objects.get(id=project01.id)
        assert project.is_admin_archive == False
        print(">> release archive by normal user PASS")


""" unit testing of the Exome CNV filtering class
"""

import unittest
from clinicalfilter.variant.cnv import CNV
from clinicalfilter.variant.cnv_exome_filter import ExomeCNV

class TestExomeCnvPy(unittest.TestCase):
    """ test the Exome CNV filters
    """

    def setUp(self):
        """ define a default VcfInfo object
        """

        chrom = "1"
        pos = "15000000"
        snp_id = "."
        ref = "A"
        alt = "<DUP>"
        filt = "PASS"

        # set up a SNV object, since SNV inherits VcfInfo
        cnv = CNV(chrom, pos, snp_id, ref, alt, filt)
        self.var = ExomeCNV(cnv)

        info = "HGNC=TEST;HGNC_ALL=TEST,OR5A1;CQ=missense_variant;CONVEX;RC50INTERNALFREQ=0.005;COMMONFORWARDS=0.000;MEANLR2=0.5;MADL2R=0.02"
        format_keys = "inheritance:DP"
        sample_values = "deNovo:50"

        self.var.cnv.add_info(info)
        self.var.cnv.add_format(format_keys, sample_values)
        self.var.cnv.set_gender("F")

    def test_add_cns_state(self):
        """ test that add_cns_state() works correctly
        """

        # meanl2r > 0 gives cns of 3
        self.var.cnv.info["MEANLR2"] = "1"
        self.var.add_cns_state()
        self.assertEqual(self.var.cnv.info["CNS"], "3")

        self.var.cnv.info["MEANLR2"] = "0"
        self.var.add_cns_state()
        self.assertEqual(self.var.cnv.info["CNS"], "3")

        self.var.cnv.info["MEANLR2"] = "-0.1"
        self.var.add_cns_state()
        self.assertEqual(self.var.cnv.info["CNS"], "1")

        self.var.cnv.info["MEANLR2"] = "-2"
        self.var.add_cns_state()
        self.assertEqual(self.var.cnv.info["CNS"], "1")

        self.var.cnv.info["MEANLR2"] = "-2.1"
        self.var.add_cns_state()
        self.assertEqual(self.var.cnv.info["CNS"], "0")

    def test_fails_mad_ratio(self):
        """ test that fails_mad_ratio() works correctly
        """

        # check that var passes when MAD ratio > 5
        self.var.cnv.info["MEANLR2"] = "0.5"
        self.var.cnv.info["MADL2R"] = "0.02"
        self.assertFalse(self.var.fails_mad_ratio())

        # check that var passes when MAD ratio == 5
        self.var.cnv.info["MEANLR2"] = "0.1"
        self.var.cnv.info["MADL2R"] = "0.02"
        self.assertFalse(self.var.fails_mad_ratio())

        # check that var fails when MAD ratio < 5
        self.var.cnv.info["MEANLR2"] = "0.09"
        self.var.cnv.info["MADL2R"] = "0.02"
        self.assertTrue(self.var.fails_mad_ratio())

        # check that var fails when trying to divide by zero
        self.var.cnv.info["MEANLR2"] = "0.2"
        self.var.cnv.info["MADL2R"] = "0"
        self.assertTrue(self.var.fails_mad_ratio())

    def test_fails_population_frequency(self):
        """ test that fails_population_frequency() works correctly
        """

        # check that var passes when RC50INTERNALFREQ < 0.01
        self.var.cnv.info["RC50INTERNALFREQ"] = "0.005"
        self.assertFalse(self.var.fails_population_frequency())

        # check that var passes when RC50INTERNALFREQ == 0.1
        self.var.cnv.info["RC50INTERNALFREQ"] = "0.01"
        self.assertFalse(self.var.fails_population_frequency())

        # check that var fails when RC50INTERNALFREQ > 0.1
        self.var.cnv.info["RC50INTERNALFREQ"] = "0.02"
        self.assertTrue(self.var.fails_population_frequency())

    def test_fails_convex_score(self):
        """ test that fails_convex_score() works correctly
        """

        # check that var passes when CONVEXSCORE > 7
        self.var.cnv.info["CONVEXSCORE"] = "8"
        self.assertFalse(self.var.fails_convex_score())

        # check that var fails when CONVEXSCORE == 7
        self.var.cnv.info["CONVEXSCORE"] = "7"
        self.assertTrue(self.var.fails_convex_score())

        # check that var fails when CONVEXSCORE < 7
        self.var.cnv.info["CONVEXSCORE"] = "6"
        self.assertTrue(self.var.fails_convex_score())

    def test_fails_commmon_forwards(self):
        """ test that fails_commmon_forwards() works correctly
        """

        # check that var passes when COMMONFORWARDS < 0.8
        self.var.cnv.info["COMMONFORWARDS"] = "0.0"
        self.assertFalse(self.var.fails_commmon_forwards())

        # check that var fails when COMMONFORWARDS == 0.8
        self.var.cnv.info["COMMONFORWARDS"] = "0.8"
        self.assertFalse(self.var.fails_commmon_forwards())

        # check that var fails when COMMONFORWARDS > 0.8
        self.var.cnv.info["COMMONFORWARDS"] = "0.9"
        self.assertTrue(self.var.fails_commmon_forwards())

    def test_fails_no_exons(self):
        """ test that fails_no_exons() works correctly
        """

        self.var.cnv.info["NUMBEREXONS"] = "1"
        self.assertFalse(self.var.fails_no_exons())

        self.var.cnv.info["NUMBEREXONS"] = "0"
        self.assertTrue(self.var.fails_no_exons())

if __name__ == '__main__':
    unittest.main()


from las.utils import *
from las.arguments import *
from las.las import *
from data_loader import *
import os
import numpy as np
import tensorflow as tf

# arguments
args = parse_args()

# init session
gpu_options = tf.GPUOptions(allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))

# define data generator
train_libri_path = './data/LibriSpeech_train/train-clean-100'
dev_libri_path = './data/LibriSpeech_dev/dev-clean'
train_texts, train_audio_path = data_preparation(train_libri_path)
dev_texts, dev_audio_path = data_preparation(dev_libri_path)

# load from previous output
try:
    print("Load features...")
    train_feats = np.load("data/features/train_feats.npy", allow_pickle=True)
    train_featlen = np.load("data/features/train_featlen.npy", allow_pickle=True)
    train_chars = np.load("data/features/train_chars.npy", allow_pickle=True)
    train_charlen = np.load("data/features/train_charlen.npy", allow_pickle=True)
    dev_feats = np.load("data/features/dev_feats.npy", allow_pickle=True)
    dev_featlen = np.load("data/features/dev_featlen.npy", allow_pickle=True)
    dev_chars = np.load("data/features/dev_chars.npy", allow_pickle=True)
    dev_charlen = np.load("data/features/dev_charlen.npy", allow_pickle=True)

    special_chars = ['<PAD>', '<SOS>', '<EOS>', '<SPACE>']
    char2id, id2char = lookup_dicts(special_chars)

# process features
except:
    print("Process train/dev features...")
    special_chars = ['<PAD>', '<SOS>', '<EOS>', '<SPACE>']
    train_chars, train_charlen, char2id, id2char = process_texts(special_chars, train_texts)
    dev_chars, dev_charlen, _, _ = process_texts(special_chars, dev_texts)
    train_feats, train_featlen = process_audio(train_audio_path,
                                               sess,
                                               prepro_batch=100,
                                               sample_rate=args.sample_rate,
                                               frame_step=args.frame_step,
                                               frame_length=args.frame_length,
                                               feat_dim=args.feat_dim,
                                               feat_type=args.feat_type)
    dev_feats, dev_featlen = process_audio(dev_audio_path,
                                           sess,
                                           prepro_batch=100,
                                           sample_rate=args.sample_rate,
                                           frame_step=args.frame_step,
                                           frame_length=args.frame_length,
                                           feat_dim=args.feat_dim,
                                           feat_type=args.feat_type)
    np.save("data/train_feats.npy", train_feats)
    np.save("data/train_featlen.npy", train_featlen)
    np.save("data/dev_feats.npy", dev_feats)
    np.save("data/dev_featlen.npy", dev_featlen)
    np.save("data/train_chars.npy", train_chars)
    np.save("data/train_charlen.npy", train_charlen)
    np.save("data/dev_chars.npy", dev_chars)
    np.save("data/dev_charlen.npy", dev_charlen)

# Clip text length to predefined decoding steps
# train
index = train_charlen < args.dec_steps
train_feats = train_feats[index]
train_featlen = train_featlen[index]
train_chars = train_chars[index]
train_charlen = train_charlen[index]
# dev
index = dev_charlen < args.dec_steps
dev_feats = dev_feats[index]
dev_featlen = dev_featlen[index]
dev_chars = dev_chars[index]
dev_charlen = dev_charlen[index]

# init model
las =  LAS(args, Listener, Speller, char2id, id2char)

# build batch iterator
print("Build batch iterator...")
train_iter, num_train_batches = batch_gen(
            train_feats, train_chars, train_featlen, train_charlen, args.batch_size, args.feat_dim, args.dec_steps, shuffle=True)
dev_iter, num_dev_batches = batch_gen(
            dev_feats, dev_chars, dev_featlen, dev_charlen, args.batch_size, args.feat_dim, args.dec_steps, shuffle=False)
train_xs, train_ys = train_iter.get_next()
dev_xs, dev_ys = dev_iter.get_next()

# build train, inference graph
print("Build train, inference graph (please wait)...")
loss, train_op, global_step, logits = las.train(train_xs, train_ys)
logits, y_hat, atten = las.inference(dev_xs, dev_ys)
sample = convert_idx_to_token_tensor(y_hat[0], id2char)

# saver
if not os.path.exists(args.save_path):
    os.makedirs(args.save_path)
if not os.path.exists(args.result_path):
    os.makedirs(args.result_path)
saver = tf.train.Saver(max_to_keep=100)
ckpt = tf.train.latest_checkpoint(args.save_path)
if ckpt is None:
    sess.run(tf.global_variables_initializer())
else:
    saver.restore(sess, ckpt)

# init iterator and graph
sess.run(train_iter.initializer)
sess.run(dev_iter.initializer)

# training
training_steps = num_train_batches * args.epoch
print("Total num train batches:", num_train_batches)
loss_ = []
for step in range(training_steps):
    batch_loss, gs, _ = sess.run([loss, global_step, train_op])
    print("num_step: {}, loss: {}".format(gs, batch_loss))
    loss_.append(batch_loss)
    if gs and gs % num_train_batches == 0:
        ave_loss = np.mean(loss_)
        e_ =  gs // num_train_batches
        print("num epoch: {}, num_step: {}, ave loss: {}, wer: {}".format(
                                                e_, gs, ave_loss, 0))
        saver.save(sess, args.save_path+"/las_E{}".format(e_), global_step=gs)

        # eval
        texts = get_texts(y_hat, sess, num_dev_batches, id2char)
        with open(args.result_path+"/texts_E{}.txt".format(e_), 'w') as fout:
            fout.write("\n".join(texts))


#Class for creating Node with data and pointing to Null
class Node:

    def __init__(self,data):
        self.data=data
        self.next=None
        self.prev=None

#Class for performing LinkedList operations
class DoubleLinkedList:
    #Initialisation of head pointer,tail pointer and size for further reference
    def __init__(self,value):
        self.head=None
        self.tail=None
        self.size=0
        self.insertElement(value)

    #Method for inserting Element at the end of LinkedList
    def insertElement(self,value):
        if(self.head==None):
            node=Node(value)
            self.head=node
            self.tail=node
            self.size+=1
            return

        temp=self.head
        while(temp):
            if(temp.next==None):
                node=Node(value)
                self.tail.next=node
                node.prev=temp
                self.tail=node
                self.size+=1
                return
            temp=temp.next

    #Method for inserting Element at the particular location of LinkedList
    def insertElementLocation(self,value,location):

        #Checking weather LinkedList exists
        if(self.head==None):
            print("Current length of Linked List is zero inserting as starting node")
            self.insertElement(value)
            return

        #Checking weather user given location should not be greater than Size+1
        if(location>self.size):
            print("Please enter correct location currently we have "+str(self.size)+" elements")
            return

        temp=self.head

        #Inserting Node at the start of LinkedList
        if(location-1==0):
            node=Node(value)
            node.next=temp
            temp.prev=node
            self.head=node
            self.size+=1
            print("\nInsertion at location "+str(location)+" successful")
            return

        #Inserting Node at the end of LinkedList
        if(location==self.size):
            node=Node(value)
            node.prev=self.tail
            self.tail.next=node
            self.tail=node
            self.size+=1
            print("\nInsertion at location "+str(location)+" successful")
            return

        #Inserting value at any location other than Start and End
        index=0
        while(temp):
            if(index==location-1):
                node=Node(value)
                node.next,temp.next,node.prev=temp.next,node,temp
                node.next.prev=node
                self.size+=1
                print("\nInsertion at location "+str(location)+" successful")
                return
            index+=1
            temp=temp.next

    #Method for searching a node
    def searchNode(self,value):
        #Checking weather LinkedList exists
        if(self.head==None):
            print("LinkedList is empty")
            return

        temp=self.head
        index=0
        found=False
        while(temp):
            if(temp.data==value):
                print("Found the node with value: "+str(value)+" at location: "+str(index+1))
                found=True
            if(temp==self.tail):
                if(found==False):
                    print("No Node with value "+str(value)+" found.")
                return
            temp=temp.next
            index+=1

    #Method for Deleting Node from LinkedList
    def deleteElement(self,location):

        #Checking weather LinkedList exists
        if(self.head==None):
            print("Current length of Linked List is 0. No Node to delete")
            return

        #Checking weather user given location should not be greater than Size
        if(location>self.size):
            print("Please enter correct location currently we have "+str(self.size)+" elements")
            return

        temp=self.head

        #Deleting Node at the start of LinkedList
        if(location-1==0):
            self.head=temp.next
            temp.next.prev=None
            self.size-=1
            if(self.size==0):
                self.head=None
                self.tail=None
            print("\nDeletion at location "+str(location)+" successful")
            return

        #Deleting a node at end
        if(location==self.size):
            temp=self.head
            while(temp):
                if(temp.next==self.tail):
                    temp.next=None
                    self.tail.prev=None
                    self.tail=temp
                    self.size-=1
                    if(self.size==0):
                        self.head=None
                        self.tail=None
                    print("\nDeletion at location "+str(location)+" successful")
                    return
                temp=temp.next

        #Deleting Node at any other location of LinkedList
        index=0
        while(temp):
            if(index+1==location-1):
                temp.next,temp.next.prev=temp.next.next,temp
                # temp.next.prev=temp
                # temp.next.prev=None
                self.size-=1
                if(self.size==0):
                    self.head=None
                    self.tail=None
                print("\nDeletion at location "+str(location)+" successful")
                return
            index+=1
            temp=temp.next

    #Method for Deleting Complete LinkedList
    def deleteLinkedList(self):
        #Checking weather LinkedList exists
        if(self.head==None):
            print("No LinkedList exists")
            return
        print("Deleting LinkedList........")
        temp=self.head
        while(temp):
            temp.prev=None
            temp=temp.next
        self.head=None
        self.tail=None
        print("Linked List deleted successfully !")

    #Method for traversing LinkedList
    def traverse(self):
        temp=self.head
        #Checking weather LinkedList exists
        if(temp==None):
            print("Single Linked List is empty")
            return
        print("Printing Single LinkedList ")
        while(temp):
            print(temp.data,end="")
            if(temp.next!=None):
                print("=>",end="")
            temp=temp.next

    #Method for reverse traversing LinkedList
    def reverseTraverse(self):
        temp=self.tail
        #Checking weather LinkedList exists
        if(self.head==None):
            print("Single Linked List is empty")
            return
        print("Printing Single LinkedList in Reverse Order ")
        while(temp):
            print(temp.data,end="")
            if(temp.prev!=None):
                print("=>",end="")
            temp=temp.prev

if __name__ == "__main__":
    doublellst=DoubleLinkedList(0)
    # singlellst.traverse()
    doublellst.insertElement(1)
    doublellst.insertElement(2)
    doublellst.insertElement(3)
    doublellst.insertElement(4)
    doublellst.insertElement(5)
    doublellst.insertElement(6)
    doublellst.insertElement(7)
    doublellst.insertElement(8)
    doublellst.traverse()
    print()
    doublellst.insertElementLocation(10,4)
    # doublellst.insertElementLocation(11,1)
    doublellst.insertElementLocation(12,1)
    doublellst.insertElementLocation(11,10)
    print()
    doublellst.traverse()
    print("\n")

    doublellst.reverseTraverse()
    print("\n")

    doublellst.searchNode(6)
    doublellst.searchNode(20)
    print()
    doublellst.traverse()
    print("\n")
    doublellst.deleteElement(1)
    doublellst.traverse()
    print()
    doublellst.deleteElement(7)
    doublellst.traverse()
    print()
    doublellst.deleteElement(9)
    doublellst.traverse()
    print()
    doublellst.deleteElement(7)
    doublellst.traverse()
    print()
    doublellst.reverseTraverse()
    print("\n")
    doublellst.deleteLinkedList()
    doublellst.deleteLinkedList()

    doublellst.traverse()