def extract_varInfo(var_raw:list):
    # rawdata form : {'job_num': '5001_Glo_Var.JOB', 'rawdata': "gi_Search=1\t'@var Stage Search Method @type Integer @use(0 : No setting, 1:Manual Searching, 2 : Auto Searching)\n"}
    class varData(): # 중복 키 생성을 위한 Class
        def __init__(self,name):
            self.name = name
        def __repr__(self):
            return self.name

    result={}

    def extract_var_name(raw_data: str) -> str:
        if '\t' in raw_data:
            raw_data.replace('\t', '')

        if '\n' in raw_data:
            raw_data.replace('\n', '')

        if "=" in raw_data:
            raw_data = raw_data.split('=')[0].strip()

        if "DIM" in raw_data:
            raw_data = raw_data.split('DIM')[1]
            raw_data = raw_data.split('AS')[0].strip()

        return  raw_data

    def extract_var_loc(raw_data: str):
        result = raw_data.split('_')[0].strip()
        return result

    for rawdata in var_raw:
        job_num=rawdata["job_num"]
        raw_data=rawdata["rawdata"]

        variable__loc=extract_var_loc(job_num)

        raw_dataList= raw_data.split('@')

        '''
        raw_dataList Form
        ["gi_Search=1\t'",
         'var Stage Search Method ',
         'type Integer ',
         'use(0 : No setting, 1:Manual Searching, 2 : Auto Searching)\n']
        '''

        variable__name = extract_var_name(raw_dataList[0]) # 필수항목

        def extract_var_property(raw_data,property):
            result=raw_data.replace(property,'')

            cnt=0
            while True:
                cnt+=1
                if ('(' not in result) and (')' not in result )and ('\n' not in result) and ('\t' not in result) and (':' not in result):
                    break

                if cnt>100:
                    break
                if '(' in result:
                    result = result.replace("(", '')
                if ')' in result:
                    result = result.replace(")", '')
                if '\n' in result:
                    result = result.replace("\n", '')
                if '\t' in result:
                    result = result.replace("\t", '')
                if ':' in result:
                    result = result.replace(":", '')
            return result.strip()

        for raw_data__data in raw_dataList[1:]:
            if 'var' in raw_data__data:
                variable__desc = extract_var_property(raw_data__data,'var')  # 필수
                try:
                    saveform = { # 필수 항목
                        "variable__name": variable__name,
                        "variable__desc": variable__desc,
                        "variable__loc": variable__loc,
                    }
                except NameError:
                    print("well, it WASN'T defined after all!")
                else:
                    pass

            if 'type' in raw_data__data:
                variable__type =  extract_var_property(raw_data__data,'type')  # 옵션
                try:
                    saveform["variable__type"]=variable__type
                except NameError:
                    pass
                else:
                    pass
            if 'use' in raw_data__data:
                variable__use =  extract_var_property(raw_data__data,'use')  # 옵션
                try:
                    saveform["variable__use"]= variable__use
                except NameError:
                    pass
                else:
                    pass

            if 'default' in raw_data__data:
                variable__default =  extract_var_property(raw_data__data,'default')  # 옵션
                try:
                    saveform["variable__default"]= variable__default
                except NameError:
                    pass
                else:
                    pass
        result[varData(variable__name)]=saveform
    return result

def extract_functionInfo(function_raw:list):
    processed_funcdata={} # return para

    def extract_property(raw_data:str,property:str):
        property_content=raw_data.split(property)[1]

        while True:
            if (":" not in property_content) and ("'" not in property_content) and ('"' not in property_content) and ('\t' not in property_content) and ('\n' not in property_content):
                break

            if "'" in property_content:
                property_content=property_content.replace("'",'')
            if '"' in property_content:
                property_content=property_content.replace('"', '')
            if '\t' in property_content:
                property_content=property_content.replace("\t", '')
            if '\n' in property_content:
                property_content=property_content.replace("\n", '')
            if ':' in property_content:
                property_content=property_content.replace(":", '')
        return property_content.strip()

    for func_data in function_raw:
        jobfile_name=func_data["job_num"]
        job_info=func_data["rawdata"]
        func_obj = {}

        func_obj["function__name"]= jobfile_name
        for property_data in job_info:
            if "@func" in property_data:
                func_obj["function__desc"]= extract_property(property_data,"@func")
            if "@ref" in property_data:
                func_obj["function__ref"]= extract_property(property_data,"@ref")
            if "@param" in property_data:
               func_obj["function__param"]= extract_property(property_data,"@param")
            if "@result" in property_data:
                func_obj["function__result"]= extract_property(property_data,"@result")

        processed_funcdata[jobfile_name]=func_obj

    return processed_funcdata

def extract_labelInfo(label_raw:list):
    pass

def extract_descripInfo(desc_raw:dict):
    result={}
    job_num=desc_raw['job_num']
    raw_data=desc_raw['rawdata']

    if 'readme' in desc_raw:
        processed_readme = []
        for readme_line in desc_raw['readme']:

            while True:
                if ("\t" not in readme_line) and ("'" not in readme_line) and ("\n" not in readme_line) \
                        and ("/@read" not in readme_line) and ("@read/" not in readme_line):
                    break

                if "\t" in readme_line:
                    readme_line = readme_line.replace("\t", "").strip()
                if "'" in readme_line:
                    readme_line = readme_line.replace("'", "").strip()
                if "\n" in readme_line:
                    readme_line = readme_line.replace("\n", "").strip()

                if "@read/" in readme_line:
                    readme_line = readme_line.replace("@read/", "")

                if "/@read" in readme_line:
                    readme_line = readme_line.replace("/@read", "")

            processed_readme.append(readme_line)

        result["readme"] = processed_readme

    for data in raw_data:
        if ('@Description' in data):
            description =data.split(':')[1].strip()
            result["description"]=description

        if ('@Version' in data):
            version =data.split(':')[1].strip()
            result["version"] = version

        if ('@Revdate' in data):
            revdate = data.split(':')[1].strip()
            result["revdate"] = revdate

        if ('@name' in data):
            name = data.split(':')[1].strip()
            result["name"] = name

        if ('@email' in data):
            email = data.split(':')[1].strip()
            result["email"] = email

        if ('phone' in data):
            phone = data.split(':')[1].strip()
            result["phone"] = phone

    return result

def extract_code_property(code_raw:dict)->dict:
    raw_data=code_raw['rawdata']

    pure_comment_part=[] # 주석만 있는 부분
    mixed_comment_part=[] # 주석이랑 코드가 섞여있는 부분
    for line in raw_data:
        if "'" in line:
            temp=line.split("'")
            code_part=temp[0]
            comment_part=temp[1]
            if (code_part.strip()==""):
                pure_comment_part.append(line)
            else:
                mixed_comment_part.append(line)
        else:
            # 주석이 아이에 없는 부분
            mixed_comment_part.append(line)

    result={
    "job_num":code_raw['job_num'].split('_')[0].strip(),
    "code_num":code_raw['code_num'],
    "raw_data":raw_data,
    "pureComment_part":pure_comment_part,
    "mixedComment_part":mixed_comment_part,
    }
    return result

def extract_codeInfo(code_list:list)->dict:
    processed_codedata = {}  # return para
    for idx,value in enumerate(code_list):
        code_num=value["code_num"]
        reference_name="code_#"+str(code_num)
        code_property=extract_code_property(value)
        processed_codedata[reference_name]=code_property

    return processed_codedata

if __name__=="__main__":
    test_func_raw=[{'job_num': '5000_Main.JOB', 'rawdata': 'DIM lt_userIn AS String \'@var User Input @type : String @use "Y","N" @default "none"\n'}, {'job_num': '5005_UDPSetting.JOB', 'rawdata': 'DIM lt_userIn AS String \'@var User Input @type : String2 @use "Y2","N2" @default "test"\n'}]
    print(extract_varInfo(test_func_raw))


from ship import Ship

class Battleship(Ship):
    """
    A class used to represent a battleship, which inherites from the Ship class

    ...

    name : str
        name of ship as a single captial letter (i.e "B" for battleship)
    spaces: int
        how many spaces on the board the ship takes up

    """

    def __init__(self, name):
        """
        Constructs the battleship object
        :param name: name of ship (i.e B for battleship)
        """
        super().__init__(name)
        self.spaces = 4


"""
N개의 정수로 이루어진 수열이 있을 때,
크기가 양수인 부분수열 중에서
그 수열의 원소를 다 더한 값이 S가 되는 경우의 수를 구하는 프로그램을 작성하시오.

8시30분까지

5 0
-7 -3 -2 5 8
"""
import sys
from collections import deque

input = sys.stdin.readline

N, S = map(int, input().strip().split())
num = list(map(int, input().strip().split()))

stack = deque([])
ans = 0

def dfs(start):
    global ans

    if sum(stack) == S and len(stack) > 0:
        ans += 1
        # return
        # return
        # return 을 쓰면 안되는 이유: 재귀 호출을 끝내면, 이 이후 수열을 더하지 않고 return 해버림

    for k in range(start, len(num)):
        stack.append(num[k])
        dfs(k + 1)
        print(stack)
        stack.pop()

dfs(0)
print(ans)


#!/usr/bin/env python
# coding=utf-8
import sys
sys.path.append("/home/jiangyy/workspace/enroll-score-ocr-parser/score/")
sys.path.append("/home/jiangyy/workspace/enroll-score-ocr-parser/enroll/")
sys.path.append("/home/jiangyy/workspace/enroll-score-ocr-parser/extractor/")
sys.path.append("/home/jiangyy/workspace/enroll-score-ocr-parser/common/")
sys.path.append("/home/jiangyy/workspace/enroll-score-ocr-parser/conf/")
import re, os
import csv
import Levenshtein
from conf.setting import ROOT_DIR, STD_WMZY_SCH_NAME_FILE

class SchNameMatcher(object):
    """
    学校名称标准化匹配
    """
    def __init__(self):

        self.file_path = os.path.join(ROOT_DIR, STD_WMZY_SCH_NAME_FILE)
        self.std_sch_names = set()
        self.__read_std_sch_names()

    def __read_std_sch_names(self):
        with(open(self.file_path, newline='')) as sch_file:
            rows = csv.reader(sch_file, delimiter=',', )
            next(rows)
            for row in rows:
                std_name = row[0]
                alias_name = row[1]
                clean_name = re.sub("[^\u4e00-\u9fa5]+", "", std_name)
                if clean_name:
                    self.std_sch_names.add(clean_name)
                clean_name = re.sub("[^\u4e00-\u9fa5]+", "", alias_name)
                if clean_name:
                    self.std_sch_names.add(clean_name)

    def match_std_sch(self, sch_raw):
        """
        按照编辑比例计算一个sen是不是学校
        """
        # 删除所有非中文
        sen = re.sub("[^\u4e00-\u9fa5]+", "", sch_raw)
        if sen in self.std_sch_names:
            return sen

        for item in self.std_sch_names:
            if sen.startswith(item) or item.startswith(sen):
                return sen
        lis = [(sch, Levenshtein.distance(sen, sch)) for sch in self.std_sch_names]

        if lis:
            candi = min(lis, key=lambda x: x[1])
            ratio = 1 - candi[1] * 1.0 / max(len(sen), len(candi[0]))  # 不用改的
        else:
            ratio = 0.0

        if ratio > 0.75:
            return candi[0]
        else:
            return None

    def is_sch_name(self, sch_name_raw):
        # 取第一个全中文子字符串
        sch_name = re.sub("[^\u4e00-\u9fa5]", " ", sch_name_raw.strip()).strip().split(" ")[0]
        #  大学或者学院
        sch_name_suffix = ["大学", "学院", "分校", "校区", "学校", "书院"]
        for item in sch_name_suffix:
            if item in sch_name:
                if item in ["大学"] and sch_name.endswith(item) and len(sch_name) > len(item) * 3:
                    return True

                std_sch = self.match_std_sch(sch_name)
                if std_sch:
                    return True

        # 对院校名称太长被分割成两行的情况进行补充, 并过滤掉一大批短名称专业
        if len(sch_name) > 5:
            std_sch = self.match_std_sch(sch_name)
            if std_sch:
                return True
        return False


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu April 16 11:19:43 2021

@author: amir
"""
from numba import jit
from numba import cuda, float32
import numpy as np
import pandas as pd
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error as mes
from sklearn.metrics import accuracy_score as acc
import time
import matplotlib.pyplot as plt

column_names = ['user-id', 'activity', 'timestamp', 'x-axis', 'y-axis', 'z-axis']
file = pd.read_csv('WISDM_at_v2.0_raw.txt', header=None, names=column_names, comment=';')
file = file.dropna(axis=0, how='any')
file = file[1:(1 * file.shape[0]) // 100]

file = file.drop(axis=1,columns=['user-id','timestamp'])

# اماده سازی دیتا

for x in range(1,(len(file)+1)):
    if(file.loc[x]['activity'] == 'Walking') :
        file.at[x,'activity'] = 1
    elif (file.loc[x]['activity'] == 'Jogging') :
        file.at[x,'activity'] = 2
    elif (file.loc[x]['activity'] == 'LyingDown') :
        file.at[x,'activity'] = 3
    elif (file.loc[x]['activity'] == 'Standing') :
        file.at[x,'activity'] = 4
    elif (file.loc[x]['activity'] == 'Sitting') :
        file.at[x,'activity'] = 5

data = file.to_numpy()

inputs = data[:,1:] # ورودی ها

outputs = data[:,0] # خروجی

epochs = 500 #  تعداد ایپک
learningrate = 0.1 # نرخ اموزش

#### شافل کردن
per_list =  np.random.permutation(len(data))

inputs_sh = []
outputs_sh = []

for i in range (len(data)):
    indexi = per_list[i]
    temp_input = inputs[i]
    temp_output = outputs[i]
    inputs_sh.append(temp_input)
    outputs_sh.append(temp_output)

inputs_sh = np.array(inputs_sh)
outputs_sh = np.array(outputs_sh)

#minvec = inputs_sh.min(axis=0)
#maxvec = inputs_sh.max(axis=0)
#inputs_sh2 = (inputs_sh - minvec ) / ( maxvec - minvec)

## Scaling the features
scaler = MinMaxScaler(feature_range=(0,1))
## Transform the data into
inputs_sh = scaler.fit_transform(inputs_sh)

trn_test_split = int (0.75 * len(inputs_sh))

X_train = inputs_sh[0:trn_test_split, :]
Y_train = outputs_sh[0 :trn_test_split]

X_val = inputs_sh[trn_test_split : , :]
Y_val = outputs_sh[trn_test_split :, ]

n0 = 3 # تعداد نرورن در لایه ورودی

n1 = 3 #تعداد نرود در لایه دوم

n2 = 3 # تعداد نرون در لایه سوم

n3 = 1 # تعداد نرون در لایه آخر

w1 = np.random.uniform(low=-10,high=+10,size=(n1,n0)) # وزن های تصادفی در لایه ورودی
w2 = np.random.uniform(low=-10,high=+10,size=(n2,n1)) # وزن های تصادفی در لایه دوم
w3 = np.random.uniform(low=-10,high=+10,size=(n3,n2)) # وزن های تصادفی در لایه سوم

@jit(nopython=True, fastmath=True)
# تابع سیکموید
def activation(x):
    y = 1/(1 + np.exp(-1*x))
    return y

@jit(nopython=True, fastmath=True)
# الگوریتم پیشرو
def feedforwardOneLayer(input_x , w):
    x = np.dot(input_x , w.T)
    y = activation(x)
    return y

@jit(nopython=True, fastmath=True)
# مشتق تابع سیکموید
def d_activation(y):
    d_y = y * ( 1 - y )
    return d_y

#@jit(nopython=True)
def diagnal(y):
    d = d_activation(y)
    return np.diagflat(d)

list_accTrain = []
list_mesTrain = []
list_accValid = []
list_mesValid = []

start = time.time()
for i in range(epochs):
    for j in range(len(X_train)):
        inputNet = X_train[j]
        inputNet = np.reshape(inputNet,newshape=(1,n0))
        targetNet = Y_train[j]
        y1 = feedforwardOneLayer(inputNet , w1)
        y2 = feedforwardOneLayer(y1 , w2)
        y3 = feedforwardOneLayer(y2 , w3)

        error = targetNet - y3

        diagf1 = diagnal(y1)
        diagf2 = diagnal(y2)
        diagf3 = d_activation(y3)

        # w1 = w1 - lr * (-2/N)*(error) * d_f3 * w3 * d_f2 * w2 * d_f1 * ...
        # ... * input

        temp2 = -2 * error * diagf3
        temp1 = temp2 * w3
        temp1 = np.dot( temp1 , diagf2 )
        temp = np.dot( temp1 , w2 )
        temp = np.dot( temp , diagf1)
        temp = temp.T
        temp = np.dot( temp , inputNet)

        w1 = w1 - learningrate * temp

        w2 = w2 - learningrate * np.dot( temp1.T , y1 )

        w3 = w3 - learningrate * np.dot( temp2.T , y2)

    NetOut_train =[]
    Target_train = []
    Rnd_Netoutput_Train = []
    for x in range(len(X_train)):
        inputNet = X_train[x]
        target = Y_train[x]

        Target_train.append(target)
        y1 = feedforwardOneLayer(inputNet , w1)
        y2 = feedforwardOneLayer(y1 , w2)
        y3 = feedforwardOneLayer(y2 , w3)

        NetOut_train.append(y3)
        Rnd_Netoutput_Train.append(np.round(y3))

    mes_train = mes(Target_train , NetOut_train)
    list_mesTrain.append(mes_train)
    acc_train = acc(Target_train ,  Rnd_Netoutput_Train)
    list_accTrain.append(acc_train)

    print('epoch' , i , ' MSE_Train = ' , mes_train , '\tAcc_train = ', acc_train)

end = time.time()

print("Elapsed (after compilation) = %s" % (end - start))

plt.figure(figsize=(20,10))
plt.plot(list_accTrain, color = 'green', label = 'accuracy Train')
plt.plot(list_mesTrain, color = 'red', label = '')
plt.title('Prediction')
plt.xlabel('Epoch')
#plt.ylabel(' ')
plt.legend()
plt.show()


# -*- coding: utf-8 -*-
"""
Created on Tue May  9 12:08:04 2023

@author: tarun
"""

from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation

# Load the 20 newsgroups dataset
newsgroups = fetch_20newsgroups(subset='all', shuffle=True, random_state=42)

# Vectorize the text data using a bag-of-words representation
vectorizer = CountVectorizer(max_df=0.95, min_df=2, stop_words='english')
X = vectorizer.fit_transform(newsgroups.data)

# Fit an LDA model to the data
lda = LatentDirichletAllocation(n_components=20, random_state=42)
lda.fit(X)

# Print the top 10 words for each topic
feature_names = vectorizer.get_feature_names_out()
for topic_idx, topic in enumerate(lda.components_):
    print("Topic %d:" % (topic_idx))
    print(" ".join([feature_names[i] for i in topic.argsort()[:-10 - 1:-1]]))
    print()

#which toipic for which document

# Get the topic distribution for each document
doc_topic_dist = lda.transform(X)

# Print the top topic for the first 10 documents
df= pd.DataFrame({"text":newsgroups.data, "label":newsgroups.target})

df["lda"]=999

for i in range(len(df)):
    df["lda"].iloc[i] = doc_topic_dist[i].argmax()
    print("Document %d: Top Topic = %d" % (i, top_topic_idx))

df.lda.value_counts()
df.label.value_counts()

#pca
import numpy as np
from sklearn.decomposition import PCA

# Generate some sample data
X = np.random.randn(100, 5)  # 100 samples with 5 features each

# Create a PCA object with 2 components
pca = PCA(n_components=4)

# Fit the PCA model to the data
pca.fit(X)

# Transform the data to the lower-dimensional space
X_pca = pca.transform(X)

# Print the explained variance ratio of each component
print(pca.explained_variance_ratio_)
sum(pca.explained_variance_ratio_)

# Access the components (i.e. eigenvectors) of the PCA model
components = pca.components_

# Print the components
print(components)


adapters = []
joltageDiff = 3

def calculateOneAndThreeDiffs(adapters):
    oneDiffs = 0
    threeDiffs = 0
    for index, adapter in enumerate(adapters[:len(adapters) - 1]):
        diff = adapters[index + 1] - adapter
        if diff == 1:
            oneDiffs += 1
        elif diff == 3:
            threeDiffs += 1
    return (oneDiffs, threeDiffs)

with open("input.txt") as input:
    for line in input:
        adapters.append(int(line))

adapters.sort()
adapters.insert(0, 0)
adapters.append(adapters[len(adapters) - 1] + joltageDiff)

diffs = calculateOneAndThreeDiffs(adapters)
print(diffs[0] * diffs[1])


from django.conf.urls.defaults import patterns, include, url
from sindeo import views
from sindeo import settings

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'sindeo.views.home', name='home'),
    # url(r'^sindeo/', include('sindeo.foo.urls')),

    # Uncomment the admin/doc line below to enable admin documentation:
    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
    	#(r'^admin/', include(admin.site.urls)),
	('^$', views.landing_page),
	(r'^hp/$', views.home_page),
	#(r'^time/plus/(\d{1,2})/$', views.hours_ahead),
	(r'^meta/$', views.req_meta),
	(r'^search/$', views.search),
	(r'^subscribe/$', views.subscribe),
	(r'^subscribe/thanks/$', views.thanku),
	(r'^static/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.STATIC_ROOT}),
)


import os
import random
from tkinter import *
from tkinter.filedialog import askdirectory

from mutagen.id3 import ID3

import pygame

#  create the window with tkinter
root = Tk()
root.minsize(300, 300)
# create list to store the mp3 data, the list of the song titles and their rating
playlist = []
titles = []

title_label = StringVar()

index = 0  # index for the song currently choosed

def smart_shuffle(event):  # function to choose the song by smart shuffle
    global index
    songs_above_rating = []
    ran = random.uniform(0.1, 1.0)  # rate limit for the new songs
    ratelist_index = -1  # manual index for the loop

    # adds all the songs to the list which are above the rating - the better the rating
    # the more possible is it to be selected
    for i in ratelist:
        ratelist_index += 1
        if i > ran:
            songs_above_rating.append(playlist[ratelist_index])

    shuffled_track = random.choice(songs_above_rating) # selects random song from the new playlist
    pygame.mixer.music.load(shuffled_track)
    pygame.mixer.music.play()
    index = playlist.index(shuffled_track)  # changes the index to the current song
    update_label()

def correct_rate(self):
    global ratelist
    self.songs.sort(key=lambda tracks: tracks.rate, reverse=True)
    correct_multi = 1.0 / self.songs[0].rate
    for l in ratelist:
        l.correct_rate(correct_multi)

def continue_song(event): #  continues the song
    pygame.mixer.music.unpause()

def next_song(event): #  plays next song
    global index
    index += 1
    if index > len(playlist)-1:
        index = 0
    pygame.mixer.music.load(playlist[index])
    pygame.mixer.music.play()
    update_label()

def prev_song(event): #  selects previous song
    global index
    index -= 1
    pygame.mixer.music.load(playlist[index])
    pygame.mixer.music.play()
    update_label()

def pause_song(event):
    pygame.mixer.music.pause()
    title_label.set

def shuffle_song(event): # selects a random song
    global index
    index = playlist.index(random.choice(playlist))
    pygame.mixer.music.load(playlist[index])
    pygame.mixer.music.play()
    update_label()

def update_label(): #  updates the variable label to show which song is playing
    global index
    title_label.set(titles[index])

def directory_chooser(): #  chooses the directory for the songs and saves it in th list
    global ratelist
    directory = askdirectory() #  method to choose the directory
    os.chdir(directory)

    for files in os.listdir(directory):
        if files.endswith('.mp3'):  # add all files which are mp3 files

            path = os.path.realpath(files)  # gets the directory for ID3
            audio = ID3(path)
            titles.append(audio.get("TIT2"))
            playlist.append(files)
    playlist.reverse() #  reverse so the song which is player is on top
    pygame.mixer.init()
    pygame.mixer.music.load(playlist[index])
    pygame.mixer.music.play()
    ratelist = [1.0] * len(playlist)  # adds a rating to every song entry

directory_chooser()

header_label = Label(root, text='Smart Shuffle Proto')
header_label.pack()

listbox = Listbox(root)
listbox.pack()

for songs in titles:
    listbox.insert(0, songs)

titles.reverse()
update_label()

unpause_button = Button(root, text='Play')
unpause_button.pack()
unpause_button.bind("<Button-1>", continue_song)

next_button = Button(root, text='Next Song')
next_button.pack()
next_button.bind("<Button-1>", next_song)

previous_button = Button(root, text='Previous Song')
previous_button.pack()
previous_button.bind("<Button-1>", prev_song)

pause_button = Button(root, text='Stop')
pause_button.pack()
pause_button.bind("<Button-1>", pause_song)

shuffle_button = Button(root, text='Shuffle')
shuffle_button.pack()
shuffle_button.bind("<Button-1>", shuffle_song)

smart_shuffle_button = Button(root, text='SmartShuffle')
smart_shuffle_button.pack()
smart_shuffle_button.bind("<Button-1>", smart_shuffle)

songTitle = Label(root, textvariable=title_label)
songTitle.pack()

root.mainloop()

#  labelsLibrary = https://www.tutorialspoint.com/python/tk_label.htm


from django.urls import path
from . import views

app_name = 'demand_manager'
urlpatterns = [
    # ex: /demand_manager/
    # path('', views.DemandTopView.as_view(), name='top'),
    path('', views.DemandTopFilterView.as_view(), name='top'),

    # ex: /demand_manager/login/
    #path('login/', views.DemandLoginView.as_view(), name='login'),
    # ex: /demand_manager/logout/
    #path('logout/', views.DemandLogoutView.as_view(), name='logout'),

    # ex: /demand_manager/create/
    path('create/', views.DemandCreateView.as_view(), name='create'),
    # ex: /demand_manager//2/
    path('<int:pk>/', views.DemandDetailView.as_view(), name='detail'),
    # ex: /demand_manager/2/update/
    path('<int:pk>/update/', views.DemandUpdateView.as_view(), name='update'),
    # ex: /demand_manager/2/delete/
    path('<int:pk>/delete/', views.DemandDeleteView.as_view(), name='delete'),
    # ex: /demand_manager/analysis/
    path('analysis/', views.DemandAnalysisView.as_view(), name='analysis'),
    # ex: /demand_manager/import/
    path('import/', views.ImportReleasedLicenseView.as_view(), name='import'),
]


import string
import random

import matplotlib.pyplot as plt
import os

import subprocess

FILENAME = "C:/Users/vyach/CLionProjects/llp-lab1/test.bin"
INSERT_TEST = "cmake-build-windows/test/test_insert.exe"
SELECT_TEST = "cmake-build-windows/test/test_select.exe"
SELECT_WHERE_TEST = "cmake-build-windows/test/test_where.exe"
DELETE_TEST = "cmake-build-windows/test/test_delete.exe"
UPDATE_TEST = "cmake-build-windows/test/test_update.exe"
READ_TEST = "cmake-build-windows/test/test_read.exe"
JOIN_TEST = "cmake-build-windows/test/test_join.exe"
CREATE_TABLE_TEST = "cmake-build-windows/test/test_createtable.exe"

def file_size():
    return os.path.getsize("C:/Users/vyach/CLionProjects/llp-lab1/test.bin")

def random_string(n: int) -> str:
    return ''.join(random.choice(string.digits + string.ascii_letters) for _ in range(n))

def random_int32() -> int:
    return random.randint(-2147483647, 2147483646)

def random_bool() -> int:
    return random.choice([0, 1])

def random_float() -> float:
    return round(random.uniform(-1, 1), 7)

def random_input() -> str:
    return f"{random_int32()} {random_string(32)} {random_bool()} {random_float()} "

def call_insert(n: int, mode: int, records: list[str], table_name: str) -> float:
    p = subprocess.Popen([INSERT_TEST, str(n), str(mode), table_name],
                         stdout=subprocess.PIPE,
                         stdin=subprocess.PIPE)
    written = []
    for i in range(n):
        written.append(records[i])
        p.stdin.write(bytes(records[i], 'ascii'))
    p.stdin.flush()
    took = float(p.stdout.read())
    ret = p.wait(None)
    assert ret == 0
    return took

def call_select() -> float:
    p = subprocess.Popen([SELECT_WHERE_TEST], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
    took = float(p.stdout.read())
    ret = p.wait(None)
    assert ret == 0
    return took

def call_join() -> float:
    p = subprocess.Popen([JOIN_TEST], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
    took = float(p.stdout.read())
    ret = p.wait(None)
    assert ret == 0
    return took

def call_delete() -> float:
    p = subprocess.Popen([DELETE_TEST], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
    took = float(p.stdout.read())
    ret = p.wait(None)
    assert ret == 0
    return took

def call_update() -> float:
    p = subprocess.Popen([UPDATE_TEST], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
    took = float(p.stdout.read())
    ret = p.wait(None)
    assert ret == 0
    return took

def call_read() -> list[str]:
    p = subprocess.Popen([READ_TEST], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
    lines = [bytes.decode(i, encoding="ascii") for i in p.stdout.readlines()]
    ret = p.wait(None)
    assert ret == 0
    return lines

def call_create_table(mode: int, table_name: str):
    p = subprocess.Popen([CREATE_TABLE_TEST, str(mode), table_name], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
    ret = p.wait(None)
    assert ret == 0

def test_insert(table_name: str):
    call_insert(0, 2, [], table_name)
    y = []
    x = []
    z = []
    batch_size = 1000
    for i in range(1000):
        timediff = call_insert(batch_size, 1, [random_input() for _ in range(batch_size)], table_name)
        size = file_size()
        # print(f"n: {i}, inserting: {batch_size}, took: {timediff}s, filesize: {size}")
        y.append(timediff)
        x.append(i)
        z.append(size)
    fig, (time_axes, size_axes) = plt.subplots(2, sharex=True)

    time_axes.set_title(f"Insert Test (batch size: {batch_size} records)")
    time_axes.plot(x, y)
    time_axes.set_ylabel("time (seconds)")

    size_axes.plot(x, z)
    size_axes.set_ylabel("file size (bytes)")
    size_axes.set_xlabel("batch number")

    plt.show()

def test_select():
    batch_size = 1000
    y = []
    x = []
    call_insert(0, 2, [], "test")
    for i in range(100):
        print(i)
        call_insert(batch_size, 1, [random_input() for _ in range(batch_size)], "test")
        timediff = call_select()
        y.append(timediff)
        x.append(i * batch_size)

    fig, time_axes = plt.subplots()

    time_axes.set_title(f"Select Test")
    time_axes.plot(x, y)
    time_axes.set_ylabel("time (seconds)")
    time_axes.set_xlabel("records number")

    plt.show()

def test_delete_time():
    batch_size = 1000
    y = []
    x = []
    for i in range(1, 101):
        print(i)
        call_insert(i * batch_size, 2, [random_input() for _ in range(batch_size * i)], "test")
        timediff = call_delete()
        y.append(timediff)
        x.append(i * batch_size)

    fig, time_axes = plt.subplots()

    time_axes.set_title(f"Delete Test")
    time_axes.plot(x, y)
    time_axes.set_ylabel("time (seconds)")
    time_axes.set_xlabel("records number")

    plt.show()

def test_update_time():
    batch_size = 1000
    y = []
    x = []
    call_insert(0, 2, [], "test")
    for i in range(0, 100):
        print(i)
        call_insert(batch_size, 1, [random_input() for _ in range(batch_size)], "test")
        timediff = call_update()
        y.append(timediff)
        x.append(i * batch_size)

    fig, time_axes = plt.subplots()

    time_axes.set_title(f"Update Test")
    time_axes.plot(x, y)
    time_axes.set_ylabel("time (seconds)")
    time_axes.set_xlabel("records number")

    plt.show()

def test_delete_size():
    batch_size = 1000
    y = []
    x = []
    call_insert(0, 2, [], "test")
    for i in range(100):
        print(i)
        call_insert(
            batch_size,
            1,
            [f"{random_int32()} {random_string(32)} {1} {random_float()} " for _ in range(batch_size)],
            "test"
        )
        call_delete()
        y.append(file_size())
        x.append(i)

    fig, time_axes = plt.subplots()

    time_axes.set_title(f"Delete-Insert Test")
    time_axes.plot(x, y)
    time_axes.set_ylabel("file size (bytes)")
    time_axes.set_xlabel("records number")

    plt.show()

def test_update_size():
    batch_size = 1000
    y = []
    x = []
    call_insert(
        batch_size,
        2,
        [f"{random_int32()} {random_string(32)} {1} {random_float()} " for _ in range(batch_size)],
        "test"
    )
    for i in range(100):
        print(i)
        call_update()
        y.append(file_size())
        x.append(i)

    fig, time_axes = plt.subplots()

    time_axes.set_title(f"Insert-Update Test")
    time_axes.plot(x, y)
    time_axes.set_ylabel("file size (bytes)")
    time_axes.set_xlabel("records number")

    plt.show()

def test_join():
    batch_size = 10
    y = []
    x = []
    call_create_table(2, "test")
    call_create_table(1, "test2")
    for i in range(50):
        print(i)
        call_insert(batch_size, 1, [random_input() for _ in range(batch_size)], "test")
        call_insert(batch_size, 1, [random_input() for _ in range(batch_size)], "test2")
        timediff = call_join()
        y.append(timediff)
        x.append(i * batch_size)

    fig, time_axes = plt.subplots()

    time_axes.set_title(f"Select-Join Test")
    time_axes.plot(x, y)
    time_axes.set_ylabel("time (seconds)")
    time_axes.set_xlabel("records number")

    plt.show()

def correctness(n: int):
    data = {i: (i, random_string(32), random_bool(), random_float()) for i in range(n)}
    call_insert(n, 2, [f"{i[0]} {i[1]} {i[2]} {i[3]} " for i in data.values()], "test")

    res = call_read()
    read_data = []
    for i in res:
        row_str = i.strip().split()
        row = (int(row_str[0]), row_str[1], int(row_str[2]), float(row_str[3]))
        read_data.append(row)

    # check correctness
    assert(len(read_data) == len(data))
    for read, real in zip(read_data, data.values()):
        assert read[0] == real[0]
        assert read[1] == real[1]
        assert read[2] == real[2]
        assert abs(read[3] - real[3]) < 0.001

if __name__ == '__main__':
    test_join()


class Solution:
    # Sol 1. Two pointers
    def searchInsert(self, nums: List[int], target: int) -> int:
        if len(nums) == 0:
            return 0
        left_bound = 0
        right_bound = len(nums) -1
        ptr = 0
        while left_bound <= right_bound:
            ptr = (left_bound+right_bound)//2
            if nums[ptr] == target:
                return ptr
            elif nums[ptr] < target:
                left_bound = ptr + 1
            else:
                right_bound = ptr - 1
        return left_bound
    # TC: O(logn),n is the length of nums.
    # SC: O(1).


# -*- coding: utf-8 -*-
import os
import sys
__root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
if __root_path not in sys.path:
    sys.path.append(__root_path)
import requests
import datetime
from tornado.httpclient import AsyncHTTPClient
import asyncio
import urllib.parse
import threading
import time

url = "http://127.0.0.1:7011/query"
data = {"sn": "34444444"}

async def asy(n):
    client = AsyncHTTPClient()
    count = 0
    begin = datetime.datetime.now()
    for i in range(n):
        resp = await client.fetch(url, method='POST', body=urllib.parse.urlencode(data))
        if resp.code == 200:
            count += 1
            # print(resp.body)
    end = datetime.datetime.now()
    delta = (end - begin).total_seconds()
    print("异步查询{}个查询耗时{}秒,成功率: {}%".format(n, delta, (count / n) * 100))

def batch_req(n):
    with requests.Session() as ses:
        count = 0
        begin = datetime.datetime.now()
        for i in range(n):
            resp = ses.post(url, data=data)
            if resp.status_code == 200:
                count += 1
        end = datetime.datetime.now()
        delta = (end - begin).total_seconds()
        print("顺序查询{}个查询耗时{}秒,成功率: {}%".format(n, delta, (count / n) * 100))

def req(m, count):
    with requests.Session() as ses:
        m = int(m)
        for i in range(m):
            resp = ses.post(url, data=data)
            if resp.status_code == 200:
                count['val'] = count['val'] + 1
    count['status'] = True

def th(n):
    begin = datetime.datetime.now()
    count1 = {"val": 0, "status": False}
    count2 = {"val": 0, "status": False}
    count3 = {"val": 0, "status": False}
    count4 = {"val": 0, "status": False}
    t1 = threading.Thread(target=req, args=(n/4, count1,), daemon=True)
    t2 = threading.Thread(target=req, args=(n/4, count2,), daemon=True)
    t3 = threading.Thread(target=req, args=(n/4, count3,), daemon=True)
    t4 = threading.Thread(target=req, args=(n/4, count4,), daemon=False)
    t1.start()
    t2.start()
    t3.start()
    t4.start()
    while not (count1['status'] and count2['status'] and count3['status'] and count4['status']):
        time.sleep(0.0000001)
    end = datetime.datetime.now()
    delta = (end - begin).total_seconds()
    count = count1['val'] + count2['val'] + count3['val'] + count4['val']
    print("4线程查询{}个查询耗时{}秒,成功率: {}%".format(n, delta, (count / n) * 100))

if __name__ == "__main__":
    # batch_req(2000)
    loop = asyncio.get_event_loop()
    loop.run_until_complete(asy(2000))
    th(2000)
    pass


import os
import sys
reload(sys)
sys.setdefaultencoding('utf8')

class GraphicResultFile:
    """文件类:用于保存最终结果."""
    _TITLES = ['全部app:   ', '第三方app: ', '系统app:   ']

    def __init__(self, dir, file):
        """初始化类 GraphicResultFile."""
        if not os.path.exists(dir):
            os.makedirs(dir)
        self.__file = dir + '/' + file

    def transfer_to_file(self, data=''):
        """将输入内容保存至文件

        参数:
            data: 要保存的数据. 默认为''.

        返回值:
            无.

        """
        output = open(self.__file, 'w')
        for i in range(len(self._TITLES)):
            output.write(self._TITLES[i] + data[i] + '\n')

        output.close()


from nltk.corpus import wordnet

def syns_test(word):
    syns = wordnet.synsets(word)
    # print(syns[0].name())
    syns_list=[]
    for i in range (0,len(syns)):
        for j in range (0,len(syns[i].lemmas())):
            syns_list.append(syns[i].lemmas()[j].name())
    syns_list.sort()
    print(syns_list[0])

if __name__=='__main__':
    syns = wordnet.synsets("./")
    syns_list=[]
    for i in range (0,len(syns)):
        for j in range (0,len(syns[i].lemmas())):
            syns_list.append(syns[i].lemmas()[j].name())
    for word in syns_list:
        syns_test(word)


import setuptools

with open("README.md", "r", encoding="utf-8") as fh:
    long_description = fh.read()

INSTALL_REQUIRES = []

setuptools.setup(
    name="rate_limiter",
    version="0.0.1",
    author="Shenglong Zhang",
    author_email="songnon@gmail.com",
    description="HTTP request rate limiting",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/",
    packages=setuptools.find_packages(),
    install_requires=INSTALL_REQUIRES,
    package_data={'rate_limiter': ['*.json', '*.ini']},
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
    python_requires='>=3.8',
)


import numpy as np
import copy

from lsst.ts.wep.bsc.WcsSol import WcsSol
from lsst.ts.wep.Utility import FilterType

class CameraData(object):

    def __init__(self, camera):
        """Initialize the camera data class.

        Parameters
        ----------
        camera : lsst.afw.cameraGeom.camera.camera.Camera
            A collection of Detectors that also supports coordinate
            transformation. (the default is None.)
        """

        self._wcs = WcsSol(camera=camera)

        # List of wavefront sensor CCD name
        self._wfsCcd = []

        # Dictionary of (x, y) coordinates of detector corners and dimensions
        # of detection. The dictonary key is the ccd name.
        self._corners = dict()
        self._dimension = dict()

    def setWfsCcdList(self, wfsCcdList):
        """Set the wavefront sensor (WFS) charge-coupled devide (CCD) list.

        Parameters
        ----------
        wfsCcdList : list
            Wavefront sensor list (e.g. ["R:2,2 S:1,1", "R:2,2 S:1,0"]).
        """

        self._wfsCcd = wfsCcdList

    def setWfsCorners(self, wfsCorners):
        """Set the wavefront sensor corner information.

        Parameters
        ----------
        wfsCorners : dict
            Wavefront corner information. The dictionary key is the CCD name
            (e.g. "R:2,2 S:1,1").
        """

        self._corners = wfsCorners

    def setCcdDims(self, ccdDims):
        """Set the charge-coupled device (CCD) dimenstions.

        Parameters
        ----------
        ccdDims : dict
            CCD dimensions. The dictionary key is the CCD name (e.g.
            "R:2,2 S:1,1").
        """

        self._dimension = ccdDims

    def getWfsCcdList(self):
        """Get the list of wavefront sensor CCD list.

        Returns
        -------
        list
            Wavefront sensor CCD list.
        """

        return self._wfsCcd

    def getWfsCorner(self, detectorName):
        """Get the wavefront sensor corner.

        Parameters
        ----------
        detectorName : str
            Detector Name (e.g. "R:2,2 S:1,1").

        Returns
        -------
        tuple
            Wavefront sensor corner.
        """

        return self._corners[detectorName]

    def getCcdDim(self, detectorName):
        """Get the CCD dimension.

        Parameters
        ----------
        detectorName : str
            Detector name (e.g. "R:2,2 S:1,1").

        Returns
        -------
        tuple
            CCD dimension in pixel.
        """

        return self._dimension[detectorName]

    def _initDetectors(self, detectorType):
        """Initializes the camera detectors.

        Parameters
        ----------
        detectorType : lsst.afw.cameraGeom.detector.detector.DetectorType
            Detector type.
        """

        for detector in self._wcs.getCamera():

            if (detector.getType() == detectorType):

                # Collect the ccd name
                detectorName = detector.getName()
                self._wfsCcd.append(detectorName)

                # Get the detector corners
                bbox = detector.getBBox()
                xmin = bbox.getMinX()
                xmax = bbox.getMaxX()
                ymin = bbox.getMinY()
                ymax = bbox.getMaxY()
                self._corners[detectorName] = \
                                (np.array([xmin, xmin, xmax, xmax]),
                                 np.array([ymin, ymax, ymin, ymax]))

                # The CCD dimension here is an estimation.
                # Based on LCA-13381, there are three types of sensors.
                # e2V CCD250: 40.04 mm x 40.96 mm
                # STA 4400: 20.00 mm x 40.72 mm
                # STA 3800C: 40.00 mm x 40.72 mm

                dim1, dim2 = bbox.getDimensions()
                self._dimension[detectorName] = (int(dim1), int(dim2))

    def setObsMetaData(self, ra, dec, rotSkyPos, mjd=59580.0):
        """Set the observation meta data.

        Parameters
        ----------
        ra : float
            Pointing ra in degree.
        dec : float
            Pointing decl in degree.
        rotSkyPos : float
            The orientation of the telescope in degrees.
        mjd : float
            Camera MJD. (the default is 59580.0.)
        """

        self._wcs.setObsMetaData(ra, dec, rotSkyPos, mjd=mjd)

    def populatePixelFromRADecl(self, stars):
        """Populates the RAInPixel and DeclInPixel coordinates to the stars.

        Parameters
        ----------
        stars : StarData
            The stars to populate.

        Returns
        -------
        StarData
            The stars with x-, y-pixel data populated.
        """

        # Do the shallow copy
        populatedStar = copy.copy(stars)

        # Populate the pixel data
        ra = populatedStar.getRA()
        decl = populatedStar.getDecl()
        chipName = np.array([populatedStar.getDetector()] * len(ra))
        raInPixel, declInPixel = self._wcs.pixelCoordsFromRaDec(
            ra, decl, chipName=chipName, epoch=2000.0, includeDistortion=True)

        populatedStar.setRaInPixel(raInPixel)
        populatedStar.setDeclInPixel(declInPixel)

        return populatedStar

    def getStarsOnDetector(self, stars, offset):
        """Get the stars on the detector according to the pixel position.

        Parameters
        ----------
        stars : StarData
            Star information.
        offset : float
            Offset to the dimension of camera. If the detector dimension is 10
            (assume 1-D), the star's position between -offset and 10+offset
            will be seem to be on the detector.

        Returns
        -------
        StarData
            The stars on the detector.
        """

        # Do the shallow copy
        starsOnDet = copy.copy(stars)

        # Get the index that will keep the data
        starsRaInPixel = starsOnDet.getRaInPixel()
        starsDeclInPixel = starsOnDet.getDeclInPixel()
        ccdDim = self.getCcdDim(starsOnDet.getDetector())

        keep = []
        for ii in range(len(starsRaInPixel)):
            if (-offset <= starsRaInPixel[ii] <= ccdDim[0] + offset and
                -offset <= starsDeclInPixel[ii] <= ccdDim[1] + offset):
                keep.append(ii)

        # Remove the stars that are not on the detector
        starsOnDet.setId(self._getKeepItem(starsOnDet.getId(), keep))
        starsOnDet.setRA(self._getKeepItem(starsOnDet.getRA(), keep))
        starsOnDet.setDecl(self._getKeepItem(starsOnDet.getDecl(), keep))
        starsOnDet.setRaInPixel(self._getKeepItem(starsRaInPixel, keep))
        starsOnDet.setDeclInPixel(self._getKeepItem(starsDeclInPixel, keep))

        for filterType in FilterType:

            if (filterType != FilterType.REF):
                magArray = starsOnDet.getMag(filterType)

                if (len(magArray) != 0):
                    starsOnDet.setMag(filterType, self._getKeepItem(magArray,
                                                                    keep))

        return starsOnDet

    def _getKeepItem(self, valArray, keep):
        """Get the keep items in array.

        Parameters
        ----------
        valArray : list or 1-D numpy.ndarray
            Value array.
        keep : list[int]
            List of keep index.

        Returns
        -------
        list
            The array that has only the keep values.
        """

        return [valArray[idx] for idx in keep]

    def getWavefrontSensor(self):
        """

        Get the corners of LSST curvature wavefront sensors in (ra, dec) based
        on the wavefront sensor list.

        Returns:
            [dict] -- (ra, dec) of four corners of each sensor with the name
            of sensor as a list. The dictionary key is the sensor name.
        """

        return self._getDetectorRaDec(self._wfsCcd)

    def _getDetectorRaDec(self, detectorList):
        """Get the (ra, dec) of CCD corners in the detector list.

        Parameters
        ----------
        detectorList : list
            List of detectors. For example, ["R:2,2 S:1,1", "R:2,2 S:0,1"].

        Returns
        -------
        dict
            This method returns a dict of list.  The dict is keyed on the name
            of the wavefront sensor.  The list contains the (RA, Dec)
            coordinates of the corners of that sensor (RA, Dec are paired as
            tuples). For example, output['R:0,0 S:2,2B'] = [(23.0, -5.0),
            (23.1, -5.0), (23.0, -5.1), (23.1, -5.1)] would mean that the
            wavefront sensor named 'R:0,0 S:2,2B' has its corners at RA 23,
            Dec -5; RA 23.1, Dec -5; RA 23, Dec -5.1; and RA 23.1, Dec -5.1
            Coordinates are in degrees.
        """

        ra_dec_out = dict()
        for detector in detectorList:

            coords = self._corners[detector]
            xPix = coords[0]
            yPix = coords[1]

            chipName = np.array([detector] * len(xPix))
            ra, dec = self._wcs.raDecFromPixelCoords(
                xPix, yPix, chipName, epoch=2000.0, includeDistortion=True)

            ra_dec_out[detector] = [(ra[0], dec[0]), (ra[1], dec[1]),
                                    (ra[2], dec[2]), (ra[3], dec[3])]

        return ra_dec_out

if __name__ == "__main__":
    pass


import sqlite3
from utility.constants import FILE

def db_loader(sql_file_path):
    try:
        connection = sqlite3.connect(FILE.DB_FILE)
        cursor = connection.cursor()
        sql_file = open(sql_file_path)
        sql_as_string = sql_file.read()
        cursor.executescript(sql_as_string)
        return True
    except (OSError, ConnectionError):
        print('Error while loading sql file')


"""
preprocess_nih
==============

Data cleaning / wrangling before ingestion of raw data,
specifically:

  * Systematically removing generic prefixes using very hard-coded logic.
  * Inferring how to correctly deal with mystery question marks,
    using very hard-coded logic.
  * Splitting of strings into arrays as indicated by JSON the ORM,
  * CAPS to Camel Case for any string field which isn't VARCHAR(n) < 10
  * Dealing consistently with null values
  * explicit conversion to datetime of relevant fields
"""
from sqlalchemy.dialects.mysql import VARCHAR
from sqlalchemy.types import JSON, DATETIME
from datetime import datetime as dt
from functools import lru_cache
import pandas as pd
import string

GENERIC_PREFIXES = ['proposal narrative', 'project narrative',
                    'relevance to public health', 'public health relevance',
                    'narrative', 'relevance', 'statement',
                    '(relevance to veterans)', 'relevance to the va',
                    'description', '(provided by applicant)',
                    'project summary', 'abstract', 'overall', 'project',
                    '? ', ' ', '/', 'administrative', '.', 'core', 'title',
                    'summary', '*', 'pilot project', '#']

@lru_cache()
def get_json_cols(orm):
    """Return the column names in the ORM which are of JSON type"""
    return {col.name for col in orm.__table__.columns
            if type(col.type) is JSON}

@lru_cache()
def get_long_text_cols(orm, min_length=10):
    """Return the column names in the ORM which are a text type,
    (i.e. TEXT or VARCHAR) and if a max length is specified, with max
    length > 10. The length requirement is because we don't want
    to preprocess ID-like or code fields (e.g. ISO codes).
    """
    return {col.name for col in orm.__table__.columns
            if col.type.python_type is str and
            not (type(col.type) == VARCHAR  and col.type.length < 10)}

@lru_cache()
def get_date_cols(orm):
    """Return the column names in the ORM which are of JSON type"""
    return {col.name for col in orm.__table__.columns
            if type(col.type) is DATETIME}

def is_nih_null(value, nulls=('', [], {}, 'N/A', 'Not Required', 'None')):
    """Returns True if the value is listed in the `nulls` argument,
    or the value is NaN, null or None."""
    try:
        iter(value)
    # Only run pd.isnull if the value is not iterable (list, dict, etc)
    except TypeError:
        is_null = pd.isnull(value)
    # Otherwise set a dummy value = False
    else:
        is_null = False
    # Then check if value is either null, or in the list of null values
    finally:
        return is_null or value in nulls

@lru_cache()
def expand_prefix_list():
    """Expand GENERIC_PREFIXES to include integers, and then a large
    numbers of permutations of additional characters, upper case
    and title case. From tests, this covers 19 out of 20 generic
    prefixes from either abstract text or the "PHR" field."""
    prefixes = []
    numbers = list(str(i) for i in range(0, 10))
    for prefix in GENERIC_PREFIXES + numbers:
        _prefixes = [prefix]
        _prefixes += [prefix+ext for ext in ['', '/', ':', '-', ' -', '.']]
        _prefixes += [ext+prefix for ext in ['?']]
        prefixes += _prefixes
    prefixes += [p+' ' for p in prefixes]
    prefixes += [p.upper() for p in prefixes]
    prefixes += [p.title() for p in prefixes]
    # Sort by the longest first
    return sorted(set(prefixes), key=lambda x: len(x), reverse=True)

def remove_generic_suffixes(text):
    """Iteratively remove any of the generic terms in `expand_prefix_list`
    from the front of the text, until none remain."""
    still_replacing = True
    while still_replacing:
        still_replacing = False
        for prefix in expand_prefix_list():  # NB: lru_cached
            if not text.startswith(prefix):
                continue
            text = text.replace(prefix, '', 1)
            still_replacing = True
            break
    return text

def remove_large_spaces(text):
    """Iteratively replace any large spaces or tabs with a single space,
    until none remain."""
    for pattern in ['\t', '  ']:
        while pattern in text:
            text = text.replace(pattern, ' ')
    while text.startswith(' '):
        text = text[1:]
    while text.endswith(' '):
        text = text[:-1]
    return text

def replace_question_with_best_guess(text):
    """Somewhere in NiH's backend, they have a unicode processing problem.
    From inspection, most of the '?' symbols have quite an intuitive origin,
    and so this function contains the hard-coded logic for inferring
    what symbol used to be in the place of each '?'.
    """
    # Straightforward find and replace
    for find, replace in [('?s', "'s"), (' ? ', ' - '),
                          ('.?', "'."), (',?', "',"),
                          ('n?t', "n't")]:
        if find in text:
            text = text.replace(find, replace)
        if find.upper() in text:
            text = text.replace(find.upper(), replace.upper())

    # Most '?' will be replaced with a single quote,
    # though some will be replaced by a hyphen.
    replace_quote, replace_hyphen = set(), set()
    # Iterate through chars in text to find those that should be changed
    for i, char in enumerate(text):
        if char != '?':
            continue
        # Ignore final char, assume is a question
        is_final_char = (i == len(text) - 1)
        # Ignore e.g. '? Title', but not '?. Title' or '? title'
        is_question = (i < len(text) - 2 and  # a char exists after '? '
                       text[i+1] == ' ' and text[i+2].isupper())
        if is_final_char or is_question:
            continue
        # The case 'sometext?somemoretext' --> 'sometext - somemoretext'
        if i > 0 and text[i-1].isalpha() and text[i+1].isalpha():
            replace_hyphen.add(i)
        # Everything else
        else:
            replace_quote.add(i)

    # Replace from back-to-front, so that the indexes don't become shuffled
    for i in reversed(range(len(text))):
        if i in replace_quote:
            text = text[:i] + "'" + text[i+1:]
        elif i in replace_hyphen:
            text = text[:i] + ' - ' + text[i+1:]
    return text

def remove_trailing_exclamation(text):
    """A lot of abstracts end with '!' and then a bunch of spaces."""
    while text.endswith('!'):
        text = text[:-1]
    return text

def upper_to_title(text, force_title=False):
    """Inconsistently, NiH has fields as all upper case.
    Convert to titlecase"""
    if text == text.upper() or force_title:
        text = string.capwords(text.lower())
    return text

def clean_text(text, suffix_removal_length=100):
    """Apply the full text-cleaning procedure."""
    operations = [remove_unspecified_unicode, remove_large_spaces,
                  remove_trailing_exclamation]
    if len(text) > suffix_removal_length:
        operations.append(remove_generic_suffixes)
    operations += [replace_question_with_best_guess, upper_to_title,
                   remove_trailing_exclamation]
    # Sequentially apply operations, and clean up spaces
    # after each operation
    for f in operations:
        text = f(text)
        text = remove_large_spaces(text)
    return text

def detect_and_split(value):
    """Split values either by colons or commas. If there are more colons than
    commas (+1), then colons are used for splitting (this takes into account
    that NiH name fields are written as
    'last_name, first_name; next_last_name, next_first_name'). Otherwise
    NiH list fields are delimeted by commas."""
    n_commas = value.count(",")
    n_colons = value.count(";")
    # e.g "last_name, first_name; next_last_name, next_first_name"
    if n_colons >= n_commas - 1:  # also includes case where n=0
        value = value.split(";")
    else:
        value = value.split(",")
    return value

def split_and_clean(col_value):
    """Apply `detect_and_split` and then apply some general cleaning."""
    values = []
    for value in detect_and_split(col_value):
        # Remove contact boilerplate
        value = value.replace('(contact)', '').strip()
        # Splitting procedure inconsistently leads to blanks
        if value == '':
            continue
        # Force all fields to be title-cased
        value = upper_to_title(value, force_title=True)
        values.append(value)
    return values

def parse_date(value):
    """Convert to date, unless the value null, or poorly formatted."""
    try:
        value = dt.strptime(value, '%m/%d/%Y')
    except ValueError:
        value = None
    finally:
        return value

def remove_unspecified_unicode(value):
    """In a very small number of cases, NiH has some pre-processed badly
    formatted unspecifed unicode characters. The following recipe seems to
    clean up all of the discovered cases."""
    # These are all spaces
    for char in ('\xa0\xa0', '\xa0 ', '\xa0'):
        value = value.replace(char, ' ')
    # This is a hyphen
    value = value.replace('-\xad', '-')
    # Clean up any bonus spaces
    while '  ' in value:
        value = value.replace('  ', ' ')
    return value

def preprocess_row(row, orm):
    """Clean text, split values and standardise nulls, as required.

    Args:
        row (dict): Row of data to clean, that should match
                    the provided ORM.
        orm (SqlAlchemy selectable): ORM from which to infer JSON
                                     and text fields.
    """
    json_cols = get_json_cols(orm)
    long_text_cols = get_long_text_cols(orm)
    date_cols = get_date_cols(orm)
    for col_name, col_value in row.copy().items():
        # If it's: TEXT-> clean it, JSON-> split it, DATE-> parse it
        if col_name in long_text_cols and not pd.isnull(col_value):
            col_value = clean_text(col_value)
        elif col_name in json_cols:
            col_value = split_and_clean(col_value)
        elif col_name in date_cols:
            col_value = parse_date(col_value)
        if is_nih_null(col_value):
            col_value = None
        row[col_name] = col_value
    return row


# -*- coding: utf-8 -*-
""" Script to test performance of components of the OA package

Log with:

    > python ptest.py > test0-ddmmyyy.txt
"""

#%% Load necessary packages
import sys
import os
import numpy as np
import time
import subprocess
import getopt
import platform

import oapackage
import oapackage.oahelper as oahelper
from oapackage import oalib
tmp = oalib.log_print(-oalib.SYSTEM, '')

#%% Define test functions

def pytest7(verbose=1):
    """ Reduced array with random transformtion to LMC form """
    al = oalib.exampleArray(3)
    adata = oalib.arraylink2arraydata(al)
    reduction = oalib.LMCreduction_t(adata)

    for ii in range(10):
        if verbose >= 2:
            print('pytest7: randomized reduction of array %d' % ii)
        reduction.transformation.randomize()
        al2 = reduction.transformation.apply(al)

        alr = al2.reduceLMC()
        c = al == alr
        if not c:
            print('pytest7: error: reduction of randomized array failed!')

#%%

def pytest6(verbose=1):
    """ Check generation of OA(32, t, 2^a) """
    N = 32
    k = 8
    t = 3
    l = [2] * 8
    rr = []
    gtrr = [3, 5, 10, 17, 33]
    s = oalib.intVector(l)
    adata = oalib.arraydata_t(s, N, t, k)

    if verbose:
        print('pytest6: run different algorithms on the same case')
    algs = [oalib.MODE_ORIGINAL, oalib.MODE_J4]
    for _, alg in enumerate(algs):
        algname = oalib.algnames(alg)
        if verbose >= 2:
            print('pytest6: running %s, alg %s' % (adata.fullidstr(), algname))
        rr = []
        tmp = oahelper.runExtend(N, k, t, l, verbose=verbose, nums=rr, algorithm=alg)
        if not rr == gtrr:
            print('pytest6: case %s' % adata.fullidstr())
            print('   algorithm %s: error: incorrect number of arrays! %s -> %s' % (algname, rr, gtrr))

#%%

def pytest3(verbose=2):
    """ Test rank and Defficiency properties """
    k = 6
    strength = 3
    N = 40
    lll = oahelper.runExtend(N, k, strength, 2, verbose=0)

    rnk0 = np.array([19, 19, 21, 21, 19, 22, 22, 22, 22])  # ground-truth
    D0 = np.array([0.0, 0.0, 0.0, 0.1703475035645109, 0.0, 0.8107439985672059, 0.8969911485707478, 0.8953919248315042, 0.9007069559712273])  # gt

    rnk = [oalib.array2xf(al).rank() for al in lll]
    D = [al.Defficiency() for al in lll]

    if (rnk - rnk0).any():
        if verbose:
            print('  rank test FAILED!')
            return False
    if not (np.abs(D0 - D) < 1e-6).all():
        if verbose:
            print('  Defficiency test FAILED!')
        return False
    return True

def pytest4(verbose=1):
    """ Check generation of OA(16,2, 4^2 2^a) """
    N = 16
    k = 6
    t = 2
    l = [4, 4, 2]
    rr = []
    oahelper.runExtend(N, k, t, l, verbose=1, nums=rr)
    if not rr == [2, 7, 17, 27]:
        print('ERROR: incorrect number of arrays! %s' % rr)

def pytest2level(verbose=1):
    """ Check generation of OA(16,2, 4^2 2^a) """
    N = 32
    k = 10
    t = 3
    l = [2]
    rr = []
    t0 = time.time()
    oahelper.runExtend(N, k, t, l, verbose=1, nums=rr, algorithm=oalib.MODE_ORIGINAL)
    dt = time.time() - t0
    t0 = time.time()
    oahelper.runExtend(N, k, t, l, verbose=1, nums=rr, algorithm=oalib.MODE_LMC_2LEVEL)
    dt2level = time.time() - t0

    ll = l * k
    s = oalib.intVector(ll)

    adata = oalib.arraydata_t(s, N, t, k)

    if verbose:
        print('case %s: 2-level method %.2f [s] -> %.2f [s]' % (adata.idstr(), dt, dt2level))
    #cases.append('runExtend(N=%d, k=%d, t=%d, l=%s)' %(N,k,t, str(l)) )

def pytest_analysis(oadir, verbose=1):
    """ Performance of analysis algorithms """
    cmd = 'export OMP_NUM_THREADS=1; oaanalyse --gwp -A %s' % os.path.join(oadir, 'testdata', 'test64.oa')
    if verbose:
        print('pytest_analysis: running oaanalyse')
    t0 = time.time()
    return_value = os.system(cmd)
    assert(return_value == 0)
    ta = (time.time() - t0)
    return ta

def pytest2(oadir, verbose=1):
    afile = os.path.join(oadir, 'testdata', 'unstable-rank.oa')
    sols = oalib.readarrayfile(afile)
    afile = os.path.join(oadir, 'testdata', 'unstable-rank-extended.oa')
    sols = oalib.readarrayfile(afile)

    for ii, al in enumerate(sols):
        vv = oalib.doubleVector(3)
        rnk = oalib.array2rank_Deff_Beff(al, vv)

        al2 = oalib.array2xf(al)
        rnk2 = al2.rank()
        if (not rnk2 == rnk) or (not rnk < 29):
            print('warning: problem?!: %d %d %d' % (ii, rnk, rnk2))
        if verbose >= 2:
            print('array %d: rank: %d %d' % (ii, rnk, rnk2))

def pytest(verbose=1):
    t0 = time.time()
    tt = []
    cases = []

    N = 48
    k = 6
    t = 3
    l = 2
    rr = []
    oahelper.runExtend(N, k, t, l, verbose=1, nums=rr)
    tt.append(time.time() - t0)
    cases.append('runExtend(N=%d, k=%d, t=%d, l=%d)' % (N, k, t, l))
    if not rr == [4, 10, 45]:
        print('ERROR: incorrect number of arrays! %s' % rr)

    N = 32
    k = 6
    t = 3
    l = 2
    rr = []
    oahelper.runExtend(N, k, t, l, verbose=1, nums=rr)
    tt.append(time.time() - t0)
    cases.append('runExtend(N=%d, k=%d, t=%d, l=%d)' % (N, k, t, l))
    if not rr == [3, 5, 10]:
        print('ERROR: incorrect number of arrays! %s' % rr)

    N = 16
    k = 16
    t = 2
    l = 2
    rr = []
    oahelper.runExtend(N, k, t, l, verbose=1, nums=rr)
    tt.append(time.time() - t0)
    cases.append('runExtend(N=%d, k=%d, t=%d, l=%d)' % (N, k, t, l))
    if not rr == [3, 5, 11, 27, 55, 80, 87, 78, 58, 36, 18, 10, 5, 0]:
        print('ERROR: incorrect number of arrays! %s' % rr)

    N = 96
    k = 8
    t = 4
    l = 2
    rr = []
    oahelper.runExtend(N, k, t, l, verbose=1, nums=rr)
    tt.append(time.time() - t0)
    cases.append('runExtend(N=%d, k=%d, t=%d, l=%d)' % (N, k, t, l))
    if not rr == [4, 9, 4, 0]:
        print('ERROR: incorrect number of arrays! %s' % rr)

    dt = time.time() - t0
    if verbose:
        print('total time %.3f [s]' % dt)

    if verbose:
        print('extra testing cases')
    N = 24
    k = 5
    t = 2
    l = [3, 2]
    rr = []
    oahelper.runExtend(N, k, t, l, verbose=1, nums=rr)
    tt.append(time.time() - t0)
    cases.append('runExtend(N=%d, k=%d, t=%d, l=%s)' % (N, k, t, str(l)))
    if not rr == [4, 29, 573]:
        print('ERROR: incorrect number of arrays! %s' % rr)

    N = 18
    k = 9
    t = 2
    # old test: l=[2,3]
    l = [3] * 8 + [2]
    rr = []
    oahelper.runExtend(N, k, t, l, verbose=1, nums=rr)
    tt.append(time.time() - t0)
    cases.append('runExtend(N=%d, k=%d, t=%d, l=%s)' % (N, k, t, str(l)))
    if not rr == [4, 12, 10, 8, 3, 0, 0]:
        print('ERROR: incorrect number of arrays! %s' % rr)

    dt2 = time.time() - t0
    return (dt, tt, cases, dt2)

class Usage(Exception):

    def __init__(self, msg):
        self.msg = msg

#%%

def testExtendBinary(verbose=1):
    repos = dict()
    repos['oatest1.txt'] = 'result-16.2-2-2-2-2-2-2-2.oa'
    repos['oatest2.txt'] = 'result-18.3-3-3-3-3.oa'
    repos['oatest3.txt'] = 'result-8.4-2-1-4-2-2.oa'
    repos['oatest4.txt'] = 'result-28.7-2-2-2.oa'
    repos['oatest5.txt'] = 'result-16.2-2-2-2-2-2-2-2-2.oa'
    repos['oatest6.txt'] = 'result-25.5-5-5-5.oa'
    repos['oatest7.txt'] = 'result-64.2-2-2-2-2-2.oa'
    repos['oatest8.txt'] = 'result-56.2-2-2-2-2.oa'

# testExtendBinary()

#%%
def main(argv=None):
    """ Main testing function """
    if platform.system() == 'Windows':
        oadir = os.path.join(os.path.split(oapackage.__file__)[0], '..')
    else:
        oadir = os.path.join(os.path.expanduser('~'), 'misc/oa/oacode/')

    print('OA performance testing: version 2.0')
    ss = oapackage.version()
    print('OAlib: version %s' % ss)
    ss = oalib.compile_information()
    print(ss)
    print('System:')
    print('  Python: ' + ' '.join(sys.version.split('\n')))
    print('  Machine: ' + ' '.join(['%s' % x for x in platform.uname()]))
    print('  Processor: %s' % (platform.processor(),))

    if argv is None:
        argv = sys.argv
    try:
        try:
            opts, args = getopt.getopt(argv[1:], "h", ["help"])
        except getopt.error as msg:
            raise Usage(msg)
        # more code, unchanged
    except Usage as err:
        print >>sys.stderr, err.msg
        print >>sys.stderr, "for help use --help"
        return 2

    print('\nRunning tests:\n')
    (dt, tt, cases, dt2) = pytest()

    pytest2(oadir)
    pytest3(verbose=0)
    pytest4(verbose=1)
    ta1 = pytest_analysis(oadir, verbose=1)
    pytest6(verbose=0)
    pytest7(verbose=0)

    pytest2level(verbose=1)

    print('-----')

    dtx = ta1

    print('\nResults:\n')
    for ii, t in enumerate(tt):
        print('%s: time %.2f [s]' % (cases[ii], t))
    version = oapackage.__version__
    print('Total time: %.3f [s], %.3f [s], %.3f [s] (%s, %s)' % (dt, dt2, dtx, platform.node(), version))
    print('   should be of order 3.408 [s], 3.494 [s], 7.313 [s] (woelmuis, 2.4.8)')
    print('   should be of order 4.4 [s], 4.6 [s], 5.9 [s] (woelmuis)')
    print('   should be of order 3.2 [s], 3.3 [s], 4.7 [s] (marmot) [v 2.0.0]')
    print('   should be of order 2.96 [s], 3.0 [s], 4.6 [s] (marmot) [v 2.0.24]')

if __name__ == "__main__":
    main()

def timeconfig(configfile='oaconfig.txt', timebin='/usr/bin/time', pdir='/home/eendebakpt/misc/oa/oacode/performancetest'):
    os.chdir(pdir)
    res = subprocess.check_output([('cd %s;ls;' % pdir) + timebin, '--format="%%E %%S %%U"', './oaextendsingle', '-c %s -l 1' % (configfile)])
    return res


class StringMatch(object):
    @classmethod
    def brute(cls, target, pattern):
        """
        穷举法 O(t_len * p_len)
        :param target:
        :param pattern:
        :return:
        """

        t_len, p_len = len(target), len(pattern)
        for i in range(t_len - p_len + 1):
            k = 0
            while k < p_len and target[i + k] == pattern[k]:
                k += 1

            if k == p_len:
                return i

        return - 1

    @classmethod
    def boyer_moore(cls, target, pattern):
        """
        镜像启发式 + 字幕跳跃启发式 O(t_len * p_len)
        :param target:
        :param pattern:
        :return:
        """

        t_len, p_len = len(target), len(pattern)
        if p_len == 0:
            return 0

        _last = {}
        for i, k in enumerate(pattern):
            _last[k] = i

        t_index = p_len - 1
        p_index = p_len - 1
        while t_index < t_len:
            if target[t_index] == pattern[p_index]:
                if p_index == 0:
                    return t_index
                else:
                    t_index -= 1
                    p_index -= 1
            else:
                _l_index = _last.get(target[t_index], -1)
                t_index += p_len - min(p_index, _l_index + 1)
                p_index = p_len - 1

        return -1

    @classmethod
    def knuth_morris_pratt(cls, target, pattern):
        """
        KMP O(t_len + p_len)
        :param target:
        :param pattern:
        :return:
        """

        t_len, p_len = len(target), len(pattern)
        if p_len == 0:
            return 0

        _fail = cls._compute_kmp_fail(pattern)
        t_index = 0
        p_index = 0
        while t_index < t_len:
            if target[t_index] == pattern[p_index]:
                if p_index == p_len - 1:
                    return t_index - p_len + 1

                t_index += 1
                p_index += 1
            elif p_index > 0:
                p_index = _fail[p_index - 1]
            else:
                t_index += 1

        return -1

    @staticmethod
    def _compute_kmp_fail(pattern):
        """
        KMP失败函数
        :param pattern:
        :return:
        """

        p_len = len(pattern)
        _fail = [0] * p_len
        _index = 1
        _c_index = 0
        while _index < p_len:
            if pattern[_index] == pattern[_c_index]:
                _fail[_index] = _c_index + 1
                _index += 1
                _c_index += 1
            elif _c_index > 0:
                _c_index = _fail[_c_index - 1]
            else:
                _index += 1

        return _fail

if __name__ == '__main__':
    print(StringMatch.brute("jasdhfkasjhdkfasdgfkjhaklsjdhflkjhasdkhf", "jhd"))
    print(StringMatch.boyer_moore("jasdhfkasjhdkfasdgfkjhaklsjdhflkjhasdkhf", "jhd"))
    print(StringMatch.knuth_morris_pratt("jasdhfkasjhdkfasdgfkjhaklsjdhflkjhasdkhf", "asdhfkasjhd"))


from setuptools import find_packages
from setuptools import setup

try:
    README = open("README.md").read() + "\n\n" + open("CHANGELOG.md").read()
except IOError:
    README = None

setup(
    name="guillotina_batch",
    version="6.0.1.dev0",
    description="batch endpoint for guillotina",
    long_description=README,
    long_description_content_type="text/markdown",
    install_requires=["guillotina>=6.0.0b5", "backoff"],
    author="Nathan Van Gheem",
    author_email="vangheem@gmail.com",
    url="https://github.com/guillotinaweb/guillotina_batch",
    packages=find_packages(exclude=["demo"]),
    package_data={"": ["*.txt", "*.rst"], "guillotina_batch": ["py.typed"]},
    include_package_data=True,
    license="BSD",
    extras_require={
        "test": [
            "pytest>=4.1.1",
            "aiohttp>=3.6.0,<4.0.0",
            "docker",
            "backoff",
            "jsonschema==2.6.0",
            "psycopg2-binary",
            "pytest-asyncio==0.10.0",
            "pytest-cov",
            "coverage==4.4.0",
            "pytest-docker-fixtures",
            "asyncpg==0.15.0",
            "async_asgi_testclient==1.4.4",
        ]
    },
    classifiers=[
        "License :: OSI Approved :: BSD License",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
        "Topic :: Software Development :: Libraries :: Python Modules",
        "Topic :: Internet :: WWW/HTTP",
        "Intended Audience :: Developers",
    ],
    entry_points={},
)


from bs4 import BeautifulSoup
import urllib.request
import time
import random

site= 'https://etherscan.io/txs?a=0xd0a6e6c54dbc68db5db3a091b171a77407ff7ccf&p='
hdr = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
       'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
       'Accept-Encoding': 'none',
       'Accept-Language': 'en-US,en;q=0.8',
       'Connection': 'keep-alive'}

f = open('workfile', 'a')

for i in range(1167, 1947):
    req = urllib.request.Request(site + str(i), headers=hdr)
    page = urllib.request.urlopen(req)
    content = page.read()
    soup = BeautifulSoup(content, "html.parser")
    f.write(soup.find('tbody').text)
    print (i)
    #time.sleep(random.random())


from django.test import TestCase
from rest_framework.exceptions import ErrorDetail, ValidationError

from courses.models.Category import Category
from courses.models.Course import Course
from courses.models.LearningFormat import LearningFormat
from courses.models.UserCourse import UserCourse
from users.models import User
from users.models.Teacher import Teacher

from ...models.Lesson import Lesson
from ...models.LessonTypeChoices import LessonTypeChoices
from ...models.UserLesson import UserLesson
from ...serializers.LessonCompleteSerializer import LessonCompleteSerializer

class LessonCompleteSerializerTestCase(TestCase):
    databases = {'master', }

    @classmethod
    def setUpTestData(cls):
        cls.tested_class = LessonCompleteSerializer

        user = User.objects.create_user(**{
            'email': 'q' * 50 + '@q.qq',
            'password': 'q' * 50,
        })

        teacher = Teacher.objects.create(**{
            'user': user,
            'description': 'q' * 50,
        })

        category = Category.objects.create(**{
            'title': 'q' * 50,
        })

        course = Course.objects.create(**{
            'teacher': teacher,
            'title': 'q' * 50,
            'price': 50.0,
            'count_lectures': 50,
            'count_independents': 50,
            'duration': 50,
            'learning_format': LearningFormat.REMOTE,
            'category': category,
            'preview': 'temporary_img',
            'short_description': 'q' * 50,
        })

        Lesson.objects.create(**{
            'id': 1,
            'course': course,
            'title': 'q' * 50,
            'description': 'q' * 50,
            'lesson_type': LessonTypeChoices.THEME,
        })

        Lesson.objects.create(**{
            'id': 2,
            'course': course,
            'title': 'q' * 50,
            'description': 'q' * 50,
            'lesson_type': LessonTypeChoices.TEST,
        })

        Lesson.objects.create(**{
            'id': 3,
            'course': course,
            'title': 'q' * 50,
            'description': 'q' * 50,
            'lesson_type': LessonTypeChoices.THEORY,
        })

        UserCourse.objects.create(**{
            'course': course,
            'user': user,
        })

        cls.data = {
            'lesson_id': 3,
        }

        cls.context = {
            'user': user,
        }

    def test_Should_IncludeDefiniteFields(self):
        expected_fields = [
            'lesson_id',
        ]
        real_fields = list(self.tested_class().get_fields())

        self.assertEqual(expected_fields, real_fields)

    def test_When_LessonTypeIsTheme_Should_ErrorNotValidLessonType(
            self):
        data = self.data
        context = self.context

        data.update({
            'lesson_id': 1,
        })

        serializer = self.tested_class(data=data, context=context)
        serializer.is_valid()

        with self.assertRaises(ValidationError) as _raise:
            serializer.save()

        expected_raise = {
            'lesson_id': [
                ErrorDetail(**{
                    'string': 'You can not complete lesson this type.',
                    'code': 'invalid',
                }),
            ],
        }
        real_raise = _raise.exception.detail

        self.assertEqual(expected_raise, real_raise)

    def test_When_LessonTypeIsTest_Should_ErrorNotValidLessonType(
            self):
        data = self.data
        context = self.context

        data.update({
            'lesson_id': 2,
        })

        serializer = self.tested_class(data=data, context=context)
        serializer.is_valid()

        with self.assertRaises(ValidationError) as _raise:
            serializer.save()

        expected_raise = {
            'lesson_id': [
                ErrorDetail(**{
                    'string': 'You can not complete lesson this type.',
                    'code': 'invalid',
                }),
            ],
        }
        real_raise = _raise.exception.detail

        self.assertEqual(expected_raise, real_raise)

    def test_When_LessonTypeIsTheory_Should_CreateUserLesson(
            self):
        data = self.data
        context = self.context

        serializer = self.tested_class(data=data, context=context)
        serializer.is_valid()

        expected_instance = None
        real_instance = serializer.save()

        expected_completed_at = None
        real_completed_at = UserLesson.objects.last().completed_at

        self.assertNotEqual(expected_instance, real_instance)
        self.assertNotEqual(expected_completed_at, real_completed_at)

    def test_When_DuplicateLessonComplete_Should_DontUpdateLessonComplete(
            self):
        data = self.data
        context = self.context

        save_serializer = self.tested_class(data=data, context=context)
        save_serializer.is_valid()
        save_serializer.save()

        serializer = self.tested_class(data=data, context=context)
        serializer.is_valid()
        serializer.save()

        expected_data = save_serializer.data
        real_data = serializer.data

        self.assertEqual(expected_data, real_data)


import random as r

def roll1():
    pair_dice =[]
    for i in range(1, 7):
        for j in range(1, 7):
            pair_dice.append((i, j))
    print(pair_dice)
    print(r.choice(pair_dice))

def roll2():
    first = r.randint(1, 6)
    second = r.randint(1, 6)
    return first, second


#!usr/bin/env python
#-*- encoding:utf-8 -*-
__author__ = "Cheung"

import requests
from urllib import parse
from lxml import etree

class imageSpider():
    def __init__(self, begin_page, end_page, kw):
        self.begin_page = int(begin_page)
        self.end_page = int(end_page)
        self.key = parse.urlencode({"kw": kw})

    def loadPage(self, url):
        '''
            作用：下载网页 HTML 源码
            url:需要爬取的url地址
        '''
        headers = {
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11"
        }
        # 获取网页html源码
        response = requests.get(url)
        html = response.text
        # print(html)
        # 解析HTML文档 为HTML.DOM 模型
        content = etree.HTML(html)
        # print(content)
        # 返回所有成功匹配的列表集合
        link_list = content.xpath('//div[@class="threadlist_lz clearfix"]//a[@class="j_th_tit "]/@href')
        for link in link_list:
            fulllink = "http://tieba.baidu.com" + link
            # print(fulllink)
            self.loadImage(fulllink)

    def loadImage(self, link):
        '''
            作用：下载页面图片
            link：图片连接
        '''
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
        }
        response = requests.get(link, headers=headers)
        html = response.text
        # 解析
        content = etree.HTML(html)
        # 取出帖子里每层层主发送的图片连接集合
        # print(content)
        link_list = content.xpath('//img[@class="BDE_Image"]/@src')
        # 取出每个图片的链接
        for link in link_list:
            self.writeImage(link)

    def writeImage(self, link):
        '''
            作用：写入图片至本地
            link：图片连接
        '''
        print("正在保存。。。")
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
        }
        response = requests.get(link, headers=headers)
        # 图片原始数据
        image = response.content
        # 取出连接后10位做为文件名
        filename = link[-10:]
        # 写入到本地文件
        with open(filename, 'wb') as f:
            f.write(image)
        print("已经成功下载" + filename)

    def startSpider(self):
        '''
            作用：控制爬虫下载
        '''
        url = "http://tieba.baidu.com/f?"
        # 拼接 Url
        tempurl = url + self.key
        # print(tempurl)
        for page in range(self.begin_page, self.end_page + 1):
            pn = (page - 1)*50
            fullurl = tempurl + "&pn=" + str(pn)
            # print(fullurl)
            self.loadPage(fullurl)
        print("Thank you!")

if __name__ == "__main__":
    kw = input("which keyword you want to search: ")
    begin_page = input("the begin page: ")
    end_page = input("the end page: ")

    spider = imageSpider(begin_page, end_page, kw)
    spider.startSpider()


import public_class as public
import numpy as np
import pandas as pd
# 清洗数据

class DataClean(object):
    """docstring for DataClean"""

    def __init__(self, uvData, apData):
        # super(DataClean, self).__init__()
        self.uvData = uvData
        self.apData = apData
    # 清洗UV数据：替换源数据中的"\N"、更改数据type，构建新特征"forward_rate"

    def uvDataClean(self):
        uvData = self.uvData
        # 判断源数据集是否为空，如果为空数据集则不做处理直接输出
        if len(uvData) > 0:
            # 把源数据集中的"\N"符号替换为0
            uvData = uvData.replace('\\N', 0).fillna(0).sort_values(
                by='log_date', ascending=False).reset_index(drop=True)
            # 将源数据集中的'hos_id', 'dhcp', 'portal', 'prelogin','login',
            # 'webforward', 'hardforward'字段的类型更改为"int32"
            uvDataGood = uvData[['hos_id', 'dhcp', 'portal', 'prelogin',
                                 'login', 'webforward', 'hardforward']].astype('int32')
            # 将源数据集中的"log_date"字段类型更改为"datetime64[ns]"
            uvDataGood['log_date'] = uvData[
                'log_date'].astype('datetime64[ns]')
            # 构建新特征"forward_rate"
            uvDataGood['forward_rate'] = uvDataGood[
                'hardforward']/uvDataGood['dhcp']
            # 备选输出："uvDataLite"。只包含'portal', 'prelogin', 'login',
            # 'webforward'字段
            uvDataLite = uvDataGood.drop(
                ['portal', 'prelogin', 'login', 'webforward'], axis=1)
        else:
            uvDataGood = uvData
        return uvDataGood
    # 清洗AP数据：替换特殊字符、更改数据type

    def apDataClean(self):
        apData = self.apData
        if len(apData) > 0:
            # 替换特殊字符
            apData = apData.replace(
                ['福建', '安徽', '福建省', '安徽省', ''], np.nan).dropna()
            # 更改数据type
            apDataGood = apData[['hos_id', 'ap_count']].astype('int64')
            apDataGood['log_date'] = apData[
                'log_date'].astype('datetime64[ns]')
        else:
            apDataGood = apData
        return apDataGood

# 提取day1天和day2的数据

class DateData(object):
    """docstring for getDateData"""

    def __init__(self, uvData, apData, day1=1, day2=8):
        # super(getDateData, self).__init__()
        self.uvData = uvData
        self.apData = apData
        self.day1 = str(public.getDate(day1))
        self.day2 = str(public.getDate(day2))
        # 中间数据
        self._uvDataDay1 = self._getDateData(self.day1)
        self._uvDataDay2 = self._getDateData(self.day2)
        # hosid的并集
        self._allHosid = self._getAllHosid()

        self.uvDay1 = self._reindexData(self._uvDataDay1)
        self.uvDay2 = self._reindexData(self._uvDataDay2)
        self.uvDay1Lite, self.uvDay2Lite = self._dataLite()
    # 获取day天的uv数据和ap数据并进行合并。输出合并后的数据uvDay

    def _getDateData(self, day):
        # print(day)
        uvData = self.uvData
        apData = self.apData
        # day = self.day2
        if len(uvData) > 0:
            uvDay = uvData[uvData['log_date'] == day].reset_index(
                drop=True).set_index('hos_id').drop('log_date', axis=1)

        if len(apData) > 0:
            apDay = apData[apData['log_date'] == day].set_index(
                'hos_id').drop('log_date', axis=1)

        uvDay = pd.merge(uvDay, apDay, left_index=True,
                         right_index=True, how='left')
        return uvDay
    # 获取day1和day2数据hosid的并集

    def _getAllHosid(self):
        return np.union1d(self._uvDataDay1.index, self._uvDataDay2.index)
    # 用hosid的并集对数据重新索引

    def _reindexData(self, data):
        dataReindex = data.reindex(self._allHosid, fill_value=0)
        if len(self.apData) > 0:
            dataReindex.insert(0, 'ap_count', dataReindex.pop('ap_count'))
        return dataReindex
    # 肢体去dhcp大于200数据

    def _dataLite(self):
        uvDay2 = self.uvDay2
        uvDay1 = self.uvDay1
        uvDay2Lite = uvDay2[uvDay2['dhcp'] > 200]
        uvDay1Lite = uvDay1.reindex(uvDay2Lite.index)
        return uvDay1Lite, uvDay2Lite


import requests
import json

import pytest

from library_registry.util.http import (
    HTTP,
    BadResponseException,
    RemoteIntegrationException,
    RequestNetworkException,
    RequestTimedOut,
)

class MockRequestsResponse:
    """
    A mock object that simulates an HTTP response from the `requests` library.
    """
    def __init__(self, status_code, headers={}, content=None, url=None):
        self.status_code = status_code
        self.headers = headers
        self.content = content
        self.url = url or "http://url/"

    def json(self):
        content = self.content
        # The queued content might be a JSON string or it might
        # just be the object you'd get from loading a JSON string.
        if isinstance(content, (bytes, str)):
            content = json.loads(self.content)
        return content

    @property
    def text(self):
        return self.content.decode("utf8")

class TestHTTP:

    def test_request_with_timeout_success(self):

        def fake_200_response(*args, **kwargs):
            return MockRequestsResponse(200, content="Success!")

        response = HTTP._request_with_timeout(
            "the url", fake_200_response, "a", "b"
        )
        assert response.status_code == 200
        assert response.content == "Success!"

    def test_request_with_timeout_failure(self):

        def immediately_timeout(*args, **kwargs):
            raise requests.exceptions.Timeout("I give up")

        with pytest.raises(RequestTimedOut) as exc:
            HTTP._request_with_timeout("http://url/", immediately_timeout, "a", "b")
        assert "Timeout accessing http://url/: I give up" in str(exc.value)

    def test_request_with_network_failure(self):

        def immediately_fail(*args, **kwargs):
            raise requests.exceptions.ConnectionError("a disaster")

        with pytest.raises(RequestNetworkException) as exc:
            HTTP._request_with_timeout("http://url/", immediately_fail, "a", "b")
        assert "Network error contacting http://url/: a disaster" in str(exc.value)

    def test_request_with_response_indicative_of_failure(self):

        def fake_500_response(*args, **kwargs):
            return MockRequestsResponse(500, content="Failure!")

        with pytest.raises(BadResponseException) as exc:
            HTTP._request_with_timeout("http://url/", fake_500_response, "a", "b")
        err_msg = r"Bad response from http://url/: Got status code 500 from external server, cannot continue."
        assert err_msg in str(exc.value)

    def test_allowed_response_codes(self):
        # Test our ability to raise BadResponseException when
        # an HTTP-based integration does not behave as we'd expect.

        def fake_401_response(*args, **kwargs):
            return MockRequestsResponse(401, content="Weird")

        def fake_200_response(*args, **kwargs):
            return MockRequestsResponse(200, content="Hurray")

        url = "http://url/"
        m = HTTP._request_with_timeout

        # By default, every code except for 5xx codes is allowed.
        response = m(url, fake_401_response)
        assert response.status_code == 401

        # You can say that certain codes are specifically allowed, and
        # all others are forbidden.
        with pytest.raises(BadResponseException) as exc:
            m(url, fake_401_response, allowed_response_codes=[201, 200])
        assert "Bad response" in str(exc.value)
        assert "Got status code 401 from external server, but can only continue on: 200, 201." in str(exc.value)

        response = m(url, fake_401_response, allowed_response_codes=[401])
        response = m(url, fake_401_response, allowed_response_codes=["4xx"])

        # In this way you can even raise an exception on a 200 response code.
        with pytest.raises(BadResponseException) as exc:
            m(url, fake_200_response, allowed_response_codes=[401])
        assert "Got status code 200 from external server, but can only continue on: 401." in str(exc.value)

        # You can say that certain codes are explicitly forbidden, and
        # all others are allowed.
        with pytest.raises(BadResponseException) as exc:
            m(url, fake_401_response, disallowed_response_codes=[401])
        assert "Got status code 401 from external server, cannot continue." in str(exc.value)

        with pytest.raises(BadResponseException) as exc:
            m(url, fake_200_response, disallowed_response_codes=["2xx", 301])
        assert "Got status code 200 from external server, cannot continue." in str(exc.value)

        response = m(url, fake_401_response,
                     disallowed_response_codes=["2xx"])
        assert response.status_code == 401

        # The exception can be turned into a useful problem detail document.
        exception = None
        try:
            m(url, fake_200_response,
              disallowed_response_codes=["2xx"])
        except Exception as e:
            exception = e
        assert exception is not None

        debug_doc = exception.as_problem_detail_document(debug=True)

        # 502 is the status code to be returned if this integration error
        # interrupts the processing of an incoming HTTP request, not the
        # status code that caused the problem.
        #
        assert debug_doc.status_code == 502
        assert debug_doc.title == "Bad response"
        assert debug_doc.detail == (
            'The server made a request to http://url/, and got an unexpected or invalid response.'
        )
        assert debug_doc.debug_message == (
            'Bad response from http://url/: Got status code 200 from external server, '
            'cannot continue.\n\nResponse content: Hurray'
        )

        no_debug_doc = exception.as_problem_detail_document(debug=False)
        assert no_debug_doc.title == "Bad response"
        assert no_debug_doc.detail == 'The server made a request to url, and got an unexpected or invalid response.'
        assert no_debug_doc.debug_message is None

    def test_unicode_converted_to_utf8(self):
        """Any Unicode that sneaks into the URL, headers or body is
        converted to UTF-8.
        """
        class ResponseGenerator:
            def __init__(self):
                self.requests = []

            def response(self, *args, **kwargs):
                self.requests.append((args, kwargs))
                return MockRequestsResponse(200, content="Success!")

        generator = ResponseGenerator()
        url = "http://foo"
        HTTP._request_with_timeout(
            url, generator.response, url, "POST",
            headers={"unicode header": "unicode value"},
            data="unicode data"
        )
        [(args, kwargs)] = generator.requests
        url, method = args
        headers = kwargs['headers']
        data = kwargs['data']

        # All the Unicode data was converted to bytes before being sent
        # "over the wire".
        for k, v in list(headers.items()):
            assert isinstance(k, bytes)
            assert isinstance(v, bytes)
        assert isinstance(data, bytes)

class TestRemoteIntegrationException:

    def test_with_service_name(self):
        """You don't have to provide a URL when creating a
        RemoteIntegrationException; you can just provide the service
        name.
        """
        exc = RemoteIntegrationException(
            "Unreliable Service",
            "I just can't handle your request right now."
        )

        # Since only the service name is provided, there are no details to
        # elide in the non-debug version of a problem detail document.
        debug_detail = exc.document_detail(debug=True)
        other_detail = exc.document_detail(debug=False)
        assert other_detail == debug_detail

        assert debug_detail == (
            'The server tried to access Unreliable Service but the third-party service experienced an error.'
        )

class TestBadResponseException:

    def test_helper_constructor(self):
        response = MockRequestsResponse(102, content="nonsense")
        exc = BadResponseException.from_response(
            "http://url/", "Terrible response, just terrible", response
        )

        # Turn the exception into a problem detail document, and it's full of useful information.
        doc, status_code, headers = exc.as_problem_detail_document(debug=True).response
        doc = json.loads(doc)

        assert doc['title'] == 'Bad response'
        assert doc['detail'] == 'The server made a request to http://url/, and got an unexpected or invalid response.'
        assert doc['debug_message'] == (
            'Bad response from http://url/: Terrible response, just terrible\n\nStatus code: 102\nContent: nonsense'
        )

        # Unless debug is turned off, in which case none of that information is present.
        doc, status_code, headers = exc.as_problem_detail_document(debug=False).response
        assert 'debug_message' not in json.loads(doc)

    def test_bad_status_code_helper(object):
        response = MockRequestsResponse(500, content="Internal Server Error!")
        exc = BadResponseException.bad_status_code("http://url/", response)
        doc, status_code, headers = exc.as_problem_detail_document(debug=True).response
        doc = json.loads(doc)

        assert "Got status code 500 from external server, cannot continue." in doc['debug_message']

    def test_as_problem_detail_document(self):
        exception = BadResponseException(
            "http://url/", "What even is this",
            debug_message="some debug info"
        )
        document = exception.as_problem_detail_document(debug=True)
        assert document.status_code == 502
        assert document.title == "Bad response"
        assert document.detail == "The server made a request to http://url/, and got an unexpected or invalid response."
        assert document.debug_message == "Bad response from http://url/: What even is this\n\nsome debug info"

class TestRequestTimedOut:

    def test_as_problem_detail_document(self):
        exception = RequestTimedOut("http://url/", "I give up")

        debug_detail = exception.as_problem_detail_document(debug=True)
        assert debug_detail.title == "Timeout"
        assert debug_detail.detail == 'The server made a request to http://url/, and that request timed out.'

        # If we're not in debug mode, we hide the URL we accessed and just
        # show the hostname.
        standard_detail = exception.as_problem_detail_document(debug=False)
        assert standard_detail.detail == "The server made a request to url, and that request timed out."

        # The status code corresponding to an upstream timeout is 502.
        document, status_code, headers = standard_detail.response
        assert status_code == 502

class TestRequestNetworkException:

    def test_as_problem_detail_document(self):
        exception = RequestNetworkException("http://url/", "Colossal failure")

        debug_detail = exception.as_problem_detail_document(debug=True)
        assert debug_detail.title == "Network failure contacting third-party service"
        assert debug_detail.detail == 'The server experienced a network error while contacting http://url/.'

        # If we're not in debug mode, we hide the URL we accessed and just
        # show the hostname.
        standard_detail = exception.as_problem_detail_document(debug=False)
        assert standard_detail.detail == "The server experienced a network error while contacting url."

        # The status code corresponding to an upstream timeout is 502.
        document, status_code, headers = standard_detail.response
        assert status_code == 502


import sys
sys.setrecursionlimit(500*500)

n = int(input())
c = list(map(int, input().split()))

c.sort()
ans_pre = 0
ans_list = []

def count(list, b0):
    return b0 - len(list)

def f(a, b):
    if(len(b) == 1):
        ans_list.append(count(a, b[0]))
        return count(a, b[0])

    if(len(b) >= 2):
        ans_list.append(count(a, b[0]))
        a.append(b[0])
        b.pop(0)
        f(a, b)

a = []
f(a, c)
for i in range(len(ans_list)):
    if(i == 0):
        ans_pre = ans_list[i]
    else:
        ans_pre *= ans_list[i]

if(ans_pre == 0):
    ans = 0
else:
    ans = ans_pre % 1000000007
print(ans)