#数据格式：  data = {'users' : [str(1, 2, 3, ..., n)], 'user_data' : {'1' : {1号user的数据}, '2' : {2号user的数据}, ..., 'n' : {n号user的数据}}}
#1, ..., n号user的数据格式为： {'x' : [m *28 *28的列表], 'y' : [m个标签]},其中m表示每个user拥有的数据

import json
import struct
import numpy as np
import math

def _data_norm(data):
    mean = np.mean(data)
    std_variance = math.sqrt(np.var(data))
    norm_data = (data - mean) / std_variance
    return norm_data
#对数据进行标准化
def data_norm(data):
    data_list = []
    for i in range(len(data)):
        norm_data_list = _data_norm(data[i]).tolist()
        data_list.append(norm_data_list)

    return data_list

#改成自己的路径
with open('../Downloads/binary_data/t10k-labels.idx1-ubyte', mode = 'rb') as test_lable_file:

    magic , num_of_test_lables_item = struct.unpack('>II', test_lable_file.read(8)) #前面8个字节不是真正的数据
    test_y_total_np = np.fromfile(test_lable_file, dtype = np.uint8)
    test_y_total_list = test_y_total_np.tolist()  #现在读的才是真正的数据，dtype = np.uint8表示数据是按照字节读取的

with open('../Downloads/binary_data/t10k-images.idx3-ubyte', mode = 'rb') as test_image_file:
    magic , num_of_test_images_item, rows, columns = struct.unpack('>IIII', test_image_file.read(16)) #前16个字节不是真正的数据
#     test_x_total = np.fromfile(test_image_file, dtype = np.uint8)
    test_x_total_np = np.fromfile(test_image_file, dtype = np.uint8).reshape(num_of_test_images_item, 28 * 28)#这里还没标准化
    test_x_total_list = data_norm(test_x_total_np)#这里进行了标准化

with open('../Downloads/binary_data/train-images.idx3-ubyte', mode = 'rb') as train_image_file:
    magic, num_of_train_images_item, _, _ = struct.unpack('>IIII', train_image_file.read(16))
#     train_x_total = np.fromfile(train_image_file, dtype = np.uint8)
    train_x_total_np = np.fromfile(train_image_file, dtype = np.uint8).reshape(num_of_train_images_item, 28 * 28)
    train_x_total_list = data_norm(train_x_total_np)

with open('../Downloads/binary_data/train-labels.idx1-ubyte', mode = 'rb') as train_label_file:
    magic, num_of_train_lables_item = struct.unpack('>II', train_label_file.read(8))
    train_y_total_np = np.fromfile(train_label_file, dtype = np.uint8)
    train_y_total_list = train_y_total_np.tolist()

#前面做的是读取文件数据并将其转化为list格式的数据，现在开始组装数据
#数据格式：  data = {'users' : [str(1, 2, 3, ..., n)], 'user_data' : {'1' : {1号user的数据}, '2' : {2号user的数据}, ..., 'n' : {n号user的数据}}}
#1, ..., n号user的数据格式为： {'x' : [m *28 *28的列表], 'y' : [m个标签]}, 其中m表示每个user拥有的数据，test数据集m = 20，train数据集m = 120
def _read_data(batch_size, user_num, data_x, data_y):
    user_num = user_num
    all_users_list = []
    batch_size = batch_size
    all_user_data = {}
    data_per_user = {}
    for i in range(user_num):
        all_users_list.append(str(i))
        data_per_user['x'] = data_x[i * batch_size : (i + 1)* batch_size]
        data_per_user['y'] = data_y[i * batch_size : (i + 1)* batch_size]
        all_user_data[str(i)] =  data_per_user
    #print(every_user_data['3'])

    #最后组装下数据
    data = {}
    data['users'] = all_users_list
    data['user_data'] = all_user_data
    #print(data_100_users.keys())
    return data

def read_data(mode, user_num = 500):  #默认user_num是500，注意test数据集共有10000条数据，因此batch_size * user_num要等于10000，同理train数据集
    if mode == "test":
        return _read_data(batch_size = 20, user_num = user_num, data_x = test_x_total_list, data_y = test_y_total_list)
    elif mode == "train":
        return _read_data(batch_size = 120, user_num = user_num, data_x = train_x_total_list, data_y = train_y_total_list)

#最后导出为json格式的文件,用的时候取消掉注释就行
# json.dump(read_data('test'), open("test_data.json", "w"))
# json.dump(read_data('train'), open("train_data.json", "w"))


import os
import time
import sys
import pandas as pd
import numpy as np
import sklearn.pipeline
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error
import matplotlib.pyplot as plt

# We can play with the constants below to see how far we can accurately forecast the growth rate of counties by state
# CONSTANTS
# ____________________________________________________________

PREDICTOR_INTERVAL = 14  # 2 weeks
FORECASTING_INTERVAL = 7  # 1 week
REGRESSION_INTERVAL = PREDICTOR_INTERVAL + FORECASTING_INTERVAL

# ______________________________________________________________

def retrieve_data(timeseries_cases_path, timeseries_deaths_path):
    if time.time() - os.path.getmtime(timeseries_cases_path) > 86400:
        print('Downloading raw data...')

        # Import Johns Hopkins time series data
        os.system('curl https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/'
                  'csse_covid_19_time_series/time_series_covid19_confirmed_US.csv'
                  ' > ' + '"' + timeseries_cases_path + '"')
        os.system('curl https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/'
                  'csse_covid_19_time_series/time_series_covid19_deaths_US.csv'
                  ' > ' + '"' + timeseries_deaths_path + '"')

        print('Finished downloading raw data.')

# TODO:
# Convert date into days since first case
# We will use RMSE to measure accuracy of the model
# (https://medium.com/making-sense-of-data/time-series-next-value-prediction-using-regression-over-a-rolling-window-228f0acae363
# The idea is that we can build regression models based on different features, or attributes, recorded for each county
# in each state or province.

def build_regression_models(timeseries_dataset):
    datasets_bystate_index = []  # Used to index data frames stored in datasets_bystate
    datasets_bystate = []
    regression_model_bystate = []  # Used to keep track random forest models for each state

    # Transforming and cleaning data. Check to see if the prediction and target intervals are available for the
    # county being examined by the for-loop. The interval of time needed to make an accurate prediction is
    # REGRESSION_INTERVAL = PREDICTOR_INTERVAL + FORECASTING_INTERVAL. If the county examined has not had cases
    # for the requisite amount of time specified by REGRESSION_INTERVAL, then it will be dropped from the model.

    for state in timeseries_dataset.Province_State.unique():
        print("Building model for state: " + state)

        # Set index and labels
        state_dataset = timeseries_dataset.loc[timeseries_dataset['Province_State'] == state].copy()
        state_dataset.index = pd.RangeIndex(len(state_dataset.index))  # Reset index for proper func. of iloc

        # Drop unneeded attributes
        state_dataset.drop(columns=['UID', 'iso2', 'iso3', 'code3', 'FIPS',
                                    'Admin2', 'Province_State', 'Country_Region', 'Lat',
                                    'Long_', 'Combined_Key'], inplace=True)
        column_names = []  # Store new column names
        for col in range(1, state_dataset.shape[1] + 1):
            column_names.append("Day " + str(col))
        state_dataset.columns = column_names

        if state_dataset.shape[0] < 6:
            print("Unable to build model for state:" + state + " due to an insufficient number counties.")

        else:
            county = 0
            while county < state_dataset.shape[0]:
                has_dropped = False  # Control variable

                # Here, we are counting zeros and adding them to lag_count to shift lag_count columns so that
                # the data is in the format of (day 1, day 2, day 3) where each day is a day that the county
                # is infected with the virus.

                lag_count = 0  # Num. of days the county has NOT been infected.
                for date in range(state_dataset.shape[1]):
                    if state_dataset.iloc[county, date] == 0:
                        lag_count += 1
                    elif state_dataset.shape[1] - lag_count < REGRESSION_INTERVAL:
                        state_dataset.drop(state_dataset.index[county], inplace=True)
                        state_dataset.index = pd.RangeIndex(len(state_dataset.index))
                        has_dropped = True
                        break
                    elif state_dataset.iloc[county, 0] == 0:
                        # Shift the day the cases began in the county to the first column of the data frame
                        for val in range(lag_count, state_dataset.shape[1]):
                            state_dataset.iloc[county, val - lag_count] = state_dataset.iloc[county, val]
                            state_dataset.iloc[county, val] = 0  # To replace shifted values with 0
                        break
                    # Check to see if we have a county full of no cases... for some reason these are in here!
                    if date == state_dataset.shape[1] - 1 and state_dataset.iloc[county].sum() == 0:
                        state_dataset.drop(state_dataset.index[county], inplace=True)
                        state_dataset.index = pd.RangeIndex(len(state_dataset.index))
                        has_dropped = True

                if not has_dropped:
                    county += 1

                if state_dataset.shape[0] < 6:
                    break

        if state_dataset.shape[0] >= 6:
            state_dataset.drop(state_dataset.iloc[:, -(state_dataset.shape[1] - REGRESSION_INTERVAL):], axis='columns',
                               inplace=True)
            datasets_bystate.append(state_dataset)
            datasets_bystate_index.append(state)

    # Building regression model via multistep recursive forecasting with RandomForestRegressor. We analyze the
    # performance of each model with RMSE.
    for state_dataset in datasets_bystate:
        train, test = train_test_split(state_dataset, test_size=.2)

    return datasets_bystate_index, datasets_bystate

def main():
    pd.set_option('display.max_rows', None)
    pd.set_option('display.max_columns', None)

    # Dataset paths
    timeseries_cases_path = os.path.join('Covid-19 Data', 'time_series_covid19_confirmed_US.csv')
    timeseries_deaths_path = os.path.join('Covid-19 Data', 'time_series_covid19_deaths_US.csv')

    # Retrieve data from GitHub of Johns Hopkins
    retrieve_data(timeseries_cases_path, timeseries_deaths_path)

    # Making data frames from csv's
    timeseries_cases_dataset = pd.read_csv(timeseries_cases_path)

    # print(timeseries_cases_dataset.groupby('Province_State').nunique())

    datasets_bystate_index, datasets_bystate = build_regression_models(timeseries_cases_dataset)

    current_removed_state_index = 0
    for i in range(0, len(datasets_bystate)):
        print(datasets_bystate_index[i])
        print(datasets_bystate[i])

main()


import os
import sys
import json
import logging
import pickle

def getPathsToExtractedLinesFiles():
    curDir = os.getcwd()
    directoryWithCharacterFiles = os.path.join(curDir, 'extractedLines')
    return [
        os.path.join(directoryWithCharacterFiles, workPath)
        for workPath in os.listdir(directoryWithCharacterFiles)
        if os.path.isfile(os.path.join(directoryWithCharacterFiles, workPath))
    ]

def getPathsToLinesFiles(pathToCharacterJson):
    """
    Get all the paths for JSON files
    Arguments:
        pathToCharacterJson - a path to where our lines files are stored
    Returns:
        An array of all absolute paths to files in that directory
    """
    curDir = os.getcwd()
    directoryWithCharacterFiles = os.path.join(curDir, pathToCharacterJson)
    return [
        os.path.join(directoryWithCharacterFiles, workPath)
        for workPath in os.listdir(directoryWithCharacterFiles)
        if os.path.isfile(os.path.join(directoryWithCharacterFiles, workPath))
    ]

def getPathsToGenderFile(pathToGenders, allLinesFilePaths):
    """
    Get the path to a specific gender file
    Arguments:
        pathToGenders - a path to where our gender files are stored
        allLinesFilePaths - an array of paths to the lines files containing characters and their lines
    Returns:
        allGenderFilePaths - an array of paths to gender lookup files, that aligns perfectly with the ordering of the json files
    """
    curDir = os.getcwd()
    directoryWithGenderFiles = os.path.join(curDir, pathToGenders)
    # Get the file name of the jsonFilePath
    allGenderFilePaths = []
    for curLinesFilePath in allLinesFilePaths:
        baseOfLinesFile = os.path.basename(curLinesFilePath)
        baseOfGenderFile = baseOfLinesFile.replace(".json", "-genders.json")
        curGenderFilePath = os.path.join(directoryWithGenderFiles, baseOfGenderFile)
        print(curGenderFilePath)
        if not os.path.isfile(curGenderFilePath):
            logging.exception(f"Error: No gender file at expected location: {curGenderFilePath}")
            return
        allGenderFilePaths.append(curGenderFilePath)
    return allGenderFilePaths

def buildVocabToNumMapping(vocab):
    """
    Create lookup tables for vocabulary used
    Parameters:
        vocab - A list of all unique words in our vocabulary
    Returns:
        vocabToNum - A dict of vocab terms to numeric values
    """
    # Index starts at 4 so we reseve 0-3 as a special characters
    index = 4
    vocabToNum = {}
    for word in vocab:
        if word not in vocabToNum:
            vocabToNum[word] = index
            index += 1
    print(f"Length of vocab: {index}")

    return (vocabToNum)

def pathsToLinesAndGenders():
    pathToLines = sys.argv[1] if len(sys.argv) > 1 else ""
    pathToGenders = sys.argv[2] if len(sys.argv) > 2 else ""
    if not pathToLines:
        pathToLines = f'json'
    if not pathToGenders:
        pathToGenders = f'genders'
    return (pathToLines, pathToGenders)

def buildLookupTablesFromFilePaths(allLinesFilePaths):
    vocab = set()
    for path in allLinesFilePaths:
        fileJson = json.load(open(path, 'r'))
        for char in fileJson:
            lines = fileJson[char]
            for line in lines:
                vocab |= set(line.split(" "))
    return buildVocabToNumMapping(vocab)

def buildGenderToNumMapping():
    genderToNum = {}
    genderToNum["M"] = 0
    genderToNum["F"] = 1
    return (genderToNum)

def lineToIntegerVector(line, vocabToNum):
    """
    Transform a string of words into an array of corresponding numbers, based on lookup table
    Parameters:
        line - a string of words, stripped of punctuation and separated by spaces
        vocabToNum - a dictionary that translates words found in the entire corpus to corresponding, unique integers
    Returns:
        numericLine - an array of numbers, where each number corresponds to a unique word in our corpus vocabulary
    """
    numericLine = []
    for word in line.split(" "):
        numericLine.append(vocabToNum[word])
    return numericLine;

def genderToInteger(gender, genderToNum):
    return genderToNum[gender]

def labelEachLineInFile(characterToLines, characterToGender, vocabToNum, genderToNum):
    """
    Get an array of tuples with lines spoken by characters and the gender of the speaker
    Parameters:
        characterToGender - A dict of each character in a work and their gender
        characterToLines - A dict of each character and an array of their lines
    REturns:
        labeledLines - A list of tuples, with each line followed by the corresponding gender
    """
    encodedLinesByFile = []
    encodedGenderByFile = []
    for char in characterToLines:
        print(char)
        gender = characterToGender[char]
        lines = characterToLines[char]
        # Get encodings by line
        encodedLinesByCharacter = []
        for l in lines:
            encodedLine = lineToIntegerVector(l, vocabToNum)
            encodedLinesByCharacter.append(encodedLine)
            if gender == 'F':
                encodedLinesByCharacter.append(encodedLine)
                encodedLinesByCharacter.append(encodedLine)
                encodedLinesByCharacter.append(encodedLine)
        # Add all line encodings to the encodings by file
        encodedGender = genderToInteger(gender, genderToNum)
        encodedGenderByCharacter = [encodedGender] * len(encodedLinesByCharacter)
        encodedGenderByFile += encodedGenderByCharacter
        encodedLinesByFile += encodedLinesByCharacter
        # Add a corresponding number of gender labellings
    return (encodedLinesByFile, encodedGenderByFile)

def labelAllLines(allLinesFilePaths, allGenderFilePaths, vocabToNum, genderToNum):
    allLinesEncoded  = []
    allGendersEncoded = []
    # Get the encodings for each file
    for index in range(0, len(allLinesFilePaths)):
        print(allLinesFilePaths[index])
        curLinesFile = json.load(open(allLinesFilePaths[index], 'r'))
        curGenderFile = json.load(open(allGenderFilePaths[index], 'r'))
        (encodedLinesByFile, encodedGendersByFile) = labelEachLineInFile(curLinesFile, curGenderFile, vocabToNum, genderToNum)
        allLinesEncoded += encodedLinesByFile
        allGendersEncoded += encodedGendersByFile
    return (allLinesEncoded, allGendersEncoded)

def pickleAllData(tupleOfData):
    curDir = os.getcwd()
    outputFilePath = os.path.join(curDir, 'output', 'output.pkl')
    outputFile = open(outputFilePath, 'wb')
    pickle.dump(tupleOfData, outputFile)

def main():
    # Get paths to lines and gender files
    (pathToLines, pathToGenders) = pathsToLinesAndGenders()
    # Get paths to each json files
    allLinesFilePaths = getPathsToLinesFiles(pathToLines)
    allGenderFilePaths = getPathsToGenderFile(pathToGenders, allLinesFilePaths)
    # Build our vocabulary and our lookup tables -- make sure to include files that were extracted and the ones we're building our model with
    (vocabToNum) = buildLookupTablesFromFilePaths(allLinesFilePaths + getPathsToExtractedLinesFiles())
    (genderToNum) = buildGenderToNumMapping()
    (allEncodedLines, allEncodedGender) = labelAllLines(allLinesFilePaths, allGenderFilePaths, vocabToNum, genderToNum)
    print("lens of allLines and allGender labels: ", len(allEncodedLines), "  ", len(allEncodedGender))
    pickleAllData((allEncodedLines, allEncodedGender, vocabToNum, genderToNum));

if __name__ == '__main__':
    main()
    print('run')


import numpy as np
import pickle
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot
import pdb
import shutil

######################################################################
#Parameters
######################################################################
threshold = 0.6

benign_ss_path  = "/home/Drive2/aditya/benign_patches_info.npy"
normal_ss_path  = "/home/Drive2/aditya/normal_patches_info.npy"
insitu_ss_path  = "/home/Drive2/aditya/insitu_patches_info.npy"
invasive_ss_path  = "/home/Drive2/aditya/invasive_patches_info.npy"

benign_ss = np.load(benign_ss_path)
insitu_ss = np.load(insitu_ss_path)
invasive_ss = np.load(invasive_ss_path)
normal_ss = np.load(normal_ss_path)

with open('/home/Drive2/aditya/Benign.txt', 'rb') as f:
    benign = pickle.load(f)
with open('/home/Drive2/aditya/InSitu.txt', 'rb') as f:
    insitu = pickle.load(f)
with open('/home/Drive2/aditya/Invasive.txt', 'rb') as f:
    invasive = pickle.load(f)
with open('/home/Drive2/aditya/Normal.txt', 'rb') as f:
    normal = pickle.load(f)

with open('/home/Drive2/aditya/Benign_conf.txt', 'rb') as f:
    benign_confidence = pickle.load(f)
with open('/home/Drive2/aditya/InSitu_conf.txt', 'rb') as f:
    insitu_confidence = pickle.load(f)
with open('/home/Drive2/aditya/Invasive_conf.txt', 'rb') as f:
    invasive_confidence = pickle.load(f)
with open('/home/Drive2/aditya/Normal_conf.txt', 'rb') as f:
    normal_confidence = pickle.load(f)

#benign = np.load('/home/Drive2/aditya/Benign.npy')
#insitu = np.load('/home/Drive2/aditya/InSitu.npy')
#invasive = np.load('/home/Drive2/aditya/Invasive.npy')
#normal = np.load('/home/Drive2/aditya/Normal.npy')
no_samples = 25
######################################################################
#pdb.set_trace()

benign_hist = np.zeros((no_samples,4))
insitu_hist = np.zeros((no_samples,4))
invasive_hist = np.zeros((no_samples,4))
normal_hist = np.zeros((no_samples,4))

confusion_matrix = np.zeros((4,4))

benign_conf = np.zeros((4,1))
insitu_conf = np.zeros((4,1))
invasive_conf = np.zeros((4,1))
normal_conf = np.zeros((4,1))

# 0: Benign, 1:InSitu, 2:Invasive, 3:Normal
for i in range(no_samples):

    benign[i][(benign_confidence[i][:]<threshold)] = -1
    benign_hist[i,0] = (benign[i][:] == 0).sum()
    benign_hist[i,1] = (benign[i][:] == 1).sum()
    benign_hist[i,2] = (benign[i][:] == 2).sum()
    benign_hist[i,3] = (benign[i][:] == 3).sum()
    benign_hist[i,:] = benign_hist[i,:]/len(benign[i][:])
    #print(np.sum(benign_hist[i,:]))
    print((benign[i][:]==-1).sum())
    if np.sum(benign_hist[i,:]) == 0:
        print("Reduce threshold requested by Benign",i)

    benign_class_temp = np.argmax(benign_hist[i,:])

    if benign_hist[i,3] == benign_hist[i,benign_class_temp]:
        benign_class = 3
    if benign_hist[i,0] == benign_hist[i,benign_class_temp]:
        benign_class = 0
    if benign_hist[i,1] == benign_hist[i,benign_class_temp]:
        benign_class = 1
    if benign_hist[i,2] == benign_hist[i,benign_class_temp]:
        benign_class = 2

    benign_conf[benign_class]+=1

    if benign_class == 1 or benign_class == 3 or benign_class == 2:
        for j in range(benign_ss[i]):
            shutil.copyfile("/home/Drive2/aditya/Aug_Data/Validation/Benign/"+str(i)+"_"+str(j)+".png","/home/Drive2/aditya/Wrong_Class/Benign/"+str(i)+"_"+str(j)+"_TC_"+str(benign_class)+"_AC_0_PC_"+str(int(benign[i][j]))+".png")
        print(" Benign " + str(i) + " incorrectly classified as " + str(benign_class))

    insitu[i][(insitu_confidence[i][:]<threshold)] = -1
    insitu_hist[i,0] = (insitu[i][:] == 0).sum()
    insitu_hist[i,1] = (insitu[i][:] == 1).sum()
    insitu_hist[i,2] = (insitu[i][:] == 2).sum()
    insitu_hist[i,3] = (insitu[i][:] == 3).sum()
    insitu_hist[i,:] = insitu_hist[i,:]/len(insitu[i][:])
    #print(np.sum(insitu_hist[i,:]))

    if np.sum(insitu_hist[i,:]) == 0:
        print("Reduce threshold requested by insitu",i)

    insitu_class_temp = np.argmax(insitu_hist[i,:])

    if insitu_hist[i,3] == insitu_hist[i,insitu_class_temp]:
        insitu_class = 3
    if insitu_hist[i,0] == insitu_hist[i,insitu_class_temp]:
        insitu_class = 0
    if insitu_hist[i,1] == insitu_hist[i,insitu_class_temp]:
        insitu_class = 1
    if insitu_hist[i,2] == insitu_hist[i,insitu_class_temp]:
        insitu_class = 2

    insitu_conf[insitu_class]+=1

    if insitu_class == 0 or insitu_class == 3 or insitu_class == 2:
        for j in range(insitu_ss[i]):
            shutil.copyfile("/home/Drive2/aditya/Aug_Data/Validation/InSitu/"+str(i)+"_"+str(j)+".png","/home/Drive2/aditya/Wrong_Class/InSitu/"+str(i)+"_"+str(j)+"_TC_"+str(insitu_class)+"_AC_1_PC_"+str(int(insitu[i][j]))+".png")
        print(" InSitu " + str(i) + " incorrectly classified as " + str(insitu_class))

    invasive[i][(invasive_confidence[i][:]<threshold)] = -1
    invasive_hist[i,0] = (invasive[i][:] == 0).sum()
    invasive_hist[i,1] = (invasive[i][:] == 1).sum()
    invasive_hist[i,2] = (invasive[i][:] == 2).sum()
    invasive_hist[i,3] = (invasive[i][:] == 3).sum()
    invasive_hist[i,:] = invasive_hist[i,:]/len(invasive[i][:])
    #print(np.sum(invasive_hist[i,:]))

    if np.sum(invasive_hist[i,:]) == 0:
        print("Reduce threshold requested by invasive",i)

    #invasive_class = np.argmax(invasive_hist[i,:])
    invasive_class_temp = np.argmax(invasive_hist[i,:])

    if invasive_hist[i,3] == invasive_hist[i,invasive_class_temp]:
        invasive_class = 3
    if invasive_hist[i,0] == invasive_hist[i,invasive_class_temp]:
        invasive_class = 0
    if invasive_hist[i,1] == invasive_hist[i,invasive_class_temp]:
        invasive_class = 1
    if invasive_hist[i,2] == invasive_hist[i,invasive_class_temp]:
        invasive_class = 2

    invasive_conf[invasive_class]+=1

    if invasive_class == 0 or invasive_class == 3 or invasive_class == 1:
        for j in range(invasive_ss[i]):
            shutil.copyfile("/home/Drive2/aditya/Aug_Data/Validation/Invasive/"+str(i)+"_"+str(j)+".png","/home/Drive2/aditya/Wrong_Class/Invasive/"+str(i)+"_"+str(j)+"_TC_"+str(invasive_class)+"_AC_2_PC_"+str(int(invasive[i][j]))+".png")

        print(" Invasive " + str(i) + " incorrectly classified as " +str(invasive_class))

    normal[i][(normal_confidence[i][:]<threshold)] = -1
    normal_hist[i,0] = (normal[i][:] == 0).sum()
    normal_hist[i,1] = (normal[i][:] == 1).sum()
    normal_hist[i,2] = (normal[i][:] == 2).sum()
    normal_hist[i,3] = (normal[i][:] == 3).sum()
    normal_hist[i,:] = normal_hist[i,:]/len(normal[i][:])
    #print(np.sum(normal_hist[i,:]))

    if np.sum(normal_hist[i,:]) == 0:
        print("Reduce threshold requested by normal",i)

    #normal_class = np.argmax(normal_hist[i,:])
    normal_class_temp = np.argmax(normal_hist[i,:])

    if normal_hist[i,3] == normal_hist[i,normal_class_temp]:
        normal_class = 3
    if normal_hist[i,0] == normal_hist[i,normal_class_temp]:
        normal_class = 0
    if normal_hist[i,1] == normal_hist[i,normal_class_temp]:
        normal_class = 1
    if normal_hist[i,2] == normal_hist[i,normal_class_temp]:
        normal_class = 2

    normal_conf[normal_class]+=1

    if normal_class == 0 or normal_class == 2 or normal_class == 1:
        for j in range(normal_ss[i]):
            shutil.copyfile("/home/Drive2/aditya/Aug_Data/Validation/Normal/"+str(i)+"_"+str(j)+".png","/home/Drive2/aditya/Wrong_Class/Normal/"+str(i)+"_"+str(j)+"_TC_"+str(normal_class)+"_AC_3_PC_"+str(int(normal[i][j]))+".png")
        print(" Normal " + str(i) + " incorrectly classified as " +str(normal_class))

print("")
print("Confusion Matrix: 1 : Benign, 2 : InSitu, 3 : Invasive, 4 : Normal")
print("")
#pdb.set_trace()
print("Beni | InSi | Inv | Norm")
confusion_matrix[:,0] = benign_conf[:,0]
confusion_matrix[:,1] = insitu_conf[:,0]
confusion_matrix[:,2] = invasive_conf[:,0]
confusion_matrix[:,3] = normal_conf[:,0]

print(confusion_matrix)
print("")

print("Benign Acc: ", benign_conf[0]/no_samples)
print("Insitu Acc: ", insitu_conf[1]/no_samples)
print("Invasive Acc: ", invasive_conf[2]/no_samples)
print("Normal Acc: ", normal_conf[3]/no_samples)


import logging

import numpy as np
import tensorflow as tf

from lib.stream.reader.image_reader import ImageReader
from lib.tf_od_api.models.research.object_detection.utils import label_map_util
from lib.tf_od_api.models.research.object_detection.utils import visualization_utils as vis_util
from tqdm import tqdm

def load_inference_graph(path):
    logging.info(f'Loading {path} model...')
    graph = tf.compat.v1.Graph()

    with graph.as_default():
        od_graph_def = tf.compat.v1.GraphDef()
        with tf.io.gfile.GFile(path, 'rb') as fid:
            serialized_graph = fid.read()
            od_graph_def.ParseFromString(serialized_graph)
            tf.import_graph_def(od_graph_def, name='')

    return graph

def get_category_index(label_map_path, classes):
    logging.info(f'Loading labels from {label_map_path}...')
    label_map = label_map_util.load_labelmap(label_map_path)
    categories = label_map_util.convert_label_map_to_categories(
        label_map,
        max_num_classes=len(classes),
        use_display_name=True
    )
    return label_map_util.create_category_index(categories)

class Model:
    def __init__(self, inference_graph_path, label_map_path, classes):
        self.graph = load_inference_graph(inference_graph_path)
        self.__category_index = get_category_index(label_map_path, classes)

    def predict_samples(self, samples, min_score=.70):
        results = []
        reader = ImageReader(paths=[s.image.path for s in samples])
        with self.graph.as_default():
            with tf.compat.v1.Session(graph=self.graph) as session:
                with tqdm(total=len(samples), position=0, leave=True, unit='f') as pbar:
                    for frame in reader:
                        results.append(self.predict_frame(session, frame, min_score))
                        pbar.update(1)
        return results

    def predict_frame(self, session, frame, min_score=.70):
        # Expand dimensions since the model expects images to have shape: [1, None, None, 3]
        image_np_expanded = np.expand_dims(frame.raw, axis=0)

        image_tensor = self.graph.get_tensor_by_name('image_tensor:0')

        # Each box represents a part of the image where a particular object was detected.
        boxes = self.graph.get_tensor_by_name('detection_boxes:0')

        # Each score represent how level of confidence for each of the objects.
        # Score is shown on the result image, together with the class label.
        scores = self.graph.get_tensor_by_name('detection_scores:0')
        classes = self.graph.get_tensor_by_name('detection_classes:0')

        num_detections = self.graph.get_tensor_by_name('num_detections:0')

        # Actual detection.
        (boxes, scores, classes, num_detections) = session.run(
            [boxes, scores, classes, num_detections],
            feed_dict={image_tensor: image_np_expanded})

        # Visualization of the results of a detection.
        vis_util.visualize_boxes_and_labels_on_image_array(
            frame.raw,
            np.squeeze(boxes),
            np.squeeze(classes).astype(np.int32),
            np.squeeze(scores),
            self.__category_index,
            use_normalized_coordinates=True,
            line_thickness=4,
            max_boxes_to_draw=50,
            min_score_thresh=min_score
        )
        return frame


# coding=utf-8
# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                        unicode_literals, with_statement)

import os

from pants.util.contextutil import temporary_dir
from pants.util.process_handler import subprocess
from pants_test.pants_run_integration_test import PantsRunIntegrationTest

class CleanAllTest(PantsRunIntegrationTest):
  def test_clean_all_on_wrong_dir(self):
    with temporary_dir() as workdir:
      self.assert_failure(self.run_pants_with_workdir(["clean-all"], workdir))
      self.assert_failure(self.run_pants_with_workdir(["clean-all", "--async"], workdir))

  # Ensure async clean-all exits normally.

  def test_clean_all_async(self):
    self.assert_success(self.run_pants(["clean-all", "--async"]))

  # The tests below check for the existence of trash directories.
  def test_empty_trash(self):
    with self.temporary_workdir() as work_dir:
      trash_dir = os.path.join(work_dir, "trash")
      subprocess.call(["touch", trash_dir + "foo.txt"])
      self.assert_success(self.run_pants_with_workdir(["clean-all"], work_dir))
      self.assertFalse(os._exists(trash_dir))

  def test_empty_trash_async(self):
    with self.temporary_workdir() as work_dir:
      trash_dir = os.path.join(work_dir, "trash")
      subprocess.call(["touch", trash_dir + "foo.txt"])
      self.assert_success(self.run_pants_with_workdir(["clean-all", "--async"], work_dir))
      self.assertFalse(os._exists(trash_dir))


def get_url_params(url):
    params = {}
    i = url.find('?')
    if i == -1:
        i = len(url)
    base = url[:i]
    url = url[i+1:]

    while url != '':
        i = url.find('&')
        if i == -1:
            i = len(url)

        pair = url[:i]
        j = pair.find('=')
        if j < 0:
            raise Exception('Expected key-value pair for URL parameter: {}'.format(pair))
        key = pair[:j]
        value = pair[j+1:]
        params[key] = value

        url = url[i+1:]
    return base, params

def get_url(base, params):
    if len(params) == 0:
        return base
    pairs = ['{}={}'.format(k, params[k]) for k in params.keys()]
    return '{}?{}'.format(base, '&'.join(pairs))


import stan

schools_code = """
data {
  int<lower=0> J;         // number of schools
  real y[J];              // estimated treatment effects
  real<lower=0> sigma[J]; // standard error of effect estimates
}
parameters {
  real mu;                // population treatment effect
  real<lower=0> tau;      // standard deviation in treatment effects
  vector[J] eta;          // unscaled deviation from mu by school
}
transformed parameters {
  vector[J] theta = mu + tau * eta;        // school treatment effects
}
model {
  target += normal_lpdf(eta | 0, 1);       // prior log-density
  target += normal_lpdf(y | theta, sigma); // log-likelihood
}
"""

def model_1():
    print("RUNNING MODEL 1")
    schools_data = {
        "J": 8,
        "y": [28, 8, -3, 7, -1, 1, 18, 12],
        "sigma": [15, 10, 16, 11, 9, 11, 10, 18],
    }

    posterior = stan.build(schools_code, data=schools_data)
    fit = posterior.sample(num_chains=4, num_samples=1000)
    print(fit)

def model_2():
    print("RUNNING MODEL 2")
    schools_data = {
        "J": 8 * 10,
        "y": [28, 8, -3, 7, -1, 1, 18, 12] * 10,
        "sigma": [15, 10, 16, 11, 9, 11, 10, 18] * 10,
    }

    posterior = stan.build(schools_code, data=schools_data)
    fit = posterior.sample(num_chains=4, num_samples=1000)
    print(fit)

def model_3():
    print("RUNNING MODEL 3")
    schools_data = {
        "J": 8 * 11,
        "y": [28, 8, -3, 7, -1, 1, 18, 12] * 11,
        "sigma": [15, 10, 16, 11, 9, 11, 10, 18] * 11,
    }

    posterior = stan.build(schools_code, data=schools_data)
    fit = posterior.sample(num_chains=4, num_samples=1000)
    print(fit)

if __name__ == "__main__":
    skip = False
    try:
        model_1()
    except:
        skip = True
        print("FAILURE 1")
    if not skip:
        try:
            model_2()
        except:
            skip = True
            print("FAILURE 2")
    if not skip:
        try:
            model_3()
        except:
            print("FAILURE 3")


#!/usr/bin/env python
# coding: utf-8

# In[5]:

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import matplotlib
import os
import sys
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42
matplotlib.rcParams['font.family'] = 'Arial'
sns.set_context('poster', font_scale=1.0)
# matplotlib.rcParams['legend.title_fontsize'] = 'xx-small'

# In[6]:

def read_param(path_home, pathtest, num):
    df = pd.read_csv(path_home + '/' + pathtest + '/fitparms-result_{}.csv'.format(num))
#     pB = ufloat(*df[df['param']=='pB'][['fitval', 'fiterror']].values[0])
#     dwB = ufloat(*df[df['param']=='dwB'][['fitval', 'fiterror']].values[0])
#     kexAB = ufloat(*df[df['param']=='kexAB'][['fitval', 'fiterror']].values[0])
#     rchi2 = ufloat(*df[df['param']=='rchi2'][['fitval', 'fiterror']].values[0])
    pB = df[df['param']=='pB']['fitval'].values[0]*100
    dwB = df[df['param']=='dwB']['fitval'].values[0]
    kexAB = df[df['param']=='kexAB']['fitval'].values[0]
    rchi2 = df[df['param']=='rchi2']['fitval'].values[0]

    return [pB, dwB, kexAB, rchi2]

path = os.getcwd()
num = num = int(path.split('_')[-1])
pb_value = []
dw_value = []
kex_value = []

foldername = [name for name in os.listdir(path) if os.path.isdir(os.path.join(path, name))]
kex_foldername = [name for name in foldername if len(name.split('_')) == 3 and name.split('_')[1] == 'kex']
pb_foldername = [name for name in foldername if len(name.split('_')) == 3 and name.split('_')[1] == 'pb']
dw_foldername = [name for name in foldername if len(name.split('_')) == 3 and name.split('_')[1] == 'dw']

# In[3]:

test_opt_value = read_param(path, 'test', num)
for i in pb_foldername:
    pb_value.append(read_param(path, i, num))
pb_value.append(test_opt_value)
pb_value = np.array(pb_value)

for i in kex_foldername:
    kex_value.append(read_param(path, i, num))
kex_value.append(test_opt_value)
kex_value = np.array(kex_value)

for i in dw_foldername:
    dw_value.append(read_param(path, i, num))
dw_value.append(test_opt_value)
dw_value = np.array(dw_value)

# In[4]:

fig, ax = plt.subplots(3, 1, figsize = (8 , 12))

ax[0].scatter(pb_value[:,0], pb_value[:,-1])
ax[1].scatter(dw_value[:,1], dw_value[:,-1])
ax[2].scatter(kex_value[:,2], kex_value[:,-1])

ax[0].scatter(test_opt_value[0], test_opt_value[-1], color = 'r', label = 'best fit')
ax[1].scatter(test_opt_value[1], test_opt_value[-1], color = 'r', label = 'best fit')
ax[2].scatter(test_opt_value[2], test_opt_value[-1], color = 'r', label = 'best fit')
for i in range(3):
    ax[i].set_xlabel(['$p$$_{ES}$ (%)', '$\Delta$$\omega$ (ppm)', '$k$$_{ex}$ (s$^{-1}$)'][i])
    ax[i].set_ylabel('r$\chi$$^2$')

name = sys.argv[1]
ax[0].set_title(name)
plt.tight_layout()
plt.savefig('{}.pdf'.format(name), dpi = 300, transparent = True)

# In[ ]:


import functools
import glob
from multiprocessing import Pool
import os
import pandas as pd
import sys
import tempfile
import traceback

from cluster_utils.io import makedirs_safely
from core_maths.scale_split import merge_split
from db_tools.ezfuncs import query
from db_queries import get_location_metadata, get_cause_metadata, get_ids
from get_draws.api import get_draws
import gbd.constants as gbd
from gbd.decomp_step import decomp_step_id_from_decomp_step
from gbd.estimation_years import gbd_round_from_gbd_round_id
from test_support.profile_support import profile

from split_models.exceptions import IllegalSplitCoDArgument
from split_models.job_classes import SplitCoDSwarm
from split_models.validate import (
    validate_decomp_step_input,
    validate_ids
)

if sys.version_info > (3,):
    long = int

REPORTING_CAUSE_SET_ID, COMPUTATION_CAUSE_SET_ID = 3, 2

# Create list of valid cause_ids
VALID_CAUSE_IDS = get_ids(table='cause').cause_id.unique()
VALID_MEIDS = get_ids(table='modelable_entity').modelable_entity_id.unique()

@profile
def _launch_cod_splits(source_cause_id, target_cause_ids, target_meids,
                       prop_meas_id, gbd_round_id, decomp_step, output_dir,
                       project):
    """
    Split the given source_cause_id given target_meid proportions, saved
    to the target_cause_ids in output_dir.

    Arguments:
        source_cause_id (int): cause_id for the draws to be split
        target_cause_ids (intlist): list of cause ids that you want the new
            outputted subcauses to be identified by
        target_meids (intlist): list of proportion models' modelable_entity_ids
            that you want the source_cause_id to be split by, to make the
            target_cause_ids. Target_cause_ids and target_me_ids must be
            specified in the same order
        prop_meas_id (int): The measure_id that identifies the proportion
            in the target_meids to use for the split.
        gbd_round_id (int): the gbd_round_id for models being split.
        decomp_step (str): Specifies which decomposition step the returned
            estimates should be from. If using interpolate for GBD round 6 and
            above, must specify one of 'step1', 'step2', 'step3', 'step4',
            'step5', or 'iterative'.
        output_dir (str): directory where you want final results stored
        project (str): The SGE project to launch split_cod_model subjobs
            to using SplitCodSwarm.

    Returns:
        A list of tuples with each location_id paired with either 0, or an
                error message. This is then parsed in the central function
                draw_ops.split_cod_model into errors or success messages
    """

    # setup years, sex restrictions, most detailed locations, etc.

    if gbd_round_id >= 6:
        cause_set_id = COMPUTATION_CAUSE_SET_ID
    else:
        cause_set_id = REPORTING_CAUSE_SET_ID
    causes = get_cause_metadata(
        cause_set_id=cause_set_id,
        gbd_round_id=gbd_round_id,
        decomp_step=decomp_step
    ).query("cause_id==@source_cause_id")
    sex_ids = []
    if causes['male'].item() != 0:
        sex_ids.append(1)
    if causes['female'].item() != 0:
        sex_ids.append(2)
    if not sex_ids:
        raise ValueError(
            "Source_cause_id {} is restricted for both males and females, "
            "according to cause metadata"
            .format(source_cause_id)
        )

    most_detailed_locs = list(
        get_location_metadata(
            35, gbd_round_id=gbd_round_id, decomp_step=decomp_step
        ).query('most_detailed==1').location_id.unique())
    meid_cause_map = dict(zip(target_meids, target_cause_ids))

    # run interpolating/extrapolating
    intermediate_dir = os.path.join(
        output_dir, 'intermediate_{}'.format(source_cause_id)
    )
    if not os.path.exists(intermediate_dir):
        makedirs_safely(intermediate_dir)

    swarm = SplitCoDSwarm(
        source_id=source_cause_id,
        proportion_ids=target_meids,
        proportion_measure_id=prop_meas_id,
        sex_ids=sex_ids,
        gbd_round_id=gbd_round_id,
        decomp_step=decomp_step,
        intermediate_dir=intermediate_dir,
        outdir=output_dir,
        project=project
    )
    swarm.add_interpolate_tasks()
    exit_code = swarm.run()
    if exit_code != 0:
        raise RuntimeError(
            "Interpolating CoD years failed. Check logs in {}."
            .format(output_dir)
        )

    # run splitting
    for cid in target_cause_ids:
        cid_dir = os.path.join(output_dir, str(cid))
        if not os.path.exists(cid_dir):
            makedirs_safely(cid_dir)
    file_list = glob.glob(os.path.join('{}/*.h5'.format(intermediate_dir)))

    # read in draws for source cause
    source = _get_draws(source_cause_id, gbd_round_id, decomp_step, sex_ids)
    # create a temporary directory to store all the draws from the source cause
    tmpdir = tempfile.TemporaryDirectory(dir=output_dir)
    # save source cause draws to temporary directory
    source.to_hdf(
        os.path.join(tmpdir.name, 'source_cause_draws.h5'),
        key='draws', mode='w', format='table',
        data_columns=['location_id', 'year_id', 'sex_id', 'age_group_id']
    )

    run_splits = functools.partial(_parallel_merge_split,
                                   meid_cause_map,
                                   file_list,
                                   output_dir,
                                   tmpdir)
    pool = Pool(30)
    res = pool.map(run_splits, most_detailed_locs)
    pool.close()
    pool.join()
    # clean up tempdir
    tmpdir.cleanup()
    return res

def _get_draws(source_cause_id, gbd_round_id, decomp_step, sex_ids):
    """Read in all the draws from the source cause id for each sex."""
    cd = []
    for sex in sex_ids:
        version_id = best_versions(source_cause_id, gbd_round_id, sex,
                                   decomp_step)
        cd.append(get_draws(gbd_id_type='cause_id', gbd_id=source_cause_id,
                            source='codem', gbd_round_id=gbd_round_id,
                            decomp_step=decomp_step, version_id=version_id))
    cd = pd.concat(cd)
    return cd

def best_versions(cause_id, gbd_round_id, sex_id, decomp_step):
    decomp_step_id = decomp_step_id_from_decomp_step(decomp_step, gbd_round_id)
    q = """
        SELECT model_version_id
        FROM cod.model_version
        WHERE cause_id={cid} AND
              sex_id = {sex} AND
              gbd_round_id={gbd} AND
              is_best = 1 AND
              model_version_type_id IN (3, 4) AND
              decomp_step_id = {decomp}
        """.format(cid=cause_id, sex=sex_id, gbd=gbd_round_id,
                   decomp=decomp_step_id)
    res = query(q, conn_def='cod')
    mvids = res.model_version_id.tolist()
    bad_mvids_msg = ("Error: Returned more than one model_version_id: "
                     "{mvids} for cause_id: {cid}. Cannot split model."
                     .format(mvids=mvids, cid=cause_id))
    if len(mvids) != 1:
        raise RuntimeError(bad_mvids_msg)
    return mvids[0]

@profile
def _parallel_merge_split(meid_cause_map, interp_files, output_dir, tmpdir,
                          location_id):
    try:
        epi_draw = []
        for f in interp_files:
            epi_draw.append(pd.read_hdf(f, 'draws',
                            where=["location_id=={}".format(location_id)]))
        epi_draws = pd.concat(epi_draw)

        cd = pd.read_hdf(
            os.path.join(tmpdir.name, 'source_cause_draws.h5'), 'draws',
            where=['location_id=={}'.format(location_id)]
        )

        draw_cols = [col for col in cd.columns if 'draw_' in col]
        epi_draws = epi_draws[
            epi_draws['age_group_id'].isin(cd['age_group_id'].unique())]

        # these columns are not needed and cause maths.merge_split to break
        drop_cols = ['measure_id', 'model_version_id', 'metric_id']
        cd.drop(drop_cols, axis=1, inplace=True, errors='ignore')
        epi_draws.drop(drop_cols, axis=1, inplace=True, errors='ignore')

        cout = merge_split(
            cd, epi_draws,
            ['year_id', 'age_group_id', 'sex_id', 'location_id'],
            draw_cols)

        cout = cout.merge(
            cd[['year_id', 'age_group_id', 'sex_id', 'location_id',
                'envelope']],
            how='left')
        cout['cause_id'] = cout['modelable_entity_id']
        cout['cause_id'] = cout['cause_id'].replace(meid_cause_map)
        cout['measure_id'] = 1
        for cid in cout.cause_id.unique():
            cid_dir = '{}/{}'.format(output_dir, int(cid))
            cid_dir = cid_dir.replace("\r", "")
            if not os.path.exists(cid_dir):
                makedirs_safely(cid_dir)
            fn = '{}/death_{}.csv'.format(cid_dir, location_id)
            cout.query('cause_id=={}'.format(cid)).to_csv(fn, index=False)
        return location_id, 0
    except Exception:
        tb_str = traceback.format_exc()
        return location_id, tb_str

@profile
def split_cod_model(source_cause_id,
                    target_cause_ids,
                    target_meids,
                    project,
                    prop_meas_id=18,
                    decomp_step=None,
                    gbd_round_id=gbd.GBD_ROUND_ID,
                    output_dir=None):

    """Returns a dataframe containing only the name of the out_dir where this
    function will save your new draws.

    Arguments:
        source_cause_id (int): the cause_id to be split into target_cause_ids.

        target_cause_ids (intlist): the cause_ids that should be produced as a
            result of the split. These should be provided in the same order as
            the target_meids which define the split proportions.

        target_meids (intlist): the modelable_entity_ids containing the
            proportion models by which to split the source_cause_id. These
            should be provided in the same order as the target_cause_ids.

        project (str): the team-specific SGE cluster project to launch the
            subjobs of the split_cod_model swarm to.

        prop_meas_id (int): The measure_id that identifies the proportion in
            the target_meids to use for the split. Defaults to measure_id 18,
            proportion.

        decomp_step (str): Specifies which decomposition step the returned
            estimates should be from. Default to None. "Allowed values are
            None, iterative', 'step1', 'step2', 'step3', 'step4', and 'step5'
            depending on the value of gbd_round_id."

        gbd_round_id (int): the gbd_round_id for models being split. Defaults
            to current GBD round id.

        output_dir (str): place where you want new draws to be saved.

    Returns:
        Pandas.DataFrame:
        The output directory where either the draws or an errors logfile
        can be found.

    Raises:
        IllegalSplitCoDArgument: If the source_cause_id, any of the
            target_cause_ids or target_meids are invalid, or if the lists
            of target causes and target meids are not one-to-one.
        RuntimeError: If any under-the-hood errors thrown directing the
            user to the log file.
    """
    # Validate all incoming arguments.
    validate_decomp_step_input(decomp_step, gbd_round_id)

    # Validate source cause_id
    validate_ids(source_cause_id, VALID_CAUSE_IDS, 'source_cause_id')
    # Validate target_cause_ids
    validate_ids(target_cause_ids, VALID_CAUSE_IDS, 'target_cause_ids')

    # Validate modelable_entity_ids
    validate_ids(target_meids, VALID_MEIDS, 'proportion_meids')

    if len(target_cause_ids) != len(target_meids):
        raise IllegalSplitCoDArgument(
            "target_cause_ids and target_meids lists must represent a 1-to-1 "
            "mapping of cause_ids to modelable_entity_ids. Received: {t_ids} "
            "target ids and {p_ids} proportion ids."
            .format(t_ids=len(target_cause_ids), p_ids=len(target_meids))
        )

    if not output_dir:
        output_dir = 'FILEPATH'

    if not os.path.exists(output_dir):
        makedirs_safely(output_dir)

    res = _launch_cod_splits(source_cause_id,
                             target_cause_ids,
                             target_meids,
                             prop_meas_id,
                             gbd_round_id,
                             decomp_step,
                             output_dir,
                             project)
    errors = [r for r in res if r[1] != 0]

    if len(errors) == 0:
        return pd.DataFrame({'output_dir': output_dir}, index=[0])
    else:
        logfile = '{}/{}_errors.log'.format(
            output_dir, str(source_cause_id))
        with open(logfile, 'w') as f:
            estr = "\n".join([str(r) for r in errors])
            f.write(estr)

        return pd.DataFrame({'error_dir': logfile}, index=[0])


import os,sys,inspect
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0,parentdir)

import q
import isovist as iso

from my_rrt import *

from numpy import atleast_2d
a2d = atleast_2d

class Tworld( object ):
    def __init__( self ):
        halls = [ [ 0.4, 0.0 ],
         [ 0.6, 0.0 ],
         [ 0.6, 0.8 ],
         [ 1.0, 0.8 ],
         [ 1.0, 1.0 ],
         [ 0.0, 1.0 ],
         [ 0.0, 0.8 ],
         [ 0.4, 0.8 ],
         [ 0.4, 0.0 ] ]

        self.locs = [ [ 0.1, 0.9 ],
         [ 0.5, 0.1 ],
         [ 0.9, 0.9 ] ]

        halls = a2d( halls )
        self.locs = a2d( self.locs )

        # for compatibility with polygons_to_segments
        tmp_halls = np.vstack(( np.mean(halls, axis=0, keepdims=True), halls ))
        rx1,ry1,rx2,ry2 = polygons_to_segments( [tmp_halls] )

        self.my_rrt = lambda start_loc, goal_loc: run_rrt( start_loc, goal_loc, rx1,ry1,rx2,ry2 )

        self.isovist = iso.Isovist( [self.listify_segs(rx1,ry1,rx2,ry2)] )

        #self.UAVLocation = ( 0.5, 0.11 )
        #self.UAVLocation = ( 0.2, 0.82 )
        self.UAVLocation = ( 0.2, 0.801 )

        self.UAVForwardVector = ( 0.0, 0.010 )

    def listify_segs( self, rx1,ry1,rx2,ry2 ):
        result = []
        for i in range(rx1.shape[0]):
            result.append( [ (rx1[i],ry1[i]), (rx2[i],ry2[i]) ] )
        return result

    def run( self, Q ):
#        sc = Q.choice( p=1.0/3.0*np.ones((1,3)), name="sloc" )
#        gc = Q.choice( p=1.0/3.0*np.ones((1,3)), name="gloc" )

        sc = Q.choice( p=a2d([0.2,0.4,0.4]), name="sloc" )
        gc = Q.choice( p=a2d([0.2,0.4,0.4]), name="gloc" )

        start_loc = a2d( self.locs[ sc ] )
        goal_loc = a2d( self.locs[ gc ] )

        rrt_path = self.my_rrt( start_loc, goal_loc )

        isIntruderFound = self.isovist.IsIntruderSeen( rrt_path,
                                                  self.UAVLocation,
                                                  self.UAVForwardVector,
                                                  UAVFieldOfVision = 45 )

        if isIntruderFound:
            p = 0.9
        else:
            p = 0.1

#        Q.flip( p=p, name="data" )

        return sc,gc,isIntruderFound


#!/usr/bin/python
#coding: utf-8

import random
import string
import time
import os
import sys

def system(cmd):
    ret = os.system(cmd)
    if ret!=0:
        sys.exit(ret)

def gen():
    inputPath = "log"
    countFile = 0
    countLine = 0
    for fd, subfds, fns in os.walk(inputPath):
        for filename in fns:
            pathname = os.path.join(fd, filename)
            with open(pathname, 'r') as src:
                countFile = countFile + 1
                for line in src:
                    countLine = countLine + 1
                    command = 'echo "'
                    command += line.rstrip('\n')
                    command += '" | '
                    command += "/usr/local/kafka/bin/kafka-console-producer.sh --topic wgame_log --broker-list 10.105.19.63:9092"
                    system(command)
                    print(command)

    print(countFile)
    print(countLine)

gen()


#/usr/bin/env python

import sys

def cmdlinearg(name, default=None):
    for arg in sys.argv:
        if arg.startswith(name + "="):
            return arg.split("=")[1]
    if default is None:
        print("missing parameter", name)
        exit(1)
    return default

max_product = int(cmdlinearg('product_max'))
h_one = int(cmdlinearg('h_one'))

split = sys.stdin.readline().split()
assert len(split) == 2
w = int(split[0])
h = int(split[1])
b = 0
s = 0
u = 0
assert w*h<=max_product
if h_one:
    assert h==1
for i in xrange(h):
    line = sys.stdin.readline().strip()
    assert len(line) == w
    for ch in line:
        if ch == 'B':
            b += 1
        elif ch == 'S':
            s += 1
        elif ch == 'U':
            u += 1
        else:
            assert ch in '.#'
line = sys.stdin.readline()
assert len(line) == 0
assert b == 1 and s == 1 and u == 1
exit(42)


# MenuTitle: Remove Hints
from GlyphsApp import TOPGHOST, STEM, BOTTOMGHOST
from jkGlyphsHelpers.forSelected import forSelectedLayers

__doc__ = """
Remove PostScript hints in selected layers.
"""

ps_hints = [
    TOPGHOST,
    STEM,
    BOTTOMGHOST,
]

def remove_postscript_hints(layer):
    delete = []
    for i, hint in enumerate(layer.hints):
        if hint.isPostScript and hint.type in ps_hints:
            delete.append(i)
    if delete:
        for i in reversed(delete):
            del layer.hints[i]

forSelectedLayers(remove_postscript_hints)


#!/usr/bin/python

import pandas as pd
import math
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from matplotlib.backends.backend_pdf import PdfPages
from matplotlib.patches import Rectangle
from matplotlib.ticker import FormatStrFormatter
import argparse
from ast import literal_eval # to convert strings to lists when reading in csv

from get_scores_df import dump_scores

BAPS = ['Enobakhare..I wax it  James Broughton','Crazy on Ayew  Richard Mudd','Heaton Mess  Timothy Williams','Giroud - Sandstorm  Andrew Foster','Großly Underrated  Alasdair Winter']
KEY_PLAYERS = ['Owe me one Kanoti  Tamuka Kanoti','Lingardium Leviosa  Katy Williams','Don’tCallMeSchürrle  Abby O\'Rourke','Enobakhare..I wax it  James Broughton','Crazy on Ayew  Richard Mudd','Heaton Mess  Timothy Williams','Giroud - Sandstorm  Andrew Foster','Großly Underrated  Alasdair Winter']

def plot_on_page(pdf,series, x_label, y_label, players_to_show=None, log_y = False):
   fig, ax = plt.subplots(figsize=(11.69,8.27))
   ax.set_prop_cycle(color=['lightcoral','orangered','gold','darkolivegreen','limegreen','darkturquoise','midnightblue'])
   if players_to_show == None:
      al_var = 0.5
   else:
      al_var = 0.5
   # print(series)
   for i, ind in enumerate(series):
      # y_values = literal_eval(ind)
      if i < 7:
         if players_to_show is None or series.index[i] in players_to_show:
            plt.plot(ind,label=series.index[i],alpha=al_var)
         else:
            plt.plot(ind,label=series.index[i],linestyle='None',alpha=al_var)
      else:
         if players_to_show is None or series.index[i] in players_to_show:
            plt.plot(ind,label=series.index[i],linestyle='--',alpha=al_var)
         else:
            plt.plot(ind,label=series.index[i],linestyle='None',alpha=al_var)

   n_weeks = len(series[0])
   ax.set_ylabel(y_label,fontsize=10)
   ax.set_xlabel(x_label,fontsize=10)
   ax.set_xticks(range(0,n_weeks,2))
   if log_y:
      ax.set_yscale('log')
   plt.legend(fontsize=8,loc='best',handlelength=4)
   plt.grid(which='major', axis='both', alpha=0.2, linewidth = 1)
   pdf.savefig()

def calculate_goal_diff(in_row):
   goal_diff = []
   for i in range(len(in_row['points_for'])):
      goal_diff.append(in_row['points_for'][i] - in_row['points_against'][i])
   return goal_diff

def add_goal_diff(df):
   df['goal_diff'] = df.apply(lambda row: calculate_goal_diff(row), axis=1)

def calculate_form(in_list):
   result_list = in_list.copy()
   for i in range(len(result_list)):
      lower_lim = max(0,i-5)
      result_list[i] = in_list[i] - in_list[lower_lim]
   return result_list

def add_form(df, form_of_what, weeks_to_consider = 5):
   form_name = form_of_what + '_form'
   df[form_name] = df.apply(lambda row: calculate_form(row[form_of_what]),axis=1)

def calculate_cumulative(in_list):
   result_list = in_list.copy()
   for i in range(len(result_list)):
      result_list[i] = result_list[max(i-1,0)] + in_list[i]
   return result_list

def add_cumulative(df, cum_of_what):
   cum_name = cum_of_what + '_cum'
   df[cum_name] = df.apply(lambda row: calculate_cumulative(row[cum_of_what]),axis=1)

def calculate_league_points(in_goal_diff):
   results = []
   for diff in in_goal_diff:
      if diff >0: # win
         results.append(3)
      elif diff < 0: # loss
         results.append(0)
      else: # draw
         results.append(1)
   return results

def add_league_points(df):
   df['league_points'] = df.apply(lambda row: calculate_league_points(row['goal_diff']), axis=1)

def turn_strings_to_lists(df): # pandas reads the csv in as strings rather than lists
   cols = df.columns
   for col in cols:
      for i in range(len(df[col])):
         df[col][i] = literal_eval(df[col][i])

def main(args):
   print("In the main")
   if args.refresh_data == True:
      dump_scores()
      print("Score dumping finished")

   df = pd.read_csv('scores_dump.csv', index_col=0)
   turn_strings_to_lists(df)

   # Make the pdf output
   with PdfPages('Output.pdf') as pdf:
      firstPage = plt.figure(figsize=(11.69,8.27))
      firstPage.clf()
      img=mpimg.imread('alpine.jpg')
      plt.axis('off')
      imgplot = plt.imshow(img)
      txt = "Andy Foster's FPL insights, silently endorsed by Xheredan Shaqiri"
      firstPage.text(0.5,0.05,txt, transform=firstPage.transFigure, size=15, ha="center")
      pdf.savefig()
      plt.close()

      add_goal_diff(df)
      add_league_points(df)

      for col in df.columns:
         add_cumulative(df,col)

      add_form(df,'league_points_cum')

      print(df.columns)

      plot_on_page(pdf,df['points_for_cum'],'Gameweek','Total Points')
      plot_on_page(pdf,df['points_against_cum'],'Gameweek','Total Points Conceded')
      plot_on_page(pdf,df['goal_diff_cum'],'Gameweek','Goal Difference')
      plot_on_page(pdf,df['league_points_cum'],'Gameweek','Real Points')
      plot_on_page(pdf=pdf,series=df['league_points_cum_form'],x_label='Gameweek',y_label='5-Week Form', players_to_show=BAPS)
      plot_on_page(pdf=pdf,series=df['league_points_cum_form'],x_label='Gameweek',y_label='5-Week Form', players_to_show=KEY_PLAYERS)
# CAN I HAVE all players being in the legend still, but then only displaying the key players

      # plot_on_page(pdf,df['total_points_for'],'Gameweek','Total Points')
      # plot_on_page(pdf,df['total_points_against'],'Gameweek','Total Points Conceeded')
      # plot_on_page(pdf,df['total_points_diff'],'Gameweek','Goal Difference')

      # add_form(df,'cum_real_points')
      # plot_on_page(pdf,df['cum_real_points_form'],'Gameweek','Form')

def parse_args():
   description = \
   '''
   Makes some sweet stats about my FPL league
   '''
   parser = argparse.ArgumentParser(description=description)
   parser.add_argument('-r', '--refresh_data', default=False, action='store_true', help='Pass to refresh the data from the FPL site. Else we just use the previously saved csv as this is lots faster.', required=False)
   args = parser.parse_args()
   return args

if __name__ == '__main__':
   args = parse_args()
   main(args)


import cv2
import numpy as np
from sklearn import mixture
import sys
import os
from matplotlib import pyplot as plt
from numba import jit

'''
Note:
1. Using 5 cluster GMM to improve pre-maxflow segmentation accuracy
2. Using F-F with the help from numba to speed up repetitive calculation
'''

####################
# Helper functions #
####################

@jit
def DS(img,i): # Downsampling

    img_small = cv2.resize(img,None,fx=1/i, fy=1/i, interpolation = cv2.INTER_CUBIC)
    dimen_1, dimen_2, _ = img_small.shape
    img_flat = np.concatenate((img_small[:,:,0].flatten().reshape(-1,1),
                               img_small[:,:,1].flatten().reshape(-1,1),
                               img_small[:,:,2].flatten().reshape(-1,1)),axis=1)

    return img_flat, dimen_1, dimen_2

@jit
def GMM(img_flat, dimen_1, dimen_2): # GMM clustering, 5 clusters in total

    gmm_tmp = mixture.GaussianMixture(n_components=5, covariance_type='full', max_iter=500, n_init=5)
    gmm_tmp.fit(img_flat)
    tmp = np.argmax(gmm_tmp.predict_proba(img_flat),axis=1).reshape(-1,1)
    tmp0 = tmp.reshape(dimen_1, dimen_2).astype('float16')

    return tmp0

@jit
def BFS(graph,e): # BFS is written in function becuase it is recursivly called
    q = [0]
    pred = [0] * len(graph) # prevent a vertice been visited twice
    pred[0]=1
    while len(q)>0:
        cur = q.pop(0)
        for index, item in enumerate(graph[cur]):
            if pred[index]==0 and index!=0 and item>0:
                pred[index]=1
                q.append(index)
                e[index]=cur # record path to sink

    if pred[len(graph)-1]==0:
        return False, pred, e
    else:
        return True, pred, e

@jit
def F_F(graph): # Maxflow

    max_flow = 0 # initialize flow
    e = [-1]*len(graph) # record all paths

    # Find a sortest path in residual graph
    path_exist, _, _ = BFS(graph,e)
    while path_exist:
        df = float('inf')

        # Find the minimum flow on the path in the residual graph
        end = num_of_vertices-1
        while end!=0:
            df = min(df, graph[e[end]][end])
            end = e[end]

        # update residual graph in the opposite direction of flow graph
        end = num_of_vertices-1
        while end!=0:
            start = e[end]
            graph[end][start] += df
            graph[start][end] -= df
            end = start

        max_flow += df
        path_exist, _, _ = BFS(graph,e)

    return graph

@jit
def pplot1(i,j,tmp0): # Merge two clusters

    tmp1=np.zeros((dimen_1, dimen_2))
    tmp1[tmp0==i]=1; tmp1[tmp0==j]=1

    return tmp1.astype('int16')

@jit
def pplot2(i,tmp0): # Merge three clusters

    tmp1=np.zeros((dimen_1, dimen_2))
    tmp1[tmp0==i]=1

    return tmp1.astype('int16')

########
# Main #
########

# Read image
img = cv2.imread(sys.argv[1])

global height, width
dimen_limit = 300

# current_path = os.getcwd()
# image_path = current_path.replace('Python','Images')
# img = cv2.imread(image_path + '\\' + 'FB' + '\\'+ '4' + '.jpg') # 4,6,9

# Find downsampling dimension for GMM
height, width, _ = img.shape
large = max(height,width)
for i in range(1,100):
    if large/i < dimen_limit:
        break

# Downsampling for GMM
img_flat, dimen_1, dimen_2 = DS(img,i)
# Five cluster GMM
tmp0 = GMM(img_flat, dimen_1, dimen_2)

# Book keeping all 15 GMM result so user can chose one of them to perform maxflow
a1 =pplot1(0,1,tmp0); a2 =pplot1(0,2,tmp0);a3 =pplot1(0,3,tmp0);a4 =pplot1(0,4,tmp0);a5 =pplot1(1,2,tmp0);a6 =pplot1(1,3,tmp0)
a7 =pplot1(1,4,tmp0);a8 =pplot1(2,3,tmp0);a9 =pplot1(2,4,tmp0);a10 =pplot1(3,4,tmp0);a11 =pplot2(0,tmp0);a12 =pplot2(1,tmp0);
a13 =pplot2(2,tmp0);a14 =pplot2(3,tmp0);a15 =pplot2(4,tmp0)
img_choice = {'1':a1,'2':a2,'3':a3,'4':a4,'5':a5,'6':a6,'7':a7,'8':a8,'9':a9,
              '10':a10,'11':a11,'12':a12,'13':a13,'14':a14,'15':a15}

# Plot all 15 GMM images for user to select the most correct one
fig,axe=plt.subplots(4,5,figsize=(20,12))
axe[0,1].imshow(img);axe[0,1].set_title('Original Image');axe[0,1].set_xticklabels([]);axe[0,1].set_yticklabels([])
axe[0,3].imshow(np.ceil(tmp0*255).astype('int32'), cmap="hot");axe[0,3].set_title('Clustered Image');
axe[0,3].set_xticklabels([]);axe[0,3].set_yticklabels([])
axe[1,0].imshow(cv2.resize(pplot1(0,1,tmp0), (width,height)), cmap="hot");axe[1,0].set_title('1');axe[1,0].set_xticklabels([])
axe[1,0].set_yticklabels([])
axe[1,1].imshow(cv2.resize(pplot1(0,2,tmp0), (width,height)), cmap="hot");axe[1,1].set_title('2');axe[1,1].set_xticklabels([])
axe[1,1].set_yticklabels([])
axe[1,2].imshow(cv2.resize(pplot1(0,3,tmp0), (width,height)), cmap="hot");axe[1,2].set_title('3');axe[1,2].set_xticklabels([])
axe[1,2].set_yticklabels([])
axe[1,3].imshow(cv2.resize(pplot1(0,4,tmp0), (width,height)), cmap="hot");axe[1,3].set_title('4');axe[1,3].set_xticklabels([])
axe[1,3].set_yticklabels([])
axe[1,4].imshow(cv2.resize(pplot1(1,2,tmp0), (width,height)), cmap="hot");axe[1,4].set_title('5');axe[1,4].set_xticklabels([])
axe[1,4].set_yticklabels([])
axe[2,0].imshow(cv2.resize(pplot1(1,3,tmp0), (width,height)), cmap="hot");axe[2,0].set_title('6');axe[2,0].set_xticklabels([])
axe[2,0].set_yticklabels([])
axe[2,1].imshow(cv2.resize(pplot1(1,4,tmp0), (width,height)), cmap="hot");axe[2,1].set_title('7');axe[2,1].set_xticklabels([])
axe[2,1].set_yticklabels([])
axe[2,2].imshow(cv2.resize(pplot1(2,3,tmp0), (width,height)), cmap="hot");axe[2,2].set_title('8');axe[2,2].set_xticklabels([])
axe[2,2].set_yticklabels([])
axe[2,3].imshow(cv2.resize(pplot1(2,4,tmp0), (width,height)), cmap="hot");axe[2,3].set_title('9');axe[2,3].set_xticklabels([])
axe[2,3].set_yticklabels([])
axe[2,4].imshow(cv2.resize(pplot1(3,4,tmp0), (width,height)), cmap="hot");axe[2,4].set_title('10');axe[2,4].set_xticklabels([])
axe[2,4].set_yticklabels([])
axe[3,0].imshow(cv2.resize(pplot2(0,tmp0), (width,height)), cmap="hot");axe[3,0].set_title('11');axe[3,0].set_xticklabels([])
axe[3,0].set_yticklabels([])
axe[3,1].imshow(cv2.resize(pplot2(1,tmp0), (width,height)), cmap="hot");axe[3,1].set_title('12');axe[3,1].set_xticklabels([])
axe[3,1].set_yticklabels([])
axe[3,2].imshow(cv2.resize(pplot2(2,tmp0), (width,height)), cmap="hot");axe[3,2].set_title('13');axe[3,2].set_xticklabels([])
axe[3,2].set_yticklabels([])
axe[3,3].imshow(cv2.resize(pplot2(3,tmp0), (width,height)), cmap="hot");axe[3,3].set_title('14');axe[3,3].set_xticklabels([]);
axe[3,3].set_yticklabels([])
axe[3,4].imshow(cv2.resize(pplot2(4,tmp0), (width,height)), cmap="hot");axe[3,4].set_title('15');axe[3,4].set_xticklabels([])
axe[3,4].set_yticklabels([])
fig.delaxes(axe[0,0]);fig.delaxes(axe[0,2]);fig.delaxes(axe[0,4])
plt.show()

#######
# F-F #
#######

# Choose the image by user
which_img = input('Which image is the most similar to the correct answer? ')
ff_input = img_choice[which_img]
downsampleto = 150
img_gmm = cv2.resize(ff_input,(downsampleto,downsampleto), interpolation = cv2.INTER_CUBIC)

# Set up flow capacity
background_cap = img_gmm * 10
background_cap_flat = background_cap.astype('int32').flatten()

# Convert graph into adjacency matrix
num_of_vertices = 2 + downsampleto**2
graph_capacity = np.zeros((num_of_vertices,num_of_vertices))
graph_capacity[0,1:downsampleto**2+1] = background_cap_flat
graph_capacity[1:downsampleto**2+1,downsampleto**2+1] = (10 - background_cap_flat)

for i in range(downsampleto):
    for j in range(downsampleto):
        tmp_img = np.zeros((downsampleto,downsampleto))
        if i-1>=0:
            tmp_img[i-1][j] = 10
        if j-1>=0:
            tmp_img[i][j-1] = 10
        if j+1<downsampleto:
            tmp_img[i][j+1] = 10
        if i+1<downsampleto:
            tmp_img[i+1][j] = 10
        graph_capacity[i*downsampleto+j+1,1:downsampleto**2+1] = tmp_img.flatten()

# residual graph started with max capasity
graph = graph_capacity.copy()

graph = F_F(graph)

# Run BFS to find the segmentations
_, pred, _ = BFS(graph, [-1]*(num_of_vertices-1))
tmp = np.asarray(pred)[1:num_of_vertices-1].reshape(downsampleto,downsampleto).astype('int16')

# Form mask to segment image
shrink_ratio = 3
img_third = cv2.resize(img, (round(width/shrink_ratio), round(height/shrink_ratio)), interpolation=cv2.INTER_CUBIC)
resized_image = cv2.resize(tmp.astype('float32'),
                           (round(width/shrink_ratio), round(height/shrink_ratio)),
                           interpolation=cv2.INTER_CUBIC).round()
mask1 = np.zeros((round(height/shrink_ratio), round(width/shrink_ratio),3))
mask2 = np.zeros((round(height/shrink_ratio), round(width/shrink_ratio),3))

mask1[:,:,0] = mask1[:,:,2] = mask1[:,:,1] = 1-resized_image
mask2[:,:,0] = mask2[:,:,2] = mask2[:,:,1] = resized_image

seg_img1 = cv2.resize(np.multiply(mask1.astype('bool'),img_third),(round(width), round(height)), interpolation=cv2.INTER_CUBIC)
seg_img2 = cv2.resize(np.multiply(mask2.astype('bool'),img_third),(round(width), round(height)), interpolation=cv2.INTER_CUBIC)

fig,axe = plt.subplots(1,2,figsize=(9,4))
axe[0].imshow(seg_img1)
axe[1].imshow(seg_img2)
plt.show()
fig.savefig('EXT2.jpg')


import torch
import re
import pandas as pd
import numpy as np
import requests
import copy
import gensim

from tqdm import tqdm
from sklearn.preprocessing import LabelEncoder

import torch.optim as optim
import torch.nn as nn
import torch.nn.functional as F

import warnings
warnings.filterwarnings(action='ignore')

def load_dataset(data_key):
    if data_key == 'news':
        news_dataset = pd.read_excel('./dataset/news_dataset_preprocessing.xlsx')
        del news_dataset['Unnamed: 0']

        lb_encoder = LabelEncoder()
        news_dataset["encode_category"] = lb_encoder.fit_transform(news_dataset["category"])
        news_dataset[["category", "encode_category"]].head()

        return news_dataset
    elif data_key == 'tweet_disaster':
        #tweet_disaster_df = pd.read_csv('./dataset/train_tweet_disaster.csv')
        tweet_disaster_df = pd.read_csv('./dataset/train_refined.csv')
        return tweet_disaster_df
    else:
        print("DATA KEY ERROR: ", data_key)
        return None

def elapsed_time(start, end):
  from datetime import datetime

  fmt = '%Y-%m-%d %H:%M:%S'
  time_start = datetime.strptime(start, fmt)
  time_end = datetime.strptime(end, fmt)

  diff = time_end - time_start

  h = int(diff.seconds/3600)
  m = int((diff.seconds%3600)/60)
  s = (diff.seconds%3600)%60

  print("elapsed time: {:1d}days {:2d}:{:2d}:{:2d}".format(diff.days, h, m, s))

def get_timestamp():
  from datetime import datetime

  fmt = '%Y-%m-%d %H:%M:%S'
  return datetime.now().strftime(fmt)

def get_BERT_vector(sentence_list):
    from transformers import BertModel, BertTokenizer

    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
    model = BertModel.from_pretrained('bert-base-uncased')

    X_concat = torch.tensor(0)
    for i, paragraph in enumerate(tqdm(sentence_list, desc='BERT embedding', mininterval=1)):
        input_ids = torch.tensor([tokenizer.encode(paragraph, add_special_tokens=True)])
        with torch.no_grad():
            outputs = model(input_ids)
        last_hidden_states = outputs[0]  # [batch, tokens, hidden_dim (768)]

        mean = torch.mean(last_hidden_states, dim=1)  # tokens
        max, _ = torch.max(last_hidden_states, dim=1)  # tokens
        concat = torch.cat((mean, max), dim=1)
        if i == 0:
            X_concat = copy.deepcopy(concat)
        else:
            X_concat = torch.cat((X_concat, concat), dim=0)
    return X_concat

def get_embedding_vector(KEY, sentence_list):
    eliminated_row = []
    total_eliminated_row = 0
    X_concat = torch.tensor(0)
    for i, sent in enumerate(tqdm(sentence_list, desc=KEY+' embedding', mininterval=1)):
        tokens = sent.split()
        vectors = []
        if KEY == 'new-embedding':
            for token in tokens:
                if token in new_word_lower:
                    # when new word, get embedding from new-embedding model
                    vector = new_embedding_dict.get(token)
                #elif token in convert_word_lower:
                #    vector = convert_embedding_dict.get(token)
                else:
                    # when not new word, get just conceptnet embedding
                    vector = embedding_dict.get(token)

                if vector is not None:
                    vector = np.fromstring(vector, dtype=float, sep=' ')  # when return: embedding
                    vectors.append(vector)
        elif KEY == 'conceptnet':
            #sent = re.sub('\[|\]|:|(|)|\*|#|@|!|\?|\.', '', sent)
            #tokens = sent.lower().split()
            #total = 0
            for token in tokens:
                if token in new_word_lower:
                    # when new word, don't get embedding from conceptnet
                    vector = None
                else:
                    # when not new word, get conceptnet embedding
                    vector = embedding_dict.get(token)

                if vector is not None:
                    #total += 1
                    vector = np.fromstring(vector, dtype=float, sep=' ')  # when return: embedding
                    vectors.append(vector)
        elif KEY == 'glove':
            for token in tokens:
                if token in new_word_lower:
                    vector = None
                else:
                    vector = glove_embedding_dict.get(token)

                if vector is not None:
                    vectors.append(vector)
        elif KEY == 'word2vec':
            for token in tokens:
                if token in new_word_lower:
                    vector = None
                else:
                    if token in word2vec_model:
                        vector = word2vec_model[token]
                    else:
                        vector = None

                if vector is not None:
                    vectors.append(vector)
        else:
            print("KEY Error: ", KEY)
            break

        if vectors == []:
            eliminated_row.append(i)
            total_eliminated_row += 1
        else:
            tensor_vec = torch.tensor(vectors)
            mean = torch.mean(tensor_vec, dim=0)  # tokens
            max, _ = torch.max(tensor_vec, dim=0)  # tokens
            concat = torch.cat((mean, max), dim=0)
            concat = concat.unsqueeze_(dim=0) # 맨앞에 차원 하나 추가 [1, 600]
            if i == 0:
                X_concat = copy.deepcopy(concat)
            else:
                X_concat = torch.cat((X_concat, concat), dim=0)

    """
    if KEY == 'new-embedding':
        df_vec = pd.DataFrame({'X_vector': X_vector})
        df_vec.to_csv('X_vector_newEmbedding.csv', index=False, header=False)
    elif KEY == 'conceptnet':
        df_vec = pd.DataFrame({'X_vector': X_vector})
        df_vec.to_csv('X_vector_conceptnet.csv', index=False, header=False)
    elif KEY == 'glove':
        df_vec = pd.DataFrame({'X_vector': X_vector})
        df_vec.to_csv('X_vector_glove.csv', index=False, header=False)
    elif KEY == 'word2vec':
        df_vec = pd.DataFrame({'X_vector': X_vector})
        df_vec.to_csv('X_vector_word2vec.csv', index=False, header=False)
    else:
        print("KEY Error: ", KEY)
    """

    print("total eliminated row: {}\n".format(total_eliminated_row))
    return X_concat, eliminated_row

def reset_category_list(category_list, eliminated_row):
    for index in sorted(eliminated_row, reverse=True):
        del category_list[index]
    return category_list

##########################################################################
def get_input_dataset(KEY, sentence_list, category_list):
    if KEY == 'bert':
        X_concat = get_BERT_vector(sentence_list)
        X_train, X_test, y_train, y_test = get_train_test_split(X_concat, category_list)
        train_loader, test_loader = get_train_test_loader(X_train, X_test, y_train, y_test)
        return train_loader, test_loader
    else:
        X_concat, eliminated_row = get_embedding_vector(KEY, sentence_list)
        category_list = reset_category_list(category_list, eliminated_row)
        X_train, X_test, y_train, y_test = get_train_test_split(X_concat, category_list)
        train_loader, test_loader = get_train_test_loader(X_train, X_test, y_train, y_test)
        return train_loader, test_loader
##########################################################################

def get_train_test_split(X_concat, category_list):
  from sklearn.model_selection import train_test_split

  y = torch.tensor(category_list)
  X_train, X_test, y_train, y_test = train_test_split(X_concat, y,
                                                      train_size=0.8,
                                                      random_state=928,
                                                      shuffle=True,
                                                      stratify=y)
  return X_train, X_test, y_train, y_test

def get_train_test_loader(X_train, X_test, y_train, y_test):
  from torch.utils.data import TensorDataset, DataLoader

  train_loader = DataLoader(TensorDataset(X_train, y_train),
                            batch_size = B,
                            shuffle = True,
                            num_workers = 2)
  test_loader = DataLoader(TensorDataset(X_test, y_test),
                          batch_size = B,
                          shuffle = True,
                          num_workers = 2)
  return train_loader, test_loader

def flat_accuracy(preds, labels):
  pred_flat = np.argmax(preds, axis=1).flatten()
  labels_flat = labels.flatten()
  return np.sum(pred_flat == labels_flat) / len(labels_flat)

def init_all_random(SET_SEED, r_seed=0, np_seed=0, torch_seed=0, cuda_seed=0):
  import random
  if SET_SEED:
      random.seed(r_seed)
      np.random.seed(np_seed)
      torch.manual_seed(torch_seed)
      torch.cuda.manual_seed_all(cuda_seed)
      torch.backends.cudnn.deterministic = True
      torch.backends.cudnn.benchmark = False
  else:
      print("random\n", random.getstate()[1][0])
      print("numpy.random\n", np.random.get_state()[0], np.random.get_state()[1][0])
      print("torch\n", torch.random.initial_seed())
      print("torch.cuda\n", torch.cuda.random.initial_seed())

def get_model(D_in, H, D_out, activation_fn='ReLU'):
    if activation_fn == 'ReLU6':
        model = nn.Sequential(nn.Linear(D_in, H),
                              nn.ReLU6(),  # ReLU ReLU6 RReLU
                              nn.Linear(H, D_out), )
    elif activation_fn == 'RReLU':
        model = nn.Sequential(nn.Linear(D_in, H),
                              nn.RReLU(),  # ReLU ReLU6 RReLU
                              nn.Linear(H, D_out), )
    else:
        model = nn.Sequential(nn.Linear(D_in, H),
                              nn.ReLU(),  # ReLU ReLU6 RReLU
                              nn.Linear(H, D_out), )

    if USE_CUDA:
        model = model.cuda()
    return model

def run_model(model, lr, loss_fn, optimizer, train_loader, test_loader):
  activation_fn = list(model.children())[1]

  print(f"\nEMBEDDING MODEL: {model_name[KEY]}")
  print(f"learning rate: {lr}\nloss function: {loss_fn.children}")
  print(f"Activation function: {activation_fn}\noptimizer: {optimizer}\n")
  best_accuracy = 0
  best_trn_accuracy = 0
  for epoch in range(500):
    trn_loss, trn_accuracy = [], []
    for X, y in train_loader:
      X = X.float()
      y = y.float()

      if USE_CUDA:
        X = X.cuda()
        y = y.cuda()

      model.train()
      model.zero_grad()

      logits = model(X)  # [batch, D_out] = [32, 4]
      logits_ = F.softmax(logits, dim=1)  # logits_ : [batch, D_out]
      y_pred, _ = torch.max(logits_, dim=1)  # y_pred : [batch]

      loss = loss_fn(y_pred, y)

      pred = logits_.cpu().detach().numpy()
      gt = y.view(-1, 1).cpu().detach().numpy()
      accuracy = flat_accuracy(pred, gt)

      trn_loss.append(loss.item())
      trn_accuracy.append(accuracy)  # for mean accuracy every batch

      loss.backward()
      optimizer.step()

      with torch.no_grad():
        model.eval()

        eval_loss, eval_accuracy = [], []
        for eval_X, eval_y in test_loader:
          eval_X = eval_X.float()
          eval_y = eval_y.float()

          if USE_CUDA:
            eval_X = eval_X.cuda()
            eval_y = eval_y.cuda()

          logits = model(eval_X)
          logits_ = F.softmax(logits, dim=1)
          y_pred, _ = torch.max(logits_, dim=1)

          loss = loss_fn(y_pred, eval_y)

          pred = logits_.cpu().detach().numpy()
          gt = eval_y.view(-1, 1).cpu().detach().numpy()
          accuracy = flat_accuracy(pred, gt)

          eval_loss.append(loss.item())
          eval_accuracy.append(accuracy)
        avg_loss = np.mean(eval_loss, axis=0)
        avg_accuracy = np.mean(eval_accuracy, axis=0)

        if best_accuracy < avg_accuracy:
          best_accuracy = avg_accuracy
          #torch.save(model.state_dict(), model_path[KEY])

    trn_avg_loss = np.mean(trn_loss, axis=0)
    trn_avg_accuracy = np.mean(trn_accuracy, axis=0)
    if best_trn_accuracy < accuracy:
        best_trn_accuracy = accuracy
    if (epoch+1) % 50 == 0:
      print("{:3d}-epoch | trn loss: {:2.2f} | trn best accuracy: {:.4f} | val loss: {:2.2f} | val best accuracy: {:.4f}".format(epoch+1, trn_avg_loss, accuracy, avg_loss, best_accuracy))
  print("BEST TRN ACCURACY: {:.4f}".format(best_trn_accuracy))
  print("BEST VAL ACCURACY: {:.4f}".format(best_accuracy))
  print("\n#####################################\n")
  return best_accuracy

if __name__ == '__main__':
    #data_df = load_dataset('news')
    data_df = load_dataset('tweet_disaster')
    #####################################
    USE_CUDA = False
    if torch.cuda.is_available():
        USE_CUDA = True
    DEVICE = torch.device("cuda:0" if USE_CUDA else "cpu")
    #####################################

    model_input = {"bert": 1536,
                   "conceptnet": 600,
                   "new-embedding": 600,
                   "glove": 600,
                   "word2vec": 600,}
    model_name = {"bert": "BERT Base Uncased",
                  "conceptnet": "ConceptNet",
                  "new-embedding": "New Embedding",
                  "glove": "Glove",
                  "word2vec": "Word2Vec"}
    model_path = {"bert": "best-bert-model.pt",
                  "conceptnet": "best-conceptnet-model.pt",
                  "new-embedding": "best-new-embedding-model.pt",
                  "glove": "best-glove-model.pt",
                  "word2vec": 'best-word2vec-model.pt'}

    #####################################
    # Make embedding dictionary
    embedding_list = pd.read_csv('wordEmbedding_en.csv', encoding='utf-8', usecols=['word', 'vec300'], sep=' ')
    embedding_dict = embedding_list.set_index('word')['vec300'].to_dict()

    # --------------------------------------------------------
    # new word embedding (prediction)
    new_embedding_list = pd.read_csv('prediction_embedding.csv')

    new_word_lower = []
    for w in new_embedding_list['new_word']:
        new_word_lower.append(w.lower())
    new_embedding_list['new_word_lower'] = new_word_lower

    # DataFrame to dictionary
    new_embedding_dict = new_embedding_list.set_index('new_word_lower')['embedding'].to_dict()

    #------------------------------------------------------------------------------------------------------------------
    # disaster tweets data: conceptnet
    for i in range(1):
        KEY = 'conceptnet'

        init_all_random(SET_SEED=True,
                        r_seed=2147483648,
                        np_seed=2147483648,
                        torch_seed=17043145292842172883 & ((1<<63)-1),
                        cuda_seed=5039337088303087)
        D_in = model_input[KEY]
        B, H, D_out = 32, 100, 2

        train_loader, test_loader = get_input_dataset(KEY, data_df['refine_text'], data_df['target'])
        #train_loader, test_loader = get_input_dataset(KEY, data_df['keyword_text'], data_df['target'].tolist())

        model = get_model(D_in, H, D_out, 'ReLU')
        loss_fn = nn.MSELoss(reduction='sum')
        optimizer = optim.Adam(model.parameters(), lr=3e-3)
        time_start = get_timestamp()
        best_accuracy = run_model(model, 3e-3, loss_fn, optimizer, train_loader, test_loader)
        time_end = get_timestamp()
        elapsed_time(start=time_start, end=time_end)
        print('\n\n')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Oct 14 16:14:31 2018

@author: owen
"""

# There are k sorted arrays nums. Find the median of the given k sorted arrays.
# The length of the given arrays may not equal to each other.
# The elements of the given arrays are all positive number.
# Return 0 if there are no elements in the array.

class Solution:
    """
    @param nums: the given k sorted arrays
    @return: the median of the given k sorted arrays
    """
    def findMedian(self, nums):
        # write your code here
        # binary search on median, time O( log range * number of arrays k * log length of each array ), range = max - min of nums
        # can apply to negative integer, but can not apply to float number arrays
        def countLessEqual(arr, target):
            length = len(arr)
            left, right = 0, length
            while left < right:
                mid = left + (right - left) // 2
                if arr[mid] <= target: # find first index > target
                    left = mid + 1
                else:
                    right = mid
            return left  # left is the count of how many elements in arr are <= target

        def findKthSmallest(nums, kth): # kth is 1-based
            start, end = self.minVal, self.maxVal
            while start < end:
                mid = start + (end - start) // 2
                cnt = 0
                for arr in nums:
                    cnt += countLessEqual(arr, mid)
                if cnt < kth:  # count how many elements are <= mid in nums
                    start = mid + 1
                else:
                    end = mid
            return start  # start is the k-th smallest

        # main
        eleNum = 0  # total number of elements in nums
        self.minVal = float('inf')
        self.maxVal = float('-inf')
        for arr in nums:
            if arr:
                eleNum += len(arr)
                self.minVal = min(self.minVal, arr[0])
                self.maxVal = max(self.maxVal, arr[-1])

        if eleNum == 0:
            return 0.0

        kth = eleNum // 2 + 1  # 1-based
        if eleNum & 1:
            return findKthSmallest(nums, kth) * 1.0
        else:
            return (findKthSmallest(nums, kth - 1) + findKthSmallest(nums, kth)) / 2.0


from dataclasses import InitVar, dataclass

@dataclass(unsafe_hash=True)
class Block:
    tag: int
    set: int
    model: InitVar["Model"]

    def __post_init__(self, model: "Model") -> None:
        self.address = (self.tag << (model.log_sets()) | self.set)

    def __repr__(self) -> str:
        return f"Block({self.tag}, {self.set}, #{self.address})"

    @staticmethod
    def from_address(model: "Model", address: int) -> "Block":
        tag = address >> (model.log_sets())
        set = (address) & ((1 << model.log_sets()) - 1)
        return Block(tag, set, model)


#A Confederação Nacional de Natação precisa de um programa que leia o ano de nascimento de um atleta e mostre sua categoria, de acordo com a idade:
#- Até 9 anos: MIRIM
#- Até 14 anos: INFANTIL
#- Até 19 anos: JÚNIOR
#- Até 25 anos: SÊNIOR
#- Acima de 25 anos: MASTER
import datetime
anoatual = datetime.datetime.today().year
nascimento = int(input('Digite o seu ano de nascimento: '))
idade = anoatual - nascimento
if anoatual > nascimento:
    if idade <= 9:
        print('Categoria: MIRIM')
    elif idade <= 14:
        print('Categoria: INFANTIL')
    elif idade <= 19:
        print('Categoria: JÚNIOR')
    elif idade <= 25:
        print('Categoria: SÊNIOR')
    else:
        print('Categoria: MASTER')
else:
    print('Erro.Tente outra vez, com valores válidos!')


from setuptools import setup

readme = open('./README.rst').read()

classifiers = [
    "License :: OSI Approved :: MIT License",
    "Programming Language :: Python :: 2",
    "Programming Language :: Python :: 3",
    "Topic :: Software Development :: Libraries",
    "Topic :: Security :: Cryptography"
]

setup(
    name="bitjws",
    version="0.5.3",
    description='JWS using Bitcoin message signing',
    long_description=readme,
    author='Guilherme Polo',
    author_email='gp@deginner.com',
    url='https://github.com/g-p-g/bitjws',
    license='MIT',
    classifiers=classifiers,
    include_package_data=True,
    packages=['bitjws'],
    setup_requires=['pytest-runner'],
    install_requires=['base58', 'secp256k1'],
    tests_require=['pytest', 'pytest-cov']
)


#!/usr/bin/env python

import  sys, random

N=256
image = []

def noise( scale ):
    for row in range(0,N):
        for col in range(0,N):
            val=image[row*N+col]
            image[row*N+col] = min(255, int(val + random.random() * scale))

def smooth():
    ngh=[ (0,0), (-1,0), (0,1), (0,-1), (0,1) , (-1,-1), (1,1) ]
    for row in range(0,N):
        for col in range(0,N):
            avg=0;
            for n in ngh:
                pos = (row + n[0])*N + col + n[1]
                if (pos >=0 and pos < N*N):
                  avg += image[pos]
            image[row*N+col] = avg/7;

def edge_smooth():
    min_edges =[]
    dirs = [(-1,0), (0,1), (0,-1), (0,1)]
    for row in range(0,N):
        for col in range(0,N):
            min_dir = [ 257, 0, 1]
            for d in dirs:
              x=col+d[0]
              y=row+d[1]
              if ( x>=0 and x<N and y>=0 and y<N):
                  diff=abs(image[row*N+col]-image[y*N+x])
                  if (diff < min_dir[0]):
                      min_dir = [diff, row*N+col, y*N+x]
            min_edges.append(min_dir)
    min_edges.sort(key=lambda dir: dir[0], reverse=True)
    for edg in min_edges:
        if (edg[0] < min_edges[-1][0]+3):
            return
        avg = (image[edg[0]] + image[edg[1]])/2
        image[edg[0]] = (image[edg[0]] + avg)/2
        image[edg[1]] = (image[edg[1]] + avg)/2

def circle():
    x=int(random.random()*N)
    y=int(random.random()*N)
    r=3+int(random.random()*N/3)
    iib=int(random.random()*50)
    iif=iib+20+int(random.random()*30)
    for row in range(0,N):
        for col in range(0,N):
            val=image[row*N+col]
            if ( (row -x)**2 + (col-y)**2 < r*r ) :
                val = (val+iif) %256
            else:
                val = (val+iib) %256
            image[row*N+col] =val

def border():
    avg=0
    for i in range(0,N):
        avg += image[i    *N  +0]
        avg += image[i    *N  +N-1]
        avg += image[0    *N  +i]
        avg += image[(N-1)*N  +i]
    avg/=4*(N-1)
    for i in range(0,N):
        image[i    *N  +0] = avg
        image[i    *N  +N-1] = avg
        image[0    *N  +i]  = avg
        image[(N-1)*N  +i] = avg

def print_image():
    for row in range(0,N):
          for col in range(0,N):
              sys.stdout.write( str(image[row*N+col]) + " ")
          sys.stdout.write("\n")
#    print

image = [0]*(N*N)

noise( 10 )
circle()
#smooth()
edge_smooth()
#border()
#noise( 1 )
border()

sys.stdout.write( str(N) )
sys.stdout.write( "\n" )
print_image()


# Python 3.7
# 105
# arr comprehensions
# basic sintax : [expr for item in arr if cond]

from aidfunctions import append_elements
from random import randint

def list_comp():

    w = 3
    x = randint(1, w)
    y = randint(1, w)
    z = randint(1, w)
    n = randint(1, w)
    print(x)
    print(y)
    print(z)
    print(n)

    ar = [[i, j] for i in range(x + 1) for j in range(y + 1) if ((i + j) != n)]
    print(ar)

    ar3 = [[i, j, k] for i in range(x + 1) for j in range(y + 1) for k in range(z + 1) if ((i + j + k) != n)]
    print(ar3)

    return append_elements('list_comp()', x, y, z, n, 'ar: ' + str(ar), 'ar3: ' + str(ar3))


import os
import time
import pandas as pd
from io import StringIO
import werkzeug
werkzeug.cached_property = werkzeug.utils.cached_property
from flask import Flask, request, send_from_directory, make_response, Blueprint, url_for
from flask_restplus import Resource, Api, reqparse, fields
from flask_cors import CORS
from process import cleaning, fuzzymatch, src_dict, df_process
from werkzeug.datastructures import FileStorage
from utils import timeit, logger

app = Flask(__name__)
app.config.SWAGGER_UI_DOC_EXPANSION = 'list'
app.config.SWAGGER_UI_OPERATION_ID = True
app.config.SWAGGER_UI_REQUEST_DURATION = True
CORS(app)

if os.path.exists('./config.ini'):
    import configparser
    config = configparser.ConfigParser()
    config.read('./config.ini')
    PORT = config.get('config','APP_PORT')
elif os.getenv('APP_PORT'):
    PORT = int(str(os.getenv('APP_PORT')))
else:
    PORT=5000

##########################
#  Documentation Sphinx  #
##########################

blueprint_doc = Blueprint('documentation', __name__,
                          static_folder='../docs/build/html/_static',
                          url_prefix='/sivnorm/docs')

@blueprint_doc.route('/', defaults={'filename': 'index.html'})
@blueprint_doc.route('/<path:filename>')
def show_pages(filename):
    return send_from_directory('../docs/build/html', filename)

app.register_blueprint(blueprint_doc)

#################
#  API SWAGGER  #
#################

class Custom_API(Api):
    @property
    def specs_url(self):
        '''
        The Swagger specifications absolute url (ie. `swagger.json`)

        :rtype: str
        '''
        return url_for(self.endpoint('specs'), _external=False)

blueprint = Blueprint('api', __name__, url_prefix='/sivnorm')
api = Custom_API(
        blueprint, doc='/swagger', version='1.0', title='SivNorm',
        description="Match flou de la marque et modèle des véhicules dans la \
                carte grise à partir d'un référentiel")
app.register_blueprint(blueprint)

@timeit
def process_row(table_ref_name, marque, modele):
    if marque and modele:
        row = dict(modele=modele, marque=marque)
        for column in ['marque', 'modele']:
            row = fuzzymatch(cleaning(row, column), column, table_ref_name)
        row['score'] = (row['score_marque'] + row['score_modele']) / 200
        return row

@timeit
def process_csv(table_ref_name, input_file):

    num_workers = 16
    filename = "%s.csv" % ('output file')

    output_file = StringIO()

    df = pd.read_csv(input_file, names=['marque', 'modele'])

    df = df.fillna("")

    t1 = time.time()
    df = df_process(df, table_ref_name, num_workers)

    sec_wl = (num_workers*(time.time() - t1))/(df.shape[0])
    logger.debug(f'{sec_wl:.2f} seconds per worker per line')

    df.sort_index().to_csv(output_file, encoding='utf-8', index=False, header=False)
    output_csv = output_file.getvalue()
    output_file.close()

    resp = make_response(output_csv)
    resp.headers["Content-Disposition"] = ("attachment; filename=%s" % filename)
    resp.headers["Content-Type"] = "text/csv"

    return resp

parser = reqparse.RequestParser()
parser.add_argument('marque', type=str, location='args', help='Vehicle brand (eg: Renault)')
parser.add_argument('modele', type=str, location='args', help='Vehicle model (eg: Clio)')

parser_table = reqparse.RequestParser()
parser_table.add_argument('file', type=FileStorage, location='files', help='CSV file with multiple brand model lines with no header. Like: \n\n renault,clio \n renault,clio2 \n audi,TTs \n renault,renault clio7 \n renault,clio7 RT/RN')

NormOutput = api.model('NormalizedOutput', {
    'modele': fields.String(description='Matched model', example='Clio'),
    'marque': fields.String(description='Matched brand', example='Renault'),
    'score_marque': fields.Integer(description='Matching score for brand', min=0, max=100, example=100),
    'score_modele': fields.Integer(description='Matching score for model', min=0, max=100, example=100),
    'score': fields.Float(description='Global matching score. Combination of score_marque and score_modele', min=0, max=1, example=1),
})

CleanOutput = api.model('CleanOutput', {
    'modele': fields.String(description='Cleaned model', example='CLIO'),
    'marque': fields.String(description='Cleaned brand', example='RENAULT'),
})

@api.route('/norm/<string:table_ref_name>')
@api.doc(params={'table_ref_name': 'Reference table [siv,cardisiac,siv_caradisiac]'})
class Normalization(Resource):
    """Docstring for MyClass. """

    @api.expect(parser)
    @api.marshal_with(NormOutput, mask=None)
    def get(self, table_ref_name):
        """Normalize a single brand and model using a defined referential table"""
        marque = request.args.get('marque', '')
        modele = request.args.get('modele', '')

        logger.debug(f'MARQUE: {marque}')
        logger.debug(f'MODELE: {modele}')
        return process_row(table_ref_name, marque, modele)

    @api.expect(parser_table)
    @api.response(200, description='CSV file containing matching brand model and score like: \n\n RENAULT,CLIO,1.0\n RENAULT,CLIO,0.945 \n AUDI, TTS,1.0 \n RENAULT,CLIO,0.945 \n RENAULT,CLIO,0.95 \n RENAULT,CLIO,1.0')
    @api.produces(['text/csv'])
    def post(self, table_ref_name):
        """Normalize a table of brand and model using a defined referential table"""
        input_file = request.files['file']
        return process_csv(table_ref_name, input_file)

@api.route('/info/<string:table_ref_name>', doc=False)
@api.doc(params={'table_ref_name': 'Reference table [siv,cardisiac,siv_caradisiac]'})
class Information(Resource):
    """Docstring for MyClass. """

    @api.expect(parser)
    def get(self, table_ref_name):
        """Get brand and model image"""
        marque = request.args.get('marque', '')
        modele = request.args.get('modele', '')

        res_def = {'src': None, 'href': None}
        res = src_dict.get(table_ref_name, {}).get((marque, modele), res_def)
        return res

@api.route('/clean')
class Clean(Resource):
    """Docstring for MyClass. """

    @api.expect(parser)
    @api.marshal_with(CleanOutput, mask=None)
    def get(self):
        """Clean brand and model field"""
        marque = request.args.get('marque', '')
        modele = request.args.get('modele', '')
        row = dict(marque=marque, modele=modele)
        for column in ['marque', 'modele']:
            res = cleaning(row, column)
        return res

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0', port=PORT)