#!/usr/bin/env python
# coding: utf-8

# In[1]:
## Ranking proteins by stepwise forward selection, bootrapped forward and bootstrapped L1.

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split, KFold
import statsmodels.api as sm
from sklearn.metrics import accuracy_score, roc_auc_score, recall_score, precision_score, confusion_matrix, log_loss
from sklearn.metrics import precision_recall_curve, roc_curve
from sklearn.linear_model import LogisticRegression, LogisticRegressionCV
from sklearn.preprocessing import QuantileTransformer, PowerTransformer
from sklearn.feature_selection import SelectKBest, SelectFromModel, RFE, RFECV, SelectPercentile, SelectFpr, SelectFdr, SelectFwe
from sklearn.feature_selection import chi2, f_classif, mutual_info_classif, SelectFdr
from sklearn.ensemble import RandomForestClassifier
from boruta import BorutaPy
import matplotlib.pyplot as plt
import pickle
from sklearn.utils import resample
# import lifelines as ll
# from lifelines.utils.sklearn_adapter import sklearn_adapter
# CoxRegression = sklearn_adapter(ll.CoxPHFitter, event_col = 'event')
import sys
sys.path.append('/odinn/users/thjodbjorge/Python_functions/')
import Predict_functions as pf
from Calculate_score import calculate_metrics

raw_data = pd.read_csv('/odinn/users/thjodbjorge/Proteomics/Data/raw_with_info.csv',index_col = 'Barcode2d' )
probe_info = pd.read_csv('/odinn/users/thjodbjorge/Proteomics/Data/probe_info.csv', index_col = 'SeqId')

pn_info = pd.read_csv('/odinn/users/thjodbjorge/Proteomics/Data/pn_info_Mor/pn_info_Mor_event.csv',index_col = 'Barcode2d' )
probes_to_skip = pd.read_csv('/odinn/users/thjodbjorge/Proteomics/Data/probes_to_skip.txt')['probe']
nopro = pd.read_csv('/odinn/users/thjodbjorge/Proteomics/Data/no_protein_probes.txt', header = None)[0] # non-proten probes that were included
probes_to_skip = set(probes_to_skip).union(set(nopro))

folder = '/odinn/users/thjodbjorge/Proteomics/Mortality2/'
feat_folder = 'Features2/'
pred_folder = 'Predictions3/'

endpoints = ['death']
# endpoints = ['death','Cdeath','Gdeath','Ideath','Jdeath','Otherdeath']
# event_date = event_date_death
time_to_event = pn_info.time_to_death
no_event_before = pn_info.no_death_before
for endpoint in endpoints:
    if endpoint == 'death':
        use_event = pn_info.event_death
        print(use_event.sum())
    elif endpoint == 'Cdeath':
        use_event = pn_info.event_death & (pn_info.ICD_group == 'C')
        print(use_event.sum())
    elif endpoint == 'Gdeath':
        use_event = pn_info.event_death & (pn_info.ICD_group == 'G')
        print(use_event.sum())
    elif endpoint == 'Ideath':
        use_event = pn_info.event_death & (pn_info.ICD_group == 'I')
        print(use_event.sum())
    elif endpoint == 'Jdeath':
        use_event = pn_info.event_death & (pn_info.ICD_group == 'J')
        print(use_event.sum())
    elif endpoint == 'Otherdeath':
        use_event = pn_info.event_death & (~(pn_info.ICD_group == 'C')&~(pn_info.ICD_group == 'G')&~(pn_info.ICD_group == 'I')&~(pn_info.ICD_group == 'J'))
        print(use_event.sum())

y = []
for i in range(1,19):
    y.append(use_event & (time_to_event <= i))

kf = KFold(n_splits=10, random_state=10, shuffle=False)
I_train_main, I_test_main = train_test_split(pn_info.index, train_size=0.7, random_state = 10)
I_val_main, I_test_main = train_test_split(I_test_main, train_size=0.5, random_state = 10)

file = open(folder+"{}_keep_samples.pkl".format('Mor'),'rb')
keep_samples_dict = pickle.load(file)

print(keep_samples_dict.keys())
# keep_samples_keys = ['Old_18105', 'Old_60105', 'Old_6080','Old_18105_C', 'Old_18105_I', 'Old_18105_J', 'Old_18105_G','Old_18105_Other']
keep_samples_keys = ['Old_60105','Old_18105']#,'Old_6080']
# keep_samples_keys = ['Old_18105_Neoplasms','Old_18105_I','Old_18105_J','Old_18105_G','Old_18105_Other']
# keep_samples_keys = ['Old_18105_J','Old_18105_G','Old_18105_Other']
# keep_samples_keys = ['Old_60105']
# keep_samples_keys = ['Old_men_18105','Old_women_18105']

# K = [4, 9]
# K =[14]
# K = [4,9,0,1,2,3,5,6,7,8,10,11,12,13,14]
# K = [9,10,11,12,13,14]
K = [4,9]
for dataset in keep_samples_keys:
    print(dataset)
    keep_samples = keep_samples_dict[dataset]

    I_train = I_train_main.intersection(keep_samples)#.intersection(have_prs)
    I_test = I_val_main.intersection(keep_samples)#.intersection(have_prs)

    print('Training set: {}, MI within 15: {}, 10: {}, 5: {}, 2: {}'.format(len(I_train),y[14][I_train].sum(),y[9][I_train].sum(),y[4][I_train].sum(),y[1][I_train].sum()))
    print('Test set: {}, MI within 15: {}, 10: {}, 5: {}, 2: {}'.format(len(I_test),y[14][I_test].sum(),y[9][I_test].sum(),y[4][I_test].sum(),y[1][I_test].sum()))

        # ### Select data and normalize
    X = np.log(raw_data.iloc[:,16:].drop(probes_to_skip,axis=1))

    all_protein = X.columns
    X['sex'] = pn_info[['sex']].values-1
    X['age'] = pn_info[['Age_at_sample_collection_2']].values

    X['age2'] = X['age']**2
#     X['age3'] = X['age']**3
    X['agesex'] = X['age']*X['sex']
    X['age2sex'] = X['age2']*X['sex']

    agesex = ['age','sex','agesex','age2','age2sex']

    X['lnage'] = np.log(X['age'])
    X['lnage2'] = X['lnage']**2

    X['PAD'] = pn_info['PAD']
    no_bmi = (X['PAD'].isna())
    no_bmi_ind = X[no_bmi].index
    X.loc[I_train.intersection(no_bmi_ind),'PAD'] = X.loc[I_train].PAD.mean()
    X.loc[I_test.intersection(no_bmi_ind),'PAD'] = X.loc[I_test].PAD.mean()

    X['CAD'] = ~pn_info.no_CAD_before
    X['MI'] = ~pn_info.no_MI_before
    X['cancer'] = pn_info.cancer_margin
    X['ApoB'] = X['SeqId.2797-56']
    X['Smoker'] = pn_info['Smoker'].astype(int).values
    X['diabetes'] = pn_info['T2D'].astype(int).values
    X['HTN_treated'] = pn_info[['HTN_treated']].astype(int).values
#     X['statin'] = pn_info['statin'].astype(int).values
    X['statin'] = pn_info['statin_estimate_unsure'].astype(int).values
    X['ApoBstatin']  = X['ApoB']*X['statin']

    X['cancer1y']  = pn_info['cancer1y']
    X['cancer5y']  = pn_info['cancer5y']

    X['GDF15'] = X['SeqId.4374-45'].copy()
    X['GDF152'] = X['GDF15']**2
    X['GDF15age']  = X['GDF15']*X['age']
    X['GDF15sex']  = X['GDF15']*X['sex']

    X['bmi'] = pn_info['bmi']

    no_bmi = (X['bmi'].isna())
    no_bmi_ind = X[no_bmi].index
#     X.loc[no_bmi_ind,'bmi'] = X.loc[I_train].bmi.mean()
    X.loc[I_train.intersection(no_bmi_ind),'bmi'] = X.loc[I_train].bmi.mean()
    X.loc[I_test.intersection(no_bmi_ind),'bmi'] = X.loc[I_test].bmi.mean()

    X['bmi2'] = X['bmi']*X['bmi']

    X['Platelets'] = pn_info['Platelets']
    no_p = (X['Platelets'].isna()); print(no_p.sum())
    no_p_ind = X[no_p].index
    X.loc[I_train.intersection(no_p_ind),'Platelets'] = X.loc[I_train].Platelets.mean()
    X.loc[I_test.intersection(no_p_ind),'Platelets'] = X.loc[I_test].Platelets.mean()
    X['Platelets2'] = X['Platelets']*X['Platelets']

    X['Creatinine'] = pn_info['Creatinine']
    no_p = (X['Creatinine'].isna()); print(no_p.sum())
    no_p_ind = X[no_p].index
    X.loc[I_train.intersection(no_p_ind),'Creatinine'] = X.loc[I_train].Creatinine.mean()
    X.loc[I_test.intersection(no_p_ind),'Creatinine'] = X.loc[I_test].Creatinine.mean()

    X['Triglycerides'] = pn_info['Triglycerides']
    no_p = (X['Triglycerides'].isna()); print(no_p.sum())
    no_p_ind = X[no_p].index
    X.loc[I_train.intersection(no_p_ind),'Triglycerides'] = X.loc[I_train].Triglycerides.mean()
    X.loc[I_test.intersection(no_p_ind),'Triglycerides'] = X.loc[I_test].Triglycerides.mean()

    X['bmiage'] = X['bmi']*X['age']
    X['bmisex'] = X['bmi']*X['sex']
    X['bmi2age'] = X['bmi2']*X['age']
    X['bmi2sex'] = X['bmi2']*X['sex']
    X['PADage'] = X['PAD']*X['age']
    X['PADsex'] = X['PAD']*X['sex']

    X['ApoBage']  = X['ApoB']*X['age']
    X['Smokerage'] = X['Smoker']*X['age']
    X['diabetesage'] = X['diabetes']*X['age']
    X['statinage'] = X['statin']*X['age']
    X['CADage'] = X['CAD']*X['age']
    X['MIage'] = X['MI'] * X['age']
    X['HTN_treatedage'] =  X['age']*X['HTN_treated']
    X['cancerage'] = X['age']*X['cancer']

    X['Plateletsage'] = X['Platelets']*X['age']
    X['Creatinineage'] = X['Creatinine']*X['age']
    X['Triglyceridesage'] = X['Triglycerides']*X['age']
    X['Platelets2age'] = X['Platelets2']*X['age']

    X['cancer1yage'] = X['cancer1y']*X['age']
    X['cancer5yage'] = X['cancer5y']*X['age']

    X['ApoBsex']  = X['ApoB']*X['sex']
    X['Smokersex'] = X['Smoker']*X['sex']
    X['diabetessex'] = X['diabetes']*X['sex']
    X['statinsex'] = X['statin']*X['sex']
    X['CADsex'] = X['CAD']*X['sex']
    X['MIsex'] = X['MI'] * X['sex']
    X['HTN_treatedsex'] =  X['sex']*X['HTN_treated']
    X['cancersex'] = X['sex']*X['cancer']

    X['Plateletssex'] = X['Platelets']*X['sex']
    X['Creatininesex'] = X['Creatinine']*X['sex']
    X['Triglyceridessex'] = X['Triglycerides']*X['sex']

    X = X.join(pd.get_dummies(pn_info['agebin'],drop_first = True,prefix='age'))
    X['ageage2'] = X['age']*X['age_2.0']
    X['ageage3'] = X['age']*X['age_3.0']
    X['ageage4'] = X['age']*X['age_4.0']

    agebins = ['age_2.0','age_3.0','age_4.0', 'ageage2','ageage3','ageage4']
    agebinssex = [s+'sex' for s in agebins]
    X[agebinssex] = (X[agebins].transpose()*X['sex']).transpose()

    PRS = ['nonHDL_prs', 'HT_prs', 'CAD_prs', 'Cancer_prs', 'Stroke2_prs', 'alz_Jansen',
       'pgc_adhd_2017', 'PD_Nalls_2018', 'edu_160125', 'dep_2018', 'bpd_2018',
       'giant_bmi', 'schizo_clozuk', 'iq_2018', 'ipsych_pgc_aut_2017',
       'pgc_Anorexia_2019']
    X[PRS] = pn_info[PRS]

    trad = ['ApoB','Smoker','diabetes','HTN_treated','statin','CAD','MI','bmi','bmi2']
    tradage = ['ApoBage','Smokerage','diabetesage','CADage','MIage','HTN_treatedage','bmiage']
    tradsex = ['ApoBsex','Smokersex','diabetessex','CADsex','MIsex','HTN_treatedsex','bmisex']

    tradcoxR = ['Smoker','Smokersex','diabetes','diabetesage','HTN_treated','HTN_treatedage','MI','MIage','CAD','bmi','bmiage','statin','statinage']
    tradextralog = ['Smokersex','diabetessex','CADsex','CADage','MIage','HTN_treatedage','bmiage','statinage','bmi2age','ApoBstatin']
    tradextralognosex = ['CADage','MIage','HTN_treatedage','bmiage']
    tradblood = ['Creatinine','Triglycerides','Platelets','Platelets2','Plateletsage','Creatinineage','Platelets2age']
    tradcancer = ['cancer','cancerage']
    extra_cancer = ['cancer1y','cancer5y','cancer1yage','cancer5yage']

    X_train = X.loc[I_train]
    X_test = X.loc[I_test]

    train_mean = X_train.mean()
    train_std = X_train.std()

    X_train = (X_train-train_mean)/train_std
    X_test = (X_test-train_mean)/train_std

        ## For survival analysis
    X_train['event'] = use_event[I_train]
    X_test['event'] = use_event[I_test]

    tte_train = time_to_event[I_train]
    tte_test = time_to_event[I_test]

    ysurv_train = pd.DataFrame()
    ysurv_train['event'] = use_event[I_train]
    ysurv_train['time_to_event'] = time_to_event[I_train]

    for k in K:
        y_train = y[k][I_train]
        y_test = y[k][I_test]

        ycsurv_train = pd.DataFrame()
        ycsurv_train['event'] = y_train
        ycsurv_train['time_to_event'] = np.where(tte_train<= (k+1),tte_train,k+1)

        X_train['y'] = y_train
        X_test['y'] = y_test

        try:

            file = open(folder+feat_folder+"{}_{}_features.pkl".format(endpoint,dataset),'rb')
            features_dict = pickle.load(file)

            boruta = sorted(features_dict['{}_boruta_y{}'.format(dataset,k)])
            boruta.remove('SeqId.4374-45')
            boruta.remove('SeqId.11388-75')
            print('features_loaded')

        except Exception as e:
            print(e)
            print('No features')

        try:
            file = open(folder+pred_folder+"{}_{}_forward_noGDF15HE4.pkl".format(endpoint,dataset),'rb')
            pred_dict = pickle.load(file)
        except:
            pred_dict = {}

        ## Forward feature selection
        if 1:
            np.random.seed(10)
            boot_protein_order = [boruta]
            for b in range(1):
    #             ind = resample(I_train,replace=True)
                ind = I_train
                y_use = y_train[ind]
                use_const = []
                use_const.extend(agesex )
#                 use_const = ['age','age2']

                feat = []
                feat.extend(boruta)
    #             num_pro = len(feat)
                num_pro = 100
                protein_order = []
                for i in range(num_pro):#len(protein)+1):
                    mmp_pred_dict = {}
                    candidates = feat
                    print(len(candidates))
                    for c in candidates:
                        use = use_const + [c]
                        X_use = X_train.loc[ind,use]
                        model = sm.Logit(y_use,sm.add_constant(X_use))
                        res = model.fit(disp=0)

    #                     mmp_pred_dict[c] = [res.pvalues[-1]]
                        mmp_pred_dict[c] = [res.llf]

                    mmp_pred = pd.DataFrame.from_dict(mmp_pred_dict).transpose()
                    mmp_pred.columns = ['log_likelihood']

                    best_candidate = [mmp_pred.sort_values('log_likelihood', ascending = False).iloc[0,:].name]
                    feat.remove(best_candidate[0])
                    protein_order.append([best_candidate[0],mmp_pred.sort_values('log_likelihood', ascending = False).iloc[0,0]])
                    use_const = use_const + best_candidate

    #             boot_protein_order.append(pd.DataFrame(protein_order).sort_values(0).index)
                boot_protein_order.append(protein_order)
            pred_dict['{}_y{}_asprotein_lr'.format(dataset,k)] = boot_protein_order

            f = open(folder+pred_folder+"{}_{}_forward_noGDF15HE4.pkl".format(endpoint,dataset),"wb")
            pickle.dump(pred_dict,f)
            f.close()

        ## Bootstrap forward order
        if 0:
            np.random.seed(10)
            boot_protein_order = pd.DataFrame(index = boruta)

            for b in range(10):
                ind = resample(I_train,replace=True)
                y_use = y_train[ind]
                use_const = []
                use_const.extend(agesex)

                feat = []
                feat.extend(boruta)
    #             num_pro = len(feat)
                num_pro = 100
                protein_order = []
                for i in range(num_pro):#len(protein)+1):
                    mmp_pred_dict = {}
                    candidates = feat
                    print(len(candidates))
                    for c in candidates:
                        use = use_const + [c]
                        X_use = X_train.loc[ind,use]
                        model = sm.Logit(y_use,sm.add_constant(X_use))
                        res = model.fit(disp=0)
    #                     mmp_pred_dict[c] = [res.pvalues[-1]]
                        mmp_pred_dict[c] = [res.llf]

                    mmp_pred = pd.DataFrame.from_dict(mmp_pred_dict).transpose()
                    mmp_pred.columns = ['log_likelihood']

                    best_candidate = [mmp_pred.sort_values('log_likelihood', ascending = False).iloc[0,:].name]
                    feat.remove(best_candidate[0])
                    protein_order.append(best_candidate[0])
                    use_const = use_const + best_candidate

                boot_protein_order.loc[protein_order,b] = pd.DataFrame(protein_order).index
            pred_dict['{}_y{}_asprotein_lr_boot'.format(dataset,k)] = boot_protein_order

            f = open(folder+pred_folder+"{}_{}_forward.pkl".format(endpoint,dataset),"wb")
            pickle.dump(pred_dict,f)
            f.close()

        ## Recursive backward elimination feature ranking.
        if 0:
            use = []
            use.extend(agesex)
            use.extend(boruta)

            model = LogisticRegression(penalty='none',n_jobs=-1, solver ='saga', max_iter=2000, random_state = 10)
            feat_selector = RFE(model, step=1, n_features_to_select = 1)
            feat_selector.fit(X_train[use], y_train)
            pred_dict['{}_y{}_asprotein_RFElr'.format(dataset,k)] = pd.DataFrame(feat_selector.ranking_,index=use)

            f = open(folder+pred_folder+"{}_{}_forward.pkl".format(endpoint,dataset),"wb")
            pickle.dump(pred_dict,f)
            f.close()

        ## Forward feature selection baseline
        if 0:
            np.random.seed(10)
            boot_protein_order = [boruta]
            for b in range(1):
    #             ind = resample(I_train,replace=True)
                ind = I_train
                y_use = y_train[ind]
                use_const = []
                use_const.extend(agesex )
                use_const.extend(trad)
                use_const.extend(tradextralog)
                use_const.extend(tradcancer)

                feat = []
                feat.extend(boruta)
    #             num_pro = len(feat)
                num_pro = 100
                protein_order = []
                for i in range(num_pro):#len(protein)+1):
                    mmp_pred_dict = {}
                    candidates = feat
                    print(len(candidates))
                    for c in candidates:
                        use = use_const + [c]
                        X_use = X_train.loc[ind,use]
                        model = sm.Logit(y_use,sm.add_constant(X_use))
                        res = model.fit(disp=0)

    #                     mmp_pred_dict[c] = [res.pvalues[-1]]
                        mmp_pred_dict[c] = [res.llf]

                    mmp_pred = pd.DataFrame.from_dict(mmp_pred_dict).transpose()
                    mmp_pred.columns = ['log_likelihood']

                    best_candidate = [mmp_pred.sort_values('log_likelihood', ascending = False).iloc[0,:].name]
                    feat.remove(best_candidate[0])
                    protein_order.append([best_candidate[0],mmp_pred.sort_values('log_likelihood', ascending = False).iloc[0,0]])
                    use_const = use_const + best_candidate

    #             boot_protein_order.append(pd.DataFrame(protein_order).sort_values(0).index)
                boot_protein_order.append(protein_order)
            pred_dict['{}_y{}_baselineprotein_lr'.format(dataset,k)] = boot_protein_order

            f = open(folder+pred_folder+"{}_{}_forward.pkl".format(endpoint,dataset),"wb")
            pickle.dump(pred_dict,f)
            f.close()

        ## Forward feature selection baseline only men or only women
        if 0:
            np.random.seed(10)
            boot_protein_order = [boruta]
            for b in range(1):
    #             ind = resample(I_train,replace=True)
                ind = I_train
                y_use = y_train[ind]
                use_const = []
                use_const.extend(['age','age2'])
                use_const.extend(trad)
                use_const.extend(tradextralognosex)
                use_const.extend(tradcancer)

                feat = []
                feat.extend(boruta)
    #             num_pro = len(feat)
                num_pro = 100
                protein_order = []
                for i in range(num_pro):#len(protein)+1):
                    mmp_pred_dict = {}
                    candidates = feat
                    print(len(candidates))
                    for c in candidates:
                        use = use_const + [c]
                        X_use = X_train.loc[ind,use]
                        model = sm.Logit(y_use,sm.add_constant(X_use))
                        res = model.fit(disp=0)

    #                     mmp_pred_dict[c] = [res.pvalues[-1]]
                        mmp_pred_dict[c] = [res.llf]

                    mmp_pred = pd.DataFrame.from_dict(mmp_pred_dict).transpose()
                    mmp_pred.columns = ['log_likelihood']

                    best_candidate = [mmp_pred.sort_values('log_likelihood', ascending = False).iloc[0,:].name]
                    feat.remove(best_candidate[0])
                    protein_order.append([best_candidate[0],mmp_pred.sort_values('log_likelihood', ascending = False).iloc[0,0]])
                    use_const = use_const + best_candidate

    #             boot_protein_order.append(pd.DataFrame(protein_order).sort_values(0).index)
                boot_protein_order.append(protein_order)
            pred_dict['{}_y{}_baselineprotein_lr'.format(dataset,k)] = boot_protein_order

            f = open(folder+pred_folder+"{}_{}_forward.pkl".format(endpoint,dataset),"wb")
            pickle.dump(pred_dict,f)
            f.close()


from operator import itemgetter

class File:
    def __init__(self, id, filename, memory, log_id):
        self.id = id
        self.filename = filename
        self.memory = memory
        self.log_id = log_id

class Catalog:
    def __init__(self, id, name):
        self.id = id
        self.name = name

class Fillog:
    def __init__(self, log_id, fillog_id):
        self.log_id = log_id
        self.fillog_id = fillog_id
def first_f(one_to_many,catalogs):
    res_1 = []
    res_1 = [(c.name, list(filename for filename, _, name in one_to_many if name == c.name)) for c in catalogs if
             c.name[0] == 'А']
    return res_1
def sec_f(one_to_many,catalogs):
    res_2 = []
    for c in catalogs:
        log_files = list(filter(lambda x: x[2] == c.name, one_to_many))
        if len(log_files) > 0:
            mem = [size for _, size, _ in log_files]
            max_mem = max(mem)
            res_2.append((c.name, max_mem))
    res_2sort = sorted(res_2, key=itemgetter(1))
    return(res_2sort)

def third_f(many_to_many, files,catalogs,file_catalogs):
    many_to_many_temp = [(f.filename, c.log_id,)
                         for f in files
                         for c in file_catalogs
                         if f.id == c.fillog_id]
    many_to_many = [(x[0], logs.name)
                    for x in many_to_many_temp
                    for logs in catalogs
                    if logs.id == x[1]]
    res_3 = sorted(many_to_many, key=itemgetter(1), reverse=True)
    return res_3

def main():
    catalogs = [
        Catalog(1, 'Путешествия'),
        Catalog(2, 'Сериалы'),
        Catalog(3, 'АСОИУ'),
        Catalog(4, 'Алиса')]

    files = [
        File(1, 'Аннотация 1-10', 36, 3),
        File(2, 'Новая Зеландия', 1560, 1),
        File(3, 'Акация', 5432, 2),
        File(4, 'Концерт в Самаре 11.09', 2054, 4),
        File(5, 'Контрольный вопрос 310', 53, 3),
        File(6, 'Абхазия', 5783, 1) ]

    file_catalogs = [
        Fillog(1, 2),
        Fillog(1, 6),
        Fillog(2, 3),
        Fillog(3, 1),
        Fillog(3, 5),
        Fillog(4, 4)]

    one_to_many = [(f.filename, f.memory, c.name)
                   for c in catalogs
                   for f in files
                   if f.log_id == c.id]

    many_to_many_temp = [(c.name, fc.log_id, fc.fillog_id)
                         for c in catalogs
                         for fc in file_catalogs
                         if c.id == fc.fillog_id]

    many_to_many = [(f.filename, f.memory, name)
                    for name, log_id, fillog_id in many_to_many_temp
                    for f in files
                    if f.id == fillog_id]

    print('Задание Г1')

    print(first_f(one_to_many,catalogs))

    print('\nЗадание Г2:')
    print((sec_f(one_to_many,catalogs)))

    print('\nЗадание Г3:')

    print(third_f(many_to_many,files,catalogs,file_catalogs))

if __name__ == '__main__':
    main()


import sqlite3
import json
from models.entry import JournalEntry

def get_all_JournalEntries():
    # Open a connection to the database
    with sqlite3.connect("./dailyjournal.sqlite3") as conn:

        # Just use these. It's a Black Box.
        conn.row_factory = sqlite3.Row
        db_cursor = conn.cursor()

        # Write the SQL query to get the information you want
        db_cursor.execute("""
        SELECT
            e.id,
            e.concept,
            e.entry,
            e.date,
            e.mood_id
        FROM JournalEntries e
        """)

        # Initialize an empty list to hold all entry representations
        journalEntries = []

        # Convert rows of data into a Python list
        dataset = db_cursor.fetchall()

        # Iterate list of data returned from database
        for row in dataset:

            # Create an entry instance from the current row.
            # Note that the database fields are specified in
            # exact order of the parameters defined in the
            # entry class above.
            entry = JournalEntry(row['id'], row['concept'], row['entry'],
                                 row['date'], row['mood_id'])

            journalEntries.append(entry.__dict__)

    # Use `json` package to properly serialize list as JSON
    return json.dumps(journalEntries)

# Function with a single parameter
def get_single_entry(id):
    with sqlite3.connect("./dailyjournal.sqlite3") as conn:
        conn.row_factory = sqlite3.Row
        db_cursor = conn.cursor()

        # Use a ? parameter to inject a variable's value
        # into the SQL statement.
        db_cursor.execute("""
        SELECT
            e.id,
            e.concept,
            e.entry,
            e.date,
            e.mood_id
        FROM JournalEntries e
        WHERE e.id = ?
        """, (id, ))

        # Load the single result into memory
        data = db_cursor.fetchone()

        # Create an entry instance from the current row
        entry = JournalEntry(data['id'], data['concept'], data['entry'],
                             data['date'], data['mood_id'])

        return json.dumps(entry.__dict__)

def delete_entry(id):
    with sqlite3.connect("./dailyjournal.sqlite3") as conn:
        db_cursor = conn.cursor()

        db_cursor.execute("""
        DELETE FROM JournalEntries
        WHERE id = ?
        """, (id, ))

def get_entries_by_search(searchTerm):

    with sqlite3.connect("./dailyjournal.sqlite3") as conn:
        conn.row_factory = sqlite3.Row
        db_cursor = conn.cursor()
        # Write the SQL query to get the information you want
        db_cursor.execute("""
          SELECT
            e.id,
            e.concept,
            e.entry,
            e.date,
            e.mood_id
        FROM JournalEntries e
        WHERE e.entry LIKE ?
        """, ( f'%{searchTerm}%', ))

        entries = []
        dataset = db_cursor.fetchall()

        for row in dataset:
            entry = JournalEntry(
                row['id'], row['concept'], row['entry'], row['date'], row['mood_id'])

            entries.append(entry.__dict__)

    return json.dumps(entries)


import math

import torch
from torch import nn

from model.BaseModules import TransformerEncoderLayer, TransformerDecoderLayer
from model.BaseModules import TemporalConvBlock
from model.BaseModules import MLP
from model.ProbModels import GaussianMLP

class BaseEncoder(nn.Module):
    def __init__(self):
        super(BaseEncoder, self).__init__()

    def forward(self, *inputs):
        raise NotImplementedError

    def register_hooks(self):
        pass

class AutoEncoder(BaseEncoder):
    encoder_arch = MLP
    decoder_arch = MLP
    encoder: MLP
    decoder: MLP

    def __init__(self, input_size, output_size, layer_sizes, unique_kwargs=None, **kwargs):
        super(AutoEncoder, self).__init__()
        if unique_kwargs is None:
            unique_kwargs = {}
        encoder_kwargs = unique_kwargs.get('Encoder_Setting', {})
        decoder_kwargs = unique_kwargs.get('Decoder_Setting', {})
        self.encoder = self.encoder_arch(input_size, output_size, layer_sizes, **encoder_kwargs, **kwargs)
        self.decoder = self.decoder_arch(output_size, input_size, layer_sizes[::-1], **decoder_kwargs, **kwargs)

    def forward(self, x, **kwargs):
        return self.encode(x, **kwargs)

    def encode(self, x, **kwargs):
        out = self.encoder(x, **kwargs)
        return out

    def decode(self, hidden_state, **kwargs):
        return self.decoder(hidden_state, **kwargs)

class VariationalAutoEncoder(AutoEncoder):
    encoder_arch = GaussianMLP
    encoder: GaussianMLP

    def encode(self, x, return_dist=False, **kwargs):
        out = self.encoder(x, **kwargs)
        if return_dist:
            return out, self.encoder.distribution
        return out

AE = AutoEncoder
VAE = VariationalAutoEncoder

class TemporalConvNet(nn.Module):
    def __init__(self, input_size, output_size, num_channels, kernel_size, dropout,
                 stride=1, time_len=24, norm_layer=None, block_arch=TemporalConvBlock):
        super(TemporalConvNet, self).__init__()
        if norm_layer is None:
            norm_layer = nn.BatchNorm1d
        self._norm_layer = norm_layer
        self.stride = stride
        self.kernel_size = kernel_size
        self.dropout = dropout
        layers = [nn.Linear(input_size, output_size)]
        num_levels = len(num_channels)
        num_channels += [output_size]
        for i in range(num_levels):
            dilation = 2 ** i
            layers.append(block_arch(num_channels[i], num_channels[i + 1], kernel_size, stride=stride,
                                     dilation=dilation, padding=(kernel_size - 1) * dilation,
                                     dropout=dropout, time_len=time_len))
        self.layers = nn.Sequential(*layers)

    def forward(self, x, **kwargs):
        """
        :param x: Float32 Tensor with shape = (*, Time_Channels, State)
        :return: FLoat32 Tensor with shape = (*, Time_Channels, State)
        """
        old_shape = x.shape
        out = x.reshape(-1, *old_shape[-2:])
        out = self.layers(out)
        out = out.reshape(*old_shape[:-2], *out.shape[-2:])
        return out

TCN = TCNEncoder = TemporalConvNet

class TCBAEncoder(BaseEncoder):
    def __init__(self, input_size, output_size, tcn_kwargs, attn_kwargs, **kwargs):
        super(TCBAEncoder, self).__init__()
        self.TempConvModule = TemporalConvNet(input_size, output_size, **tcn_kwargs)
        self.SelfAttentionModule = TransformerEncoderLayer(output_size, **attn_kwargs)
        self.register_hooks()

    def forward(self, x, **kwargs):
        out = self.TempConvModule(x)
        out = self.SelfAttentionModule(out)
        return out

    def register_hooks(self):
        def attn_pre_fwd_hook(module, inputs):
            inputs = list(inputs)
            inputs[0] = inputs[0].permute(2, 1, 0, 3)[-1]
            return inputs

        def attn_fwd_hook(module, inputs, output):
            return output.transpose(0, 1)

        self.SelfAttentionModule.register_forward_pre_hook(attn_pre_fwd_hook)
        self.SelfAttentionModule.register_forward_hook(attn_fwd_hook)
        super(TCBAEncoder, self).register_hooks()

class BATCEncoder(TCBAEncoder):
    def __init__(self, input_size, output_size, tcn_kwargs, attn_kwargs, **kwargs):
        super(BATCEncoder, self).__init__(output_size, output_size, tcn_kwargs, attn_kwargs, **kwargs)
        self.linear = nn.Linear(input_size, output_size)

    def forward(self, x, **kwargs):
        # x.shape = Batch, Building, Time, State
        out = self.linear(x.transpose(0, 1))
        seq_len, other_dims, state_dim = out.shape[0], out.shape[1:-1], out.shape[-1]
        out = out.reshape(seq_len, -1, state_dim)
        out = self.SelfAttentionModule(out)
        out = out.reshape(seq_len, *other_dims, state_dim).transpose(0, 1)
        out = self.TempConvModule(out)
        return out

class HRLEncoder(BaseEncoder):
    def __init__(self, input_size, output_size, tcn_kwargs, att_kwargs, **kwargs):
        super(HRLEncoder, self).__init__()
        self.GlobalTCNModule = TemporalConvNet(input_size, output_size, **tcn_kwargs)
        self.Hour2DayModule = TransformerDecoderLayer(output_size, **att_kwargs)
        self.Day2WeekModule = TransformerDecoderLayer(output_size, **att_kwargs)
        self.linear_in = nn.Linear(input_size, output_size)
        self.linear_out = nn.Linear(output_size * 3, output_size)

    def forward(self, x, **kwargs):
        """
        :param x: (batch, building, time=168, state=37)
        :return:
        """
        time_length = 12
        assert x.shape[-2] % time_length == 0
        local_state = self.linear_in(x[:, :, -1])  # (batch, building, state)
        hour_states = self.GlobalTCNModule(x)
        hour_chunks = torch.stack(hour_states.split(time_length, -2))  # (Chunks, Batch, Building, Time_Length, State)

        # <---------------- Hour2Day Attention Module --------------------->
        # Use the last time & last chunk for output state
        # -> (1, Batch, Building, Chunks, State)
        daily_states = self._temporal_abstraction(hour_chunks, self.Hour2DayModule)
        day_state = daily_states[-1, :, :, -1]

        # <----------------- Day2Week Attention Module --------------------->
        # -> (1, Batch, Building, 1, State)
        weekly_states = self._temporal_abstraction(daily_states, self.Day2WeekModule)
        week_state = weekly_states[-1, :, :, -1]

        # <---------------------- output layer ----------------------------->
        out = self.linear_out(torch.cat((local_state, day_state, week_state), -1))
        return out

    @staticmethod
    def _temporal_abstraction(x, attn_module):
        """

        :param x: FloatTensor with shape = (Chunks, Batch, Building, Time_Length, State)
        :param attn_module: attention module
        :return: FloatTensor with shape = (1, Batch, Building, Chunks, State)
        """
        x = x.transpose(0, -2)  # (Time_Length, Batch, Building, Chunks, State)
        old_shape = x.shape[1:-1]
        src = x.reshape(x.size(0), -1, x.size(-1))
        tgt = src[-1:]
        out = attn_module(tgt, src)
        out = out.reshape(-1, *old_shape, out.size(-1))
        return out

class ROMAEncoder(BaseEncoder):
    # Add kwargs if needed
    __RNNArch__ = nn.GRU

    def __init__(self, input_size, output_size, role_size, embed_kwargs=None, infer_kwargs=None, rnn_kwarg=None,
                 **other_kwargs):
        super(ROMAEncoder, self).__init__()
        if embed_kwargs is None:
            embed_kwargs = {}
        if infer_kwargs is None:
            infer_kwargs = {}
        if rnn_kwarg is None:
            rnn_kwarg = {}

        default = dict(
            norm_layer=nn.BatchNorm1d,
            activation=nn.LeakyReLU(inplace=True),
            out_activation=nn.LeakyReLU(inplace=True),
            std_lbound=math.log(2e-3),
            std_ubound=+math.inf
        )

        for k in default:
            embed_kwargs.setdefault(k, default[k])
            infer_kwargs.setdefault(k, default[k])

        self.RoleEncoder = GaussianMLP(input_size, role_size, **embed_kwargs)
        self.PosteriorEstimator = GaussianMLP(input_size + output_size, role_size, **infer_kwargs)
        self.LinearEncoder = MLP(input_size, output_size, out_activation=nn.ReLU(inplace=True))
        self.RNNUnit = self.__RNNArch__(output_size, output_size, **rnn_kwarg)

        self._reset_rnn_params()

    def _reset_rnn_params(self):
        for p in self.RNNUnit.parameters():
            if p.ndim > 1:
                torch.nn.init.orthogonal_(p)

    def forward(self, x, h_in=None, train_mode=False, deterministic=False, **other_kwargs):
        """

        :param x: (*, Time, Input_Size)
        :param h_in: (1, *, Output_Size), default=None, if None, use zero vector instead.
        :param train_mode: See return for details.
        :param deterministic: Whether to sample from distribution
        :param other_kwargs:
        :return: (output, h_out, info)
            output: (*, Time, Output_Size)
            h_out: (1, **, Output_Size)
            info: If train_mode is False, info = role_embed
                               otherwise, info = (role_embed, role_infer, dist_embed, dist_infer)
            role_embed, role_infer: (*, Time, role_size)
        """
        def to_seq_first(tensor):
            tensor = tensor.unsqueeze(0).transpose(0, -2)
            return tensor.reshape(tensor.size(0), -1, tensor.size(-1))

        def undo_seq_first(tensor, lead_dims):
            tensor = tensor.transpose(0, -2)
            return tensor.reshape(*lead_dims, *tensor.shape[-2:])

        lead_dim = x.shape[:-2]
        role_embed = self.RoleEncoder(x, deterministic=deterministic)
        dist_embed = self.RoleEncoder.distribution

        h_s = self.LinearEncoder(to_seq_first(x))
        h_s, h_out = self.RNNUnit(h_s, h_in)
        out = undo_seq_first(h_s, lead_dim)
        if train_mode:
            if h_in is None:
                h_init = torch.zeros_like(h_s[-1], device=h_s.device).reshape(1, -1, h_s.size(-1))
            else:
                h_init = self.get_hidden_init(h_in)
            h_infer = torch.cat((h_init, h_s[:-1]))
            h_infer = undo_seq_first(h_infer, lead_dim)
            role_infer = self.PosteriorEstimator(torch.cat((x, h_infer), -1), deterministic=deterministic)
            dist_infer = self.PosteriorEstimator.distribution
            info = (role_embed, role_infer, dist_embed, dist_infer)
        else:
            info = role_embed

        return out, h_out, info

    def get_hidden_init(self, h_in):
        time_idx = -2 if self.RNNUnit.bidirectional else -1
        if isinstance(self.RNNUnit, nn.LSTM):
            h_in = h_in[0]
        return h_in[time_idx].unsqueeze(0)

class ROMALSTMEncoder(ROMAEncoder):
    __RNNArch__ = nn.LSTM


from copy import deepcopy
import csv
from datetime import datetime
import glob
import operator
from data import getStocks, getData
from objects import Indicator, ReqParam, Stock, DayTrades
from talib_processes import getIndicatorValues, getPeriodStr, getRequirementData
import pandas as pd
import numpy as np
import util
import time
from util import removeNaN

indicatorMap = {}
resMap = {}
adxMap = {}
chaikinMap = {}
histMap = {}
volMap = {}
natrMap = {}

def getRes(indicator: Indicator, max_buy_val, min_sell_val):
    stocks = getStocks()
    for count, ticker in enumerate(stocks):
        df = pd.DataFrame()
        df[indicator.name] = indicatorMap.get(ticker + getPeriodStr(indicator))
        df['Res'] = np.where(df[indicator.name] <= max_buy_val, 1, np.where(df[indicator.name] >= min_sell_val, -1, 0))
        resMap[ticker + getPeriodStr(indicator) + '_mbv_' + str(max_buy_val) + '_msv_' + str(min_sell_val)] = util.removeNaN(df['Res'].values)

def checkRequirement(req: ReqParam, position: int=None, vals: list=None) -> bool:
    if req.condition == 0:
        return True
    elif req.condition != 0 and len(vals) == 0:
        return False
    elif req.condition == 1:
        if position is None or vals is None or req.adx_min is None:
            raise ValueError('Need values for position, adx and adx_min')
        return vals[position] > req.adx_min
    elif req.condition == 2:
        if position is None or vals is None:
            raise ValueError('Need values for position and chaikin')
        return vals[position] > 0
    elif req.condition == 3:
        if position is None or vals is None or req.isBuy is None:
            raise ValueError('Need values for position, isBuy and hist')
        return vals[position] > 0 if req.isBuy else vals[position] < 0
    elif req.condition == 4:
        if position is None or vals is None:
            raise ValueError('Need values for position and vol')
        return vals[position] > 0
    elif req.condition == 5:
        if position is None or vals is None or req.isBuy is None:
            raise ValueError('Need values for position, isBuy and vals')
        return vals[position - 1] <= vals[position] if req.isBuy else vals[position - 1] >= vals[position]
    elif req.condition == 6:
        if position is None or vals is None or req.natr_min is None:
            raise ValueError('Need values for position, adx and adx_min')
        return vals[position] > req.natr_min
    else:
        raise ValueError('Need condition needs to be between 0 & 6')

def getRequirementValues(req: ReqParam, ticker, indicator: Indicator):
    if req.condition == 0:
        return []
    elif req.condition == 1:
        return adxMap[ticker + '_' + str(req.indicatorPeriod)]
    elif req.condition == 2:
        return chaikinMap[ticker + '_' + str(req.fastPeriod) + '_' + str(req.slowPeriod)]
    elif req.condition == 3:
        return histMap[ticker + '_' + str(req.indicatorPeriod) + '_' + str(req.fastPeriod) + '_' + str(req.slowPeriod)]
    elif req.condition == 4:
        return volMap[ticker + '_' + str(req.rollingPeriod)]
    elif req.condition == 5:
        return indicatorMap[ticker + getPeriodStr(indicator)].values
    elif req.condition == 6:
        return natrMap[ticker + '_' + str(req.indicatorPeriod)]

def addTrade(trades, ticker, date, val, volume=0, indicatorVal=0, isBuy=True):
    s = Stock()
    s.symbol = ticker
    s.wap = val

    if isBuy:
        s.indicatorVal = indicatorVal
        s.max_quantity = volume // 10

    if trades.get(date) is not None:
        ndt = trades.get(date)
        if isBuy:
            ndt.buys.append(s)
        else:
            ndt.sells.append(s)
        trades[date] = ndt

    else:
        dt = DayTrades()
        dt.date = date
        if isBuy:
            dt.buys.append(s)
        else:
            dt.sells.append(s)
        trades[date] = dt

def doCalculation(file, max_buy_val, min_sell_val, indicator, req1=ReqParam(), req2=ReqParam(), req3=ReqParam(), req4=ReqParam()):
    trades = {}
    total_res = np.zeros(15)
    stocks = getStocks(file)
    for count, ticker in enumerate(stocks):
        df = deepcopy(data[ticker])

        dates = df.index.tolist()
        vals = df['VWAP'].values
        volume = df['Volume'].values
        res = resMap[ticker + getPeriodStr(indicator) + '_mbv_' + str(max_buy_val) + '_msv_' + str(min_sell_val)]
        indicatorVals = removeNaN(indicatorMap[ticker + getPeriodStr(indicator)].values)

        r1 = getRequirementValues(req1, ticker, indicator)
        r2 = getRequirementValues(req2, ticker, indicator)
        r3 = getRequirementValues(req3, ticker, indicator)
        r4 = getRequirementValues(req4, ticker, indicator)

        max_len = util.getMaxDataPoints(res, indicatorVals, r1, r2, r3, r4)
        dates = util.getUsableDataPoints(dates, max_len)
        vals = util.getUsableDataPoints(vals, max_len)
        volume = util.getUsableDataPoints(volume, max_len)
        res = util.getUsableDataPoints(res, max_len)
        indicatorVals = util.getUsableDataPoints(indicatorVals, max_len)
        r1 = util.getUsableDataPoints(r1, max_len)
        r2 = util.getUsableDataPoints(r2, max_len)
        r3 = util.getUsableDataPoints(r3, max_len)
        r4 = util.getUsableDataPoints(r4, max_len)

        amt = np.zeros(len(res))
        gain = np.zeros(len(res))
        day_diff = np.zeros(len(res)).astype(int)
        sell_days = np.zeros(len(res)).astype(int)
        buy_signal = 0
        sell_signal = 0

        i = 1
        while i < len(res) - 1:
            if res[i] > buy_signal and checkRequirement(req1, i, r1) and checkRequirement(req2, i, r2):
                addTrade(trades, ticker, dates[i + 1][1], vals[i + 1], indicatorVal=indicatorVals[i],
                         volume=volume[i + 1])
                for j in range(i + 1, len(res) - 1):
                    if res[j] < sell_signal and checkRequirement(req3, j, r3) and checkRequirement(req4, j, r4):
                        addTrade(trades, ticker, dates[j + 1][1], vals[j + 1], isBuy=False)
                        amt[j] = vals[j] - vals[i]
                        gain[j] = (vals[j] - vals[i]) * 100 / vals[i]
                        day_diff[j] = j - i
                        i = j
                        break

                    if j == len(res) - 2:
                        i = j

            i += 1

        # to match the df with the values of the arrays
        df.drop(df.head(len(df) - max_len).index, inplace=True)

        df['Amt'] = amt[:]
        df['Gain'] = gain[:]
        df['Day_Diff'] = day_diff[:]
        df['Sell_Days'] = sell_days[:]

        total_res[0] += len(df['Gain'])
        total_res[1] += len(df['Gain'][df['Gain'] > 0])
        total_res[2] += len(df['Gain'][df['Gain'] < 0])
        total_res[3] += df['Gain'][(df['Gain'] > 0) | (df['Gain'] < 0)].sum()
        total_res[4] += df['Gain'][df['Gain'] > 0].sum()
        total_res[5] += df['Gain'][df['Gain'] < 0].sum()
        total_res[6] += df['Day_Diff'][(df['Gain'] > 0) | (df['Gain'] < 0)].sum()
        total_res[7] += df['Day_Diff'][df['Gain'] > 0].sum()
        total_res[8] += df['Day_Diff'][df['Gain'] < 0].sum()
        total_res[9] += len(df['Gain'][(df['Gain'] > 0) | (df['Gain'] < 0)])
        total_res[10] += len(df['Sell_Days'][df['Sell_Days'] > 0])

    result = np.zeros(13)
    result[0] = total_res[0]
    result[1] = total_res[1]
    result[2] = total_res[2]
    result[3] = total_res[1]/total_res[9] if total_res[1] > 100 else 0
    result[4] = total_res[3] / total_res[9]
    result[5] = total_res[4] / total_res[1]
    result[6] = total_res[5] / total_res[2]
    result[7] = total_res[6] / total_res[9]
    result[8] = total_res[7] / total_res[1]
    result[9] = total_res[8] / total_res[2]
    result[10] = total_res[9]

    group = file[32: len(file) - 8]

    result[11] = backTestStrategy(trades, group == '500')
    result[12] = backTestStrategy(trades, group == '500', date='2017-01-01')

    row = [indicator.name, group, max_buy_val, min_sell_val, indicator.period, indicator.fastperiod, indicator.slowperiod, indicator.timeperiod1, indicator.timeperiod2, indicator.timeperiod3, req1.__repr__(), req2.__repr__(), req3.__repr__(),
           req4.__repr__()]

    # check for diff append and extend https://stackoverflow.com/a/252711/5512020
    row.extend([element for element in result])
    # without newline = '' , a row is skipped in csv
    with open('./analyses/analyses.csv', 'a', newline='') as f:
        writer = csv.writer(f)
        writer.writerow(row)
    f.close()

def backTestStrategy(trades, isNifty500, date=None):
    buyDict = {}
    iniVal = 250000
    money = deepcopy(iniVal)
    nav = deepcopy(iniVal)

    if date is None:
        tradeSubset = trades
    else:
        # to analyse trades only for 2017, use filter
        timeStamp = datetime.strptime(date, "%Y-%m-%d").timestamp()
        tradeSubset = dict(filter(lambda x: datetime.strptime(x[0], "%Y-%m-%d").timestamp() > timeStamp, trades.items()))

    for key, dayTrade in sorted(tradeSubset.items(), key=lambda t: datetime.strptime(t[0], "%Y-%m-%d").timestamp()):
        for stock in dayTrade.sells:
            # cases where date is not None, buyDict will return none for early dates
            if buyDict.get(stock.symbol) is not None:
                # .4% loss due to brockerage and taxes
                money += buyDict.get(stock.symbol)[0] * stock.wap * 0.997
                nav += buyDict.get(stock.symbol)[0] * (stock.wap * 0.997 - buyDict.get(stock.symbol)[1])
                buyDict[stock.symbol] = (0, 0)

        # // gives int division so no decimal
        max_stocks = 15 if isNifty500 else 4
        max_per_stock = nav // max_stocks
        sortedBuys = sorted(dayTrade.buys, key=operator.attrgetter('indicatorVal'))
        for s in sortedBuys:
            amount = min(max_per_stock, money)
            quantity = min(amount // (s.wap * 1.003), s.max_quantity)
            buyDict[s.symbol] = (quantity, s.wap)
            money -= amount - amount % s.wap

    return nav - iniVal

def analyseIndicator(count):
    periods = np.array([6, 8, 10, 12, 14, 16, 18])
    maxBuyVals = np.array([-85, -80, -75, -70])
    minSellVals = np.array([-25, -20, -15])
    adx_periods = np.array([6, 8, 10, 12, 14])
    adx_mins = np.array([25, 30, 35])

    for i in range(len(periods)):
        indicator = Indicator()
        indicator.period = periods[i]
        indicator.name = 'WILLR'

        req2 = ReqParam()
        req2.condition = 5
        req2.indicatorPeriod = periods[i]
        req2.isBuy = True

        req3 = ReqParam()
        req3.condition = 5
        req3.indicatorPeriod = periods[i]
        req3.isBuy = False

        getIndicatorValues(indicator, indicatorMap)

        for j in range(len(maxBuyVals)):
            for k in range(len(minSellVals)):
                getRes(indicator, maxBuyVals[j], minSellVals[k])

                for l in range(len(adx_periods)):
                    for m in range(len(adx_mins)):
                        req1 = ReqParam()
                        req1.condition = 1
                        req1.indicatorPeriod = adx_periods[l]
                        req1.adx_min = adx_mins[m]
                        getRequirementData(req1, data, adxMap)

                        for f in allFiles:
                            count += 1
                            print("Analyses: ", f[32: len(f) - 8], count, periods[i], maxBuyVals[j],
                                  minSellVals[k], adx_periods[l], adx_mins[m])

                            doCalculation(f, max_buy_val=maxBuyVals[j], min_sell_val=minSellVals[k], indicator=indicator, req1=req1, req2=req2, req3=req3)

pd.options.mode.chained_assignment = None

st = int(round(time.time() * 1000))
count = 0

path = r'F:\data\nse500\indices'
allFiles = glob.glob(path + "\*.csv")

data = getData()
analyseIndicator(count)
totalTime = int(round(time.time() * 1000)) - st
print('Total Time:', totalTime)


from pytest import approx

import numpy as np
import funcy as fn

from redax.module import Interface, CompositeInterface
from redax.spaces import DynamicCover
from redax.synthesis import ControlPre, DecompCPre, ReachGame, SafetyGame, ReachAvoidGame
from redax.visualizer import scatter2D, plot3D, plot3D_QT, pixel2D
from redax.utils.overapprox import bloatbox

from redax.predicates.dd import BDD
mgr = BDD()
mgr.configure(reordering=True)

ts = .2
k = .1
g = 9.8

def dynamics(p, v, a):
    vsign = 1 if v > 0 else -1
    return p + v * ts, v + a*ts - vsign*k*(v**2)*ts - g*ts

pspace = DynamicCover(-10, 10)
vspace = DynamicCover(-16, 16)
aspace = DynamicCover(0, 20)

# Smaller component modules
pcomp = Interface(mgr,
                 {'p': pspace,
                  'v': vspace},
                 {'pnext': pspace}
        )
vcomp = Interface(mgr,
                 {'v': vspace,
                  'a': aspace},
                 {'vnext': vspace}
        )

# Declare grid precision
p_precision = 7
v_precision = 7
precision = {'p': p_precision, 'v': v_precision, 'a': 7, 'pnext': p_precision, 'vnext': v_precision}
bittotal = sum(precision.values())
outorder = {0: 'pnext', 1: 'vnext'}
possible_transitions = (pcomp * vcomp).count_io_space(bittotal)

np.random.seed(1337)

for numapplied in range(6000):

    # Shrink window widths over time
    width = 20 * 1/np.log10(2*numapplied+10)

    # Generate random input windows
    f_width = {'p': np.random.rand()*.5*width,
               'v': np.random.rand()*width,
               'a': np.random.rand()*.5*width}
    f_left  = {'p': -10 +  np.random.rand() * (20 - f_width['p']),
               'v': -16 + np.random.rand() * (32 - f_width['v']),
               'a': 0 + np.random.rand() * (20 - f_width['a'])}
    f_right = {k: f_width[k] + f_left[k] for k in f_width}
    iobox   = {k: (f_left[k], f_right[k]) for k in f_width}

    # Generate output overapproximation
    ll = dynamics(**f_left)
    ur = dynamics(**f_right)
    outbox = {outorder[i]: (ll[i], ur[i]) for i in range(2)}
    iobox.update(outbox)

    # Apply constraint to parallel updates
    pcomp = pcomp.io_refined({k: v for k, v in iobox.items() if k in pcomp.vars}, nbits=precision)
    vcomp = vcomp.io_refined({k: v for k, v in iobox.items() if k in vcomp.vars}, nbits=precision)

def test_safe_control():
    composite = CompositeInterface((pcomp, vcomp))
    dcpre = DecompCPre(composite, (('p', 'pnext'), ('v', 'vnext')), ('a'))

    safe = pspace.conc2pred(mgr, 'p', [-8,8], 6, innerapprox=True)
    safesink = Interface(mgr, {'p': pspace, 'v': vspace}, {},  guar = mgr.true, assum=safe)

    # Solve game and plot 2D invariant region
    game = SafetyGame(dcpre, safesink)
    dinv, _, controller = game.run()

    system = pcomp * vcomp
    cpre = ControlPre(system, (('p', 'pnext'), ('v', 'vnext')), ('a'))
    game = SafetyGame(cpre, safesink)
    inv, _, _ = game.run(verbose=True)

    assert dinv == inv

#     assert dinv.count_nb(p_precision + v_precision) == approx(5988)

    # Simulate for initial states
    state_box = fn.first(controller.winning_states())
    assert state_box is not None
    state = {k: .5*(v[0] + v[1]) for k, v in state_box.items()}
    for step in range(30):
        u = fn.first(controller.allows(state))
        assert u is not None
        picked_u = {'a': u['a'][0]} # Pick lower bound of first allowed control voxel

        state.update(picked_u)
        nextstate = dynamics(**state)
        state = {'p': nextstate[0], 'v': nextstate[1]}

def test_reach_control():
    composite = CompositeInterface((pcomp, vcomp))
    dcpre = DecompCPre(composite, (('p', 'pnext'), ('v', 'vnext')), ('a'))

    target = pspace.conc2pred(mgr, 'p', [-2,2], 6, innerapprox=True)
    targetint = Interface(mgr, {'p': pspace, 'v': vspace}, {},  guar = mgr.true, assum=target)

    # Solve game and plot 2D invariant region
    game = ReachGame(dcpre, targetint)
    dbasin, _, _ = game.run()

    system = pcomp * vcomp
    cpre = ControlPre(system, (('p', 'pnext'), ('v', 'vnext')), ('a'))
    game = ReachGame(cpre, targetint)
    basin, _, _ = game.run()

    assert dbasin == basin

#     assert dbasin.count_nb(p_precision + v_precision) == approx(5964)

def test_reachavoid():
    composite = CompositeInterface((pcomp, vcomp))
    dcpre = DecompCPre(composite, (('p', 'pnext'), ('v', 'vnext')), ('a'))

    target =  pspace.conc2pred(mgr, 'p', [-3,3], 6, innerapprox=True)
    target &= vspace.conc2pred(mgr, 'v', [0,2], 6, innerapprox=True)
    targetint = Interface(mgr, {'p': pspace, 'v': vspace}, {},  guar = mgr.true, assum=target)

    safe =  pspace.conc2pred(mgr, 'p', [-8,8], 6, innerapprox=True)
    safe &=  ~pspace.conc2pred(mgr, 'p', [-.4,.4], 6, innerapprox=True)
    safe &= vspace.conc2pred(mgr, 'v', [-4,4], 6, innerapprox=True)
    safeint = Interface(mgr, {'p': pspace, 'v': vspace}, {},  guar = mgr.true, assum=safe)

    game = ReachAvoidGame(dcpre, safeint, targetint)
    basin, _, _ = game.run()

#     scatter2D(mgr, ('p', pspace), ('v', vspace),
#                  basin.pred,
#                  fname = "reachavoid_doubleint.png"
#                  )


import time
import requests

_head = {
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3423.2 Safari/537.36"
}

_api = "https://www.sojson.com/open/api/weather/json.shtml?city=%s"

class Weather():

    def __init__(self, address):
        self.address = address
        self.weather = self.format_weather()

    def link_api(self, address):
        count = 0
        while True:
            res = requests.get(_api % address, headers=_head, timeout=10)
            count += 1
            if res.status_code == 200:
                return res.json()
            if count > 5:
                return None
            time.sleep(4)

    def format_weather(self):
        source = self.link_api(self.address)
        if source is None:
            return None
        weather_dict = {
            "今日天气": source["data"]["forecast"][0]["type"],
            "平均温度": source["data"]["wendu"],
            "最高温度": source["data"]["forecast"][0]["high"],
            "最低温度": source["data"]["forecast"][0]["low"],
            "湿度": source["data"]["shidu"],
            "PM2.5": source["data"]["pm25"],
            "空气质量": source["data"]["quality"],
            "风力": source["data"]["forecast"][0]["fl"],
            "风向": source["data"]["forecast"][0]["fx"]
        }
        return weather_dict

    def get_weather(self):
        return self.weather

if __name__ == "__main__":

    """以下是实验"""

    data = Weather("成都")
    print(data.weather)

    dictionary = {'city': '成都', 'date': '20180526',
    'data':
    {'ganmao': '儿童、老年人及心脏、呼吸系统疾病患者人群应减少长时间或高强度户外锻炼', 'pm25': 33.0, 'wendu': '24', 'quality': '轻度污染', 'pm10': 63.0,
    'yesterday':
    {'aqi': 118.0,'notice': '阴晴之间，谨防紫外线侵扰', 'date': '25日星期五', 'low': '低温 18.0℃', 'sunset': '19:57', 'fx': '无持续风向', 'type': '多云', 'fl': '<3级', 'sunrise': '06:05', 'high': '高温 29.0℃'},
    'forecast':
    [
        {'aqi': 54.0, 'notice': '雨虽小，注意保暖别感冒', 'date': '26日星期六', 'low': '低温 17.0℃', 'sunset': '19:57', 'fx': '无持续风向', 'type': '小雨', 'fl': '<3级', 'sunrise': '06:04', 'high': '高温 26.0℃'},
        {'aqi': 58.0, 'notice': '阴晴之间，谨防紫外线侵扰', 'date': '27日星期日', 'low': '低温 16.0℃', 'sunset': '19:58', 'fx': '无持续风向', 'type': '多云', 'fl': '<3级', 'sunrise': '06:04', 'high': '高温 29.0℃'},
        {'aqi': 66.0, 'notice': '阴晴之间，谨防紫外线侵扰', 'date': '28日星期一', 'low': '低温 17.0℃', 'sunset': '19:59', 'fx': '无持续风向', 'type': '多云', 'fl': '<3级', 'sunrise': '06:04', 'high': '高温 28.0℃'},
        {'aqi': 71.0, 'notice': '阴晴之间，谨防紫外线侵扰', 'date': '29日星期二', 'low': '低温 17.0℃', 'sunset': '19:59', 'fx': '无持续风向', 'type': '多云', 'fl': '<3级', 'sunrise': '06:03', 'high': '高温 27.0℃'},
        {'aqi': 68.0, 'notice': '阴晴之间，谨防紫外线侵扰', 'date': '30日星期三', 'low': '低温 18.0℃', 'sunset': '20:00', 'fx': '无持续风向', 'type': '多云', 'fl': '<3级', 'sunrise': '06:03', 'high': '高温 25.0℃'}
        ],
    'shidu': '78%'}, 'status': 200, 'message': 'Success !', 'count': 1102}


from bayes_opt import BayesianOptimization, SequentialDomainReductionTransformer
from catboost import CatBoostRegressor
from sklearn import model_selection

def catboost_cv(learning_rate, l2_leaf_reg, bagging_temperature, depth, x_train, y_train):
    """
    Function to perform cross validation of an CatBoost model
    :param learning_rate:
    :param l2_leaf_reg:
    :param bagging_temperature:
    :param depth:
    :param x_train:
    :param y_train:
    :return:
    """

    params = {
        'loss_function': 'RMSE',
        'eval_metric': 'RMSE',
        'iterations': 999,
        'learning_rate': learning_rate,
        'random_seed': 42,
        'l2_leaf_reg': l2_leaf_reg,
        'bagging_temperature': bagging_temperature,
        'depth': int(depth),
        'early_stopping_rounds': 10,
        'od_pval': 10e-2,
        'task_type': 'GPU',
        'logging_level': 'Silent',
        'boosting_type': 'Plain'
    }

    catboost = CatBoostRegressor(**params)

    cv_result = model_selection.cross_validate(estimator=catboost, X=x_train, y=y_train)

    mse = cv_result['test_score'] ** 2

    return -1.0 * mse

def optimize_catboost(x_train, y_train, pbounds, n_iter, init_points):
    """
    Performs Bayesian Optimization of an CatBoost Model
    :param x_train:
    :param y_train:
    :param pbounds:
    :param n_iter:
    :param init_points:
    :return:
    """

    def catboost_crossval(learning_rate, l2_leaf_reg, bagging_temperature, depth):
        """
        Wrapper for catboost_cv
        :param learning_rate:
        :param l2_leaf_reg:
        :param bagging_temperature:
        :param depth:
        :return:
        """
        return catboost_cv(learning_rate, l2_leaf_reg, bagging_temperature, depth, x_train, y_train)

    optimizer = BayesianOptimization(
        f=catboost_crossval,
        pbounds=pbounds,
        bounds_transformer=SequentialDomainReductionTransformer(),
        random_state=42,
        verbose=2
    )

    optimizer.maximize(n_iter=n_iter, init_points=init_points, acq='ei')

    return optimizer


import boto3
import uuid
import json

def lambda_handler(event, context):
    print(json.dumps(event))

    record = event['Records'][0]
    #collecting bucket name and key
    s3bucket = record['s3']['bucket']['name']
    s3object = record['s3']['object']['key']

    #extracting the file name from the key
    file = s3object.split("/")[3]
    s3Path = f's3://{s3bucket}/{s3object}'
    jobName = f'{file}-{str(uuid.uuid4())}'
    #setting the output path (transcripts will be stored under "s3://transcripts/" folder)
    outputKey = f'transcripts/{file}-transcript.json'

    client = boto3.client('transcribe')

    response = client.start_transcription_job(
        TranscriptionJobName=jobName,
        LanguageCode='fr-FR',
        Media={'MediaFileUri': s3Path},
        OutputBucketName=s3bucket,
        OutputKey=outputKey
    )

    print(json.dumps(response, default=str))

    return {
        'TranscriptionJobName': response['TranscriptionJob']['TranscriptionJobName']
    }


import sys

# 길이가 N인 두 배열 A, B가 있다.
# 최대 K번 A와 B의 원소를 바꿔 A의 모든 원소합이 최대로 되게 할 때 A의 원소합의 최대값은?
# 전략> A 오름차순 정렬, B 내림차순 정렬.
#      1) 앞에서부터 A < B이면 교체. (K번까지 가능)
#      2) A == B면 그냥 continue. (count안하고.)
#      3) 만약 번까지 안갔는데 A > B면 종료.

N, K = map(int, input().split())
A = list(map(int, sys.stdin.readline().rsplit()))
B = list(map(int, sys.stdin.readline().rsplit()))

A = sorted(A)
B = sorted(B, reverse=True)

swapCnt = 0
answer = 0
for n in range(N):
    if swapCnt == K:
        break

    if A[n] < B[n]:
        A[n], B[n] = B[n], A[n]
        swapCnt += 1
    elif A[n] == B[n]:
        continue
    else:
        break

answer = sum(A)
print(answer)

4-6.두 배열의 원소 교체.py


import functools
from sklearn.utils.class_weight import compute_class_weight
import numpy as np
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()
from tensorflow.compat.v1  import set_random_seed
from tensorflow.python.framework import ops
from tensorflow.python.client import device_lib
print('device_lib.list_local_devices()',device_lib.list_local_devices())

BATCH_SIZE = 128
EPOCHS = 12
KEEP_PROB = 0.8
REGULARIZATION = 0.001
NUM_HIDDEN = [512, 512]
RANDOM = 0
NUM_CLASSES = 4

def lazy_property(function):
    attribute = '_' + function.__name__

    @property
    @functools.wraps(function)
    def wrapper(self):
        if not hasattr(self, attribute):
            setattr(self, attribute, function(self))
        return getattr(self, attribute)
    return wrapper

class VariableSequenceLabelling:

    def __init__(self, data, target, dropout_prob, reg, num_hidden, class_weights):
        self.data = data
        self.target = target
        self.dropout_prob = dropout_prob
        self.reg = reg
        self._num_hidden = num_hidden
        self._num_layers = len(num_hidden)
        self.num_classes = len(class_weights)
        self.attn_length = 0
        self.class_weights = class_weights
        self.prediction
        self.error
        self.optimize

    @lazy_property
    def make_rnn_cell(self,
                      attn_length=0,
                      base_cell=tf.compat.v1.nn.rnn_cell.BasicRNNCell):

        attn_length = self.attn_length
        input_dropout = self.dropout_prob
        output_dropout = self.dropout_prob

        cells = []
        for num_units in self._num_hidden:
            cell = base_cell(num_units)
            cell = tf.nn.rnn_cell.DropoutWrapper(cell, input_keep_prob=input_dropout, output_keep_prob=output_dropout,
                                                seed=RANDOM)
            cells.append(cell)

        cell = tf.nn.rnn_cell.MultiRNNCell(cells)

        return cell

    # predictor for slices
    @lazy_property
    def prediction(self):

        cell = self.make_rnn_cell

        # Recurrent network.
        output, final_state = tf.nn.dynamic_rnn(cell,
            self.data,
            dtype=tf.float32
        )

        with tf.variable_scope("model") as scope:
            tf.get_variable_scope().reuse_variables()

            # final weights
            num_classes = self.num_classes
            weight, bias = self._weight_and_bias(self._num_hidden[-1], num_classes)

            # flatten + sigmoid
            if self.attn_length > 0:
                logits = tf.matmul(final_state[0][-1][-1], weight) + bias
            else:
                logits = tf.matmul(final_state[-1][-1], weight) + bias

            prediction = tf.nn.softmax(logits)

            return logits, prediction

    @lazy_property
    def cross_ent(self):
        predictions = self.prediction[0]
        real = tf.cast(tf.squeeze(self.target), tf.int32)
        weights = tf.gather(self.class_weights, real)

        xent = tf.losses.sparse_softmax_cross_entropy(labels=real, logits=predictions, weights=weights)
        loss = tf.reduce_mean(xent) #shape 1
        ce = loss
        l2 = self.reg * sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables())
        ce += l2
        return ce

    @lazy_property
    def optimize(self):
        learning_rate = 0.0003
        optimizer = tf.train.AdamOptimizer(learning_rate)
        return optimizer.minimize(self.cross_ent)

    @lazy_property
    def error(self):
        prediction = tf.argmax(self.prediction[1], 1)
        real = tf.cast(self.target, tf.int32)
        prediction = tf.cast(prediction, tf.int32)
        mistakes = tf.not_equal(real, prediction)
        mistakes = tf.cast(mistakes, tf.float32)
        mistakes = tf.reduce_sum(mistakes, reduction_indices=0)
        total = 128
        mistakes = tf.divide(mistakes, tf.to_float(total))
        return mistakes

    @staticmethod
    def _weight_and_bias(in_size, out_size):
        weight = tf.truncated_normal([in_size, out_size], stddev=0.01)
        bias = tf.constant(0.1, shape=[out_size])
        return tf.Variable(weight), tf.Variable(bias)

    @lazy_property
    def summaries(self):
        tf.summary.scalar('loss', tf.reduce_mean(self.cross_ent))
        tf.summary.scalar('error', self.error)
        merged = tf.summary.merge_all()
        return merged

def LSTM_Model(x_train, y_train, x_test, x_val):
    class_weight = compute_class_weight('balanced', classes=np.unique(y_train), y=y_train)
    print('class_weight', class_weight)
    classes = np.unique(y_train)
    print('classes', classes)
    print('LSTM starting')
    ops.reset_default_graph()
    set_random_seed(RANDOM)
    config = tf.compat.v1.ConfigProto(allow_soft_placement = True)

    sess = tf.compat.v1.Session(config = config)
    with  sess, tf.device('/cpu:0'):
        _, length, num_features = x_train.shape
        num_data_cols = num_features

        data = tf.placeholder(tf.float32, [None, length, num_data_cols])
        target =  tf.placeholder(tf.float32, [None])
        dropout_prob =  tf.placeholder(tf.float32)
        reg =  tf.placeholder(tf.float32)

        # initialization
        model = VariableSequenceLabelling(data, target, dropout_prob, reg, num_hidden=NUM_HIDDEN, class_weights=class_weight)
        sess.run(tf.global_variables_initializer())

        batch_size = BATCH_SIZE
        dp = KEEP_PROB
        rp = REGULARIZATION
        train_samples = x_train.shape[0]
        indices = list(range(train_samples))
        num_classes = NUM_CLASSES

        # for storing results
        test_data = x_test
        val_data = x_val

        val_aucs = []
        test_aucs = []
        val_aucs_macro = []
        test_aucs_macro = []
        test_accuracys = []
        test_f1s = []
        test_auprcs = []

    epoch = -1


#!/usr/bin/python3
import numpy as np
from time import sleep

def main():
    try:
        range_n = np.arange(80)
        bar = []

        for x in range(80):
            bar.append('▒')

        print()
        for x in range_n:
            print("|{} \r".format("".join(bar)), end='')
            bar[x]  = "█"
            bar[-1] = " [{}%/{}%]".format(str(x), 78)
            sleep(.1)

        print()

    except Exception as err:
        print(err)

if __name__ == '__main__':
    main()


from teeth_overlord import tests

from teeth_overlord.jobs import base as jobs_base
from teeth_overlord import models

class TestModelMock(tests.TeethMockTestUtilities):
    """Tests for the mock utilities in the TeethUnitTest base class."""

    def test_mock_class(self):

        mock = self.add_mock(jobs_base.JobClient)
        mock.submit_job.return_value = 42

        # re-import JobClient so the patch is picked up. You probably shouldn't
        # do this if you can help it.
        from teeth_overlord.jobs import base as test_jobs_base
        client = test_jobs_base.JobClient("stuff")
        ret = client.submit_job("jobstuff")

        mock.submit_job.assert_called_once_with("jobstuff")
        self.assertEqual(ret, 42)

    def test_mock_method(self):

        mock = self.add_mock(models.Flavor, "save", "save_return")

        model = models.Flavor(name="flavor")
        ret = model.save()

        self.assertEqual(ret, "save_return")
        mock.assert_called_once_with()

    def test_mock_model(self):

        self.add_mock(models.Flavor, return_value=["filter_return"])

        mock = self.get_mock(models.Flavor, "objects")

        model = models.Flavor(name="flavor")
        ret = list(model.objects.filter())
        ret2 = list(model.objects.limit(100).filter(1))

        self.assertTrue(ret == ret2 == ["filter_return"])
        mock.assert_called_once_with("filter", 1)
        mock.assert_called_once_with("filter")

    def test_add_mock_model_objects(self):

        self.add_mock(models.JobRequest, "get", "get_return")

        mock = self.get_mock(models.JobRequest, "get")

        model = models.JobRequest(job_type="jobrequest")
        ret = model.get()

        self.assertEqual(ret, "get_return")
        mock.assert_called_once_with()

    def test_add_mock_superclass_method(self):

        mock = self.add_mock(models.JobRequest, "save")
        mock.return_value = "save_return"

        model = models.JobRequest(job_type="jobrequest")
        ret = model.save()

        self.assertEqual(ret, "save_return")
        self.get_mock(models.JobRequest, "save").assert_called_once_with()

    def test_add_mock_instance_method(self):

        mock = self.add_mock(models.JobRequest, "touch")
        mock.return_value = "touch_return"

        model = models.JobRequest(job_type="jobrequest")
        ret = model.touch()

        self.assertEqual(ret, "touch_return")
        self.get_mock(models.JobRequest, "touch").assert_called_once_with()

    def test_mock_missing_attribute_fails(self):

        self.assertRaises(AttributeError,
                          self.add_mock,
                          models.JobRequest,
                          "foo.bar")

        self.assertRaises(AttributeError,
                          self.add_mock,
                          models.JobRequest,
                          "foobar")


#*************************************************************************************************
#                               GERENCIADOR DE LAYOUT PACK
#*************************************************************************************************
#O gerenciador de layout pack agrupa os widgets conforme determinadas propriedades
#INCLUDES
from tkinter import*
#CONSTANTES E DEFINIÇÕES
resolucao = '300x300+200+200'

#GUI
janela = Tk()
janela.geometry(resolucao)
bt1 = Button(janela, width=20, text='Botão 1')
bt2 = Button(janela, width=20, text='Botão 2')
bt3 = Button(janela, width=20, text='Botão 3')
bt4 = Button(janela, width=20, text='Botão 4')
#LAYOUT
#Definindo gerenciador de Layout pack, a ordem de agrupamento será a ordem da sequencia de cada
#linha de comando, no caso vem bt3-bt2-bt1-bt4
bt4.pack(side=TOP)      #Alinha o label 4 na parte superior da tela
bt3.pack(side=RIGHT)    #Alinha a direita. OBS: O valor da distância TOP é default.
bt2.pack(side=BOTTOM)   #Alinha abaixo.
bt1.pack(side=LEFT)     #Alinha a esquerda, a propriedade side pode ser: TOP BOTTOM LEFT RIGHT

janela.mainloop()


#!/usr/bin/env python
#-------------------------------------------#
#Tittle: mailroom_part_04, PYTHON210 - Assignment 05
#Desc: Mailroom Part 4
#Change Log: (Who, When, What)
#Brent Kieszling, <2020-Dec-13>, Created file
#Brent Kieszling, <2020-Dec-19>, updated display_donors() to return a list and display_report() to return a string
#Brent Kieszling, <2020-Dec-20>, moved operations in option_3_mm to create_thank_you_file
#-------------------------------------------#

import pickle
#DATA---------------------------------------

main_menu = "Main Menu\n\n[1] Send a thank you letter.\n[2] Create a report.\
\n[3] Write thank you letters for all donors.\n[4] Exit program.\n"

ty_menu = "Thank You Menu\n\n[1] Create thank you for existing donor.\n[2] Create thank you for new donor.\n\
[3] View current donor list.\n[4] Return to main menu.\n"

data_file = "donor_history.dat"

#PROCESS------------------------------------
def menu_selection(menu, dispatch):
    """Navigate a menu.

        Args:
            menu(str): Display menu
            dispatch(dict): Dictionary controlling menu options
    """
    while True:
        response = input(menu)
        try:
            response = int(response)
        except ValueError:
            print("Please only use numbers.")
        else:
            if dispatch.get(response)() == "Exit":
                break

def option_1_mm():
    """Write thank you"""
    dict_ty_menu = {1: option_1_ty, 2: option_2_ty, 3: option_3_ty, 4: option_exit}
    menu_selection(ty_menu, dict_ty_menu)

def option_2_mm():
    """Display report"""
    donors = donor_sort(load_donors(data_file))
    print(display_report(donors))

def option_3_mm():
    """Send a letter to each donor."""
    donors = load_donors(data_file)
    create_thank_you_file(donors)

def option_exit():
    return "Exit"

def option_1_ty():
    """Create thank you for existing donor."""
    donors = load_donors(data_file)
    while True:
        name = input("Please enter the donor's name: ")
        donor = find_donor(donors, name)
        if name == "exit":
            return
        elif donor == None:
            print("Donor not found. Please use an existing donor. Or type\
 'exit' to return to the Thank You Menu.")
        else:
            break
    while True:
        donation = input("Enter the new donation ammount: ")
        try:
            donation = float(donation)
        except ValueError:
            print("Please use a numerical entry (ex: 1000.00)")
        else:
            break
    #Update donor list with new information
    donors = update_donors(donors, name, donation)
    #Save updated donor history
    save_donors(donors, data_file)
    #Display thank you
    print(thank_you(donor))

def option_2_ty():
    """Create a thank you for a new donor."""
    name = input("Please enter the donor's name: ")
    while True:
        donation = input("Enter the new donation ammount: ")
        try:
            donation = float(donation)
        except ValueError:
            print("Please use a numerical entry (ex: 1000.00)")
        else:
            break
    donors = load_donors(data_file)
    #Create new donor and at to the donor list
    donors.append(new_donor(name, donation))
    #Save updated donor history
    save_donors(donors, data_file)
    #Display thank you
    print(thank_you(find_donor(donors, name)))

def option_3_ty():
    """Display current donors."""
    donors = load_donors(data_file)
    print("Current donors are:")
    individuals = display_donors(donors)
    print(individuals)

def save_donors(lst_group, save_file_b):
    """Save the current list of donors to a binary file

        Args:lst_group(list): List of dictionaries containing donors and their contribution.
            save_file_b(str): Data file name
    """
    with open(save_file_b, 'wb') as fileObj:
        pickle.dump(lst_group, fileObj)

def load_donors(save_file_b):
    """Load donor history from file

        Args:
            save_file(str): Data file name

        Returns:
            lst_group(list): List of dictionaries containing donors and their contribution.
    """
    try:
        with open(save_file_b, 'rb') as fileObj:
            lst_group = pickle.load(fileObj)
    #If nothing is saved to file return an empty list
    except FileNotFoundError:
        lst_group = []
    return lst_group

def create_thank_you_file(lst_group):
    """Create a file containing a thank you letter for each donor in lst_group

        Args:
            lst_group(list): List of dictionaries containing donors and their contribution.
        Returns:
            None
    """
    lst_files = [(donor["Name"].replace(" ", "_") + ".txt") for donor in lst_group]
    n = 0
    for item in lst_group:
        #Get personalized thank you letter
        letter = thank_you(item)
        with open(lst_files[n], 'w') as file:
            file.write(letter)
        n += 1

def new_donor(person, money):
    """Create a new dictionary containing a donor and donation history

        Args:
            person(str): Name of donor.
            money(float): Donation ammount.

        Returns:
            a_new_donor(dict): Dictionary containing a donor and donation history
    """
    a_new_donor = {"Name": person, "Donation": [money], "Total": money, "Average": money, "Gifts": 1}
    return a_new_donor

def update_donors(lst_group, person, money):
    """Update a list of donors with a new donation

        Args:
            lst_group(list): List of dictionaries containing donors and their contribution.
            person(str): Name of donor.
            money(float): Most recent donation.

        Returns:
            lst_group(list): Updated list of donors.
    """
    for item in lst_group:
        if item["Name"] == person:
            item["Donation"].append(money)
            item["Total"] += money
            item["Gifts"] += 1
            item["Average"] = item["Total"]/item["Gifts"]
    return lst_group

def thank_you(donor_specs):
    """Display a thank you letter for a donor.

        Args:
            donor_specs(dict): Dictionary containing a donor and their donation information
        Returns:
            letter(str): Thank you letter for a donor.
    """
    letter = """
    Dear {donor},

    Thank you for your most recent donation of ${last:.2f}. We are humbled by your
    lifetime contribution of ${total:.2f}.

    Sincerly,
    Making Good Things Happen"""
    return letter.format(donor = donor_specs["Name"], last = donor_specs["Donation"][-1], total = donor_specs["Total"])

def find_donor(lst_group, person):
    """Update a list of donors with a new donation

        Args:
            lst_group(list): List of lists containing donors and their contribution.
            person(str): Name of donor.

        Returns:
            lst_person(list): List containing a donor and their donations.
    """
    for item in lst_group:
        if person == item["Name"]:
            dict_person = item
            return dict_person

def display_donors(lst_group):
    """Display all donors on record.

        Args:
            lst_group(list): List of dictionaries containing a donors and their donations.

        Returns:
            display_names(string): String listing the names of donors.
    """
    names = []
    for item in lst_group:
        names.append(item["Name"])
    display_names ="{}\n" *len(names)
    display_names = display_names.format(*names)
    return display_names

def display_report(lst_people):
    """Format a list of dictionaries containing donor metrics.

    Args:
        lst_people (list): List of lists containing a donor's name, total
            contribution, number of contributions, and anverage contribution.

    Returns:
        report (string): Formatted report of donors
    """
    top ="==================== Donor Report: ==========================\n"
    formatted_header = "{name:<20} | {total:<10} | {gifts:<9} | {average:<10}\n"\
        .format(name = "Donor Name", total = "Total Given", gifts = "Num Gifts",\
                average = "Average Gift")
    formatted_row = "{Name:<20} $ {Total:<10.2f}   {Gifts:^9n}  $ {Average:>10.2f}\n"
    bot = "==============================================================\n"

    report = top + formatted_header
    for item in lst_people:
        report += formatted_row.format(**item)
    report += bot
    return report

def donor_sort(lst_group):
    """Sort a list of dictionaries about key "Total" in high to low order.

        Args:
            lst_group(list): List of dictionaries containing donors and their contribution.

        Returns:
            lst_group(list): Sorted list of dictionaries containing donors and their contribution.
    """
    lst_group.sort(reverse = True, key = lambda y: y["Total"])
    return lst_group

#PRESENTATION INPUT/OUTPUT------------------

dict_main_menu = {1: option_1_mm, 2: option_2_mm, 3: option_3_mm, 4: option_exit}

if __name__ == '__main__':
    menu_selection(main_menu, dict_main_menu)


#!/bin/env python3
import json

import sseclient

messages = sseclient.SSEClient("https://www.blaseball.com/events/streamData")

last_data = ""
last_game = {}
for m in messages:
    if last_data == m.data:
        print("DUPLICATE DATA")
    else:
        print("changed data")

    j = json.loads(m.data)

    schedule = j["value"]["games"]["schedule"]

    g = next(s for s in schedule if s["awayTeamNickname"] == "Lift" or s["homeTeamNickname"] == "Lift")

    if last_game == g:
        print("DUPLICATE GAME\n\n")
    else:
        print("changed game\n\n")

    last_game = g
    last_data = m.data


from django.db import models

# Create your models here.

from django.db import models
from datetime import datetime
from ckeditor.fields import RichTextField
from multiselectfield import MultiSelectField

# Create your models here.
class Car(models.Model):

    city_choice = (
        ('Antipolo', 'Antipolo'),
        ('BGC', 'BGC'),
        ('Makati', 'Makati'),
        ('Marikina', 'Marikina'),
        ('Pasig', 'Pasig'),
        ('Quezon City', 'Quezon City'),
        ('San Juan', 'San Juan'),
        ('Taguig', 'Taguig'),
    )

    year_choice = []
    for r in range(2000, (datetime.now().year+1)):
        year_choice.append((r,r))

    features_choices = (
        ('Cruise Control', 'Cruise Control'),
        ('Audio Interface', 'Audio Interface'),
        ('Airbags', 'Airbags'),
        ('Air Conditioning', 'Air Conditioning'),
        ('Seat Heating', 'Seat Heating'),
        ('Alarm System', 'Alarm System'),
        ('ParkAssist', 'ParkAssist'),
        ('Power Steering', 'Power Steering'),
        ('Reversing Camera', 'Reversing Camera'),
        ('Direct Fuel Injection', 'Direct Fuel Injection'),
        ('Auto Start/Stop', 'Auto Start/Stop'),
        ('Wind Deflector', 'Wind Deflector'),
        ('Bluetooth Handset', 'Bluetooth Handset'),
    )

    door_choices = (
        ('2', '2'),
        ('3', '3'),
        ('4', '4'),
        ('5', '5'),
        ('6', '6'),
    )

    transmission_choices = (
        ('Automatic', 'Automatic'),
        ('Manual', 'Manual'),
        ('Hybrid', 'Hybrid'),
    )

    body_choices = (
        ('SUV', 'SUV'),
        ('Crossover', 'Crossover'),
        ('Sedan', 'Sedan'),
        ('Hatchback', 'Hatchback'),
        ('Coupe', 'Coupe'),
        ('Convertible', 'Convertible'),
        ('Sports', 'Sports'),
        ('Truck', 'Truck'),
        ('Van', 'Van'),
    )

    passengers_choices = (
        ('2', '2'),
        ('4', '4'),
        ('6', '6'),
        ('8', '8'),
        ('10', '10'),
        ('12', '12'),
    )

    fuel_choices = (
        ('Gasoline', 'Gasoline'),
        ('Diesel', 'Diesel'),
        ('Bio-diesel', 'Bio-diesel'),
        ('Ethanol', 'Ethanol'),
    )

    car_title = models.CharField(max_length=255)
    city = models.CharField(choices=city_choice, max_length=100)
    color = models.CharField(max_length=100)
    model = models.CharField(max_length=100)
    year = models.IntegerField(('year'), choices=year_choice)
    condition = models.CharField(max_length=100)
    price = models.IntegerField()
    description = RichTextField()
    car_photo = models.ImageField(upload_to='photos/%Y/%m/%d/')
    car_photo_1 = models.ImageField(upload_to='photos/%Y/%m/%d/', blank=True)
    car_photo_2 = models.ImageField(upload_to='photos/%Y/%m/%d/', blank=True)
    car_photo_3 = models.ImageField(upload_to='photos/%Y/%m/%d/', blank=True)
    car_photo_4 = models.ImageField(upload_to='photos/%Y/%m/%d/', blank=True)
    features = MultiSelectField(choices=features_choices)
    body_style = models.CharField(choices=body_choices, max_length=100)
    engine = models.CharField(max_length=100)
    transmission = models.CharField(choices=transmission_choices, max_length=50)
    interior = models.CharField(max_length=100)
    miles = models.IntegerField()
    doors = models.CharField(choices=door_choices, max_length=10)
    passengers = models.CharField(choices=passengers_choices, max_length=10)
    vin_no = models.CharField(max_length=100)
    milage = models.IntegerField()
    fuel_type = models.CharField(choices=fuel_choices, max_length=50)
    no_of_owners = models.CharField(max_length=100)
    is_featured = models.BooleanField(default=False)
    created_date = models.DateTimeField(default=datetime.now, blank=True)

    def __str__(self):
        return self.car_title


#! /usr/bin/python

import string, os
import pygame
pygame.init()

font = pygame.font.SysFont("Courier", 100)

for char in string.printable:
	if char != " " and char in string.whitespace:
		continue
	index = ord(char)
	img = font.render(char, True, (255, 255, 255))#.convert(24)
	pygame.image.save(img, "%i.png" % index)


import numpy as np
import sys
import pathlib

current_dir = pathlib.Path(__file__).resolve().parent
sys.path.append(str(current_dir) + "/../")
from common.physics_model import PhysicsModel

# state: (X, Y)
# X and Y are defined in inertial coordinate system.

# input: (x_velocity, y_velocity)
# The direction of x_force and y_force are defined in body coordinate system.

class Dot2D(PhysicsModel):
    def __init__(self, init_state, **kwargs):
        # set default param values
        self.name = "Dot2D"

        init_input = (0.0, 0.0)

        super().__init__(init_state, init_input, **kwargs)

    def dynamics(self, states, u):
        X_dot = u[0]
        Y_dot = u[1]

        return np.array([X_dot, Y_dot])

    def get_param(self):
        return {}

    def set_param(self, **kwargs):
        for key in kwargs:
            raise TypeError(
                "The required key {key!r} " "are not in kwargs".format(key=key)
            )

    def get_sensor_data(self):
        X_dot = self.last_state_dot[0]
        Y_dot = self.last_state_dot[1]
        return {"velocity": [X_dot, Y_dot]}


import telebot
from config import keys, TOKEN
from extensions import CryptoConverter, ConvertionException

bot = telebot.TeleBot(TOKEN)

@bot.message_handler(commands=['start', 'help'])
def hepl(message: telebot.types.Message):
    text = "Чтобы начать работу введите команду боту в следующем формате: \n <имя валюты цену которой хотите узнать> \
    <имя валюты в которой надо узнать цену первой валюты> \
    <количество первой валюты> \n Увидеть список всех возможных влют: /values"
    bot.reply_to(message, text)

@bot.message_handler(commands=['values'])
def values(message: telebot.types.Message):
    text = "Доступные валюты"
    for key in keys.keys():
        text = '\n'.join((text, key, ))
    bot.reply_to(message, text)

@bot.message_handler(content_types=['text',])
def convert(message: telebot.types.Message):
    try:
        values = message.text.split(' ')

        if len(values) != 3:
            raise ConvertionException("Запрос введен некорректно. Слишком много/мало параметров.")
        quote, base, amount = values
        quote = quote.upper()
        base = base.upper()
        amount = amount.replace(",", ".")
        cost = CryptoConverter.convert(quote, base, amount)
    except ConvertionException as e:
        bot.reply_to(message, f"Ошибка пользователя. \n {e}")
    except Exception as e:
        bot.reply_to(message, f"Не удалось обработать команду. \n {e}")
    else:
        text = f"Цена 1 {quote} - {cost.get('single')} {base}.\n Стоимость {amount} {quote} = {cost.get('total')} {base}"
        bot.send_message(message.chat.id, text)

bot.polling()


from luggfinder.db_models import Process, db
from flask import render_template, Blueprint, request, redirect, url_for

user_bp = Blueprint('user', __name__)

@user_bp.route('/user')
def home():
    query = Process.query.all()
    suppliers = {"", "Gv Express", "Três Company", "Voucher", "Retirou"}

    return render_template("user_pages/user.html",
                            title="User",
                            query=query,
                            suppliers=suppliers)

@user_bp.route('/update_process', methods=['POST'])
def update_process():
    json_data = request.get_json()

    for i in json_data:
        update_process = Process.query.filter_by(process=json_data[i]["process"]).first();

        if update_process:
            update_process.date = json_data[i]["date"]
            update_process.name = json_data[i]["name"]
            update_process.pnr = json_data[i]["pnr"]
            update_process.process_type = json_data[i]["process_type"]
            update_process.fault_station = json_data[i]["fault_station"]
            update_process.process_reason = json_data[i]["process_reason"]
            update_process.cost = json_data[i]["cost"]
            update_process.supplier = json_data[i]["supplier"]

            db.session.commit()

    return "received"

# @user_bp.route('/user/new_process')
# def new_process():
#     add_process = Process(
#             process = json_data[i]["process"],
#             date = json_data[i]["date"],
#             name = json_data[i]["name"],
#             pnr = json_data[i]["pnr"],
#             process_type = json_data[i]["process_type"],
#             fault_station = json_data[i]["fault_station"],
#             process_reason = json_data[i]["process_reason"],
#             cost = json_data[i]["cost"],
#             supplier = json_data[i]["supplier"]
#             )

#     db.session.add(add_process)

@user_bp.route('/delete/<process>')
def delete_process(process):
    to_delete = Process.query.filter_by(process=process.upper()).first();
    print(to_delete)
    db.session.delete(to_delete)
    db.session.commit()

    return redirect(url_for('user.home'))


import pandas as pd
import tensorflow as tf
import numpy as np

def get_data():
    train_data = pd.read_csv('./data/train.csv').values
    test_data = pd.read_csv('./data/test.csv').values

    eval_x, eval_y, train_x, train_y = split_train_set(train_data, test_size=0.1)
    test_x = test_data.reshape(-1, 28, 28, 1)

    # datagen = tf.keras.preprocessing.image.ImageDataGenerator(
    #     rotation_range=10,
    #     zoom_range=0.1,
    #     width_shift_range=0.1,
    #     height_shift_range=0.1
    # )
    # datagen.fit()

    return train_x/255.0, train_y, eval_x/255.0, eval_y, test_x/255.0

def split_train_set(train_data, test_size):
    np.random.shuffle(train_data)
    index = int((1 - test_size) * train_data.shape[0])
    train_x = train_data[:index, 1:].reshape(-1, 28, 28, 1)
    train_y = train_data[:index, 0].reshape(-1, 1)
    eval_x = train_data[index:, 1:].reshape(-1, 28, 28, 1)
    eval_y = train_data[index:, 0].reshape(-1, 1)
    return eval_x, eval_y, train_x, train_y

def cnn_model(features, labels, mode):
    features = tf.to_float(features)
    input_layer = tf.reshape(features, [-1, 28, 28, 1])
    conv1 = tf.layers.conv2d(inputs=input_layer, filters=32, kernel_size=[5, 5], padding='same', activation=tf.nn.relu)
    pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2)
    conv2 = tf.layers.conv2d(inputs=pool1, filters=64, kernel_size=[5, 5], padding='same', activation=tf.nn.relu)
    pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2)
    conv3 = tf.layers.conv2d(inputs=pool2, filters=128, kernel_size=[5, 5], padding='same',activation=tf.nn.relu)
    pool3 = tf.layers.max_pooling2d(inputs=conv3, pool_size=[2, 2], strides=2)
    pool3_flat = tf.reshape(pool3, [-1, 3 * 3 * 128])
    dense1 = tf.layers.dense(inputs=pool3_flat, units=512, activation=tf.nn.relu)
    dropout = tf.layers.dropout(inputs=dense1, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN)
    logits = tf.layers.dense(inputs=dropout, units=10)
    predictions = {
        'number': tf.argmax(input=logits, axis=1),
        'probabilities': tf.nn.softmax(logits=logits, name='softmax_tensor'),
    }

    if mode == tf.estimator.ModeKeys.PREDICT:
        return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)

    loss =tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)

    if mode == tf.estimator.ModeKeys.EVAL:
        eval_metris_ops = {
            'accuracy': tf.metrics.accuracy(labels=labels, predictions=predictions['number'])
        }
        return tf.estimator.EstimatorSpec(mode=mode, eval_metric_ops=eval_metris_ops, loss=loss)

    assert mode == tf.estimator.ModeKeys.TRAIN
    optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
    train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step())
    return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)

def main(arg):
    tf.logging.set_verbosity(tf.logging.INFO)

    mnist_classifier = tf.estimator.Estimator(
        model_fn=cnn_model,
        model_dir='./output',
    )

    train_x, train_y, eval_x, eval_y, test_x = get_data()

    # train_log = {'probabilities': 'softmax_tensor'}
    # logging_hook = tf.train.LoggingTensorHook(tensors=train_log, every_n_iter=20)

    train_input_fn = tf.estimator.inputs.numpy_input_fn(x=train_x, y=train_y, batch_size=128, num_epochs=1000, shuffle=True)
    mnist_classifier.train(input_fn=train_input_fn, steps=200, max_steps=10000)

    eval_input_fn = tf.estimator.inputs.numpy_input_fn(x=eval_x, y=eval_y, shuffle=False)
    eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn)
    print(eval_results)

    predict_input_fn = tf.estimator.inputs.numpy_input_fn(x=test_x, shuffle=False)
    predictions = mnist_classifier.predict(input_fn=predict_input_fn)
    predictions_array = []
    for pred in predictions:
        predictions_array.append(pred['number'])
    submission = pd.DataFrame(data=predictions_array, index=np.arange(1, test_x.shape[0] + 1))
    submission.to_csv('./data/submission', header=['Label'], index_label='ImageId')

if __name__ == '__main__':
    tf.app.run()


import sys
from collections import Counter

n4_offsets = [(0, -1), (0, 1), (-1, 0), (1, 0)]
n8_offsets = [(1, 0), (1, -1), (0, -1), (-1, -1), (-1, 0), (-1, 1), (0, 1), (1, 1)]
dircheck_offsets = [
        [o for o in n8_offsets if o[1] == -1],
        [o for o in n8_offsets if o[1] == 1],
        [o for o in n8_offsets if o[0] == -1],
        [o for o in n8_offsets if o[0] == 1]
    ]

def propose(pos, startdir):
    x, y = pos
    if any((x + o[0], y + o[1]) in elves for o in n8_offsets):
        for dir_offset in range(4):
            dir = (startdir + dir_offset) % 4
            if all((x + o[0], y + o[1]) not in elves for o in dircheck_offsets[dir]):
                return (x + n4_offsets[dir][0], y + n4_offsets[dir][1]), dir
    return pos, None

elves = set()
for y, line in enumerate(sys.stdin.read().splitlines()):
    for x, c in enumerate(line):
        if c == '#':
            elves.add((x, y))

startdir = 0
for _ in range(10):
    counters = Counter()
    targets = {}
    for pos in elves:
        target, dir = propose(pos, startdir)
        counters[target] += 1
        targets.update({target : pos})
    for target, pos in targets.items():
        if counters[target] == 1:
            elves.remove(pos)
            elves.add(target)
    startdir = (startdir + 1) % 4

xmin = min(p[0] for p in elves)
xmax = max(p[0] for p in elves)
ymin = min(p[1] for p in elves)
ymax = max(p[1] for p in elves)
print((xmax-xmin+1) * (ymax-ymin+1) - len(elves))


import logging
import time
import numpy as np
from numpy import array, ones, sqrt
from numpy.random import rand
from .pyutils import is_number
from .core import ttm, nvecs, norm

__all__ = [
    'hooi',
    'hosvd',
]

_log = logging.getLogger('TUCKER')
__DEF_MAXITER = 500
__DEF_INIT = 'nvecs'
__DEF_CONV = 1e-7

def hooi(X, rank, **kwargs):
    """
    Compute Tucker decomposition of a tensor using Higher-Order Orthogonal
    Iterations.

    Parameters
    ----------
    X : tensor_mixin
        The tensor to be decomposed
    rank : array_like
        The rank of the decomposition for each mode of the tensor.
        The length of ``rank`` must match the number of modes of ``X``.
    init : {'random', 'nvecs'}, optional
        The initialization method to use.
            - random : Factor matrices are initialized randomly.
            - nvecs : Factor matrices are initialzed via HOSVD.
        default : 'nvecs'

    Examples
    --------
    Create dense tensor

    >>> T = np.zeros((3, 4, 2))
    >>> T[:, :, 0] = [[ 1,  4,  7, 10], [ 2,  5,  8, 11], [3,  6,  9, 12]]
    >>> T[:, :, 1] = [[13, 16, 19, 22], [14, 17, 20, 23], [15, 18, 21, 24]]
    >>> T = dtensor(T)

    Compute Tucker decomposition of ``T`` with n-rank [2, 3, 1] via higher-order
    orthogonal iterations

    >>> Y = hooi(T, [2, 3, 1], init='nvecs')

    Shape of the core tensor matches n-rank of the decomposition.

    >>> Y['core'].shape
    (2, 3, 1)
    >>> Y['U'][1].shape
    (3, 2)

    References
    ----------
    .. [1] L. De Lathauwer, B. De Moor, J. Vandewalle: On the best rank-1 and
           rank-(R_1, R_2, \ldots, R_N) approximation of higher order tensors;
           IEEE Trans. Signal Process. 49 (2001), pp. 2262-2271
    """
    # init options
    ainit = kwargs.pop('init', __DEF_INIT)
    maxIter = kwargs.pop('maxIter', __DEF_MAXITER)
    conv = kwargs.pop('conv', __DEF_CONV)
    dtype = kwargs.pop('dtype', X.dtype)
    if not len(kwargs) == 0:
        raise ValueError('Unknown keywords (%s)' % (kwargs.keys()))

    ndims = X.ndim
    if is_number(rank):
        rank = rank * ones(ndims)

    normX = norm(X)

    U = __init(ainit, X, ndims, rank, dtype)
    fit = 0
    exectimes = []
    for itr in range(maxIter):
        tic = time.clock()
        fitold = fit

        for n in range(ndims):
            Utilde = ttm(X, U, n, transp=True, without=True)
            U[n] = nvecs(Utilde, n, rank[n])

        # compute core tensor to get fit
        core = ttm(Utilde, U, n, transp=True)

        # since factors are orthonormal, compute fit on core tensor
        normresidual = sqrt(normX ** 2 - norm(core) ** 2)

        # fraction explained by model
        fit = 1 - (normresidual / normX)
        fitchange = abs(fitold - fit)
        exectimes.append(time.clock() - tic)

        _log.debug(
            '[%3d] fit: %.5f | delta: %7.1e | secs: %.5f'
            % (itr, fit, fitchange, exectimes[-1])
        )
        if itr > 1 and fitchange < conv:
            break
    return core, U

def hosvd(X, rank, dims=None, dtype=None, compute_core=True):
    U = [None for _ in range(X.ndim)]
    if dims is None:
        dims = range(X.ndim)
    if dtype is None:
        dtype = X.dtype
    for d in dims:
        U[d] = array(nvecs(X, d, rank[d]), dtype=dtype)
    if compute_core:
        core = X.ttm(U, transp=True)
        return U, core
    else:
        return U

def __init(init, X, N, rank, dtype):
    # Don't compute initial factor for first index, gets computed in
    # first iteration
    Uinit = [None]
    if isinstance(init, list):
        Uinit = init
    elif init == 'random':
        for n in range(1, N):
            Uinit.append(array(rand(X.shape[n], rank[n]), dtype=dtype))
    elif init == 'nvecs':
        Uinit = hosvd(X, rank, range(1, N), dtype=dtype, compute_core=False)
    return Uinit