from flask import Flask, render_template, jsonify, request
from flask_pymongo import PyMongo
from flask_cors import CORS, cross_origin

import json
import copy
import warnings
import re
import random
import math
import pandas as pd
pd.set_option('use_inf_as_na', True)
import numpy as np
import multiprocessing
from operator import itemgetter
from rulelist import RuleList
import umap

from joblib import Memory
from joblib import Parallel, delayed

from sklearn import preprocessing

from sklearn.model_selection import cross_validate
from sklearn.model_selection import cross_val_predict

from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix

from sklearn.model_selection import RandomizedSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.model_selection import train_test_split

from sklearn import metrics
from sklearn.cluster import DBSCAN

# this block of code is for the connection between the server, the database, and the client (plus routing)

# access MongoDB
app = Flask(__name__)

app.config["MONGO_URI"] = "mongodb://localhost:27017/mydb"
mongo = PyMongo(app)

cors = CORS(app, resources={r"/data/*": {"origins": "*"}})

@cross_origin(origin='localhost',headers=['Content-Type','Authorization'])
@app.route('/data/Reset', methods=["GET", "POST"])
def reset():
    global DataRawLength
    global DataResultsRaw
    global previousState
    previousState = []

    global filterActionFinal
    filterActionFinal = ''

    global keySpecInternal
    keySpecInternal = 1

    global RANDOM_SEED
    RANDOM_SEED = 42

    global keyData
    keyData = 0

    global ModelSpaceUMAP
    ModelSpaceUMAP = []

    global keepOriginalFeatures
    keepOriginalFeatures = []

    global XData
    XData = []

    global roundValueSend
    roundValueSend = 15

    global XDataNorm
    XDataNorm = []

    global yData
    yData = []

    global XDataNoRemoval
    XDataNoRemoval = []

    global XDataNoRemovalOrig
    XDataNoRemovalOrig = []

    global finalResultsData
    finalResultsData = []

    global detailsParams
    detailsParams = []

    global algorithmList
    algorithmList = []

    global ClassifierIDsList
    ClassifierIDsList = ''

    global RetrieveModelsList
    RetrieveModelsList = []

    global crossValidation
    crossValidation = 3

    global parametersSelData
    parametersSelData = []

    global target_names
    target_names = []

    global keyFirstTime
    keyFirstTime = True

    global target_namesLoc
    target_namesLoc = []

    global featureCompareData
    featureCompareData = []

    global columnsKeep
    columnsKeep = []

    global columnsNewGen
    columnsNewGen = []

    global columnsNames
    columnsNames = []

    global fileName
    fileName = []

    global results
    results = []

    global scoring
    scoring = {'accuracy': 'accuracy', 'precision_macro': 'precision_macro', 'recall_macro': 'recall_macro'}

    global names_labels
    names_labels = []

    global randomSearchVar
    randomSearchVar = 10

    global paramsRF
    paramsRF = {}

    global paramsAB
    paramsAB = {}

    return 'The reset was done!'

# retrieve data from client and select the correct data set
@cross_origin(origin='localhost',headers=['Content-Type','Authorization'])
@app.route('/data/ServerRequest', methods=["GET", "POST"])
def retrieveFileName():
    global DataRawLength
    global DataResultsRaw
    global DataResultsRawTest
    global DataRawLengthTest

    global fileName
    fileName = []
    fileName = request.get_data().decode('utf8').replace("'", '"')
    print(fileName)
    global keySpecInternal
    keySpecInternal = 1

    global filterActionFinal
    filterActionFinal = ''

    global dataSpacePointsIDs
    dataSpacePointsIDs = []

    global RANDOM_SEED
    RANDOM_SEED = 42

    global keyData
    keyData = 0

    global ModelSpaceUMAP
    ModelSpaceUMAP = []

    global keepOriginalFeatures
    keepOriginalFeatures = []

    global XData
    XData = []

    global roundValueSend
    roundValueSend = 15

    global XDataNorm
    XDataNorm = []

    global XDataNoRemoval
    XDataNoRemoval = []

    global XDataNoRemovalOrig
    XDataNoRemovalOrig = []

    global previousState
    previousState = []

    global scoring
    scoring = {'accuracy': 'accuracy', 'precision_macro': 'precision_macro', 'recall_macro': 'recall_macro'}

    global yData
    yData = []

    global finalResultsData
    finalResultsData = []

    global ClassifierIDsList
    ClassifierIDsList = ''

    global algorithmList
    algorithmList = []

    global detailsParams
    detailsParams = []

    # Initializing models

    global RetrieveModelsList
    RetrieveModelsList = []

    global resultsList
    resultsList = []

    global resultsRF
    global resultsAB

    resultsRF = []
    resultsAB = []

    global crossValidation
    crossValidation = 3

    global parametersSelData
    parametersSelData = []

    global target_names

    target_names = []

    global keyFirstTime
    keyFirstTime = True

    global target_namesLoc
    target_namesLoc = []

    global names_labels
    names_labels = []

    global featureCompareData
    featureCompareData = []

    global columnsKeep
    columnsKeep = []

    global columnsNewGen
    columnsNewGen = []

    global columnsNames
    columnsNames = []

    global randomSearchVar
    randomSearchVar = 10

    global paramsRF
    paramsRF = {}

    global paramsAB
    paramsAB = {}

    DataRawLength = -1
    DataRawLengthTest = -1
    data = json.loads(fileName)
    if data['fileName'] == 'HeartC':
        CollectionDB = mongo.db.HeartC.find()
        target_names.append('Healthy')
        target_names.append('Diseased')
    elif data['fileName'] == 'biodegC':
        StanceTest = True
        CollectionDB = mongo.db.biodegC.find()
        CollectionDBTest = mongo.db.biodegCTest.find()
        CollectionDBExternal = mongo.db.biodegCExt.find()
        target_names.append('Non-biodegr.')
        target_names.append('Biodegr.')
    elif data['fileName'] == 'breastC':
        CollectionDB = mongo.db.breastC.find()
        target_names.append('Malignant')
        target_names.append('Benign')
    elif data['fileName'] == 'TitanicC':
        CollectionDB = mongo.db.TitanicC.find()
        target_names.append('Survived')
        target_names.append('Died')
    elif data['fileName'] == 'CreditC':
        CollectionDB = mongo.db.CreditC.find()
        target_names.append('Rejected')
        target_names.append('Accepted')
    elif data['fileName'] == 'HappinessC':
        CollectionDB = mongo.db.HappinessC.find()
        target_names.append('HS-Level-1')
        target_names.append('HS-Level-2')
        target_names.append('HS-Level-3')
    elif data['fileName'] == 'diabetesC':
        CollectionDB = mongo.db.diabetesC.find()
        target_names.append('Positive')
        target_names.append('Negative')
    elif data['fileName'] == 'ContraceptiveC':
        CollectionDB = mongo.db.ContraceptiveC.find()
        target_names.append('Short-term')
        target_names.append('Long-term')
        target_names.append('No-use')
    elif data['fileName'] == 'VehicleC':
        CollectionDB = mongo.db.VehicleC.find()
        target_names.append('Van')
        target_names.append('Car')
        target_names.append('Bus')
    elif data['fileName'] == 'WineC':
        CollectionDB = mongo.db.WineC.find()
        target_names.append('Fine')
        target_names.append('Superior')
        target_names.append('Inferior')
    else:
        CollectionDB = mongo.db.IrisC.find()
    DataResultsRaw = []
    for index, item in enumerate(CollectionDB):
        item['_id'] = str(item['_id'])
        item['InstanceID'] = index
        DataResultsRaw.append(item)
    DataRawLength = len(DataResultsRaw)

    DataResultsRawTest = []

    dataSetSelection()

    return 'Everything is okay'

def dataSetSelection():

    global AllTargets
    global target_names
    global paramsRF
    global paramsAB
    global sendHyperRF
    global sendHyperAB

    target_namesLoc = []

    DataResults = copy.deepcopy(DataResultsRaw)

    for dictionary in DataResultsRaw:
        for key in dictionary.keys():
            if (key.find('*') != -1):
                target = key
                continue
        continue

    DataResultsRaw.sort(key=lambda x: x[target], reverse=True)
    DataResults.sort(key=lambda x: x[target], reverse=True)

    for dictionary in DataResults:
        del dictionary['_id']
        del dictionary['InstanceID']
        del dictionary[target]

    AllTargets = [o[target] for o in DataResultsRaw]
    AllTargetsFloatValues = []

    global fileName
    data = json.loads(fileName)

    previous = None
    Class = 0
    for i, value in enumerate(AllTargets):
        if (i == 0):
            previous = value
            if (data['fileName'] == 'IrisC' or data['fileName'] == 'BreastC'):
                target_names.append(value)
            else:
                pass
        if (value == previous):
            AllTargetsFloatValues.append(Class)
        else:
            Class = Class + 1
            if (data['fileName'] == 'IrisC' or data['fileName'] == 'BreastC'):
                target_names.append(value)
            else:
                pass
            AllTargetsFloatValues.append(Class)
            previous = value

    ArrayDataResults = pd.DataFrame.from_dict(DataResults)

    global XData, yData, RANDOM_SEED
    XData, yData = ArrayDataResults, AllTargetsFloatValues

    global keepOriginalFeatures

    for col in XData.columns:
        keepOriginalFeatures.append(col)

    warnings.simplefilter('ignore')

    executeSearch()

    return 'Everything is okay'

# Initialize every model for each algorithm
@cross_origin(origin='localhost',headers=['Content-Type','Authorization'])
@app.route('/data/ServerRequestSelParameters', methods=["GET", "POST"])
def executeSearch():

    global fileName
    global crossValidation
    global XData
    global XDataNorm
    global yData
    global RANDOM_SEED
    global target_names
    global keepOriginalFeatures
    global sendHyperRF
    global sendHyperAB
    global paramsRF
    global paramsAB
    global allParametersPerformancePerModel
    allParametersPerformancePerModel = []
    global randomSearchVar
    global X_train
    global Order
    global X_test
    global y_train
    global y_test
    global resultsRF
    global resultsAB
    global roundValueSend

    resultsRF = []
    resultsAB = []
    pd.set_option("display.max_rows", None, "display.max_columns", None)
    print(XData)

    x = XData.values #returns a numpy array
    min_max_scaler = preprocessing.MinMaxScaler()
    x_scaled = min_max_scaler.fit_transform(x)
    XDataNorm = pd.DataFrame(x_scaled)

    featureNamesLocal = []
    for col in XData.columns:
        featureNamesLocal.append(col+'_minLim')

    for col in XData.columns:
        featureNamesLocal.append(col+'_maxLim')

    for ind, value in enumerate(target_names):
        featureNamesLocal.append(str(ind))

    data = json.loads(fileName)

    if data['fileName'] == 'TitanicC' or data['fileName'] == 'CreditC':
        X_train, X_test, y_train, y_test = train_test_split(XDataNorm, yData, test_size=0.1, stratify=yData, random_state=32)
    else:
        X_train, X_test, y_train, y_test = train_test_split(XDataNorm, yData, test_size=0.1, stratify=yData, random_state=RANDOM_SEED)
    print(X_test)
    X_train = X_train.reset_index(drop=True)
    X_test = X_test.reset_index(drop=True)

    algorithms = ['RF', 'AB']

    X_train.columns = keepOriginalFeatures

    countNumberOfFeatures = 0
    for col in XData.columns:
        countNumberOfFeatures = countNumberOfFeatures + 1
    rootSQ = int(math.sqrt(countNumberOfFeatures))

    paramsRF = {'n_estimators': list(range(2, 21)), 'max_depth': list(range(10, 26)), 'min_samples_leaf': list(range(1,11)), 'max_features':list(range(rootSQ,countNumberOfFeatures)) } # (2,21) (10,26) and (1,11)
    sendHyperRF = paramsRF
    paramsAB = {'n_estimators': list(range(2, 21)), 'base_estimator__max_depth': list(range(10, 26)), 'learning_rate': list(np.arange(0.1,0.4,0.1)), 'base_estimator__min_samples_leaf':list(range(1,11))}
    sendHyperAB = paramsAB

    for eachAlgor in algorithms:
        if (eachAlgor) == 'RF':
            clf = RandomForestClassifier(random_state=RANDOM_SEED)
            AlgorithmsIDsEnd = 0
            resultsLocalRF = randomSearchRF(XData, X_train, y_train, X_test, clf, paramsRF, eachAlgor, AlgorithmsIDsEnd, crossValidation, randomSearchVar, RANDOM_SEED, roundValueSend)
        else:
            clf = AdaBoostClassifier(base_estimator=DecisionTreeClassifier(), random_state=RANDOM_SEED)
            AlgorithmsIDsEnd = randomSearchVar
            resultsLocalAB = randomSearchAB(XData, X_train, y_train, X_test, clf, paramsAB, eachAlgor, AlgorithmsIDsEnd, crossValidation, randomSearchVar, RANDOM_SEED, roundValueSend)

    allParametersPerformancePerModel = resultsLocalRF + resultsLocalAB
    jsonFileRF = []
    jsonFileRF = json.loads(allParametersPerformancePerModel[6])
    frameRF = []
    frameRF = pd.read_json(jsonFileRF)

    jsonFileGB = []
    jsonFileGB = json.loads(allParametersPerformancePerModel[23])
    frameGB = pd.DataFrame()
    frameGB = pd.read_json(jsonFileGB)

    predValueRF = []
    predValueRF = json.loads(allParametersPerformancePerModel[10])
    predValueRFRead = pd.DataFrame()
    predValueRFRead = pd.read_json(predValueRF)

    predValueGB = []
    predValueGB = json.loads(allParametersPerformancePerModel[27])
    predValueGBRead = pd.DataFrame()
    predValueGBRead = pd.read_json(predValueGB)

    together = pd.DataFrame()
    together = pd.concat([frameRF, frameGB])
    together = together.reset_index(drop=True)

    togetherPredValue = pd.DataFrame()
    togetherPredValue = pd.concat([predValueRFRead, predValueGBRead])
    togetherPredValue = togetherPredValue.reset_index(drop=True)

    epsilon = 0.3
    minSamples = range(5,51)
    storePosition = -1
    findMax = 0
    for i in minSamples:
        db = DBSCAN(eps=epsilon, min_samples=i, n_jobs=-1).fit(together)
        core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
        core_samples_mask[db.core_sample_indices_] = True
        labels = db.labels_
        if ((metrics.adjusted_rand_score(togetherPredValue['predicted_value'], labels)) > findMax):
            findMax = metrics.adjusted_rand_score(togetherPredValue['predicted_value'], labels)
            storePosition = i

    db = DBSCAN(eps=epsilon, min_samples=storePosition, n_jobs=-1).fit(together)
    core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
    core_samples_mask[db.core_sample_indices_] = True
    labels = db.labels_
    n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
    #n_noise_ = list(labels).count(-1)

    print('Estimated number of clusters: %d' % n_clusters_)
    #print('Estimated number of noise points: %d' % n_noise_)
    print(findMax)

    if (n_clusters_ > 100):
        n_clusters_ = 100
    elif (n_clusters_ == 1):
        n_clusters_ = 2
    else:
        pass

    global ModelSpaceUMAP
    ModelSpaceUMAP = FunUMAP(together, n_clusters_)

    SendEachClassifiersPerformanceToVisualize()

    return 'Everything Okay'

location = './cachedir'
memory = Memory(location, verbose=0)

@memory.cache
def randomSearchRF(XData, X_train, y_train, X_test, clf, params, eachAlgor, AlgorithmsIDsEnd, crossValidation, randomS, RANDOM_SEED, roundValue):
    print('insideRFNow!!!')
    # this is the grid we use to train the models
    randSear = RandomizedSearchCV(
        estimator=clf, param_distributions=params, n_iter=randomS,
        cv=crossValidation, refit='accuracy', scoring=scoring,
        verbose=0, n_jobs=-1, random_state=RANDOM_SEED)

    # fit and extract the probabilities
    randSear.fit(X_train, y_train)

    # process the results
    cv_results = []
    cv_results.append(randSear.cv_results_)
    df_cv_results = pd.DataFrame.from_dict(cv_results)

    number_of_models = []
    # number of models stored
    number_of_models = len(df_cv_results.iloc[0][0])

    # initialize results per row
    df_cv_results_per_row = []

    modelsIDs = []
    for i in range(number_of_models):
        number = AlgorithmsIDsEnd+i
        modelsIDs.append(eachAlgor+str(number))
        df_cv_results_per_item = []
        for column in df_cv_results.iloc[0]:
            df_cv_results_per_item.append(column[i])
        df_cv_results_per_row.append(df_cv_results_per_item)

    df_cv_results_classifiers = pd.DataFrame()
    # store the results into a pandas dataframe
    df_cv_results_classifiers = pd.DataFrame(data = df_cv_results_per_row, columns= df_cv_results.columns)

    # copy and filter in order to get only the metrics
    metrics = df_cv_results_classifiers.copy()
    metrics = metrics.filter(['mean_test_accuracy', 'mean_test_precision_macro', 'mean_test_recall_macro',])

    parametersPerformancePerModel = pd.DataFrame()
    # concat parameters and performance
    parametersPerformancePerModel = pd.DataFrame(df_cv_results_classifiers['params'])
    parametersPerformancePerModel = parametersPerformancePerModel.to_json(double_precision=15)

    parametersLocal = json.loads(parametersPerformancePerModel)['params'].copy()
    Models = []
    for index, items in enumerate(parametersLocal):
        Models.append(str(index))

    parametersLocalNew = [ parametersLocal[your_key] for your_key in Models ]

    perModelProb = []
    confuseFP = []
    confuseFN = []
    featureImp = []
    collectDecisionsPerModel = pd.DataFrame()
    collectDecisions = []
    collectLocationsAll = []
    collectDecisionsMod = []
    collectStatistics = []
    collectStatisticsMod = []
    collectStatisticsPerModel = []
    collectInfoPerModel = []
    yPredictTestList = []
    perModelPrediction = []
    storeTrain = []
    storePredict = []

    featureNames = []
    featureNamesDuplicated = []

    for col in XData.columns:
        featureNames.append(col)
        featureNamesDuplicated.append(col+'_minLim')
    for col in XData.columns:
        featureNamesDuplicated.append(col+'_maxLim')

    counterModels = 1
    for eachModelParameters in parametersLocalNew:
        collectDecisions = []
        collectLocations = []
        collectStatistics = []
        sumRes = 0
        clf.set_params(**eachModelParameters)
        np.random.seed(RANDOM_SEED) # seeds
        clf.fit(X_train, y_train)
        yPredictTest = clf.predict(X_test)
        yPredictTestList.append(yPredictTest)

        feature_importances = clf.feature_importances_
        feature_importances[np.isnan(feature_importances)] = 0
        featureImp.append(list(feature_importances))

        yPredict = cross_val_predict(clf, X_train, y_train, cv=crossValidation)
        yPredict = np.nan_to_num(yPredict)
        perModelPrediction.append(yPredict)

        yPredictProb = cross_val_predict(clf, X_train, y_train, cv=crossValidation, method='predict_proba')
        yPredictProb = np.nan_to_num(yPredictProb)
        perModelProb.append(yPredictProb.tolist())

        storeTrain.append(y_train)
        storePredict.append(yPredict)
        cnf_matrix = confusion_matrix(y_train, yPredict)
        FP = cnf_matrix.sum(axis=0) - np.diag(cnf_matrix)
        FN = cnf_matrix.sum(axis=1) - np.diag(cnf_matrix)
        FP = FP.astype(float)
        FN = FN.astype(float)
        confuseFP.append(list(FP))
        confuseFN.append(list(FN))
        for tree_idx, est in enumerate(clf.estimators_):
            decisionPath = extractDecisionInfo(est,counterModels,tree_idx,X_train,y_train,featureNamesDuplicated,eachAlgor,feature_names=featureNames,only_leaves=True)
            if (roundValue == -1):
                pass
            else:
                decisionPath[0] = decisionPath[0].round(roundValue)
            collectDecisions.append(decisionPath[0])
            collectStatistics.append(decisionPath[1])
            sumRes = sumRes + decisionPath[2]
            collectLocations.append(decisionPath[3])
        collectDecisionsMod.append(collectDecisions)
        collectStatisticsMod.append(collectStatistics)
        collectInfoPerModel.append(sumRes)
        collectLocationsAll.append(collectLocations)
        counterModels = counterModels + 1

    collectInfoPerModelPandas = pd.DataFrame(collectInfoPerModel)

    totalfnList = []
    totalfpList = []
    numberClasses = [y_train.index(x) for x in set(y_train)]
    if (len(numberClasses) == 2):
        for index,nList in enumerate(storeTrain):
            fnList = []
            fpList = []
            for ind,el in enumerate(nList):
                if (el==1 and storePredict[index][ind]==0):
                    fnList.append(ind)
                elif (el==0 and storePredict[index][ind]==1):
                    fpList.append(ind)
                else:
                    pass
            totalfpList.append(fpList)
            totalfnList.append(fnList)
    else:
        for index,nList in enumerate(storeTrain):
            fnList = []
            class0fn = []
            class1fn = []
            class2fn = []
            for ind,el in enumerate(nList):
                if (el==0 and storePredict[index][ind]==1):
                    class0fn.append(ind)
                elif (el==0 and storePredict[index][ind]==2):
                    class0fn.append(ind)
                elif (el==1 and storePredict[index][ind]==0):
                    class1fn.append(ind)
                elif (el==1 and storePredict[index][ind]==2):
                    class1fn.append(ind)
                elif (el==2 and storePredict[index][ind]==0):
                    class2fn.append(ind)
                elif (el==2 and storePredict[index][ind]==1):
                    class2fn.append(ind)
                else:
                    pass
            fnList.append(class0fn)
            fnList.append(class1fn)
            fnList.append(class2fn)
            totalfnList.append(fnList)
        for index,nList in enumerate(storePredict):
            fpList = []
            class0fp = []
            class1fp = []
            class2fp = []
            for ind,el in enumerate(nList):
                if (el==0 and storeTrain[index][ind]==1):
                    class0fp.append(ind)
                elif (el==0 and storeTrain[index][ind]==2):
                    class0fp.append(ind)
                elif (el==1 and storeTrain[index][ind]==0):
                    class1fp.append(ind)
                elif (el==1 and storeTrain[index][ind]==2):
                    class1fp.append(ind)
                elif (el==2 and storeTrain[index][ind]==0):
                    class2fp.append(ind)
                elif (el==2 and storeTrain[index][ind]==1):
                    class2fp.append(ind)
                else:
                    pass
            fpList.append(class0fp)
            fpList.append(class1fp)
            fpList.append(class2fp)
            totalfpList.append(fpList)

    # for index, nList in enumerate(totalfpList):
    #     if ((index+1) != len(totalfpList)):
    #         confuseFPCommon.append(len(list(set(nList).intersection(totalfpList[index+1]))))
    # for index, nList in enumerate(totalfnList):
    #     if ((index+1) != len(totalfnList)):
    #         confuseFNCommon.append(len(list(set(nList).intersection(totalfnList[index+1]))))

    summarizeResults = []
    summarizeResults = metrics.sum(axis=1)
    summarizeResultsFinal = []
    for el in summarizeResults:
        summarizeResultsFinal.append(round(((el * 100)/3),2))

    indices, L_sorted = zip(*sorted(enumerate(summarizeResultsFinal), key=itemgetter(1)))
    indexList = list(indices)

    collectDecisionsSorted = []
    collectStatisticsSorted = []
    collectLocationsAllSorted = []
    for el in indexList:
        for item in collectDecisionsMod[el]:
            collectDecisionsSorted.append(item)
        for item2 in collectStatisticsMod[el]:
            collectStatisticsSorted.append(item2)
        for item3 in collectLocationsAll[el]:
            collectLocationsAllSorted.append(item3)

    collectDecisionsPerModel = pd.concat(collectDecisionsSorted)
    collectStatisticsPerModel = pd.concat(collectStatisticsSorted)
    collectLocationsAllPerSorted = pd.DataFrame(collectLocationsAllSorted)
    collectDecisionsPerModel = collectDecisionsPerModel.reset_index(drop=True)
    collectStatisticsPerModel = collectStatisticsPerModel.reset_index(drop=True)
    collectLocationsAllPerSorted = collectLocationsAllPerSorted.reset_index(drop=True)

    collectDecisionsPerModel = collectDecisionsPerModel.to_json(double_precision=15)
    collectStatisticsPerModel = collectStatisticsPerModel.to_json(double_precision=15)
    collectInfoPerModelPandas = collectInfoPerModelPandas.to_json(double_precision=15)
    collectLocationsAllPerSorted = collectLocationsAllPerSorted.to_json(double_precision=15)

    perModelPredPandas = pd.DataFrame(perModelPrediction)
    perModelPredPandas = perModelPredPandas.to_json(double_precision=15)

    yPredictTestListPandas = pd.DataFrame(yPredictTestList)
    yPredictTestListPandas = yPredictTestListPandas.to_json(double_precision=15)

    perModelProbPandas = pd.DataFrame(perModelProb)
    perModelProbPandas = perModelProbPandas.to_json(double_precision=15)

    metrics = metrics.to_json(double_precision=15)
    # gather the results and send them back
    resultsRF.append(modelsIDs) # 0
    resultsRF.append(parametersPerformancePerModel) # 1
    resultsRF.append(metrics) # 2
    resultsRF.append(json.dumps(confuseFP)) # 3
    resultsRF.append(json.dumps(confuseFN)) # 4
    resultsRF.append(json.dumps(featureImp)) # 5
    resultsRF.append(json.dumps(collectDecisionsPerModel)) # 6
    resultsRF.append(perModelProbPandas) # 7
    resultsRF.append(json.dumps(perModelPredPandas)) # 8
    resultsRF.append(json.dumps(target_names)) # 9
    resultsRF.append(json.dumps(collectStatisticsPerModel)) # 10
    resultsRF.append(json.dumps(collectInfoPerModelPandas)) # 11
    resultsRF.append(json.dumps(keepOriginalFeatures)) # 12
    resultsRF.append(json.dumps(yPredictTestListPandas)) # 13
    resultsRF.append(json.dumps(collectLocationsAllPerSorted)) # 14
    resultsRF.append(json.dumps(totalfpList)) # 15
    resultsRF.append(json.dumps(totalfnList)) # 16

    return resultsRF

location = './cachedir'
memory = Memory(location, verbose=0)

@memory.cache
def randomSearchAB(XData, X_train, y_train, X_test, clf, params, eachAlgor, AlgorithmsIDsEnd, crossValidation, randomS, RANDOM_SEED, roundValue):
    print('insideABNow!!!')
    # this is the grid we use to train the models
    randSear = RandomizedSearchCV(
        estimator=clf, param_distributions=params, n_iter=randomS,
        cv=crossValidation, refit='accuracy', scoring=scoring,
        verbose=0, n_jobs=-1, random_state=RANDOM_SEED)

    # fit and extract the probabilities
    randSear.fit(X_train, y_train)

    # process the results
    cv_results = []
    cv_results.append(randSear.cv_results_)
    df_cv_results = pd.DataFrame.from_dict(cv_results)

    number_of_models = []
    # number of models stored
    number_of_models = len(df_cv_results.iloc[0][0])

    # initialize results per row
    df_cv_results_per_row = []

    modelsIDs = []
    for i in range(number_of_models):
        number = AlgorithmsIDsEnd+i
        modelsIDs.append(eachAlgor+str(number))
        df_cv_results_per_item = []
        for column in df_cv_results.iloc[0]:
            df_cv_results_per_item.append(column[i])
        df_cv_results_per_row.append(df_cv_results_per_item)

    df_cv_results_classifiers = pd.DataFrame()
    # store the results into a pandas dataframe
    df_cv_results_classifiers = pd.DataFrame(data = df_cv_results_per_row, columns= df_cv_results.columns)

    # copy and filter in order to get only the metrics
    metrics = df_cv_results_classifiers.copy()
    metrics = metrics.filter(['mean_test_accuracy', 'mean_test_precision_macro', 'mean_test_recall_macro',])

    parametersPerformancePerModel = pd.DataFrame()
    # concat parameters and performance
    parametersPerformancePerModel = pd.DataFrame(df_cv_results_classifiers['params'])
    parametersPerformancePerModel = parametersPerformancePerModel.to_json(double_precision=15)

    parametersLocal = json.loads(parametersPerformancePerModel)['params'].copy()
    Models = []
    for index, items in enumerate(parametersLocal):
        Models.append(str(index))

    parametersLocalNew = [ parametersLocal[your_key] for your_key in Models ]

    perModelProb = []
    confuseFP = []
    confuseFN = []
    featureImp = []
    collectDecisionsPerModel = pd.DataFrame()
    collectDecisions = []
    collectDecisionsMod = []
    collectLocationsAll = []
    collectStatistics = []
    collectStatisticsMod = []
    collectStatisticsPerModel = []
    collectInfoPerModel = []
    yPredictTestList = []
    perModelPrediction = []
    storeTrain = []
    storePredict = []

    featureNames = []
    featureNamesDuplicated = []

    for col in XData.columns:
        featureNames.append(col)
        featureNamesDuplicated.append(col+'_minLim')
    for col in XData.columns:
        featureNamesDuplicated.append(col+'_maxLim')

    counterModels = 1
    for eachModelParameters in parametersLocalNew:
        collectDecisions = []
        collectLocations = []
        collectStatistics = []
        sumRes = 0
        clf.set_params(**eachModelParameters)
        np.random.seed(RANDOM_SEED) # seeds
        clf.fit(X_train, y_train)
        yPredictTest = clf.predict(X_test)
        yPredictTestList.append(yPredictTest)

        feature_importances = clf.feature_importances_
        feature_importances[np.isnan(feature_importances)] = 0
        featureImp.append(list(feature_importances))

        yPredict = cross_val_predict(clf, X_train, y_train, cv=crossValidation)
        yPredict = np.nan_to_num(yPredict)
        perModelPrediction.append(yPredict)

        yPredictProb = cross_val_predict(clf, X_train, y_train, cv=crossValidation, method='predict_proba')
        yPredictProb = np.nan_to_num(yPredictProb)
        perModelProb.append(yPredictProb.tolist())

        storeTrain.append(y_train)
        storePredict.append(yPredict)
        cnf_matrix = confusion_matrix(y_train, yPredict)
        FP = cnf_matrix.sum(axis=0) - np.diag(cnf_matrix)
        FN = cnf_matrix.sum(axis=1) - np.diag(cnf_matrix)
        FP = FP.astype(float)
        FN = FN.astype(float)
        confuseFP.append(list(FP))
        confuseFN.append(list(FN))
        for tree_idx, est in enumerate(clf.estimators_):
            decisionPath = extractDecisionInfo(est,counterModels,tree_idx,X_train,y_train,featureNamesDuplicated,eachAlgor,feature_names=featureNames,only_leaves=True)
            if (roundValue == -1):
                pass
            else:
                decisionPath[0] = decisionPath[0].round(roundValue)
            collectDecisions.append(decisionPath[0])
            collectStatistics.append(decisionPath[1])
            sumRes = sumRes + decisionPath[2]
            collectLocations.append(decisionPath[3])
        collectDecisionsMod.append(collectDecisions)
        collectStatisticsMod.append(collectStatistics)
        collectInfoPerModel.append(sumRes)
        collectLocationsAll.append(collectLocations)
        counterModels = counterModels + 1

    collectInfoPerModelPandas = pd.DataFrame(collectInfoPerModel)

    totalfnList = []
    totalfpList = []
    numberClasses = [y_train.index(x) for x in set(y_train)]
    if (len(numberClasses) == 2):
        for index,nList in enumerate(storeTrain):
            fnList = []
            fpList = []
            for ind,el in enumerate(nList):
                if (el==1 and storePredict[index][ind]==0):
                    fnList.append(ind)
                elif (el==0 and storePredict[index][ind]==1):
                    fpList.append(ind)
                else:
                    pass
            totalfpList.append(fpList)
            totalfnList.append(fnList)
    else:
        for index,nList in enumerate(storeTrain):
            fnList = []
            class0fn = []
            class1fn = []
            class2fn = []
            for ind,el in enumerate(nList):
                if (el==0 and storePredict[index][ind]==1):
                    class0fn.append(ind)
                elif (el==0 and storePredict[index][ind]==2):
                    class0fn.append(ind)
                elif (el==1 and storePredict[index][ind]==0):
                    class1fn.append(ind)
                elif (el==1 and storePredict[index][ind]==2):
                    class1fn.append(ind)
                elif (el==2 and storePredict[index][ind]==0):
                    class2fn.append(ind)
                elif (el==2 and storePredict[index][ind]==1):
                    class2fn.append(ind)
                else:
                    pass
            fnList.append(class0fn)
            fnList.append(class1fn)
            fnList.append(class2fn)
            totalfnList.append(fnList)
        for index,nList in enumerate(storePredict):
            fpList = []
            class0fp = []
            class1fp = []
            class2fp = []
            for ind,el in enumerate(nList):
                if (el==0 and storeTrain[index][ind]==1):
                    class0fp.append(ind)
                elif (el==0 and storeTrain[index][ind]==2):
                    class0fp.append(ind)
                elif (el==1 and storeTrain[index][ind]==0):
                    class1fp.append(ind)
                elif (el==1 and storeTrain[index][ind]==2):
                    class1fp.append(ind)
                elif (el==2 and storeTrain[index][ind]==0):
                    class2fp.append(ind)
                elif (el==2 and storeTrain[index][ind]==1):
                    class2fp.append(ind)
                else:
                    pass
            fpList.append(class0fp)
            fpList.append(class1fp)
            fpList.append(class2fp)
            totalfpList.append(fpList)

    summarizeResults = []
    summarizeResults = metrics.sum(axis=1)
    summarizeResultsFinal = []
    for el in summarizeResults:
        summarizeResultsFinal.append(round(((el * 100)/3),2))

    indices, L_sorted = zip(*sorted(enumerate(summarizeResultsFinal), key=itemgetter(1)))
    indexList = list(indices)

    collectDecisionsSorted = []
    collectStatisticsSorted = []
    collectLocationsAllSorted = []
    for el in indexList:
        for item in collectDecisionsMod[el]:
            collectDecisionsSorted.append(item)
        for item2 in collectStatisticsMod[el]:
            collectStatisticsSorted.append(item2)
        for item3 in collectLocationsAll[el]:
            collectLocationsAllSorted.append(item3)

    collectDecisionsPerModel = pd.concat(collectDecisionsSorted)
    collectStatisticsPerModel = pd.concat(collectStatisticsSorted)
    collectLocationsAllPerSorted = pd.DataFrame(collectLocationsAllSorted)

    collectDecisionsPerModel = collectDecisionsPerModel.reset_index(drop=True)
    collectStatisticsPerModel = collectStatisticsPerModel.reset_index(drop=True)
    collectLocationsAllPerSorted = collectLocationsAllPerSorted.reset_index(drop=True)
    collectDecisionsPerModel = collectDecisionsPerModel.to_json(double_precision=15)
    collectStatisticsPerModel = collectStatisticsPerModel.to_json(double_precision=15)
    collectInfoPerModelPandas = collectInfoPerModelPandas.to_json(double_precision=15)
    collectLocationsAllPerSorted = collectLocationsAllPerSorted.to_json(double_precision=15)

    perModelPredPandas = pd.DataFrame(perModelPrediction)
    perModelPredPandas = perModelPredPandas.to_json(double_precision=15)

    yPredictTestListPandas = pd.DataFrame(yPredictTestList)
    yPredictTestListPandas = yPredictTestListPandas.to_json(double_precision=15)

    perModelProbPandas = pd.DataFrame(perModelProb)
    perModelProbPandas = perModelProbPandas.to_json(double_precision=15)

    metrics = metrics.to_json(double_precision=15)
    # gather the results and send them back
    resultsAB.append(modelsIDs) # 0 17
    resultsAB.append(parametersPerformancePerModel) # 1 18
    resultsAB.append(metrics) # 2 19
    resultsAB.append(json.dumps(confuseFP)) # 3 20
    resultsAB.append(json.dumps(confuseFN)) # 4 21
    resultsAB.append(json.dumps(featureImp)) # 5 22
    resultsAB.append(json.dumps(collectDecisionsPerModel)) # 6 23
    resultsAB.append(perModelProbPandas) # 7 24
    resultsAB.append(json.dumps(perModelPredPandas)) # 8 25
    resultsAB.append(json.dumps(target_names)) # 9 26
    resultsAB.append(json.dumps(collectStatisticsPerModel)) # 10 27
    resultsAB.append(json.dumps(collectInfoPerModelPandas)) # 11 28
    resultsAB.append(json.dumps(keepOriginalFeatures)) # 12 29
    resultsAB.append(json.dumps(yPredictTestListPandas)) # 13 30
    resultsAB.append(json.dumps(collectLocationsAllPerSorted)) # 14 31
    resultsAB.append(json.dumps(totalfpList)) # 15 32
    resultsAB.append(json.dumps(totalfnList)) # 16 33

    return resultsAB

def extractDecisionInfo(decision_tree,counterModels,tree_index,X_train,y_train,feature_names_duplicated,eachAlgor,feature_names=None,only_leaves=True):
    '''return dataframe with node info
    '''
    decision_tree.fit(X_train, y_train)
    # extract info from decision_tree
    n_nodes = decision_tree.tree_.node_count
    children_left = decision_tree.tree_.children_left
    children_right = decision_tree.tree_.children_right
    feature = decision_tree.tree_.feature
    threshold = decision_tree.tree_.threshold
    impurity = decision_tree.tree_.impurity
    value = decision_tree.tree_.value
    n_node_samples = decision_tree.tree_.n_node_samples

    whereTheyBelong = decision_tree.apply(X_train)

    # cast X_train as dataframe
    df = pd.DataFrame(X_train)
    if feature_names is not None:
        df.columns = feature_names

    # indexes with unique nodes
    idx_list = df.assign(
        leaf_id = lambda df: decision_tree.apply(df)
    )[['leaf_id']].drop_duplicates().index

    # test data for unique nodes
    X_test = df.loc[idx_list,].to_numpy()
    # decision path only for leaves
    dp = decision_tree.decision_path(X_test)
    # final leaves for each data
    leave_id = decision_tree.apply(X_test)
    # values for each data
    leave_predict = decision_tree.predict(X_test)
    # dictionary for leave_id and leave_predict
    dict_leaves = {k:v for k,v in zip(leave_id,leave_predict)}

    # create decision path information for all nodes
    dp_idxlist = [[ini, fin] for ini,fin in zip(dp.indptr[:-1],dp.indptr[1:])]
    dict_decisionpath = {}
    for idxs in dp_idxlist:
        dpindices = dp.indices[idxs[0]:idxs[1]]
        for i,node in enumerate(dpindices):
            if node not in dict_decisionpath.keys():
                dict_decisionpath[node] = dpindices[:i+1]

    # initialize number of columns and output dataframe
    n_cols = df.shape[-1]
    df_thr_all = pd.DataFrame()

    # predict for samples
    for node, node_index in dict_decisionpath.items():
        l_thresh_max = np.ones(n_cols) * np.nan
        l_thresh_min = np.ones(n_cols) * np.nan

        # decision path info for each node
        for i,node_id in enumerate(node_index):
            if node == node_id:
                continue

            if children_left[node_id] == node_index[i+1]: #(X_test[sample_id, feature[node_id]] <= threshold[node_id]):
                l_thresh_max[feature[node_id]] = threshold[node_id]
            else:
                l_thresh_min[feature[node_id]] = threshold[node_id]
        # append info to df_thr_all
        df_thr_all = df_thr_all.append(
            [[thr_min for thr_max,thr_min in zip(l_thresh_max,l_thresh_min)]
             + [thr_max for thr_max,thr_min in zip(l_thresh_max,l_thresh_min)]
             + [
                 node,
                 counterModels,
                 tree_index,
                 np.nan if node not in dict_leaves.keys() else dict_leaves[node],
                 #value[node],
                 impurity[node],
                 n_node_samples[node]
               ]
            ]
        )
    # rename columns and set index
    if feature_names is not None:
        df_thr_all.columns = feature_names_duplicated + ['node','counterModels','tree_index','predicted_value','impurity','samples']
    else:
        df_thr_all.columns = ['X_{}'.format(i) for i in range(n_cols)] + ['node','counterModels','tree_index','predicted_value','impurity','samples']
    #df_thr_all = df_thr_all.set_index('decision')
    #df_thr_all = df_thr_all.reset_index(drop=True)
    if only_leaves:
        df_thr_all = df_thr_all[~df_thr_all['predicted_value'].isnull()]
        df_thr_all['impurity'].loc[df_thr_all['impurity'] < 0] = 0
        # df_thr_all['impurity'].loc[df_thr_all['impurity'] >= 0.5] = 0.8

    # del df_thr_all['decision']
    # del df_thr_all['predicted_value']

    #df_thr_all.reset_index()

    df_thr_all = df_thr_all.replace(np.nan,2) # nan mapped as value 2

    #df_thr_all = df_thr_all.sort_index()

    copy_df_thr_all = df_thr_all.copy()

    del df_thr_all['node']
    del df_thr_all['counterModels']
    del df_thr_all['tree_index']
    del df_thr_all['predicted_value']
    del df_thr_all['impurity']
    del df_thr_all['samples']

    copy_df_thr_all = copy_df_thr_all[['node','counterModels','tree_index','predicted_value', 'impurity', 'samples']]
    return [df_thr_all,copy_df_thr_all,len(df_thr_all),whereTheyBelong]

# Sending each model's results to frontend
@cross_origin(origin='localhost',headers=['Content-Type','Authorization'])
@app.route('/data/PerformanceForEachModel', methods=["GET", "POST"])
def SendEachClassifiersPerformanceToVisualize():

    loopEmpty = []
    initialStates = []
    for x in range(randomSearchVar*2):
        if (x < (randomSearchVar*2/2)):
            loopEmpty.append(x+1)
        initialStates.append(1)
    response = {
        'PerformancePerModel': allParametersPerformancePerModel,
        'ModelSpaceUMAPSend': ModelSpaceUMAP,
        'X_train': X_train.to_json(double_precision=15),
        'y_train': y_train,
        'X_test': X_test.to_json(double_precision=15),
        'y_test': y_test,
        'HypRF': sendHyperRF,
        'HypAB': sendHyperAB,
        'NumberOfModels': loopEmpty,
        'States': initialStates
    }
    return jsonify(response)

def FunUMAP (data, neighbors):
    trans = umap.UMAP(n_neighbors=neighbors, min_dist=0.1, random_state=RANDOM_SEED, transform_seed=RANDOM_SEED).fit(data)
    Xpos = trans.embedding_[:, 0].tolist()
    Ypos = trans.embedding_[:, 1].tolist()
    return [Xpos,Ypos]

@cross_origin(origin='localhost',headers=['Content-Type','Authorization'])
@app.route('/data/SearchingAgain', methods=["GET", "POST"])
def requestedNewModels():

    RetrievedModel = request.get_data().decode('utf8').replace("'", '"')
    RetrievedModel = json.loads(RetrievedModel)

    limitationRF = RetrievedModel['RFNewModels']
    limitationAB = RetrievedModel['ABNewModels']
    thresholdMain = RetrievedModel['thresholdMain']

    global paramsRF
    paramsRF = limitationRF

    global paramsAB
    paramsAB = limitationAB

    global randomSearchVar
    randomSearchVar = thresholdMain

    executeSearch()

    return 'Everything is okay'

@cross_origin(origin='localhost',headers=['Content-Type','Authorization'])
@app.route('/data/updateRounding', methods=["GET", "POST"])
def updateRoundingFun():

    RetrievedModel = request.get_data().decode('utf8').replace("'", '"')
    RetrievedModel = json.loads(RetrievedModel)

    roundingValueLocal = RetrievedModel['roundingValue']

    global roundValueSend
    roundValueSend = roundingValueLocal

    executeSearch()

    return 'Everything is okay'

@cross_origin(origin='localhost',headers=['Content-Type','Authorization'])
@app.route('/data/askToRemoveAFeature', methods=["GET", "POST"])
def askToRemoveAFeatureFun():

    RetrievedModel = request.get_data().decode('utf8').replace("'", '"')
    RetrievedModel = json.loads(RetrievedModel)

    removingFeature = RetrievedModel['whoWasRemoved']

    global XData
    global keepOriginalFeatures
    keepOriginalFeatures = []

    XData = XData.drop(columns=[removingFeature])
    for col in XData.columns:
        keepOriginalFeatures.append(col)

    executeSearch()

    return 'Everything is okay'


#PYTHON 3
import urllib,json, time
from bs4 import BeautifulSoup
exec(open('/home/fbbot/cfb/sload.py').read())

confs_site=BeautifulSoup(urllib.urlopen('http://espn.go.com/college-football/standings?t='+str(time.time())),"html5lib")
conferences=confs_site.findAll('div',{'class':'responsive-table-wrap'})
confs={}
for conf in conferences:
	conf_name=conf.find('span',{'class':'long-caption'}).getText()
	confs[conf_name]={}
	subconf=False
	rows=conf.findAll(['thead','tr'])
	c=0
	for row in rows:
		if str(row).count('class="standings-categories"') == 1:
			headings=row.findAll('span')
			for heading in headings:
				heading=heading.getText()
				if heading not in ['Conference','Overall','Polls','CONF','PF','PA','OVER','HOME','ROAD','STRK','AP','USA']:
					subconf=heading
					confs[conf_name][subconf]={}
					c=0
		team=row.find('span',{'class':'team-names'})
		if team != None:
			c+=1
			if subconf: confs[conf_name][subconf][str(c)]=unicode(team.getText()).encode('ascii',errors='ignore').replace('\r','').replace('\n','').replace('San Jos State','San Jose State')
			else: confs[conf_name][str(c)]=unicode(team.getText()).encode('ascii',errors='ignore').replace('\r','').replace('\n','').replace('San Jos State','San Jose State')
sql.unique_set('data','conf_standings',json.dumps(confs))


from tkinter import *
from tkinter import messagebox
root=Tk()
def veiw1():
    if(e1.get()=="HOPE"):
        import user
    else:
        messagebox.showinfo("VEIW",("Sorry,Wrong input"))
canvas = Canvas(root,width = 500, height = 200, bg = 'blue')
canvas.pack()
img = PhotoImage(file = 'c3.png')
canvas.create_image(50, 10, image = img, anchor = NW)
l1=Label(root,text="Enter The Text in image:")
l1.pack()
e1=Entry(root,bd=4,bg="light blue",fg="black")
e1.pack()
b1=Button(root,text="NEXT",command=veiw1)
b1.pack()
root.mainloop()


import numpy as np
from scipy.stats import multivariate_normal
from sklearn.cluster import KMeans
import cv2
np.random.seed(0)

def EM_Segmentation(data, parameters, epsilon):
    mean_1 = parameters['mean_1']
    mean_2 = parameters['mean_2']
    covariance_1 = parameters['covariance_1']
    covariance_2 = parameters['covariance_2']
    mixing_coefficient_1 = parameters['mixing_coefficient_1']
    mixing_coefficient_2 = parameters['mixing_coefficient_2']
    # initialize loglikelihood
    N_1 = multivariate_normal.pdf(data, mean=mean_1, cov=covariance_1, allow_singular=True)
    N_2 = multivariate_normal.pdf(data, mean=mean_2, cov=covariance_2, allow_singular=True)
    sum_ = mixing_coefficient_1 * N_1 + mixing_coefficient_2 * N_2
    last_loglikelihood = sum(np.log(sum_))

    while(1):
        # Expectation Step
        N_1 = multivariate_normal.pdf(data, mean=mean_1, cov=covariance_1, allow_singular=True)
        N_2 = multivariate_normal.pdf(data, mean=mean_2, cov=covariance_2, allow_singular=True)
        sum_ = mixing_coefficient_1 * N_1 + mixing_coefficient_2 * N_2
        responsibility_1 = mixing_coefficient_1 * N_1 / sum_
        responsibility_2 = mixing_coefficient_2 * N_2 / sum_

        # Maximization Step
        mean_1 = np.sum(responsibility_1.reshape([-1,1]) * data, axis=0) / np.sum(responsibility_1)
        mean_2 = np.sum(responsibility_2.reshape([-1,1]) * data, axis=0) / np.sum(responsibility_2)
        covariance_1 = ((data - mean_1).T).dot((data - mean_1) * responsibility_1.reshape([-1,1])) / np.sum(responsibility_1)
        covariance_2 = ((data - mean_2).T).dot((data - mean_2) * responsibility_2.reshape([-1,1])) / np.sum(responsibility_2)
        mixing_coefficient_1 = np.sum(responsibility_1) / len(data)
        mixing_coefficient_2 = np.sum(responsibility_2) / len(data)

        # Evaluate
        N_1 = multivariate_normal.pdf(data, mean=mean_1, cov=covariance_1, allow_singular=True)
        N_2 = multivariate_normal.pdf(data, mean=mean_2, cov=covariance_2, allow_singular=True)
        sum_ = mixing_coefficient_1 * N_1 + mixing_coefficient_2 * N_2
        loglikelihood = sum(np.log(sum_))
        if (np.abs(loglikelihood - last_loglikelihood) < epsilon):
            break
        else:
            last_loglikelihood = loglikelihood

    parameters['mean_1'] = mean_1
    parameters['mean_2'] = mean_2
    parameters['covariance_1'] = covariance_1
    parameters['covariance_2'] = covariance_2
    parameters['mixing_coefficient_1'] = mixing_coefficient_1
    parameters['mixing_coefficient_2'] = mixing_coefficient_2

    return parameters

def segmentation(data, parameters):
    N_1 = multivariate_normal.pdf(data,
                                  mean=parameters['mean_1'],
                                  cov=parameters['covariance_1'],
                                  allow_singular=True)
    N_2 = multivariate_normal.pdf(data,
                                  mean=parameters['mean_2'],
                                  cov=parameters['covariance_2'],
                                  allow_singular=True)
    responsibility_1 = parameters['mixing_coefficient_1'] * N_1
    responsibility_2 = parameters['mixing_coefficient_2'] * N_2

    return responsibility_1 < responsibility_2

def writeToFile(data, indices, height, width):
    foreground_data = data.copy()
    background_data = data.copy()
    mask_data = data.copy()
    # assume the foreground' size is smaller than background's size
    if (sum(indices) > len(indices)/2.):
        fore_index = [not id for id in indices]
        back_index = indices.copy()
    else:
        back_index = [not id for id in indices]
        fore_index = indices.copy()

    background_data[fore_index] = 0.
    foreground_data[back_index] = 0.
    mask_data[fore_index] = 0.
    mask_data[back_index] = 255.

    print("start write foreground into image......")
    cv2.imwrite('data/foreground.jpg', foreground_data.reshape([height, width, -1]))
    print("succeed in saving foreground image into \"./data/foreground.jpg\"")

    print("start write background into image......")
    cv2.imwrite('data/background.jpg', background_data.reshape([height, width, -1]))
    print("succeed in saving background image into \"./data/background.jpg\"")

    print("start write mask into image......")
    cv2.imwrite('data/mask.jpg', mask_data.reshape([height, width, -1]))
    print("succeed in saving mask image into \"./data/mask.jpg\"")

def initParameters(data):
    kmeans = KMeans(n_clusters=2, random_state=0).fit(data)
    clusters = kmeans.labels_
    centroids = kmeans.cluster_centers_
    parameters = {}
    parameters['mean_1'] = centroids[0]
    parameters['mean_2'] = centroids[1]
    parameters['covariance_1'] = np.cov(data[clusters == 0].transpose())
    parameters['covariance_2'] = np.cov(data[clusters == 1].transpose())
    parameters['mixing_coefficient_1'] = 0.4
    parameters['mixing_coefficient_2'] = 0.6
    return parameters

def Process(data):
    height = data.shape[0]
    width = data.shape[1]
    data = data.reshape([-1, 3])
    init_parameters = initParameters(data)
    final_parameters = EM_Segmentation(data, init_parameters, epsilon=1e-4)
    indices = segmentation(data, final_parameters)
    writeToFile(data, indices, height, width)

if __name__ == "__main__":
    image = cv2.imread("data/cow.jpg", cv2.IMREAD_COLOR)
    print("image is loaded.")
    Process(image)


import tweepy
import configparser
from datetime import datetime
import json

config = configparser.ConfigParser()
config.read('./twitter.ini')

ACCESS_TOKEN = config.get('twitter', 'ACCESS_TOKEN')
ACCESS_TOKEN_SECRET = config.get('twitter', 'ACCESS_TOKEN_SECRET')
API_KEY = config.get('twitter', 'API_KEY')
API_TOKEN = config.get('twitter', 'API_TOKEN')

auth = tweepy.OAuthHandler(API_KEY, API_TOKEN)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
api = tweepy.API(auth, wait_on_rate_limit=True)

#procurar por covid, tweets que não foram replies ou retweets
search = "covid -filter:replies -filter:retweets"

results = tweepy.Cursor(api.search, q=search, since="2020-03-10",  result_type="mixed").items(1000)

tweets = []
count = 0
for result in results:
    tweet = {
        "created_at": datetime.timestamp(result.created_at),
        "hashtags": result.entities['hashtags'],
        "favorite_count": result.favorite_count,
        "retweet_count": result.retweet_count,
        "text": result.text,
        "id": result.id,
        "geo": result.geo,
        "lang": result.lang
    }
    count += 1
    tweets.append(tweet)

print(count)
with open('../data/tweets.json', 'w') as jsonFile:
    json.dump(tweets, jsonFile)


import cv2
import os
import glob
import numpy as np

image_path=np.array(sorted(glob.glob(r"C:\Data\img\*.png")))
save_path=r"C:\Data\result"

def load_t(x):

    img = cv2.imread(x,-1)
    basename=os.path.basename(x)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    grad = grade(gray)
    savepath=os.path.join(save_path,"55"+basename)
    cv2.imwrite(savepath,255*grad)

def grade(img):
    x = cv2.Sobel(img, cv2.CV_32F, 1, 0, ksize=5)
    y = cv2.Sobel(img, cv2.CV_32F, 0, 1, ksize=5)
    absX = cv2.convertScaleAbs(x)
    absY = cv2.convertScaleAbs(y)
    dst = cv2.addWeighted(absX, 0.5, absY, 0.5, 0)
    mi=np.min(dst)
    ma=np.max(dst)
    return (dst-mi)/(ma-mi)
for name in image_path:
    load_t(name)


# Accessing an element in an array
array = [9,4,5,7,0]

print (array[3])
# output = 7

# print (array[9])---> This will print  "list index out of range" since the index at 9 is not available.

# Insertion operation in an array
# One can add one or more element in an array at the end, beginning or any given index
# Insertion takes two arguments, the index to insert the element and the element , i,e insert(i, element)
array = [9,4,5,7,0]

array.insert(0,3)  # the zero(0) is the first index and 3 is the element

print (array)  #  output --> [3, 9, 4, 5, 7, 0]

array.insert(4,8)

print (array) # output --> [3, 9, 4, 5, 8, 7, 0]

# Deletion operation in an array
# Remove an exixting element in an array with the python inbult function remove()

array = [9,4,5,7,0]
array.remove(7)

print(array) # removes the number 7 specified in the function and re-organizes the array

# Search operation in an array
# the search operation searches an element based on the index given or its value
# It uses in-built python function index()
array = [9,4,5,7,0]

print (array.index(4)) # output ---> this returns 1 since index of 4 is 1
# print (array.index(3))  # output ---> this returns an ValueError since 3 is not in the array list

# Update operation in an array
# this operation updates an exixting element in the array by re-assigning a new value to an element by index.
array = [9,4,5,7,0]
array[0] = 3

print (array)  # output --->[3, 4, 5, 7, 0] upadates value at index 0 to 3 and removes 9

array = [9,4,5,7,0]
# array[5] = 3 This will return an error since the index at 5 is out of range...index goes upto 4
# print (array) - IndexError: list assignment index out of range


from datetime import datetime

from consulta.models import Agendamento
from consulta.ext.database import db

class AgendamentoService:
    @staticmethod
    def agendar_consulta(dados):
        dados['data'] = datetime.strptime(dados['data'], '%Y-%m-%d %H:%M:%S')
        agendamento = Agendamento(**dados)
        db.session.add(agendamento)
        db.session.commit()
        db.session.close()
        return True

    @staticmethod
    def listar_agendamentos():
        agendamentos = Agendamento.query.all()
        return {"agendamentos": [agendamento.to_dict() for agendamento in agendamentos]}


from scipy.interpolate import interp1d
import numpy as np
import pyglet
from board_colors import board_colors
from board_graphics import BoardBackground, BoardForeground

class ProbabilityDisplay:
  def __init__(self, window_size, board_size, initial_probabilities):
    self.window_size = window_size
    self.board_size = board_size
    self.window = ProbabilityDisplay.make_window(window_size)
    highest_probability_range = [1/len(board_colors), 1]
    self.highest_probability_to_opacity = interp1d(highest_probability_range, BoardForeground.opacity_range)
    self.update(initial_probabilities)

  def has_closed(self):
    return self.window.has_exit

  def update(self, probabilities):
    self.window.dispatch_events()
    background = BoardBackground(self.window_size)
    background.draw()
    self.draw_foreground(probabilities)
    self.window.flip()

  def draw_foreground(self, probabilities):
    foreground = BoardForeground(self.window_size, self.board_size)
    for x in range(self.board_size):
      for y in range(self.board_size):
        dominant_color = np.argmax(probabilities[x, y])
        opacity = self.highest_probability_to_opacity(np.max(probabilities[x, y]))
        foreground.add_color(dominant_color, x, y, opacity)
    foreground.draw()

  @staticmethod
  def make_window(window_size):
    try:
      config = pyglet.gl.Config(sample_buffers = 1, samples = 4)
      window = pyglet.window.Window(window_size, window_size, config = config)
    except pyglet.window.NoSuchConfigException:
      window = pyglet.window.Window(window_size, window_size)
    def on_close():
      window.has_exit = True
      window.close()
    window.on_close = on_close
    return window


import sys
input = sys.stdin.readline
def BOJ_1992(x,y,size):
    if size == 0 : return

    cnt = 0
    for i in range(x,x+size):
        cnt += sum(lst[i][y:y+size])
    if cnt == 0:
        print(0,end="")
    elif cnt == size*size:
        print(1,end="")
    else:
        print('(',end="")
        BOJ_1992(x,y,size//2)
        BOJ_1992(x,y+size//2,size//2)
        BOJ_1992(x+size//2,y,size//2)
        BOJ_1992(x+size//2,y+size//2,size//2)
        print(')',end="")
    return

N= int(input())
lst = [ list(map(int,list(input().rstrip('\n')))) for _ in range(N)]
BOJ_1992(0,0,N)


from collections import defaultdict
from utils import timeit

@timeit
def get_data():
    data = []
    with open('input.txt') as input_file:
        for line in input_file:
            value = line.strip().split()
            data.append(value)
    return data

def execute(program, default_registers=None):

    # Do not modify the original program
    program = list(tuple(instruction) for instruction in program)

    registers = defaultdict(int)
    if default_registers is not None:
        registers.update(default_registers)
    instruction_pointer = 0

    while 0 <= instruction_pointer < len(program):

        instruction, *values = program[instruction_pointer]

        if instruction == 'cpy':
            x, y = values
            try:
                x = int(x)
            except ValueError:
                x = registers[x]
            registers[y] = x

        elif instruction == 'inc':
            x = values[0]
            registers[x] += 1

        elif instruction == 'dec':
            x = values[0]
            registers[x] -= 1

        elif instruction == 'jnz':
            x, y = values
            try:
                x = int(x)
            except ValueError:
                x = registers[x]
            if x != 0:
                try:
                    y = int(y)
                except ValueError:
                    y = registers[y]
                instruction_pointer += y - 1
        elif instruction == 'tgl':
            x = values[0]
            try:
                x = int(x)
            except ValueError:
                x = registers[x]

            if 0 <= instruction_pointer + x < len(program):
                instruction_, *values_ = program[instruction_pointer + x]
                if instruction_ in ('inc', 'dec', 'tgl'):
                    instruction_ = 'dec' if instruction_ == 'inc' else 'inc'
                elif instruction_ in ('cpy', 'jnz'):
                    instruction_ = 'cpy' if instruction_ == 'jnz' else 'jnz'

                program[instruction_pointer + x] = (instruction_, *values_)

        elif instruction == 'nop':
            pass

        elif instruction == 'mul':
            x, y, z = values
            try:
                x = int(x)
            except ValueError:
                x = registers[x]
            try:
                y = int(y)
            except ValueError:
                y = registers[y]
            registers[z] = x*y
        else:
            raise ValueError(f"Unknown instruction {(instruction, *values)} line {instruction_pointer}.")

        instruction_pointer += 1

    return registers

@timeit
def part_1(data):
    registers = execute(program=data, default_registers={'a': 7})
    return registers['a']

@timeit
def part_2(data):
    registers = execute(program=data, default_registers={'a': 12})
    return registers['a']

def main():
    data = get_data()

    data[3:10] = [
        ('mul', 'b', 'd', 'a'),
        ('cpy', '0', 'c'),
        ('cpy', '0', 'd'),
        ('nop',),
        ('nop',),
        ('nop',),
        ('nop',),
    ]

    part_1(data)
    part_2(data)

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
from django.contrib.sites.models import Site
from django.db import models
from django.db.models import Q
from django.utils import six

from cms.cache.permissions import get_permission_cache, set_permission_cache
from cms.exceptions import NoPermissionsException
from cms.models.query import PageQuerySet
from cms.publisher import PublisherManager
from cms.utils import get_cms_setting
from cms.utils.i18n import get_fallback_languages

class PageManager(PublisherManager):
    """Use draft() and public() methods for accessing the corresponding
    instances.
    """

    def get_queryset(self):
        """Change standard model queryset to our own.
        """
        return PageQuerySet(self.model)

    def drafts(self):
        return super(PageManager, self).drafts()

    def public(self):
        return super(PageManager, self).public()

    # !IMPORTANT: following methods always return access to draft instances,
    # take care on what you do one them. use Page.objects.public() for accessing
    # the published page versions

    # Just some of the queryset methods are implemented here, access queryset
    # for more getting more supporting methods.

    # TODO: check which from following methods are really required to be on
    # manager, maybe some of them can be just accessible over queryset...?

    def on_site(self, site=None):
        return self.get_queryset().on_site(site)

    def published(self, site=None):
        return self.get_queryset().published(site=site)

    def get_home(self, site=None):
        return self.get_queryset().get_home(site)

    def search(self, q, language=None, current_site_only=True):
        """Simple search function

        Plugins can define a 'search_fields' tuple similar to ModelAdmin classes
        """
        from cms.plugin_pool import plugin_pool

        qs = self.get_queryset()
        qs = qs.public()

        if current_site_only:
            site = Site.objects.get_current()
            qs = qs.filter(site=site)

        qt = Q(title_set__title__icontains=q)

        # find 'searchable' plugins and build query
        qp = Q()
        plugins = plugin_pool.get_all_plugins()
        for plugin in plugins:
            cmsplugin = plugin.model
            if not (
                hasattr(cmsplugin, 'search_fields') and
                hasattr(cmsplugin, 'cmsplugin_ptr')
            ):
                continue
            field = cmsplugin.cmsplugin_ptr.field
            related_query_name = field.related_query_name()
            if (
                related_query_name and
                related_query_name != '+'
            ):
                for field in cmsplugin.search_fields:
                    qp |= Q(**{
                        'placeholders__cmsplugin__{0}__{1}__icontains'.format(
                            related_query_name,
                            field,
                        ): q})
        if language:
            qt &= Q(title_set__language=language)
            qp &= Q(cmsplugin__language=language)

        qs = qs.filter(qt | qp)

        return qs.distinct()

class TitleManager(PublisherManager):
    def get_title(self, page, language, language_fallback=False):
        """
        Gets the latest content for a particular page and language. Falls back
        to another language if wanted.
        """
        try:
            title = self.get(language=language, page=page)
            return title
        except self.model.DoesNotExist:
            if language_fallback:
                try:
                    titles = self.filter(page=page)
                    fallbacks = get_fallback_languages(language)
                    for lang in fallbacks:
                        for title in titles:
                            if lang == title.language:
                                return title
                    return None
                except self.model.DoesNotExist:
                    pass
            else:
                raise
        return None

    # created new public method to meet test case requirement and to get a list of titles for published pages
    def public(self):
        return self.get_queryset().filter(publisher_is_draft=False, published=True)

    def drafts(self):
        return self.get_queryset().filter(publisher_is_draft=True)

    def set_or_create(self, request, page, form, language):
        """
        set or create a title for a particular page and language
        """
        base_fields = [
            'slug',
            'title',
            'meta_description',
            'page_title',
            'menu_title'
        ]
        advanced_fields = [
            'redirect',
        ]
        cleaned_data = form.cleaned_data
        try:
            obj = self.get(page=page, language=language)
        except self.model.DoesNotExist:
            data = {}
            for name in base_fields:
                if name in cleaned_data:
                    data[name] = cleaned_data[name]
            data['page'] = page
            data['language'] = language
            if page.has_advanced_settings_permission(request):
                overwrite_url = cleaned_data.get('overwrite_url', None)
                if overwrite_url:
                    data['has_url_overwrite'] = True
                    data['path'] = overwrite_url
                else:
                    data['has_url_overwrite'] = False
                for field in advanced_fields:
                    value = cleaned_data.get(field, None)
                    data[field] = value
            return self.create(**data)
        for name in base_fields:
            if name in form.base_fields:
                value = cleaned_data.get(name, None)
                setattr(obj, name, value)
        if page.has_advanced_settings_permission(request):
            if 'overwrite_url' in cleaned_data:
                overwrite_url = cleaned_data.get('overwrite_url', None)
                obj.has_url_overwrite = bool(overwrite_url)
                obj.path = overwrite_url
            for field in advanced_fields:
                if field in form.base_fields:
                    value = cleaned_data.get(field, None)
                    setattr(obj, field, value)
        obj.save()
        return obj

################################################################################
# Permissions
################################################################################

class BasicPagePermissionManager(models.Manager):
    """Global page permission manager accessible under objects.

    !IMPORTANT: take care, PagePermissionManager and GlobalPagePermissionManager
    both inherit from this manager
    """

    def with_user(self, user):
        """Get all objects for given user, also takes look if user is in some
        group.
        """
        return self.filter(Q(user=user) | Q(group__user=user))

    def with_can_change_permissions(self, user):
        """Set of objects on which user haves can_change_permissions. !But only
        the ones on which is this assigned directly. For getting reall
        permissions use page.permissions manager.
        """
        return self.with_user(user).filter(can_change_permissions=True)

class GlobalPagePermissionManager(BasicPagePermissionManager):

    def user_has_permission(self, user, site_id, perm):
        """
        Provide a single point of entry for deciding whether any given global
        permission exists.
        """
        # if the user has add rights to this site explicitly
        this_site = Q(**{perm: True, 'sites__in': [site_id]})
        # if the user can add to all sites
        all_sites = Q(**{perm: True, 'sites__isnull': True})
        return self.with_user(user).filter(this_site | all_sites)

    def user_has_add_permission(self, user, site_id):
        return self.user_has_permission(user, site_id, 'can_add')

    def user_has_change_permission(self, user, site_id):
        return self.user_has_permission(user, site_id, 'can_change')

    def user_has_view_permission(self, user, site_id):
        return self.user_has_permission(user, site_id, 'can_view')

class PagePermissionManager(BasicPagePermissionManager):
    """Page permission manager accessible under objects.
    """

    def subordinate_to_user(self, user):
        """Get all page permission objects on which user/group is lover in
        hierarchy then given user and given user can change permissions on them.

        !IMPORTANT, but exclude objects with given user, or any group containing
        this user - he can't be able to change his own permissions, because if
        he does, and removes some permissions from himself, he will not be able
        to add them anymore.

        Example:
                                       A
                                    /    \
                                  user    B,E
                                /     \
                              C,X     D,Y

            Gives permission nodes C,X,D,Y under user, so he can edit
            permissions if he haves can_change_permission.

        Example:
                                      A,Y
                                    /    \
                                  user    B,E,X
                                /     \
                              C,X     D,Y

            Gives permission nodes C,D under user, so he can edit, but not
            anymore to X,Y, because this users are on the same level or higher
            in page hierarchy. (but only if user have can_change_permission)

        Example:
                                        A
                                    /      \
                                  user     B,E
                                /     \      \
                              C,X     D,Y    user
                                            /    \
                                           I      J,A

            User permissions can be assigned to multiple page nodes, so merge of
            all of them is required. In this case user can see permissions for
            users C,X,D,Y,I,J but not A, because A user in higher in hierarchy.

        If permission object holds group, this permission object can be visible
        to user only if all of the group members are lover in hierarchy. If any
        of members is higher then given user, this entry must stay invisible.

        If user is superuser, or haves global can_change_permission permissions,
        show him everything.

        Result of this is used in admin for page permissions inline.
        """
        from cms.models import GlobalPagePermission, Page

        if user.is_superuser or \
                GlobalPagePermission.objects.with_can_change_permissions(user):
            # everything for those guys
            return self.all()

        # get user level
        from cms.utils.permissions import get_user_permission_level

        try:
            user_level = get_user_permission_level(user)
        except NoPermissionsException:
            return self.none()
            # get current site
        site = Site.objects.get_current()
        # get all permissions
        page_id_allow_list = Page.permissions.get_change_permissions_id_list(user, site)

        # get permission set, but without objects targeting user, or any group
        # in which he can be
        qs = self.filter(
            page__id__in=page_id_allow_list,
            page__level__gte=user_level,
        )
        qs = qs.exclude(user=user).exclude(group__user=user)
        return qs

    def for_page(self, page):
        """Returns queryset containing all instances somehow connected to given
        page. This includes permissions to page itself and permissions inherited
        from higher pages.

        NOTE: this returns just PagePermission instances, to get complete access
        list merge return of this function with Global permissions.
        """
        # permissions should be managed on the draft page only
        page = page.get_draft_object()
        from cms.models import (ACCESS_DESCENDANTS, ACCESS_CHILDREN,
            ACCESS_PAGE_AND_CHILDREN, ACCESS_PAGE_AND_DESCENDANTS, ACCESS_PAGE)

        if page.depth is None or page.path is None or page.numchild is None:
            raise ValueError("Cannot use unsaved page for permission lookup, missing MPTT attributes.")

        paths = [
            page.path[0:pos]
            for pos in range(0, len(page.path), page.steplen)[1:]
        ]
        parents = Q(page__path__in=paths) & (Q(grant_on=ACCESS_DESCENDANTS) | Q(grant_on=ACCESS_PAGE_AND_DESCENDANTS))
        direct_parents = Q(page__pk=page.parent_id) & (Q(grant_on=ACCESS_CHILDREN) | Q(grant_on=ACCESS_PAGE_AND_CHILDREN))
        page_qs = Q(page=page) & (Q(grant_on=ACCESS_PAGE_AND_DESCENDANTS) | Q(grant_on=ACCESS_PAGE_AND_CHILDREN) |
                                  Q(grant_on=ACCESS_PAGE))
        query = (parents | direct_parents | page_qs)
        return self.filter(query).order_by('page__depth')

class PagePermissionsPermissionManager(models.Manager):
    """Page permissions permission manager.

    !IMPORTANT: this actually points to Page model, not to PagePermission.
    Seems this will be better approach. Accessible under permissions.

    Maybe this even shouldn't be a manager - it mixes different models together.
    """
    # we will return this in case we have a superuser, or permissions are not
    # enabled/configured in settings
    GRANT_ALL = 'All'

    def get_publish_id_list(self, user, site):
        """
        Give a list of page where the user has publish rights or the string "All" if
        the user has all rights.
        """
        return self.__get_id_list(user, site, "can_publish")

    def get_change_id_list(self, user, site):
        """
        Give a list of page where the user has edit rights or the string "All" if
        the user has all rights.
        """
        return self.__get_id_list(user, site, "can_change")

    def get_add_id_list(self, user, site):
        """
        Give a list of page where the user has add page rights or the string
        "All" if the user has all rights.
        """
        return self.__get_id_list(user, site, "can_add")

    def get_delete_id_list(self, user, site):
        """
        Give a list of page where the user has delete rights or the string "All" if
        the user has all rights.
        """
        return self.__get_id_list(user, site, "can_delete")

    def get_advanced_settings_id_list(self, user, site):
        """
        Give a list of page where the user can change advanced settings or the
        string "All" if the user has all rights.
        """
        return self.__get_id_list(user, site, "can_change_advanced_settings")

    def get_change_permissions_id_list(self, user, site):
        """Give a list of page where the user can change permissions.
        """
        return self.__get_id_list(user, site, "can_change_permissions")

    def get_move_page_id_list(self, user, site):
        """Give a list of pages which user can move.
        """
        return self.__get_id_list(user, site, "can_move_page")

    def get_view_id_list(self, user, site):
        """Give a list of pages which user can view.
        """
        return self.__get_id_list(user, site, "can_view")

    def get_restricted_id_list(self, site):
        from cms.models import (GlobalPagePermission, PagePermission,
            MASK_CHILDREN, MASK_DESCENDANTS, MASK_PAGE)

        global_permissions = GlobalPagePermission.objects.all()
        if global_permissions.filter(Q(sites__in=[site]) | Q(sites__isnull=True)
                ).filter(can_view=True).exists():
            # user or his group are allowed to do `attr` action
            # !IMPORTANT: page permissions must not override global permissions
            from cms.models import Page

            return Page.objects.filter(site=site).values_list('id', flat=True)
            # for standard users without global permissions, get all pages for him or
        # his group/s
        qs = PagePermission.objects.filter(page__site=site, can_view=True).select_related('page')
        qs.order_by('page__path')
        # default is denny...
        page_id_allow_list = []
        for permission in qs:
            if permission.grant_on & MASK_PAGE:
                page_id_allow_list.append(permission.page_id)
            if permission.grant_on & MASK_CHILDREN:
                page_id_allow_list.extend(permission.page.get_children().values_list('id', flat=True))
            elif permission.grant_on & MASK_DESCENDANTS:
                page_id_allow_list.extend(permission.page.get_descendants().values_list('id', flat=True))
                # store value in cache
        return page_id_allow_list

    def __get_id_list(self, user, site, attr):
        if site and not isinstance(site, six.integer_types):
            site = site.pk
        from cms.models import (GlobalPagePermission, PagePermission,
            MASK_PAGE, MASK_CHILDREN, MASK_DESCENDANTS)

        if attr != "can_view":
            if not user.is_authenticated() or not user.is_staff:
                return []
        if user.is_superuser or not get_cms_setting('PERMISSION'):
            # got superuser, or permissions aren't enabled? just return grant
            # all mark
            return PagePermissionsPermissionManager.GRANT_ALL
            # read from cache if possible
        cached = get_permission_cache(user, attr)
        if cached is not None:
            return cached
            # check global permissions
        global_perm = GlobalPagePermission.objects.user_has_permission(user, site, attr).exists()
        if global_perm:
            # user or his group are allowed to do `attr` action
            # !IMPORTANT: page permissions must not override global permissions
            return PagePermissionsPermissionManager.GRANT_ALL
            # for standard users without global permissions, get all pages for him or
        # his group/s
        qs = PagePermission.objects.with_user(user)
        qs.filter(**{'page__site_id': site}).order_by('page__path').select_related('page')
        # default is denny...
        page_id_allow_list = []
        for permission in qs:
            if getattr(permission, attr):
                # can add is special - we are actually adding page under current page
                if permission.grant_on & MASK_PAGE or attr is "can_add":
                    page_id_allow_list.append(permission.page_id)
                if permission.grant_on & MASK_CHILDREN and not attr is "can_add":
                    page_id_allow_list.extend(permission.page.get_children().values_list('id', flat=True))
                elif permission.grant_on & MASK_DESCENDANTS:
                    page_id_allow_list.extend(permission.page.get_descendants().values_list('id', flat=True))
                    # store value in cache
        set_permission_cache(user, attr, page_id_allow_list)
        return page_id_allow_list


import numpy

import pandas as pd

from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression

from PIL import Image
import PIL.ImageOps

X= numpy.load("image.numpyz")['arr_0']
y = pd.read_csv("labels.csv")["labels"]

classes = ['A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z']

nclasses = len(classes)

X_train,X_test,y_train,y_test = train_test_split(X,y,train_size = 3500,test_size=500,random_state=42)

X_train_scaled = X_train/255
X_test_scaled = X_test/255

clf = LogisticRegression(solver="saga",multi_class="multinomial").fit(X_train_scaled,y_train)

def get_prediction(image):
    image_pil = Image.open(image)
    image_bw = image_pil.convert("L")
    image_bw_resized = image_bw.resize((28,28),Image.ANTIALIAS)
    pixel_filter = 20
    min_pixel = numpy.percentile(image_bw_resized,pixel_filter)
    image_bw_resized_scaled = numpy.clip(image_bw_resized-min_pixel,0,255)
    max_pixel = numpy.max(image_bw_resized)
    image_bw_resized_scaled = numpy.asarray(image_bw_resized_scaled)/max_pixel
    test_sample = numpy.array(image_bw_resized_scaled).reshape(1,660)
    predict = clf.predict(test_sample)
    return predict[0]


from reverse_linked_list_iterative import reverse_linked_list
from test_framework import generic_test

def is_linked_list_a_palindrome(L):
    data_list = []
    orig = L
    while L:
        data_list.append(L.data)
        L = L.next
    end = len(data_list)-1
    for start in range(len(data_list)//2):
        if data_list[start] != data_list[end]:
            return False
        end -= 1
    return True

if __name__ == '__main__':
    exit(
        generic_test.generic_test_main("is_list_palindromic.py",
                                       'is_list_palindromic.tsv',
                                       is_linked_list_a_palindrome))


#!/usr/bin/env python
# coding: utf-8

import pyqrcode
import pandas as pd
import os

# set variable(s)
QR_folder = "test_data/QRs"
student_CSV = "test_data/sj_sample.csv"
last_name_row = "Last name"
first_name_row = "First name"
student_id_row = "Student ID"

def createQR(student):

    qr = pyqrcode.create(student)
    qr.png(QR_folder + "/" + student + ".png", scale=20)

# create folder to store QRs
os.mkdir(QR_folder)

# import student data
students = pd.read_csv(student_CSV)

# create QR for every student
for index, row in students.iterrows():
    createQR(row[last_name_row] + row[first_name_row] + str(row[student_id_row]))


import rclpy
from rclpy.node import Node

import serial
import sys
import threading
import glob

from std_msgs.msg import String

class SerialRelay(Node):
    def __init__(self):
        # Initalize node with name
        super().__init__("serial_publisher")

        # Create a publisher to publish any output the pico sends
        self.publisher = self.create_publisher(String, '/pico/output', 10)

        # Create a subscriber to listen to any commands sent for the pico
        self.subscriber = self.create_subscription(String, '/pico/command', self.send, 10)

        # Loop through all serial devices on the computer to check for the pico
        self.port = None
        ports = SerialRelay.list_serial_ports()
        for port in ports:
            try:
                # connect and send a ping command
                ser = serial.Serial(port, timeout=1)
                ser.write(b"ping\n")
                response = ser.read_until("\n")

                # if pong is in response, then we are talking with the pico
                if b"pong" in response:
                    self.port = port
                    print(f"Found pico at {self.port}!")
                    break
            except:
                pass

        if self.port is None:
            print("Unable to find pico... please make sure it is connected.")
            sys.exit(1)

        self.ser = serial.Serial(self.port, 115200)

    def run(self):
        # This thread makes all the update processes run in the background
        thread = threading.Thread(target=rclpy.spin, args={self}, daemon=True)
        thread.start()

        try:
            while rclpy.ok():
                # Check the pico for updates
                self.read_pico()

        except KeyboardInterrupt:
            sys.exit(0)

    def read_pico(self):
        output = str(self.ser.readline(), "utf8")
        # If received output
        if output:
            print(f"[Pico] {output}", end="")
            # Create a string message object
            msg = String()

            # Set message data
            msg.data = output

            # Publish data
            self.publisher.publish(msg)

    def send(self, msg):
        command = msg.data + '\n'
        print(f"[Sys] {command}", end="")

        # Send command to pico
        self.ser.write(bytes(command, "utf8"))

    @staticmethod
    def list_serial_ports():
        # This code only works on linux systems
        # See here for other OSes: https://stackoverflow.com/a/14224477
        return glob.glob("/dev/tty[A-Za-z]*")

def main(args=None):
    rclpy.init(args=args)

    serial_pub = SerialRelay()
    serial_pub.run()

if __name__ == '__main__':
    main()


from .utilities.navigable import make_navigable

__converters = {}

def conversion(output_type, *mime_types):
    """Decorator: registers the decorated function as the converter of each of
    the specified MIME types to the specified OutputType."""
    def _conversion(f):
        def _register_converter(output_type, mime_type):
            k = (output_type, mime_type)
            if k in __converters:
                raise ValueError(
                        "BUG: can't register two handlers for"
                        " the same (OutputType, MIME type) pair!", k)
            else:
                __converters[k] = f
        if mime_types:
            for m in mime_types:
                _register_converter(output_type, m)
        else:
            _register_converter(output_type, None)
        return f
    return _conversion

def convert(resource, output_type, mime_override=None):
    """Tries to convert a Resource to the specified OutputType by using the
    database of registered conversion functions.

    Raises a KeyError if no conversion exists."""
    mime_type = resource.compute_type() if not mime_override else mime_override
    try:
        converter = __converters[(output_type, mime_type)]
    except KeyError as e:
        try:
            converter = __converters[(output_type, None)]
        except KeyError:
            # Raise the original, more specific, exception
            raise KeyError("No converters registered for "
                           "{0}".format(e)) from e
    value = converter(resource)
    if value is not None and not hasattr(value, 'parent'):
        value = make_navigable(value)
    return value


import csv

content_pages = {}
count = 0
# with open('resources/Dummy Data_final_310123 - with full content samples.xlsx - Full Content Samples.csv'
#         , newline='') as csvfile:
with open('resources/MA Database 220523.xlsx - topics.csv'
        , newline='') as csvfile:

    reader = csv.reader(csvfile, delimiter=',')
    next(reader, None)

    topics_h = set()
    topics_w = set()
    topics_c = set()
    topics_l = set()
    for row in reader:
        topic = row[7]
        pillar = row[0]
        # print(pillar)
        if pillar == "Health":
            for t in topic.split(","):
                topics_h.add(t)
        if pillar == "Wealth":
            for t in topic.split(","):
                topics_w.add(t)
        if pillar == "Community":
            for t in topic.split(","):
                topics_c.add(t)
        if pillar == "Learning":
            for t in topic.split(","):
                topics_l.add(t)

    print("--- HEALTH")
    for t in sorted(topics_h):
        print(t)
    print("")
    print("--- WEALTH")
    for t in sorted(topics_w):
        print(t)
    print("")
    print("--- COMMUNITY")
    for t in sorted(topics_c):
        print(t)
    print("")
    print("--- LEARNING")
    for t in sorted(topics_l):
        print(t)
    print("")


import os

import torch
from typing import Dict, List, Tuple, Optional
from dataclasses import dataclass

from src.dataType import State, Action
from src.animator.animator import Animator
from src.const import VNF_SELECTION_IN_DIM, VNF_PLACEMENT_IN_DIM

@dataclass
class DebugInfo:
    timestamp: str
    episode: int
    mean_100_step: int
    std_100_step: int
    mean_100_change_slp_srv: float
    std_100_change_slp_srv: float
    mean_100_init_slp_srv: float
    std_100_init_slp_srv: float
    mean_100_final_slp_srv: float
    std_100_final_slp_srv: float
    srv_n: int
    mean_100_change_sfc_in_same_srv: float
    std_100_change_sfc_in_same_srv: float
    mean_100_init_sfc_in_same_srv: float
    std_100_init_sfc_in_same_srv: float
    mean_100_final_sfc_in_same_srv: float
    std_100_final_sfc_in_same_srv: float
    sfc_n: int
    mean_100_exploration: float
    std_100_exploration: float
    mean_100_reward: float
    std_100_reward: float

def setup_mp_env():
    os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
    os.environ['CUDA_VISIBLE_DEVICES'] = ''
    os.environ['OMP_NUM_THREADS'] = '1'

def print_debug_info(debug_info: DebugInfo, refresh: bool = False):
    debug_msg = "[{}] Episode {:05}, Step {:04.2f}\u00B1{:04.2f}, #SleepSrv ({:02.3f}\u00B1{:02.3f})({:02.3f}\u00B1{:02.3f}->{:02.3f}\u00B1{:02.3f}/{}), #SFCinSameSrv ({:02.3f}\u00B1{:02.3f})({:02.3f}\u00B1{:02.3f}->{:02.3f}\u00B1{:02.3f}/{}), Exploration: {:.3f}\u00B1{:02.3f}, Rewards: {:02.3f}\u00B1{:02.3f}".format(
        debug_info.timestamp, debug_info.episode, debug_info.mean_100_step, debug_info.std_100_step,
        debug_info.mean_100_change_slp_srv, debug_info.std_100_change_slp_srv, debug_info.mean_100_init_slp_srv, debug_info.mean_100_init_slp_srv, debug_info.mean_100_final_slp_srv, debug_info.std_100_final_slp_srv, debug_info.srv_n,
        debug_info.mean_100_change_sfc_in_same_srv, debug_info.std_100_change_sfc_in_same_srv, debug_info.mean_100_init_sfc_in_same_srv, debug_info.std_100_init_sfc_in_same_srv, debug_info.mean_100_final_sfc_in_same_srv, debug_info.std_100_final_sfc_in_same_srv, debug_info.sfc_n,
        debug_info.mean_100_exploration, debug_info.std_100_exploration, debug_info.mean_100_reward, debug_info.std_100_reward,
    )
    print(debug_msg, end='\r', flush=True)
    if refresh:
        print('\x1b[2K' + debug_msg, flush=True)

def convert_state_to_vnf_selection_input(state: State, max_vnf_num: int) -> torch.Tensor:
    vnf_selection_input = torch.zeros(max_vnf_num, VNF_SELECTION_IN_DIM, dtype=torch.float32)
    for vnf in state.vnfs:
        vnf_selection_input[vnf.id] = torch.tensor([
            vnf.cpu_req, vnf.mem_req, vnf.sfc_id,
            state.srvs[vnf.srv_id].cpu_cap, state.srvs[vnf.srv_id].mem_cap,
            state.srvs[vnf.srv_id].cpu_load, state.srvs[vnf.srv_id].mem_load,
            state.edge.cpu_cap, state.edge.mem_cap,
            state.edge.cpu_load, state.edge.mem_load,
        ])
    return vnf_selection_input

def convert_state_to_vnf_placement_input(state: State, vnf_id: int) -> torch.Tensor:
    vnf_placement_input = torch.zeros(len(state.srvs), VNF_PLACEMENT_IN_DIM, dtype=torch.float32)
    for srv in state.srvs:
        vnf_placement_input[srv.id] = torch.tensor([
            srv.cpu_cap, srv.mem_cap, srv.cpu_load, srv.mem_load,
            state.vnfs[vnf_id].cpu_req, state.vnfs[vnf_id].mem_req, state.vnfs[vnf_id].sfc_id,
            state.edge.cpu_cap, state.edge.mem_cap, state.edge.cpu_load, state.edge.mem_load
        ])
    return vnf_placement_input

def get_possible_actions(state: State, max_vnf_num: int) -> Dict[int, List[int]]:
    '''return possible actions for each state

    Args:
        state (State): state

    Returns:
        Dict[int, List[int]]: possible actions
                                    ex) {vnfId: [srvId1, srvId2, ...], vnfId2: [srvId1, srvId2, ...], ...}
    '''
    possible_actions = {}
    for vnf_idx in range(max_vnf_num):
        possible_actions[vnf_idx] = []
        if len(state.vnfs) <= vnf_idx: continue
        vnf = state.vnfs[vnf_idx]
        for srv in state.srvs:
            # 이미 설치된 srv로 이동하는 action 막기
            if vnf.srv_id == srv.id: continue
            # capacity 확인
            if srv.cpu_cap - srv.cpu_load < vnf.cpu_req or srv.mem_cap - srv.mem_load < vnf.mem_req: continue
            possible_actions[vnf.id].append(srv.id)
    return possible_actions

def get_info_from_logits(logits: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    probs = logit_to_prob(logits)
    dist = torch.distributions.Categorical(probs=probs)
    actions = dist.sample().to(torch.int32)
    logpas = dist.log_prob(actions)
    is_exploratory = actions != torch.argmax(logits, dim=1)
    return actions, logpas, is_exploratory

def logit_to_prob(logits: torch.Tensor) -> torch.Tensor:
    probs = torch.zeros_like(logits)
    # 0인 값은 prob을 0으로 유지하고, 나머지 값을 확률로 변경
    for i in range(logits.shape[0]):
        probs[i, logits[i] != 0] = torch.softmax(logits[i, logits[i] != 0], dim = 0)
    return probs

def get_device():
    return torch.device("cuda" if torch.cuda.is_available() else "cpu")

def get_zero_util_cnt(state: State) -> int:
    cnt = 0
    for srv in state.srvs:
        if len(srv.vnfs) == 0:
            cnt += 1
    return cnt

def get_sfc_cnt_in_same_srv(state: State) -> int:
    cnt = 0
    for sfc in state.sfcs:
        if len(sfc.vnfs) == 0:
            continue
        cnt += 1
        srv_id = sfc.vnfs[0].srv_id
        for vnf in sfc.vnfs:
            if srv_id != vnf.srv_id:
                cnt -= 1
                break
    return cnt

def save_animation(
        srv_n: int, sfc_n: int, vnf_n: int,
        srv_mem_cap: int, srv_cpu_cap: int,
        history: List[Tuple[State, Optional[Action]]],
        path=f'./result/anim.mp4'):
    animator = Animator(srv_n=srv_n, sfc_n=sfc_n, vnf_n=vnf_n,
                        srv_mem_cap=srv_mem_cap, srv_cpu_cap=srv_cpu_cap, history=history)
    animator.save(path)


import cv2 as cv
import numpy as np
import os

def dummy_resize(img, size):
    _sz = np.shape(img)[:-1]
    field = np.zeros((max(_sz), max(_sz), 3), dtype=np.uint8)
    if _sz[0] > _sz[1]:
        field[:, (_sz[0]-_sz[1])//2:(_sz[0]+_sz[1])//2+(_sz[1] % 2), :] = img
    else:
        field[(_sz[1]-_sz[0])//2:(_sz[1]+_sz[0])//2+(_sz[0] % 2), :, :] = img
    return resize(field, size)

def is_image(name):
    image_formats = ['jpg', 'png', 'tif', 'bmp']
    return os.path.splitext(name)[1][1:].lower() in image_formats

def put_on_color_field(images, background):
    shape = np.shape(images[0])
    if background == 'black':
        field = np.zeros(shape, dtype=np.uint8)
    elif background == 'white':
        field = 255*np.ones(shape, dtype=np.uint8)
    elif background == 'gray':
        field_color = np.mean([np.mean(img[img != 0]) for img in images])
        field = field_color * np.ones(shape, dtype=np.uint8)
    else:
        raise NameError('incorrect background name!')
    result = []
#    print(np.mean(field))
    for img in images:
        img[img < 3] = field[img < 3]
        result.append(np.uint8(img))
    return result

def put_on_square_field(img, field_size, x0, y0, size):
    result = np.zeros((field_size, field_size, 3), dtype=np.uint8)
    resized_img = cv.resize(img, (size, size))
    result[x0:x0+size, y0:y0+size] = resized_img
    return result

def read_images(path):
    names = np.sort([name for name in os.listdir(path) if is_image(name)])
    images = [cv.imread(os.path.join(path, name)) for name in names]
    return images, names

def read_one_image(path, name):
    return cv.imread(os.path.join(path, name))

def read_images_names(path):
    return np.sort([name for name in os.listdir(path) if is_image(name)])

def remove_chromakey(img, sens=35, chanel_to_remove='green'):
    reds = img[:, :, 2]
    greens = img[:, :, 1]
    blues = img[:, :, 0]
    # Refactoring in future
    if chanel_to_remove == 'green':
        mask = (greens < sens) | (reds > greens) | (blues > greens)
    elif chanel_to_remove == 'blue':
        mask = (blues < sens) | (reds > blues) | (greens > blues)
    elif chanel_to_remove == 'red':
        mask = (reds < sens) | (blues > reds) | (greens > reds)
    else:
        raise TypeError("Chanel {} not found!".format(chanel_to_remove))
    empty_img = np.zeros_like(img)
    empty_img[mask, 0] = 255
    result = np.zeros_like(img)
    for index in range(3):
        result[:, :, index] = img[:, :, index]*np.where(mask, 1, 0)
    return result

def resize(img, size):
    return cv.resize(img, (size, size), interpolation=cv.INTER_CUBIC)

def save_images(path, images, names):
    [cv.imwrite(os.path.join(path, nm), img) for nm, img in zip(names, images)]
    return 0

def save_one_image(path, image, name):
    cv.imwrite(os.path.join(path, name), image)
    return 0

def select_square(img0, return_params=False):
    img_size = np.shape(img0)[:-1]
    max_size = max(img_size)
    img = np.zeros((max_size, max_size, 3), dtype=np.uint8)
    img[max_size // 2 - img_size[0] // 2:
        max_size // 2 + img_size[0] // 2 + img_size[0] % 2,
        max_size // 2 - img_size[1] // 2:
        max_size // 2 + img_size[1] // 2 + img_size[1] % 2] = img0
    mask = cv.medianBlur(img[:, :, 0], 3) > 15
    minmax = np.argwhere(mask == 1)
    x_min, y_min = np.min(minmax, axis=0)
    x_max, y_max = np.max(minmax, axis=0)
    sz = [x_max-x_min, y_max-y_min]
#    Info: will be removed soon
#    print(x_min, x_max, y_min, y_max, sz)
    if sz[0] > sz[1]:
        x0 = x_min
        y0 = y_min-(sz[0]-y_max+y_min)//2
        _size = sz[0]
    else:
        x0 = x_min-(sz[1]-x_max+x_min)//2
        y0 = y_min
        _size = sz[1]
    if return_params:
        return {'x0': x0, 'y0': y0, 'size': _size}
    else:
        return img[x0:x0+_size, y0:y0+_size, :]


import unittest.mock as mock

from tensortrade.oms.instruments import ExchangePair, BTC, USD

@mock.patch('tensortrade.exchanges.Exchange')
def test_valid_init(mock_exchange):

    exchange = mock_exchange.return_value
    exchange.name = "bitfinex"

    exchange_pair = ExchangePair(exchange, USD/BTC)
    assert exchange_pair
    assert exchange_pair.pair.base == USD
    assert exchange_pair.pair.quote == BTC

@mock.patch('tensortrade.exchanges.Exchange')
def test_str(mock_exchange):
    exchange = mock_exchange.return_value
    exchange.name = "bitfinex"

    exchange_pair = ExchangePair(exchange, USD/BTC)

    assert str(exchange_pair) == "bitfinex:USD/BTC"