import re

from .struct import TypeInfo

class SchemaLinker:

    def __init__(self):
        pass

    def find_in_list(self, comlist, words):
        result = False
        for com in comlist:
            if words in com:
                result = True
                break
        return result

    def get_continue_score(self, pstr, tstr):
        comlist = []
        minlen = min(len(pstr), len(tstr))
        for slen in range(minlen, 1, -1):
            for ts in range(0, len(tstr), 1):
                if ts + slen > len(tstr):
                    continue
                words = tstr[ts:ts + slen]
                if words in pstr and not self.find_in_list(comlist, words):
                    comlist.append(words)

        comlen = 0
        for com in comlist:
            comlen += len(com) * len(com)
        weight = comlen / (len(tstr) * len(tstr) + 0.001)
        if weight > 1.0:
            weight = 1.0

        return weight

    def get_match_score(self, ptokens, ttokens):
        pset = set(ptokens)
        tset = set(ttokens)
        comset = pset & tset
        allset = pset | tset
        weight2 = len(comset) / (len(allset) + 0.001)
        weight3 = self.get_continue_score(''.join(ptokens), ''.join(ttokens))
        return 0.4 * weight2 + 0.6 * weight3

    def is_number(self, s):
        try:
            float(s)
            return True
        except ValueError:
            pass

        try:
            import unicodedata
            unicodedata.numeric(s)
            return True
        except (TypeError, ValueError):
            pass

        return False

    def get_match_phrase(self, query, target):
        if target in query:
            return target, 1.0

        qtokens = []
        for i in range(0, len(query), 1):
            qtokens.append(query[i:i + 1])
        ttokens = []
        for i in range(0, len(target), 1):
            ttokens.append(target[i:i + 1])
        ttok_set = set(ttokens)

        phrase = ''
        score = 0.0
        for qidx, qword in enumerate(qtokens):
            if qword not in ttok_set:
                continue

            eidx = (qidx + 2 * len(ttokens)) if (
                len(qtokens) > qidx + 2 * len(ttokens)) else len(qtokens)
            while eidx > qidx:
                ptokens = qtokens[qidx:eidx]
                weight = self.get_match_score(ptokens, ttokens)
                if weight + 0.001 > score:
                    score = weight
                    phrase = ''.join(ptokens)
                eidx -= 1

        if self.is_number(target) and phrase != target:
            score = 0.0
        if len(phrase) > 1 and phrase in target:
            score *= (1.0 + 0.05 * len(phrase))

        return phrase, score

    def allfindpairidx(self, que_tok, value_tok, weight):
        idxs = []
        for i in range(0, len(que_tok) - len(value_tok) + 1, 1):
            s = i
            e = i
            matched = True
            for j in range(0, len(value_tok), 1):
                if value_tok[j].lower() == que_tok[i + j].lower():
                    e = i + j
                else:
                    matched = False
                    break
            if matched:
                idxs.append([s, e, weight])

        return idxs

    def findnear(self, ps1, pe1, ps2, pe2):
        if abs(ps1 - pe2) <= 2 or abs(pe1 - ps2) <= 2:
            return True
        return False

    def get_column_type(self, col_idx, table):
        colType = table['header_types'][col_idx]
        if 'number' in colType or 'duration' in colType or 'real' in colType:
            colType = 'real'
        elif 'date' in colType:
            colType = 'date'
        elif 'bool' in colType:
            colType = 'bool'
        else:
            colType = 'text'

        return colType

    def add_type_all(self, typeinfos, index, idxs, label, linktype, value,
                     orgvalue):
        for idx in idxs:
            info = TypeInfo(label, index, linktype, value, orgvalue, idx[0],
                            idx[1], idx[2])
            flag = True
            for i, typeinfo in enumerate(typeinfos):
                if info.pstart < typeinfo.pstart:
                    typeinfos.insert(i, info)
                    flag = False
                    break

            if flag:
                typeinfos.append(info)

        return typeinfos

    def save_info(self, tinfo, sinfo):
        flag = True
        if tinfo.pstart > sinfo.pend or tinfo.pend < sinfo.pstart:
            pass
        elif tinfo.pstart >= sinfo.pstart and \
                tinfo.pend <= sinfo.pend and tinfo.index == -1:
            flag = False
        elif tinfo.pstart == sinfo.pstart and sinfo.pend == tinfo.pend and \
                abs(tinfo.weight - sinfo.weight) < 0.01:
            pass
        else:
            if sinfo.label == 'col' or sinfo.label == 'val':
                if tinfo.label == 'col' or tinfo.label == 'val':
                    if (sinfo.pend
                            - sinfo.pstart) > (tinfo.pend - tinfo.pstart) or (
                                sinfo.weight > tinfo.weight
                                and sinfo.index != -1):
                        flag = False
                else:
                    flag = False
            else:
                if (tinfo.label == 'op' or tinfo.label == 'agg'):
                    if (sinfo.pend - sinfo.pstart) > (
                            tinfo.pend
                            - tinfo.pstart) or sinfo.weight > tinfo.weight:
                        flag = False

        return flag

    def normal_type_infos(self, infos):
        typeinfos = []
        for info in infos:
            typeinfos = [x for x in typeinfos if self.save_info(x, info)]
            flag = True
            for i, typeinfo in enumerate(typeinfos):
                if not self.save_info(info, typeinfo):
                    flag = False
                    break
                if info.pstart < typeinfo.pstart:
                    typeinfos.insert(i, info)
                    flag = False
                    break
            if flag:
                typeinfos.append(info)
        return typeinfos

    def findnear_typeinfo(self, info1, info2):
        return self.findnear(info1.pstart, info1.pend, info2.pstart,
                             info2.pend)

    def find_real_column(self, infos, table):
        for i, vinfo in enumerate(infos):
            if vinfo.index != -1 or vinfo.label != 'val':
                continue
            eoidx = -1
            for j, oinfo in enumerate(infos):
                if oinfo.label != 'op':
                    continue
                if self.findnear_typeinfo(vinfo, oinfo):
                    eoidx = j
                    break
            for j, cinfo in enumerate(infos):
                if cinfo.label != 'col' or table['header_types'][
                        cinfo.index] != 'real':
                    continue
                if self.findnear_typeinfo(cinfo, vinfo) or (
                        eoidx != -1
                        and self.findnear_typeinfo(cinfo, infos[eoidx])):
                    infos[i].index = cinfo.index
                    break

        return infos

    def filter_column_infos(self, infos):
        delid = []
        for i, info in enumerate(infos):
            if info.label != 'col':
                continue
            for j in range(i + 1, len(infos), 1):
                if infos[j].label == 'col' and \
                        info.pstart == infos[j].pstart and \
                        info.pend == infos[j].pend:
                    delid.append(i)
                    delid.append(j)
                    break

        typeinfos = []
        for idx, info in enumerate(infos):
            if idx in set(delid):
                continue
            typeinfos.append(info)

        return typeinfos

    def filter_type_infos(self, infos, table):
        infos = self.filter_column_infos(infos)
        infos = self.find_real_column(infos, table)

        colvalMp = {}
        for info in infos:
            if info.label == 'col':
                colvalMp[info.index] = []
        for info in infos:
            if info.label == 'val' and info.index in colvalMp:
                colvalMp[info.index].append(info)

        delid = []
        for idx, info in enumerate(infos):
            if info.label != 'val' or info.index in colvalMp:
                continue
            for index in colvalMp.keys():
                valinfos = colvalMp[index]
                for valinfo in valinfos:
                    if valinfo.pstart <= info.pstart and \
                            valinfo.pend >= info.pend:
                        delid.append(idx)
                        break

        typeinfos = []
        for idx, info in enumerate(infos):
            if idx in set(delid):
                continue
            typeinfos.append(info)

        return typeinfos

    def get_table_match_score(self, nlu_t, schema_link):
        match_len = 0
        for info in schema_link:
            scale = 0.6
            if info['question_len'] > 0 and info['column_index'] != -1:
                scale = 1.0
            else:
                scale = 0.5
            match_len += scale * info['question_len'] * info['weight']

        return match_len / (len(nlu_t) + 0.1)

    def get_entity_linking(self,
                           tokenizer,
                           nlu,
                           nlu_t,
                           tables,
                           col_syn_dict,
                           table_id=None,
                           history_sql=None):
        """
        get linking between question and schema column
        """
        typeinfos = []
        numbers = re.findall(r'[-]?\d*\.\d+|[-]?\d+|\d+', nlu)

        if table_id is not None and table_id in tables:
            tables = {table_id: tables[table_id]}

        # search schema link in every table
        search_result_list = []
        for tablename in tables:
            table = tables[tablename]
            trie_set = None
            if 'value_trie' in table:
                trie_set = table['value_trie']

            typeinfos = []
            for ii, column in enumerate(table['header_name']):
                column = column.lower()
                column_new = column
                cphrase, cscore = self.get_match_phrase(
                    nlu.lower(), column_new)
                if cscore > 0.3 and cphrase.strip() != '':
                    phrase_tok = tokenizer.tokenize(cphrase)
                    cidxs = self.allfindpairidx(nlu_t, phrase_tok, cscore)
                    typeinfos = self.add_type_all(typeinfos, ii, cidxs, 'col',
                                                  'column', cphrase, column)
                if cscore < 0.8 and column_new in col_syn_dict:
                    columns = list(set(col_syn_dict[column_new]))
                    for syn_col in columns:
                        if syn_col not in nlu.lower() or syn_col == '':
                            continue
                        phrase_tok = tokenizer.tokenize(syn_col)
                        cidxs = self.allfindpairidx(nlu_t, phrase_tok, 1.0)
                        typeinfos = self.add_type_all(typeinfos, ii, cidxs,
                                                      'col', 'column', syn_col,
                                                      column)

            for ii, trie in enumerate(trie_set):
                ans = trie.match(nlu.lower())
                for cell in ans.keys():
                    vphrase = cell
                    vscore = 1.0
                    phrase_tok = tokenizer.tokenize(vphrase)
                    if len(phrase_tok) == 0 or len(vphrase) < 2:
                        continue
                    vidxs = self.allfindpairidx(nlu_t, phrase_tok, vscore)
                    linktype = self.get_column_type(ii, table)
                    typeinfos = self.add_type_all(typeinfos, ii, vidxs, 'val',
                                                  linktype, vphrase, ans[cell])

            for number in set(numbers):
                number_tok = tokenizer.tokenize(number.lower())
                if len(number_tok) == 0:
                    continue
                nidxs = self.allfindpairidx(nlu_t, number_tok, 1.0)
                typeinfos = self.add_type_all(typeinfos, -1, nidxs, 'val',
                                              'real', number, number)

            newtypeinfos = self.normal_type_infos(typeinfos)

            newtypeinfos = self.filter_type_infos(newtypeinfos, table)

            final_question = [0] * len(nlu_t)
            final_header = [0] * len(table['header_name'])
            for typeinfo in newtypeinfos:
                pstart = typeinfo.pstart
                pend = typeinfo.pend + 1
                if typeinfo.label == 'op' or typeinfo.label == 'agg':
                    score = int(typeinfo.linktype[-1])
                    if typeinfo.label == 'op':
                        score += 6
                    else:
                        score += 11
                    for i in range(pstart, pend, 1):
                        final_question[i] = score

                elif typeinfo.label == 'col':
                    for i in range(pstart, pend, 1):
                        final_question[i] = 4
                    if final_header[typeinfo.index] % 2 == 0:
                        final_header[typeinfo.index] += 1

                elif typeinfo.label == 'val':
                    if typeinfo.index == -1:
                        for i in range(pstart, pend, 1):
                            final_question[i] = 5
                    else:
                        for i in range(pstart, pend, 1):
                            final_question[i] = 2
                        final_question[pstart] = 1
                        final_question[pend - 1] = 3
                        if final_header[typeinfo.index] < 2:
                            final_header[typeinfo.index] += 2

            # collect schema_link
            schema_link = []
            for sl in newtypeinfos:
                if sl.label in ['val', 'col']:
                    schema_link.append({
                        'question_len':
                        max(0, sl.pend - sl.pstart + 1),
                        'question_index': [sl.pstart, sl.pend],
                        'question_span':
                        ''.join(nlu_t[sl.pstart:sl.pend + 1]),
                        'column_index':
                        sl.index,
                        'column_span':
                        table['header_name'][sl.index]
                        if sl.index != -1 else '空列',
                        'label':
                        sl.label,
                        'weight':
                        round(sl.weight, 4)
                    })

            # get the match score of each table
            match_score = self.get_table_match_score(nlu_t, schema_link)

            # cal table_score
            if history_sql is not None and 'from' in history_sql:
                table_score = int(table['table_id'] == history_sql['from'][0])
            else:
                table_score = 0

            search_result = {
                'table_id': table['table_id'],
                'question_knowledge': final_question,
                'header_knowledge': final_header,
                'schema_link': schema_link,
                'match_score': match_score,
                'table_score': table_score
            }
            search_result_list.append(search_result)

        search_result_list = sorted(
            search_result_list,
            key=lambda x: (x['match_score'], x['table_score']),
            reverse=True)[0:1]

        return search_result_list


import numpy as np
import pandas as pd
import os
import datetime
from scipy import stats

"""genera le features degli utenti a partire dalla cartella dei file; costruisce un file utente-features"""

path = "filtered_subrs/"
selected_usernames = pd.read_csv("selected_usernames", names=["name"])
list_df = []

# legge i file nella cartella e inserisce i dati in un df

for filename in os.listdir(path):
    print("Working on {}".format(filename))

    df_small = pd.read_json(path + filename)
    df_small = df_small[df_small["author"].isin(list(selected_usernames["name"]))]

    if filename.split("_")[1] == "submission":
        df_small["is_submission"] = True
    else:
        df_small["is_submission"] = False

    list_df.append(df_small)
    print("{} completed! Df-size = {}".format(filename, df_small.size))

df = pd.concat(list_df)
df = df.fillna(0)
df["text_length"] = df["body"].apply(lambda x: len(x))

def get_month(tstamp):
    return int(datetime.datetime.fromtimestamp(int(tstamp)).strftime('%m'))

df["month"] = df["created_utc"].apply(lambda x: get_month(x))

class utente():

    def __init__(self, name):

        self.name = name
        self.distrib_month = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
        self.distrib_subr = {}
        self.n_controversial = 0
        self.n_gilded = 0
        self.n_comments = 0
        self.n_submission = 0
        self.tot_activities = 0
        self.all_scores = []
        self.tot_text_length = 0

    def extract_features(self, controversiality, gilded, is_submission, score, subreddit, text_length, month):
        # iterativamente aggiorna i campi

        if controversiality:
            self.n_controversial += 1
        if gilded:
            self.n_gilded += 1

        if is_submission:
            self.n_submission += 1
        else:
            self.n_comments += 1

        self.tot_activities += 1

        self.all_scores.append(int(score))
        self.tot_text_length += text_length

        if subreddit not in self.distrib_subr:
            self.distrib_subr[subreddit] = 1
        else:
            self.distrib_subr[subreddit] += 1

        self.distrib_month[int(month) - 1] += 1

    def transform_features(self):
        # costruisce nuove features

        self.fraction_controversial = self.n_controversial / self.tot_activities
        self.fraction_gilded = self.n_gilded / self.tot_activities
        self.fraction_submission = self.n_submission / self.tot_activities
        self.avg_text_length = self.tot_text_length / self.tot_activities

        self.all_scores = sorted(self.all_scores)
        self.min_score = self.all_scores[0]
        self.max_score = self.all_scores[-1]
        self.avg_score = np.mean(self.all_scores)
        self.std_score = np.std(self.all_scores)

        self.avg_attivita_mese = np.mean(self.distrib_month)
        self.n_mesi_inattivita = self.distrib_month.count(0)
        self.std_attivita_mese = np.std(self.distrib_month)
        self.month_kurtosis = stats.kurtosis(self.distrib_month)
        self.month_skewedness = stats.skew(self.distrib_month)

        self.subr_list = sorted(list(self.distrib_subr.values()))
        self.n_attivita_favourite_subr = self.subr_list[-1]
        self.n_subr = len(self.subr_list)
        self.avg_attivita_subr = np.mean(self.subr_list)
        self.std_attivita_subr = np.std(self.subr_list)
        self.subr_kurtosis = stats.kurtosis(self.subr_list)
        self.subr_skewedness = stats.skew(self.subr_list)

    def get_user_features(self):

        return (self.name, self.tot_activities, self.fraction_submission, self.fraction_controversial,
                self.fraction_gilded, self.avg_text_length, self.min_score, self.max_score, self.avg_score,
                self.std_score, self.avg_attivita_mese, self.n_mesi_inattivita, self.std_attivita_mese,
                self.month_kurtosis, self.month_skewedness, self.n_attivita_favourite_subr, self.n_subr,
                self.avg_attivita_subr, self.std_attivita_subr, self.subr_kurtosis, self.subr_skewedness)

diz_users = {}

for index, row in df.iterrows():

    if row["author"] not in diz_users:
        diz_users[row["author"]] = utente(row["author"])

    diz_users[row["author"]].extract_features(row["controversiality"], row["gilded"], row["is_submission"],
                                              row["score"],
                                              row["subreddit"], row["text_length"], row["month"])

new_rows = []

for user in diz_users.values():
    user.transform_features()
    new_rows.append(user.get_user_features())

new_cols = ["name", "tot_activities", "%_submission", "%_controversial", "%_gilded", "avg_text_length", "min_score",
            "max_score", "avg_score", "std_score", "avg_attivita_mese", "n_mesi_inattivita", "std_attivita_mese",
            "month_kurt", "month_skew", "n_attivita_fav_subr", "n_subr", "avg_attivita_subr", "std_attivita_subr",
            "subr_kurt", "subr_skew"]

print("FINISHED!")

df_new = pd.DataFrame(new_rows, columns=new_cols)

df_new.to_csv("utenti_features.csv", index=False)


from typing import Optional
from dataclasses import dataclass
import webbrowser

import pygame as pg
import pygame_gui as pg_gui
from pygame_gui.core import ObjectID
from pygame_gui import PackageResource

from . import base_window
from .. import settings
from .. import data_base

@dataclass
class SKey:
    success: bool = True
    reson: str = ''
    key: Optional[str] = None

class SetKeyWindow(base_window.Window):
    def __init__(self) -> None:
        them_pack = PackageResource(
            package='data.theme',
            resource='test_theme.json'
        )
        super().__init__('Set Key', 400, 200, them_pack, settings.RGB_WHITE)
        with data_base.DataBase(settings.DATA_PATH) as db:
            self.data_base = db
        self.skey = SKey()
        self.label = pg_gui.elements.UILabel(
            pg.Rect(0, 0, 400, 50),
            'engtry iqair key',
            manager=self.manager,
            object_id=ObjectID(object_id='#lbt2', class_id=None)
        )
        self.text_entry = pg_gui.elements.UITextEntryLine(
            pg.Rect(0, 55, 400, 45),
            manager=self.manager,
            object_id=ObjectID(object_id='#tel', class_id=None)
        )
        self.link_text = pg_gui.elements.UITextBox(
            'Get Key',
            pg.Rect(160, 100, 80, 45),
            manager=self.manager,
            object_id=ObjectID(object_id='#tb', class_id=None)
        )
        key_str = self.data_base.get_data('key')
        if key_str is not None:
            self.text_entry.set_text(key_str)

    def handle_event(self, key_event: pg.event.Event) -> None:
        super().handle_event(key_event)
        if key_event.type == pg.KEYDOWN:
            if key_event.key == pg.K_RETURN:
                self.is_running = False
        if key_event.type == pg_gui.UI_TEXT_BOX_LINK_CLICKED:
            webbrowser.open(key_event.link_target)

    def update(self, dt: float) -> None:
        super().update(dt)

    def run(self) -> SKey:
        super().run()
        key_str = self.text_entry.get_text()
        if key_str != '':
            with self.data_base as db:
                db.set_data('key', key_str, 0)
        self.skey.key = key_str
        return self.skey


# -*- coding: utf-8 -*-
"""
Created on Thu Apr  4 19:29:53 2019

@author: Rajesh D Borate
"""

import re
ip=input()
l=((re.findall('\d+', ip)))
l=list(map(int,l))
print(max(l),end='')


import utils.constants as cs
import numpy as np
import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt
import os

from feature_extraction.weighted_pca.feature_statistics import compute_kde, find_kde_bw

def plot_clusters(title, X, labels=None, marker='o', plot=True, save=False, filename=""):
    """
    Plots and optionally saves the plot of the FIRST two dimensions of the dataset X.

    @param title: string, title of the plot
    @param X: 2D array of shape (n_samples, n_features), stores the data
    @param labels: 1D array, clustering label of each data sample
    @param marker: type of the plotting marker
    @param plot: Boolean, True - show plot
    @param save: Boolean, True - save plot
    @param filename: string, location to save the plot
    """

    plt.figure()
    plt.title(title)

    if labels is None:
        plt.scatter(X[:, 0], X[:, 1], marker=marker, edgecolors='k')
    else:
        try:
            label_color = [cs.LABEL_COLOR_MAP[l] for l in labels]
            plt.scatter(X[:, 0], X[:, 1], c=label_color, marker='o', alpha=1, edgecolors='k')
        except KeyError:
            print('Too many labels! Using default colors...\n')

        plt.ylabel('Magnitude (mV)')
        plt.xlabel('Magnitude (mV)')

        if save:
            plt.savefig(filename)
        if plot:
            plt.show()

        plt.close()

def plot_more_dims(X, labels, save=False, filename=""):
    """
    Plots and optionally saves the dataset X, creating a matrix of shape (n_features, n_features) of plots.
    Each matrix cell represents a plot of two dimensions.

    @param X: 2D array of shape (n_samples, n_features), stores the data
    @param labels: 1D array, clustering label of each data sample
    @param save: Boolean, True - save the plot
    @param filename: string, location to save the plot
    """

    df = pd.DataFrame(X)
    df['cluster'] = labels
    sns.pairplot(df, diag_kind='kde', hue='cluster', height=3)

    if save:
        plt.savefig(filename)

    plt.show()

def plot_distribution_one_feature(X, feature_nr, hist=False, save=False, filename=""):
    """
    Plots the histogram or the kernel density estimator of a feature from dataset X.

    @param X: 2D array of shape (n_samples, n_features), stores the data
    @param feature_nr: integer, index of the feature used for plotting
    @param hist: Boolean, True - plot histogram, False - plot kernel density estimator
    @param save: Boolean, True - save the plot
    @param filename: string, location to save the plot
    """

    spikes = X[:, feature_nr]

    if hist:
        sns.displot(spikes, kde=False, discrete=False, element="bars", stat="count", common_norm=False)
    else:
        sns.displot(spikes, kind="kde")

    plt.xlabel('magnitude (mV)')

    if hist:
        plt.ylabel('count')
    else:
        plt.ylabel('probability density')

    if save:
        plt.savefig(filename)
    # plt.show()

def two_features_plot(X, f1, bw1, f2, bw2, sim):
    """
    Plot the kde of two features in the same plot.

    @param X: 2D array of shape (n_samples, n_features), stores the data
    @param f1: integer, index of the first feature
    @param bw1: bandwidth of kde of feature 1
    @param f2: integer, index of the second feature
    @param bw2: bandwidth of kde of feature 2
    @param sim: simulation number
    """

    x1 = np.linspace(X[:, f1].min(), X[:, f1].max(), X[:, f1].shape[0])[:, np.newaxis]
    kde1 = compute_kde(X[:, f1], bw1)
    pdf1 = np.exp(kde1.score_samples(x1))
    label = 'f' + str(f1)
    dc1, = plt.plot(x1, pdf1, label=label)

    x2 = np.linspace(X[:, f2].min(), X[:, f2].max(), X[:, f2].shape[0])[:, np.newaxis]
    kde2 = compute_kde(X[:, f2], bw2)
    pdf2 = np.exp(kde2.score_samples(x2))
    label = 'f' + str(f2)
    dc2, = plt.plot(x2, pdf2, label=label)

    plt.xlabel('magnitude', fontsize=15)
    plt.ylabel('density', fontsize=15)
    plt.legend(handles=[dc1, dc2])

    fig_name = 's' + str(sim) + 'f' + str(f1) + 'f' + str(f2)
    plt.savefig(fig_name)

    plt.show()

def plot_two_distributions(p, q, figname=""):
    """
    Plot the kde of two distributions in the same plot
    """

    bw1 = find_kde_bw(p)
    bw2 = find_kde_bw(q)

    x1 = np.linspace(p.min(), p.max(), p.shape[0])[:, np.newaxis]
    kde1 = compute_kde(p, bw1)
    pdf1 = np.exp(kde1.score_samples(x1))
    label = 'before'
    dc1, = plt.plot(x1, pdf1, label=label)

    x2 = np.linspace(q.min(), q.max(), q.shape[0])[:, np.newaxis]
    kde2 = compute_kde(q, bw2)
    pdf2 = np.exp(kde2.score_samples(x2))
    label = 'after'
    dc2, = plt.plot(x2, pdf2, label=label)

    plt.xlabel('magnitude', fontsize=15)
    plt.ylabel('density', fontsize=15)
    plt.legend(handles=[dc1, dc2])

    plt.title = "before/after weighting f" + figname
    plt.savefig(figname)
    plt.close()

    # plt.show()

def one_feature_statistics(f1, figname):
    from feature_extraction.weighted_pca.feature_statistics import compute_kde, find_kde_bw

    bw1 = find_kde_bw(f1)

    x1 = np.linspace(f1.min(), f1.max(), len(f1))[:, np.newaxis]
    kde1 = compute_kde(f1, bw1)
    pdf1 = np.exp(kde1.score_samples(x1))
    label = 'old'
    dc1, = plt.plot(x1, pdf1, label=label)

    plt.xlabel('magnitude')
    plt.ylabel('density')
    plt.legend(handles=[dc1])

    old_min = round(f1.min(), 3)
    old_max = round(f1.max(), 3)
    old_var = round(f1.var(), 3)

    textstr = 'range: [%.3f, %.3f]\nnew var:%.3f' % (old_min, old_max, old_var)

    plt.text(old_max + 0.1, 0, textstr, fontsize=8)
    # plt.title(title)
    # plt.grid(True)
    plt.subplots_adjust(right=0.75)

    # if os.path.isdir(folder_name) is False:
    #     os.mkdir(folder_name)
    #
    # plt.savefig(folder_name + '/' + title)
    # plt.close()
    #
    plt.savefig(figname)
    # plt.show()
    plt.close()

def two_features_statistics(f1, f2, title, folder_name, w):
    """
    Plot the kde of two features in the same plot
    """

    bw1 = find_kde_bw(f1)
    bw2 = find_kde_bw(f2)

    x1 = np.linspace(f1.min(), f1.max(), len(f1))[:, np.newaxis]
    kde1 = compute_kde(f1, bw1)
    pdf1 = np.exp(kde1.score_samples(x1))
    label = 'old'
    dc1, = plt.plot(x1, pdf1, label=label)

    x2 = np.linspace(f2.min(), f2.max(), len(f2))[:, np.newaxis]
    kde2 = compute_kde(f2, bw2)
    pdf2 = np.exp(kde2.score_samples(x2))
    label = 'new'
    dc2, = plt.plot(x2, pdf2, label=label)

    plt.xlabel('magnitude')
    plt.ylabel('density')
    plt.legend(handles=[dc1, dc2])

    # fig_name = 's'+str(sim)+'f' + str(f1) + 'f' + str(f2)
    # plt.savefig(fig_name)

    new_min = round(f2.min(), 3)
    new_max = round(f2.max(), 3)
    old_min = round(f1.min(), 3)
    old_max = round(f1.max(), 3)
    new_var = round(f2.var(), 3)
    old_var = round(f1.var(), 3)

    # print(new_min, new_max, old_min, old_max, new_var, old_var)

    textstr = 'w:%.3f\nold range: [%.3f, %.3f]\nnew range: [%.3f, %.3f]\nold var:%.3f\nnew var:%.3f' % (
        w, old_min, old_max, new_min, new_max, old_var, new_var)

    plt.text(max(old_max, new_max) + 0.1, 0, textstr, fontsize=8)
    plt.title(title)
    # plt.grid(True)
    plt.subplots_adjust(right=0.75)

    if os.path.isdir(folder_name) is False:
        os.mkdir(folder_name)

    plt.savefig(folder_name + '/' + title)
    plt.close()

    # plt.show()

def plot_heatmap(matrix, title="", figure_name=""):
    # y = np.arange(15, 50)
    # x = [0, 1]
    # sns.heatmap(matrix, xticklabels=x, yticklabels=y, annot=True, cmap="YlGnBu")

    sns.heatmap(matrix, annot=False, cmap="YlGnBu")

    plt.title(title)
    plt.xlabel("principal components")
    plt.ylabel("features")

    if figure_name != "":
        plt.savefig(figure_name)
        plt.show()

def plot_individual_clusters(X, y):
    c1 = np.zeros((202, 2))
    c2 = np.zeros((531, 2))
    c3 = np.zeros((1127, 2))

    index = 0
    index2 = 0
    index3 = 0

    for j, i in enumerate(y):
        if i == 1:
            c1[index] = X[j]
            index += 1
        if i == 2:
            c2[index2] = X[j]
            index2 += 1
        if i == 3:
            c3[index3] = X[j]
            index3 += 1

    labels = np.full(len(c1), 1)
    plot_clusters("c1", c1, labels)

    labels = np.full(len(c2), 2)
    plot_clusters("c2", c2, labels)

def plot_two_data(title, X1, X2, labels=None, marker='o', plot=True, save=False, filename=""):
    plt.figure()
    plt.title(title)

    label_color_1 = [cs.LABEL_COLOR_MAP[2] for i in np.arange(1, 10)]
    label_color_2 = [cs.LABEL_COLOR_MAP[6] for i in np.arange(1, 10)]

    # first
    plt.scatter(X1[:9, 0], X1[:9, 1], c=label_color_1, marker='o', alpha=1, edgecolors='k')
    plt.scatter(X1[9:18, 0], X1[9:18, 1], c=label_color_1, marker='x', alpha=1, edgecolors='k')

    # second
    plt.scatter(X2[:9, 0], X2[:9, 1], c=label_color_2, marker='o', alpha=1, edgecolors='k')
    plt.scatter(X2[9:18, 0], X2[9:18, 1], c=label_color_2, marker='x', alpha=1, edgecolors='k')

    # draw lines between them
    for i in range(9):
        # plt.plot([X1[i][0], X2[i][0]], [X1[i][1], X2[i][1]], c='lightblue')
        plt.annotate("",
                     xy=(X2[i][0], X2[i][1]), xycoords='data',
                     xytext=(X1[i][0], X1[i][1]), textcoords='data',
                     arrowprops=dict(arrowstyle="->",
                                     color="0.5",
                                     shrinkA=5, shrinkB=5,
                                     patchA=None,
                                     patchB=None,
                                     ),
                     )

    for i in range(9, 18):
        # plt.plot([X1[i, 0], X2[i][0]], [X1[i, 1], X2[i][1]], c='lavender')
        plt.annotate("",
                     xy=(X2[i][0], X2[i][1]), xycoords='data',
                     xytext=(X1[i][0], X1[i][1]), textcoords='data',
                     arrowprops=dict(arrowstyle="->",  # linestyle="dashed",
                                     color="0.5",
                                     shrinkA=5, shrinkB=5,
                                     patchA=None,
                                     patchB=None,
                                     ),
                     )

    for i_x, i_y in zip(X1[:, 0], X1[:, 1]):
        plt.text(i_x, i_y, '({}, {})'.format(round(i_x, 2), round(i_y, 2)), fontsize="x-small")

    for i_x, i_y in zip(X2[:, 0], X2[:, 1]):
        plt.text(i_x, i_y, '({}, {})'.format(round(i_x, 2), round(i_y, 2)), fontsize="x-small")

    plt.ylabel('mV')
    plt.xlabel('mV')

    if save:
        plt.savefig(filename)
    if plot:
        plt.show()

    plt.close()


from handler.config.data import Data
from handler.logging.log import Log
import os

class CommandController:
    data = Data()
    command_log = Log().create(__name__, data.config["commandLog"])
    total_loaded = 0

    def __init__(self, bot):
        self.bot = bot
        self.command_log.info("Initing commands")

    def load(self):
        if self.data.config["enableDevCommands"]:
            self.set_command_state(self.data.config["devCog"], 'load')
        if self.data.config["enableAdminCommands"]:
            self.set_command_state(self.data.config["adminCog"], 'load')
        if self.data.config["enableModCommands"]:
            self.set_command_state(self.data.config["modCog"], 'load')
        if self.data.config["enableUserCommands"]:
            self.set_command_state(self.data.config["userCog"], 'load')
        self.command_log.info(f"Total commands loaded: {self.total_loaded}")

    def set_command_state(self, module, state, command=None):
        path_string = self.data.config["cogPath"] + '.' + module
        if command and isinstance(command, str):
            arg = {path_string + '.' + command}
            getattr(self.bot, "%s_extension" % state)(*arg)
        else:
            for cmd in os.listdir(self.data.config["cogPath"].replace('.', '/') + '/' + module):
                if cmd.endswith('.py'):
                    arg = {path_string + '.' + cmd[:-3]}
                    getattr(self.bot, "%s_extension" % state)(*arg)
                    self.total_loaded += 1

    def get_command_ranks(self) -> list:
        path = self.data.config["cogPath"].replace(".", "/")
        ranks = []
        for dir in os.listdir(path):
            ranks.append(dir)
        return ranks

    def get_command_list(self, rank=None) -> list:
        path = self.data.config["cogPath"].replace(".", "/")
        commands = []
        if rank:
            for file in os.listdir(path + "/" + rank):
                if file.endswith(".py"):
                    commands.append(file.replace(".py", ""))
            return commands
        if rank is None:
            for dir in os.listdir(path):
                for file in os.listdir(path + "/" + dir):
                    if file.endswith(".py"):
                        commands.append(file.replace(".py", ""))
            return commands


from flask import Blueprint, render_template, request
from geopy.geocoders import Nominatim
from geopy import distance
import pandas as pd
from shapely.geometry import Polygon
from shapely.geometry import Point
import logging

# Constants
MKAD_LOCATION = (55.8277252, 37.6387268)

# Nominatim geolocation service
geolocator = Nominatim(user_agent="myapp.py")

# Creating the Blueprint
distance_bp = Blueprint('distance_bp', __name__)

# Configure the logging
logging.basicConfig(filename = 'result.log', level=logging.DEBUG)

# Functions
def create_polygon():
    """Return a polygon created with the set of coordinates that delimit the
    MKAD area.
    """
    df = pd.read_csv(r'mkad_coordinates.csv')
    df = df.rename(columns = {'37.842762': 'lat', "55.774558" : 'long'})
    polygon_matrix = [[df['long'][ind], df['lat'][ind]] for ind in df.index]
    return Polygon(polygon_matrix)

def get_distance(lat, long):
    """If the point related to lat and long coordinates is inside MKAD
    area, distance is not calculated else, it assigns the geodesic distance.
    """
    if(create_polygon().contains(Point(lat, long))):
        dist = "Address is inside of MKAD"
        logging.debug("distance: {}".format(dist))
    else:
        # Use geopy to get the geodesic distance
        dist = distance.distance(MKAD_LOCATION, (lat, long))
        logging.debug("distance: {}".format(dist))
    return dist

def get_location_parameters(loc):
    """If the address is found with geopy, it obtains the lat, long, address and
    distance. Otherwise it assigns null values to the previous
    mentioned variables.
    """
    location = geolocator.geocode(loc)
    if(location is not None): # If the address exists in database
        latitude = location.latitude
        longitude = location.longitude
        formatted_address = location.address
        dist = get_distance(latitude, longitude)
    else:
        latitude, longitude, formatted_address, dist = null_address()

    return latitude, longitude, formatted_address, dist

def no_input():
    """Fills variables for null address."""
    return "Null input", "Null input", "Null input","Null input"

def null_address():
    """Fills variables for an unknown address."""
    return "Null address", "Null address", "Null address", "Null address"

@distance_bp.route('/result',methods = ['POST', 'GET'])
def result():
    """The blueprint finds the distance from Moscow Ring Road to an addressed
    passed to the application in an HTTP request. If the address is located
    inside the MKAD, the distance is stated this way; if the geolocator is
    unable to find the address the results are stated as null address; and if
    the input address is null, the results are stated as null input.
    """
    if request.method == 'POST':
        location = request.form["location"]
        if(location.strip()): # If location is not null
            (latitude,
             longitude,
             formatted_address,
             dist) = get_location_parameters(location)
        else:
            (latitude,
            longitude,
            formatted_address,
            dist) = no_input()

    # Write address and distance into .log file
    logging.debug("{} distance to Moscow Ring Road: {}".format(formatted_address,
                                                        dist))

    return render_template("result.html",
                          result = formatted_address,
                          Latitude=latitude,
                          longitude=longitude,
                          distance=dist)


import numpy as np

def read_gt(gt, size):
    y_true = np.zeros(size)
    for el in gt:
        y_true[int(el[1])-1, int(el[0])-1] = 1
    return y_true

def compute_aveP(retrieved, gt):
    """
    computes average precision for a given query
    Parameters
        ----------
        retrieved : numpy array
            vector of similarities scores between the query and the documents
        gt : numpy array,
            ground truth vector for the given query. Ones denote the position of relevant documents

    Returns
    -------
    ap_score : float
        average precision
    """

    num = 0.0
    rel = 0
    for i, r in enumerate(retrieved):
        if gt[r] == 1:
            rel += 1
            num += float(rel)/(i+1)
    if rel > 0:
        return num/rel
    else:
        return 0

def evaluate_retrieval(sim, gt, verbose):
    """
    computes mean average precision
    Parameters
        ----------
        sim : numpy array
            matrix of similarities scores between documents (rows) and queries (columns)
        gt : list,
            ground truth list. Each entry is a tuple, where the first element
            indicates a query and the second a relevant document
        verbose : if average precision score for each query must be printed

    Returns
    -------
    map_score : float
        mean average precision
    """

    y_true = read_gt(gt, sim.shape)
    avps = []
    for q in range(y_true.shape[1]):
        avp = compute_aveP(np.argsort(-sim[:, q]), y_true[:, q])
        if verbose:
            print ('Query:', q+1, 'AveP:', avp)
        avps.append(avp)
    map_score = np.mean(avps)
    print ('MAP:', map_score)
    return map_score


import streamlit as st
import gspread as gs
import pandas as pd
import numpy as np
import yfinance as yf

import warnings
warnings.simplefilter(action='ignore', category=pd.errors.PerformanceWarning)

def build_data():
    print("""
    # ----------------------------------------------
    # Get data, concate, merge and extract metrics
    # for portfolio Dashboard
    # ----------------------------------------------
    """)

    gc = gs.service_account_from_dict(st.secrets['gcp_service_account'])
    ss = gc.open_by_key(st.secrets['portfolio'].spreadsheet_key)
    operation = pd.DataFrame(ss.worksheet('Operations').get_all_records()).astype({'Date': 'datetime64[ns]'}).sort_values('Date').set_index('Date')

    dict = pd.DataFrame(ss.worksheet('Dict').get_all_records())
    assets = dict.set_index('Asset')
    print(assets)

    conf = pd.DataFrame(ss.worksheet('Conf').get_all_records())
    ext_data = pd.DataFrame()
    for (_, sheet, columns) in conf.itertuples():
        print(sheet, columns)
        ext_data = pd.concat([ext_data, pd.DataFrame(ss.worksheet(sheet).get_all_records()).astype({'Date': 'datetime64[ns]'}).sort_values('Date').set_index('Date')[eval(columns)]], axis=1)

    market = yf.download(' '.join([*assets['Forex'], *assets['Download']]), start='2021-01-01')['Close']
    market = pd.concat([market, ext_data], axis=1).ffill().bfill()
    print(market.columns)

    # operation.to_csv('.out/operation.csv')
    # assets.to_csv('.out/assets.csv')
    # market.to_csv('.out/market.csv')
    # operation = pd.read_csv('.out/operation.csv')
    # assets = pd.read_csv('.out/assets.csv')
    # market = pd.read_csv('.out/market.csv')

    df = pd.DataFrame(index=market.index, columns=pd.MultiIndex(levels=[[],[],[],[]], codes=[[],[],[],[]]))

    for portfolio in np.unique(operation['Portfolio']):
        print(portfolio)
        dfp: pd.DataFrame = operation[operation['Portfolio'] == portfolio]

        print(f"\tAssets")
        for asset in np.unique(dfp['Asset']):
            classs = assets.loc[asset]['Class']
            print(f"\t\t{classs}:\t{asset}")

            dfa: pd.DataFrame = dfp[dfp['Asset'] == asset]
            dfa = dfa.groupby('Date').agg({'Quantity': 'sum', 'Operation': 'sum'}).reindex(df.index).fillna(0)

            df[portfolio, classs, asset, 'Quantity'] = dfa['Quantity']
            df[portfolio, classs, asset, 'Operation'] = dfa['Operation']
            df[portfolio, classs, asset, 'Market'] = market[assets.loc[asset]['Market']]
            df[portfolio, classs, asset, 'Forex'] = market[assets.loc[asset]['Forex']]

        print(f"\tCurrency")
        for currency in np.unique(dfp['Currency']):
            classs = assets.loc[currency]['Class']
            print(f"\t\t{classs}:\t{currency}")

            dfc: pd.DataFrame = dfp[dfp['Currency'] == currency]
            dfc = dfc.groupby('Date').agg({'Quantity': 'sum', 'Operation': 'sum'}).reindex(df.index).fillna(0)

            if (portfolio, classs, currency, 'Quantity') not in df.columns:
                df[portfolio, classs, currency, 'Quantity'] = 0
            if (portfolio, classs, currency, 'Operation') not in df.columns:
                df[portfolio, classs, currency, 'Operation'] = 0
            if (portfolio, classs, currency, 'Market') not in df.columns:
                df[portfolio, classs, currency, 'Market'] = market[assets.loc[currency]['Market']]

            df[portfolio, classs, currency, 'Quantity'] -= dfc['Operation']
            df[portfolio, classs, currency, 'Operation'] -= dfc['Operation'] * df[portfolio, classs, currency, 'Market']

        print(f"\tBuild datatable ...")
        for asset in np.unique([*dfp['Asset'], *dfp['Currency']]):
            classs = assets.loc[asset]['Class']
            print(f"\t\t{asset}")

            toEUR = market[assets.loc[asset]['Forex']]
            df[portfolio, classs, asset, 'OperationEUR'] = df[portfolio, classs, asset, 'Operation'] / toEUR

            position, invested, investedEUR = pd.Series(dtype=float), pd.Series(dtype=float), pd.Series(dtype=float)
            for idx in df.index:
                position[idx] = np.sum(df.loc[:idx][portfolio, classs, asset, 'Quantity'])
                invested[idx] = np.sum(df.loc[:idx][portfolio, classs, asset, 'Operation'])
                investedEUR[idx] = np.sum(df.loc[:idx][portfolio, classs, asset, 'OperationEUR'])

            df = pd.concat([df, pd.DataFrame({
                    (portfolio, classs, asset, 'Position'): position,
                    (portfolio, classs, asset, 'Invested'): invested,
                    (portfolio, classs, asset, 'InvestedEUR'): investedEUR,
                })], axis=1)

            df[portfolio, classs, asset, 'PRU'] = df[portfolio, classs, asset, 'Invested'] / df[portfolio, classs, asset, 'Position']
            df[portfolio, classs, asset, 'Value'] = df[portfolio, classs, asset, 'Position'] * df[portfolio, classs, asset, 'Market']
            df[portfolio, classs, asset, 'PnL'] = df[portfolio, classs, asset, 'Value'] - df[portfolio, classs, asset, 'Invested']

            df[portfolio, classs, asset, 'ValueEUR'] = df[portfolio, classs, asset, 'Value'] / toEUR
            df[portfolio, classs, asset, 'PnLEUR'] = df[portfolio, classs, asset, 'ValueEUR'] - df[portfolio, classs, asset, 'InvestedEUR']

        for classs in np.unique(assets['Class']):
            if classs not in df[portfolio].columns.get_level_values(0):
                df[portfolio, classs, '.', 'ValueEUR'] = 0
                df[portfolio, classs, '.', 'InvestedEUR'] = 0
                df[portfolio, classs, '.', 'PnLEUR'] = 0
                print('\t\t.', classs, 'missing')

    print(f"\t... OK")

    return df, assets