import  Database.Data_manage_SQLalchemy as db
from sqlalchemy.orm import sessionmaker

from sqlalchemy.ext.declarative import declarative_base
from Database.Stock import Real_values, StockTwits,Real_values_days,StockTwits_preprocessed,Fundamental_information,Financial_News_Finbert_preprocessed

from Prices_extractor.Price_extractor import *
from StockTwits.StockTwits_extractor import *
from StockTwits.StockTwits_preprocesses import *
from Work.Technical_Analysis import *
from Fundamental_Analisis.FinBERT.predict import finbert_prediction
import random
import pytz

##StockTwits data functions
#from Fundamental_Analysis.load_data import LoadData, get_stocktwits_preprocessed_table
# from Fundamental_Analisis.sentiment_analysis import SentimentAnalysis
#
# from sklearn.externals import joblib
import datetime
import feedparser
import nltk
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import time
from Database.Stock import Stock,Financial_News,Financial_News_preprocessed
from Financial_News.Web_scrapping.StockNEWS.Scrapper import goose_article_information,full_scrap_finviz,full_scrap_NASDAQ,raw_scraper
from Financial_News.news_preprocessing import get_score

## Clase para manejar una base de datos
class Manage_StockDatabase():

    def __init__(self):
        self.engine = db.engine
        self.Session = sessionmaker(bind=self.engine)
        self.session = self.Session()
        self.Base = declarative_base()
        # 2 - generate database schema
        self.Base.metadata.create_all(self.engine)

    def Real_values_days(self,stock):
        prices=self.session.query(Real_values_days).filter_by(stock=stock).all()
        if len(prices)==0:
             self.create_StockDatabase_1d(stock=stock)
        else:

            if stock.last_day == None :
                dataframe = self.get_Stock_to_dataframe(stock=stock, Table_info=Real_values_days, index=['Time'])
                last_day=dataframe.tail(1).index[0]
                print(last_day)
                stock.last_day=last_day
                self.session.commit()

            self.update_prices_yahoo_1d(stock=stock)

    ## Crea una base de datos de un indice en particular (probado con el Down Jones)
    def create_StockDatabase_1m(self,list_stocks):
        session=self.Session()
        for ticker in list_stocks:
            new_stock = Stock(ticker)
            prices = get_yfinance_m(new_stock.name, '7d')
            prices.dropna(inplace=True)
            if prices.empty:
                print('no hay valores en {}'.format(ticker))
                continue
            cont = 0
            # all_real_value=[]
            for time in prices.index:
                # print(time)
                row = prices.iloc[cont]
                Open = row['Open']
                Close = row['Close']
                High = row['High']
                Volume = row['Volume']
                Low = row['Low']
                values = Real_values_days(new_stock)
                values.add_value(time=time, open=Open, close=Close, high=High, low=Low, volume=Volume)
                session.add(values)
                cont += 1
                new_stock.last_time = time
                session.add(new_stock)
                session.commit()
        session.close()

    ## Crea una base de datos con intervalos de 1 dia de un indice en particular (probado con el Down Jones)
    def create_StockDatabase_1d(self,stock):
        session = self.Session()
        #new_stock = stock
        prices = get_yfinance_d(stock.name)
        print(prices)
        prices.dropna(inplace=True)
        if prices.empty:
            print('no hay valores en {}'.format(stock.name))
            return
        # all_real_value=[]
        cont=0
        for day in prices.index:
            # print(time)
            row = prices.iloc[cont]
            Open = row['open']
            Close = row['close']
            High = row['high']
            Volume = row['volume']
            Low = row['low']
            Adjclose=row['adjclose']
            values = Real_values_days(stock)
            values.add_value(time=day, open=Open, close=Close, high=High, low=Low, volume=Volume,adjclose=Adjclose)
            local_object = session.merge(values)
            session.add(local_object)
            session.commit()
            if cont==[len(prices.index)-1]:
                session2 = self.Session()
                # print(last_update_time_datetime)
                stock2 = session2.query(Stock).filter_by(name=stock.name).first()
                stock2.last_day = day
                session2.commit()
                session2.close()
            #session.add(values)
            cont += 1
            #session.add(new_stock)

        session.close()
        return

## funcion que actualiza los valores de los precios por minuto desde yahoo finance
    def update_prices_yahoo(self,stock):
        print(stock.name)
        prices = get_yfinance_m(stock.name, '7d')
        if prices.empty:
            return
        prices.dropna(inplace=True)
        prices_last=prices.tail(2)
        print(prices_last)
        if len(prices_last)<2:
            last_yahoo_time = prices_last.index[0]
        else:
            last_yahoo_time=prices_last.index[1]
        last_yahoo_time_string = last_yahoo_time.strftime("%Y-%m-%d %H:%M:%S")
        last_yahoo_time_datetime = datetime.datetime.strptime(last_yahoo_time_string, "%Y-%m-%d %H:%M:%S")

        #cont = 0
        last_time_string = stock.last_time.strftime("%Y-%m-%d %H:%M:%S")
        last_time= datetime.datetime.strptime(last_time_string, "%Y-%m-%d %H:%M:%S")

        print(last_yahoo_time_datetime)
        print(last_time)
        self.fill_prices(t1=last_time,t2=last_yahoo_time_datetime,stock=stock,prices=prices)
        last_update_time = prices_last.index[0]
        last_update_time_string = last_update_time.strftime("%Y-%m-%d %H:%M:%S")
        last_update_time_datetime = datetime.datetime.strptime(last_update_time_string, "%Y-%m-%d %H:%M:%S")
        print('new last update')
        session=self.Session()
        print(last_update_time_datetime)
        stock2=session.query(Stock).filter_by(name=stock.name).first()
        stock2.last_time=last_update_time_datetime
        session.commit()
        session.close()

    def update_prices_yahoo_1d(self,stock):
        print(stock.name)
        prices = get_yfinance_d(stock.name)
        if prices.empty:
            return
        prices.dropna(inplace=True)
        prices_last=prices.tail(2)
        print(prices_last)
        if len(prices_last)<2:
            last_yahoo_time = prices_last.index[0]
        else:
            last_yahoo_time=prices_last.index[1]
        last_yahoo_time_string = last_yahoo_time.strftime("%Y-%m-%d %H:%M:%S")
        last_yahoo_time_datetime = datetime.datetime.strptime(last_yahoo_time_string, "%Y-%m-%d %H:%M:%S")

        #cont = 0
        last_time_string = stock.last_day.strftime("%Y-%m-%d %H:%M:%S")
        last_time= datetime.datetime.strptime(last_time_string, "%Y-%m-%d %H:%M:%S")

        print(last_yahoo_time_datetime)
        print(last_time)
        self.fill_prices_1d(t1=last_time,t2=last_yahoo_time_datetime,stock=stock,prices=prices)
        last_update_time = prices_last.index[0]
        last_update_time_string = last_update_time.strftime("%Y-%m-%d %H:%M:%S")
        last_update_time_datetime = datetime.datetime.strptime(last_update_time_string, "%Y-%m-%d %H:%M:%S")
        print('new last update')
        session=self.Session()
        print(last_update_time_datetime)
        stock2=session.query(Stock).filter_by(name=stock.name).first()
        stock2.last_day=last_update_time_datetime
        session.commit()
        session.close()

    def fill_holes(self,stock):
        session = self.Session()
        dataframe= pd.read_sql(session.query(Real_values).filter_by(stock=stock).statement, con=self.engine)
        session.close()
        for val in dataframe.index:
            if val < len(dataframe.index) - 1:
                time = dataframe.iloc[val]['Time']
                next_time = dataframe.iloc[val + 1]['Time']
                difference = next_time.minute - time.minute
                if difference > 1:
                    prices = get_yfinance_m(stock.name, '7d')
                    self.fill_prices(t1=time, t2=next_time, stock=stock,prices=prices)
            else:
                print('ultimo valor')
    def fill_prices(self,t1, t2, stock,prices):

        cont1 = 0
        cont_t1 = 0
        cont_t2 = 0
        for date in prices.index:
            time_string = date.strftime("%Y-%m-%d %H:%M:%S")
            date_val = datetime.datetime.strptime(time_string, "%Y-%m-%d %H:%M:%S")
            if date_val == t1:
                cont_t1 = cont1
            if date_val == t2:
                cont_t2 = cont1
            cont1 += 1
        hole = prices.iloc[cont_t1:cont_t2]
        cont=0
        session = self.Session()
        for time in hole.index:
            time_string = time.strftime("%Y-%m-%d %H:%M:%S")
            t_value = datetime.datetime.strptime(time_string, "%Y-%m-%d %H:%M:%S")
            lastest = max((t1, t_value))  # Si es t1, significa que no es necesario agregar
            if lastest == t_value and lastest!=t1:
                row=hole.iloc[cont]
                Open = row['Open']
                Close = row['Close']
                High = row['High']
                Volume = row['Volume']
                Low = row['Low']
                values = Real_values(stock)
                values.add_value(time=t_value, open=Open, close=Close, high=High, low=Low, volume=Volume)
                local_object = session.merge(values)
                session.add(local_object)
                session.commit()
            cont+=1
        session.close()

    def fill_prices_1d(self,t1, t2, stock,prices):
        cont1 = 0
        cont_t1 = 0
        cont_t2 = 0
        for date in prices.index:
            time_string = date.strftime("%Y-%m-%d %H:%M:%S")
            date_val = datetime.datetime.strptime(time_string, "%Y-%m-%d %H:%M:%S")
            if date_val == t1:
                cont_t1 = cont1
            if date_val == t2:
                cont_t2 = cont1
            cont1 += 1
        hole = prices.iloc[cont_t1:cont_t2]
        #print(hole)
        cont=0
        session = self.Session()
        for time in hole.index:
            time_string = time.strftime("%Y-%m-%d %H:%M:%S")
            t_value = datetime.datetime.strptime(time_string, "%Y-%m-%d %H:%M:%S")
            lastest = max((t1, t_value))  # Si es t1, significa que no es necesario agregar
            if lastest == t_value and lastest!=t1:
                row=hole.iloc[cont]
                Open = row['open']
                Close = row['close']
                High = row['high']
                Volume = row['volume']
                Low = row['low']
                Adjclose=row['adjclose']
                values_days = Real_values_days(stock)
                values_days.add_value(time=t_value, open=Open, close=Close, high=High, low=Low, volume=Volume,adjclose=Adjclose)
                local_object = session.merge(values_days)
                session.add(local_object)
                session.commit()
            cont+=1
        session.close()

    def delete_repeted_values(self,stock):
        session=self.Session()
        prices=session.query(Real_values).filter_by(stock=stock)
        for p in prices:
            repeted=session.query(Real_values).filter_by(stock=stock).filter_by(Time=p.Time).filter_by(Open=p.Open).filter_by(Close=p.Close).filter_by(High=p.High).filter_by(Low=p.Low).all()
            c=0
            if len(repeted)>1:
                first=True
                for r in repeted:
                    if first:
                        r.repr()
                        first=False
                    else:
                        r.repr()
                        local_object = session.merge(r)
                        session.delete(local_object)
                        session.commit()
                        c+=1
            print('Elimino {} precios'.format(c))
        session.close()
    def export_1d_data(self,save_dir):
        import numpy as np
        import os
        session=self.Session()
        stocks=session.query(Stock).all()
        for stock in stocks:
            data=self.get_Stock_to_dataframe(stock=stock,Table_info=Real_values_days)
            data=data[['Time','Open','Close','High','Low','Volume','AdjClose']]
            data=data.sort_values(by=['Time'])
            data=data.drop_duplicates(subset=['Time'])
            data=data.rename(columns={'Time':'Date'})
            data['serial_number']=np.linspace(0,len(data.values)-1,len(data.values))
            if not os.path.exists(save_dir):
                os.mkdir(save_dir)
            file_name='{}.csv'.format(stock.name)
            save_path = os.path.join(save_dir, file_name)
            data.to_csv(save_path)
        session.close()

    def export_1m_data(self,path_dir,n_val=None):
        import numpy as np
        session=self.Session()
        stocks=session.query(Stock).all()
        for stock in stocks:
            stock.repr()
            data=self.get_Stock_to_dataframe(stock=stock,Table_info=Real_values)
            data=data[['Time','Open','Close','High','Low','Volume']]
            data=data.sort_values(by=['Time'])
            data=data.drop_duplicates(subset=['Time'])
            #data['Time']=data['Time'].apply(lambda x: )
            #print(data['Time'].values)
            data=data.rename(columns={'Time':'datetime'})
            data['minutes'] = data['datetime'].apply(lambda x: x.time())
            data['date'] = data['datetime'].apply(lambda x: x.date())
            #data['serial_number'] = np.linspace(0, len(data.values) - 1, len(data.values))
            data_final=pd.DataFrame(columns=['datetime','Open','Close','High','Low','Volume','serial_number'])
            for data_day in data.groupby(['date'], sort=True):
                data_day=data_day[1]
                data_day['serial_number']=np.linspace(0, len(data_day.values) - 1, len(data_day.values))
                data_final=data_final.append(data_day)
            if n_val!=None:
                data_final=data_final.tail(n_val)
            #print(data_final['serial_number'])
            data_final.to_csv(path_dir+'/{}.csv'.format(stock.name))

    def export_1m_AT_data(self):

            session = self.Session()
            stocks = session.query(Stock).all()
            for stock in stocks:
                stock.repr()
                data = self.get_Stock_to_dataframe(stock=stock, Table_info=Real_values)
                data = data[['Time', 'Open', 'Close', 'High', 'Low', 'Volume']]
                data = data.sort_values(by=['Time'])
                data = data.drop_duplicates(subset=['Time'])
                ###ADD Technical inciators
                data=get_technical_indicator(DF=data)
                print(data)
                data = data.rename(columns={'Time': 'datetime'})
                data['minutes'] = data['datetime'].apply(lambda x: x.time())
                data['date'] = data['datetime'].apply(lambda x: x.date())
                # data['serial_number'] = np.linspace(0, len(data.values) - 1, len(data.values))
                data_final = pd.DataFrame(
                    columns=['datetime', 'Open', 'Close', 'High', 'Low', 'Volume', 'serial_number'])
                for data_day in data.groupby(['date'], sort=True):
                    data_day = data_day[1]
                    data_day['serial_number'] = np.linspace(0, len(data_day.values) - 1, len(data_day.values))
                    data_final = data_final.append(data_day)
                data_final.to_csv(
                    'C:/Users/56979/PycharmProjects/TradingGym/trading_env/test/data/1minute_data/DOWJONES/AT_data/{}.csv'.format(
                        stock.name))

#Convierte  una tabla de sql en dataframe (pensada para tener la tabla stock de la base de datos y luego poder iterar en todos los stocks dentro de esa base
#Para nuestro caso table_name puede ser stock o Real_value por ahora

    def sql_to_dataframe(self,table_name='Stock'):
        table_df = pd.read_sql_table(table_name,con=self.engine,index_col='stock_id')
        return table_df

#Entrega un dataframe de un stock en particular, solo necesita el nombre del stock y una session de la base de datos
    def get_Stock_to_dataframe(self,stock,Table_info,start=datetime.datetime(1900, 1, 30, 0, 0, 0),end=datetime.datetime.now(),index=None):
            session = self.Session()
            #stock=session.query(Stock).filter(Stock.name == stock).first()

            if index==None:
                stock_pd= pd.read_sql(session.query(Table_info).filter(Table_info.stock_id == stock.id).statement, con=self.engine)
            else:
                stock_pd = pd.read_sql(session.query(Table_info).filter(Table_info.stock_id == stock.id).statement, con=self.engine,index_col=index)
            session.close()
            return stock_pd

    def input_gaphs(self,stock_name,start_time,end_time):
        pd = self.get_Stock_to_dataframe(stock_name=stock_name, start=start_time, end=end_time)
        pd = pd.sort_values('Time')
        cont = 0
        candle_stick = []
        volume=[]
        for val in pd.index:
            row = pd.iloc[cont]
            time = row['Time']
            time_string = time.strftime("%Y-%m-%d %H:%M:%S")
            open = row['Open']
            high = row['High']
            close = row['Close']
            low = row['Low']
            vol = row['Volume']
            dicc = {'t': time_string, 'o': open, 'h': high, 'l': low, 'c': close}
            candle_stick.append(dicc)
            volume.append(vol)
            cont += 1
        return candle_stick,volume

#
# ###StockTwits Tables#############################
    def get_prev_StockTwits_from_db(self,stock):
        session = self.Session()
        prev_messages = []
        for m in session.query(StockTwits).filter_by(stock=stock).all():
            prev_messages.append(m.message_id)
        session.close()
        return prev_messages

    def get_fresh_new_Twits(self,stock):
        print('entro al get refresh')
        old_data = self.get_prev_StockTwits_from_db(stock)
        twits,Time_out=extractor(old_messages_id=old_data,SYMBOL=stock.name)
        if not twits.empty:
            ##Add new Twits to database
            utc = pytz.UTC
            session_twits=self.Session()
            for i in twits.index:
                message=twits.iloc[i]
                message_time = message['datetime']
                message_id = message['message_id']
                dt = datetime.datetime.strptime(message_time, "%Y-%m-%dT%H:%M:%SZ")
                dt = dt.replace(tzinfo=utc)
                twits_message = StockTwits(stock=stock)
                sentiment=message['sentiment']
                twits_message.add_sentiment(sentiment=sentiment)
                text = message['message']
                text_array = text.split()
                text2 = ' '
                for a in text_array:
                    s = a.encode('utf-8')
                    #print(s)
                    v = s.decode("utf-8", errors="replace").replace("\x00", "\uFFFD")
                    #print(v)
                    text2 = text2 + ' ' + v
                user = message['user']
                twits_message.add_value(time=dt, user=int(user), message=text2, message_id=int(message_id))
                local_object = session_twits.merge(twits_message)
                session_twits.add(local_object)
                # StockTwits Preprocessing
                message = clean2(message=text2)
                message_preprocessed = StockTwits_preprocessed(stock=stock)
                message_preprocessed.add_value(time=dt, message=message, user=int(user),
                                               message_id=int(message_id),
                                               sentiment=sentiment)

                # message_preprocessed.add_sentiment_score(sentiment_score=score)
                message_preprocessed.repr()
                # message_preprocessed.add_naive_bayes_sentiment(naive_sentiment=naive)
                local_object_2 = session_twits.merge(message_preprocessed)
                session_twits.add(local_object_2)
                # local_object2 = session.merge(m)
                local_object.preprocessed = True
                session_twits.commit()
            session_twits.close()
        return twits,Time_out

    def Delete_repeted_StockTwits_preprocesses(self):
        from sqlalchemy import func,and_
        print('Elimina los StockTwits preprocesados que  estan repetido')
        #Table = StockTwits
        session=self.Session()
        # helper subquery: find first row (by primary key) for each unique date
        subq = (session.query(StockTwits_preprocessed.message_id, func.min(StockTwits_preprocessed.id).label("min_id")).group_by(StockTwits_preprocessed.message_id)).subquery('message_id_min_id')
        sq = (session.query(StockTwits_preprocessed.id).join(subq, and_(StockTwits_preprocessed.message_id == subq.c.message_id,StockTwits_preprocessed.id != subq.c.min_id) )).subquery("subq")
        dq = (session.query(StockTwits_preprocessed).filter(StockTwits_preprocessed.id.in_(sq))).delete(synchronize_session=False)
        session.commit()
        session.close()

    def Delete_repeted_StockTwits(self):
        from sqlalchemy import func,and_
        print('Elimina StockTwits  repetidos')
        #Table = StockTwits
        session=self.Session()
        # helper subquery: find first row (by primary key) for each unique date
        subq = (session.query(StockTwits.message_id, func.min(StockTwits.id).label("min_id")).group_by(StockTwits.message_id)).subquery('message_id_min_id')
        sq = (session.query(StockTwits.id).join(subq, and_(StockTwits.message_id == subq.c.message_id,StockTwits.id != subq.c.min_id) )).subquery("subq")
        dq = (session.query(StockTwits).filter(StockTwits.id.in_(sq))).delete(synchronize_session=False)
        session.commit()
        session.close()

    def Repeted_StockTwits(self,stock,time_out):
        print('checkea si esta repetido')
        #Table = StockTwits
        session=self.Session()
        StockTwit=session.query(StockTwits).filter_by(stock=stock).filter(StockTwits.exist_one==None).all()
        print(len(StockTwit))
        contador=0
        timeout = time.time() + time_out  # execute_time seconds times
        for m in StockTwit:
            #m.repr()
            if time.time() <= timeout:
                repeated=session.query(StockTwits).filter_by(stock=stock).filter_by(message_id=m.message_id).all()
                local_object = session.merge(m)
                if len(repeated)==1:

                    local_object.exist_one = True
                else:
                    local_object.exist_one = False

                session.commit()
                contador+=1

            else:
                session.close()
                return len(StockTwit)-contador
        session.close()
        return len(StockTwit)-contador

    def delete_repeted_StockTwits(self,stock):
        print('Entra al delete')
        session=self.Session()
        Twits=session.query(StockTwits).filter_by(stock=stock).filter_by(exist_one=False).all()
        print(len(Twits))
        for m in Twits:
            array=session.query(StockTwits).filter_by(stock=stock).filter_by(Time=m.Time).filter_by(User=m.User).filter_by(Message=m.Message).filter_by(message_id=m.message_id).all()
            print(len(array))
            first=True
            cont=0
            for sm in array:
                if first:
                    sm.repr()
                    local_object = session.merge(sm)
                    local_object.exist_one=True
                    session.commit()
                    first=False
                else:
                    local_object2 = session.merge(sm)
                    session.delete(local_object2)
                    session.commit()
                    cont+=1
            print('elimino {}'.format(cont))

    def check_Preprocesses_StockTwits(self,stock,time_out):
        print('checkea si esta preprocesado')
        session=self.Session()
        StockTwit=session.query(StockTwits).filter_by(stock=stock).filter(StockTwits.preprocessed!=True).all()
        print(len(StockTwit))
        timeout = time.time() + time_out  # execute_time seconds times
        contador=0
        for m in StockTwit:
            if time.time() <= timeout:
                preprocesses=session.query(StockTwits_preprocessed).filter_by(stock=stock).filter_by(message_id=m.message_id).all()
                print(len(preprocesses))
                local_object = session.merge(m)
                local_object.repr()
                if len(preprocesses)==0:
                    print('Este es false')
                    local_object.preprocessed = False
                else:
                    local_object.preprocessed = True
                session.commit()
                contador+=1
            else:
                session.close()
                return len(StockTwit)-contador
        session.close()
        return len(StockTwit)-contador

    def labelled_StockTwits(self):
        session=self.Session()
        Twits_pd = pd.read_sql(session.query(StockTwits).filter(StockTwits.sentiment!=None).statement, con=self.engine)
        stocks=pd.read_sql(session.query(Stock).statement,con=self.engine,index_col=['id'])
        stocks_name=stocks['name'].sort_index()
        feature=['message_id','Message','sentiment','stock_id']
        Twits_pd=Twits_pd[feature]
        file_csv_location = 'C:/Users/56979/PycharmProjects/Fundamental_Analisis/StockTwits_classifiers/All_labelled_data/DowJones_labelled_data.csv'
        feature=['message_id','Message','sentiment','stock_id']
        Twits_pd=Twits_pd[feature]
        # for m in Twits_pd['stock_id'].values:
        #     print(m)
        #     print(stocks_name.iloc[m-1])
        # print('se acabo ')
        Twits_pd['stock_id']=Twits_pd['stock_id'].apply(lambda x:stocks_name.iloc[x-1] )
        Twits_pd.to_csv(file_csv_location)

        Twits_pd.to_csv(file_csv_location)
        session.close()
        return Twits_pd

    def Preprocess_Stocktwits(self,stock):
        print('Entra al preprocess')
        session=self.Session()
        StockTwits_messages=session.query(StockTwits).filter_by(stock=stock).filter_by(preprocessed=False).all()
        print(len(StockTwits_messages))
        for m in StockTwits_messages:
                #print('se proprosesa')
                time = m.Time
                message = m.Message
                message=clean2(message=message)
                ####Sentiment score####
                #score=get_sentiword_score(message=message)
                ####Naive Sentiment
                #naive=tweet_classifier.predict([message])
                #naive=naive[0]
                user = int(m.User)
                sentiment = m.sentiment
                message_id = int(m.message_id)
                message_preprocessed = StockTwits_preprocessed(stock=stock)
                message_preprocessed.add_value(time=time, message=message, user=user,
                                                   message_id=message_id,
                                                   sentiment=sentiment)

                #message_preprocessed.add_sentiment_score(sentiment_score=score)
                #message_preprocessed.repr()
                #message_preprocessed.add_naive_bayes_sentiment(naive_sentiment=naive)
                local_object = session.merge(message_preprocessed)
                m.preprocessed=True
                session.add(local_object)
                session.commit()
        session.close()

        return

    def export_sentiment_stocktwits(self,stock,dir_path):
            data = self.get_Stock_to_dataframe(stock=stock, Table_info=StockTwits_preprocessed)
            data = data[['datetime','sentiment title','sentiment summary','finbert sentiment summary','finbert sentiment title','finbert sentiment text']]
            #data = data.drop_duplicates(subset=['title'])
            data.to_csv(dir_path+'/{}.csv'.format(stock.name))

####Financial news #########
    def add_Financial_new(self,new_data_list, stock):
        print('entra a guardar')
        print(new_data_list.index)
        for i in new_data_list.index:
            new_data=new_data_list.iloc[i]
            #print(new_data)
            guid = new_data['guid']
            published = new_data['published']
            summary = new_data['summary']
            split_summary = str(summary).split(' ')
            title = new_data['title']
            text = new_data['text']
            split_text = str(text).split(' ')
            if len(split_summary) < 2 or len(split_text) < 2:
                print('sale por falta de summary o texto ')
                continue

            source = new_data['fuente']
            url = new_data['url']
            p_date = new_data['p_date']
            image = new_data['image']
            keywords = new_data['keywords']
            if type(keywords) == float or keywords == '':
                keywords = ['No', 'keywords']
            session=self.Session()
            exist_new = session.query(Financial_News).filter_by(stock=stock).filter_by(text=text).filter_by(
                         title=title).filter_by(url=url).filter_by(summary=summary).all()
            #exist_new=[]
            print(len(exist_new))
            if len(exist_new) != 0:
                print('esta wea ya existia ')
                continue
            else:
                new = Financial_News(stock=stock)

                new.add_value(guid=guid, published=published, summary=summary, title=title, text=text, source=source,
                                      url=url, image=image, p_date=p_date)
                sentiment_summary = new_data['sentiment_summary']
                sentiment_title = new_data['sentiment_title']
                new.add_sentiment(sentiment_title=sentiment_title, sentiment_summary=sentiment_summary)
                new.add_keywords(keywords)
                new.repr()
                local_object = session.merge(new)
                session.add(local_object)
                #session.add(new)
                session.commit()
                session.close()
        return

    def read_YAHOO(self, stock):

            session=self.Session()
            yahoo_old_news= pd.read_sql(session.query(Financial_News).filter_by(stock=stock).filter(Financial_News.guid!='No_guid').statement,con=self.engine)
            #source_page='https://finance.yahoo.com'
            YAHOO_URL = 'https://feeds.finance.yahoo.com/rss/2.0/headline?s=%s&region=US&lang=en-US'
            # """Download VADER"""
            try:
                nltk.data.find('vader_lexicon')
            except LookupError:
                nltk.download('vader_lexicon', quiet=True)

            ##Busca en Yahoo finance
            while True:
                try:
                    feed = feedparser.parse(YAHOO_URL % stock.name)
                    break
                except Exception as ex:
                    print(ex.args)
                    ##Web scrapping method ### Sirve para que parezca mas humano scraper
                    time_request = random.randint(0, 50)
                    print('DORMIRA POR {} SEGUNDOS'.format(time_request))
                    time.sleep(time_request)

            yahoo_news=pd.DataFrame(columns=['guid','author','title','published','text','summary','keywords','image','fuente','url','p_date','sentiment_summary','sentiment_title'])
            for entry in feed.entries:
                if entry.guid in yahoo_old_news['guid'].values:
                    print('Esta noticia de yahoo ya existe')
                    continue

                info = goose_article_information(tmp=entry.link)
                if info==None:
                    continue
                """Find url and skip if exists"""

                if len(info['keywords'])==0 or len(info['author'])==0 or  info['image']==None:
                        #url_article = entry.link
                        image,keywords,author = raw_scraper(info['url'],memoize=False)
                        if keywords!=None:
                            if len(keywords)>0:
                                print(keywords)
                                info['keywords']=keywords
                        if image!=None:
                            info['image']=image

                        if author!=None:
                            if len(author)>0:
                                info['author']=author

                """Analyze the sentiment"""
                sia = SentimentIntensityAnalyzer()
                _summary = sia.polarity_scores(entry.summary)['compound']
                _title = sia.polarity_scores(entry.title)['compound']

                """Parse the date"""
                p_date = '%s_%s' % (stock.name,
                                        datetime.datetime.strptime(entry.published, '%a, %d %b %Y %H:%M:%S +0000').strftime(
                                            "%Y-%m-%d"))
                new_data = {'guid': entry.guid,
                                'author': info['author'],
                                'title': entry.title,
                                'published': entry.published,
                                'text': info['text'],
                                'summary': entry.summary,
                                'keywords': info['keywords'],
                                'image': info['image'],
                                'fuente': info['fuente'],
                                'url': info['url'],
                                'p_date': p_date,
                                'sentiment_summary': _summary,
                                'sentiment_title': _title}

                data = pd.DataFrame([new_data])
                yahoo_news = yahoo_news.append(data, ignore_index=True)

                #yahoo_news.append(new_data)
            self.add_Financial_new(new_data_list=yahoo_news, stock=stock)
            ###Cierro session para no molestar
            session.close()

    def read_finviz(self,stock):
            session=self.Session()
            finviz_old_news= pd.read_sql(session.query(Financial_News).filter_by(stock=stock).filter(Financial_News.guid =='No_guid').statement,con=self.engine)
            finviz_old_news2 = pd.read_sql(
                session.query(Financial_News).filter_by(stock=stock).filter(Financial_News.guid == 'No_guid_finviz').statement,con=self.engine)
            #print(finviz_old_news)
            # """Download VADER"""
            try:
                 nltk.data.find('vader_lexicon')
            except LookupError:
                    nltk.download('vader_lexicon', quiet=True)
            cont=0
            articles = full_scrap_finviz(stock_name=stock.name)
            finviz_news=pd.DataFrame(columns=['guid','author','title','published','text','summary','keywords','image','fuente','url','p_date','sentiment_summary','sentiment_title'])
            for link in articles:
                if link in finviz_old_news['url'].values:
                    old_new=session.query(Financial_News).filter_by(url=link).first()
                    if old_new.guid=='No_guid':
                        old_new.guid='No_guid_finviz'
                        session.commit()
                    print('Esta noticia de finviz ya esta en la base de datos')
                    continue
                if link in finviz_old_news2['url'].values:
                    #old_new=session.query(Financial_News).filter_by(url=link).first()
                    #if old_new.guid=='No_guid':
                    #    old_new.guid='No_guid_finviz'
                    #    session.commit()
                    print('Esta noticia de finviz ya esta en la base de datos')
                    continue
                print(link)
                info = goose_article_information(tmp=link)
                if info ==None:
                    continue
                    ###Cierro session para no molestar
                if len(info['keywords'])==0 or len(info['author'])==0 or  info['image']==None :
                        #url_article = entry.link
                        image,keywords,author = raw_scraper(info['url'],memoize=False)
                        if keywords!=None:
                            if len(keywords)>0:
                                print(keywords)
                                info['keywords']=keywords
                        if image!=None:
                            info['image']=image
                        if author!=None:
                            if len(author)>0:
                                info['author']=author

                summary = info['summary']
                title = info['title']
                published = info['published']
                """Analyze the sentiment"""
                sia = SentimentIntensityAnalyzer()
                _summary = sia.polarity_scores(summary)['compound']
                _title = sia.polarity_scores(title)['compound']
                # strftime
                """Parse the date"""
                p_date = '{}_%{}'.format(stock.name, str(published))
                new_data = {'guid': 'No_guid_finviz',
                                'author': info['author'],
                                'title': title,
                                'published': published,
                                'text': info['text'],
                                'summary': summary,
                                'keywords': info['keywords'],
                                'image': info['image'],
                                'fuente': info['fuente'],
                                'url': info['url'],
                                'p_date': p_date,
                                'sentiment_summary': _summary,
                                'sentiment_title': _title}
                data=pd.DataFrame([new_data])
                finviz_news=finviz_news.append(data,ignore_index=True)
            self.add_Financial_new(new_data_list=finviz_news, stock=stock)
            session.close()

    def read_nasdaq(self, stock):
        session = self.Session()
        nasdaq_old_news = pd.read_sql(
            session.query(Financial_News).filter_by(stock=stock).filter(Financial_News.guid == 'No_guid').statement,
            con=self.engine)
        nasdaq_old_news2 = pd.read_sql(
            session.query(Financial_News).filter_by(stock=stock).filter(
                Financial_News.guid == 'No_guid_nasdaq').statement, con=self.engine)
        # print(finviz_old_news)
        # """Download VADER"""
        try:
            nltk.data.find('vader_lexicon')
        except LookupError:
            nltk.download('vader_lexicon', quiet=True)

        articles = full_scrap_NASDAQ(stock=stock.name,MAX_VALUE=100)
        nasdaq_news = pd.DataFrame(columns=['guid','author','title','published','text','summary','keywords','image','fuente','url','p_date','sentiment_summary','sentiment_title'])
        for link in articles:
            if link in nasdaq_old_news['url'].values:
                old_new = session.query(Financial_News).filter_by(url=link).first()
                if old_new.guid == 'No_guid':
                    old_new.guid = 'No_guid_nasdaq'
                    session.commit()
                print('Esta noticia de nasdaq.com ya esta en la base de datos')
                continue
            if link in nasdaq_old_news2['url'].values:
                print('Esta noticia de nasdaq.com ya esta en la base de datos')
                continue
            info = goose_article_information(tmp=link)
            if info == None:
                print('No se puede sacar nada del articulo')
                continue

            ###Cierro session para no molestar
            if len(info['keywords']) == 0 or len(info['author']) == 0 or info['image'] == None:
                # url_article = entry.link
                image, keywords, author = raw_scraper(info['url'], memoize=False)
                if keywords != None:
                    if len(keywords) > 0:
                        print(keywords)
                        info['keywords'] = keywords
                if image != None:
                    info['image'] = image

                if author != None:
                    if len(author) > 0:
                        info['author'] = author

            summary = info['summary']
            title = info['title']
            published = info['published']
            """Analyze the sentiment"""
            sia = SentimentIntensityAnalyzer()
            _summary = sia.polarity_scores(summary)['compound']
            _title = sia.polarity_scores(title)['compound']
            # strftime
            """Parse the date"""
            p_date = '{}_%{}'.format(stock.name, str(published))
            new_data = {'guid': 'No_guid',
                        'author': info['author'],
                        'title': title,
                        'published': published,
                        'text': info['text'],
                        'summary': summary,
                        'keywords': info['keywords'],
                        'image': info['image'],
                        'fuente': info['fuente'],
                        'url': info['url'],
                        'p_date': p_date,
                        'sentiment_summary': _summary,
                        'sentiment_title': _title}
            data = pd.DataFrame([new_data])
            nasdaq_news = nasdaq_news.append(data, ignore_index=True)

        self.add_Financial_new(new_data_list=nasdaq_news, stock=stock)
        session.close()
    def add_datetime_news(self):
        import arrow
        dict_mese = {'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'May': '05', 'Jun': '06', 'Jul': '07',
                     'July': '07', 'Aug': '08', 'Sep': '09', 'Sept': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'}
        session = self.Session()
        stocks = session.query(Stock).all()
        print('\n### All Stocks:')
        file = open('Total_news.txt', 'w')
        file.write('Symbol : Cantidad \n')
        for stock in stocks:
            stock.repr()
            news = session.query(Financial_News).filter_by(stock=stock).filter_by(datetime_published=None).all()
            for n in news:
                try:
                    date_split = n.published.split(',')
                    if len(date_split) == 1:
                        date_time_str = date_split[0]
                        utc = arrow.get(date_time_str)

                    elif len(date_split) == 2:
                        d2 = date_split[0].split(' ')
                        if len(d2) >= 2:
                            d1 = date_split[1].split(' ')
                            if len(d1) <= 4:
                                año = d1[1]
                                hora = d1[2]
                                try:
                                    match = re.search('AM', hora)
                                    s = match.start()
                                    e = match.end()
                                    # print('Found "{}"\nin "{}"\nfrom {} to {} ("{}")'.format(
                                    #     match.re.pattern, match.string, s, e, hora[s:e]))
                                    hora = hora[0:s]
                                except:
                                    match = re.search('PM', hora)
                                    s = match.start()
                                    e = match.end()
                                    # print('Found "{}"\nin "{}"\nfrom {} to {} ("{}")'.format(
                                    #     match.re.pattern, match.string, s, e, hora[s:e]))
                                    hora = hora[0:s]
                                    hora_split = hora.split(':')
                                    hora_12 = int(hora_split[0]) + 12
                                    if hora_12 >= 24:
                                        hora_12 = int(hora_split[0])
                                    hora = str(hora_12) + ':' + hora_split[1]
                                # print('año:{}, hora:{}'.format(año,hora))
                                # print(d2)
                                mes = dict_mese[d2[0]]
                                dia = d2[1]
                                # print('mes: {} , dia: {}'.format(mes,dia))
                                # date_time_str = '2018-06-29 08:15:27.243860'
                                date_time_str = año + '-' + mes + '-' + dia + ' ' + hora
                                # utc = arrow.get(date_time_str)
                                # print(date_time_str)
                                # date_time_obj = datetime.datetime.strptime(date_time_str, '%Y-%m-%d %H:%M')
                                date = datetime.datetime.strptime(date_time_str, '%Y-%m-%d %H:%M')
                                utc = arrow.get(date)
                            else:
                                # print('except')
                                # print(d2)
                                # print(d1)
                                año = d1[1]
                                hora = d1[2]
                                m_t = d1[3]
                                # print(m_t)
                                if m_t == 'pm':
                                    # print(m_t)
                                    # print(d1)
                                    hora_split = hora.split(':')
                                    hora_12 = int(hora_split[0]) + 12
                                    if hora_12 >= 24:
                                        hora_12 = int(hora_split[0])
                                    hora = str(hora_12) + ':' + hora_split[1]
                                # print(año)
                                # print(hora)
                                dia = d2[1]
                                mes = d2[0].strip('.')
                                mes = dict_mese[mes]
                                # print(mes)
                                date_time_str = año + '-' + mes + '-' + dia + ' ' + hora
                                # print(date_time_str)
                                # utc = arrow.get(date_time_str)
                                datetime_obj = datetime.datetime.strptime(date_time_str, '%Y-%m-%d %H:%M')
                                utc = arrow.get(datetime_obj)

                        else:
                            # print('segundo except')
                            second_split = date_split[1].split(' ')
                            # print(second_split)
                            dia = second_split[1]
                            mes = second_split[2]
                            mes = dict_mese[mes]
                            año = second_split[3]
                            hora = second_split[4]
                            date_time_str = año + '-' + mes + '-' + dia + ' ' + hora
                            utc = arrow.get(date_time_str)

                except:
                    print('este except')
                    print(n.published)
                    if n.published != None:
                        #print(date_split)
                        second_split = date_split[0].split(':')
                        #print(second_split)
                        tercer_split = second_split[1].split(' ')
                        #print(tercer_split)
                        cuarto_split = tercer_split[1].split('.')
                        #print(cuarto_split)
                        mes = cuarto_split[0]
                        mes = dict_mese[mes]
                        dia = tercer_split[2]
                        quinto_split = date_split[1].split(' ')
                        año = quinto_split[1]
                        hora = quinto_split[3]
                        m_t = quinto_split[4]
                        #print(m_t)
                        if m_t == 'p.m.':
                            # print(d1)
                            hora_split = hora.split(':')
                            hora_12 = int(hora_split[0]) + 12
                            if hora_12 >= 24:
                                hora_12 = int(hora_split[0])
                            hora = str(hora_12) + ':' + hora_split[1]
                        date_time_str = año + '-' + mes + '-' + dia + ' ' + hora
                        date = datetime.datetime.strptime(date_time_str, '%Y-%m-%d %H:%M')
                        utc = arrow.get(date)
                        # time.sleep(3)
                    else:
                        continue
                date_time_obj = utc.to('America/New_York')
                print(date_time_obj.datetime)
                n.datetime_published=date_time_obj.datetime
                session.commit()
        session.close()
    def Finbert_preprocess_news(self,stock,start):
        #import csv
        print('Entra al preprocess')
        session = self.Session()
        News = session.query(Financial_News).filter_by(stock=stock).filter(Financial_News.datetime_published>=start).all()
        Finbert_dataframe = pd.read_sql(session.query(Financial_News_Finbert_preprocessed).statement,con=self.engine)
        print(len(News))
        for m in News:
            new_id = m.id
            if new_id in Finbert_dataframe['financial_new_id'].values:
                continue
            print('se preprosesa')
            #time = m.datetime_published
            text = m.text
            title=m.title
            summary = m.summary
            text_result,summary_result,title_result=finbert_prediction(text=text,summary=summary,title=title)
            finbert_preprocesse=Financial_News_Finbert_preprocessed(financial_new=m)
            #print(finbert_preprocesse.financial_new_id)
            title_save_path='C:/Users/56979/PycharmProjects/DOW_JONES_BOTS/Preprocessing_BOTS/News/news_preprocesses/title.csv'
            summary_save_path = 'C:/Users/56979/PycharmProjects/DOW_JONES_BOTS/Preprocessing_BOTS/News/news_preprocesses/summary.csv'
            text_save_path = 'C:/Users/56979/PycharmProjects/DOW_JONES_BOTS/Preprocessing_BOTS/News/news_preprocesses/text.csv'
            title_result.to_csv(title_save_path)
            summary_result.to_csv(summary_save_path)
            text_result.to_csv(text_save_path)
            try:
                file_title = open(title_save_path, "r")
                title_lines=file_title.readlines()
                print(title_lines)
                file_summary = open(summary_save_path, "r")
                summary_lines = file_summary.readlines()
                print(summary_lines)
                file_text = open(text_save_path, "r")
                text_lines = file_text.readlines()
                print(text_lines)
                finbert_preprocesse.add_value(summary=summary_lines, title=title_lines, text=text_lines)
                session.commit()
            except:
                continue

        session.close()

        return

    def add_Financial_new_preprocessed(self):
        print('Entra al get_new_score')
        session = self.Session()
        News_preprocess = session.query(Financial_News_Finbert_preprocessed).all()
        Financial_new_preprocessed_dataframe = pd.read_sql(session.query(Financial_News_preprocessed).statement, con=self.engine)
        print(Financial_new_preprocessed_dataframe['id'].values)
        #print(len(News_preprocess))
        for new in News_preprocess:
            #print('entra al loop')
            new_id=new.financial_new_id
            if new_id in Financial_new_preprocessed_dataframe['id'].values:
                continue
            financial_new=session.query(Financial_News).filter_by(id=new_id).first()
            if financial_new!=None:
                #financial_new.repr()
                time=financial_new.datetime_published
                #print(time)
                title_list=new.sentiment_title
                summary_list=new.sentiment_summary
                text_list=new.sentiment_text
                if title_list!=None and summary_list!=None and text_list!=None:
                    title_score,summary_score,text_score=get_score(title_list=title_list,text_list=text_list,summary_list=summary_list)
                    preprocess_new=Financial_News_preprocessed(stock=financial_new.stock)
                    preprocess_new.add_value(id=new_id,guid=financial_new.guid,published=time,summary=financial_new.summary,title=financial_new.title,text=financial_new.text,source=financial_new.source,url=financial_new.url)
                    preprocess_new.add_sentiment(sentiment_title=financial_new.sentiment_title,sentiment_summary=financial_new.sentiment_summary)
                    preprocess_new.add_keywords(list=financial_new.keywords)
                    preprocess_new.add_finbert_sentiment(sentiment_summary=summary_score,sentiment_title=title_score,sentiment_text=text_score)
                    session.commit()

        session.close()

    def export_sentiment_news(self,stock,dir_path):
            data = self.get_Stock_to_dataframe(stock=stock, Table_info=Financial_News_preprocessed)
            data = data[['datetime','sentiment title','sentiment summary','finbert sentiment summary','finbert sentiment title','finbert sentiment text']]
            #data = data.drop_duplicates(subset=['title'])
            data.to_csv(dir_path+'/{}.csv'.format(stock.name))

    def add_Fundamental_information(self,new_data, stock):
        Fundamental = Fundamental_information(stock=stock)
        name=str(new_data['Name'].values[0])
        print(str(name))
        sector=str(new_data['Sector'].values[0])
        # #sector = new_data['Sector']
        price = float(new_data['Price'].values[0])
        price_earnings = float(new_data['Price/Earnings'].values[0])
        dividend_yields = float(new_data['Dividend Yield'].values[0])
        earnings_share = float(new_data['Earnings/Share'].values[0])
        week_52_low = float(new_data['52 Week Low'].values[0])
        week_52_high = float(new_data['52 Week High'].values[0])
        marketcap = float(new_data['Market Cap'].values[0])
        ebitda = float(new_data['EBITDA'].values[0])
        price_sales = float(new_data['Price/Sales'].values[0])
        price_book = float(new_data['Price/Book'].values[0])
        sec_filings = str(new_data['SEC Filings'].values[0])
        Fundamental.add_value(name=name,sector=sector,price=price,
                               price_earnings=price_earnings,
                               dividend_yield=dividend_yields,
                               earnings_share=earnings_share,weak52_low=week_52_low,
                               weak52_high=week_52_high,market_cap=marketcap,ebitda=ebitda,
                               price_sales=price_sales,price_book=price_book,sec_filings_url=sec_filings)
        self.session.add(Fundamental)
        Fundamental.repr()
        self.session.commit()
        return

# #####Manage Financial_News Database##########################
# class Manage_NewsDatabase():
#     def __init__(self):
#         self.engine = db.engine_webscrapping_latest_news
#         self.Session = sessionmaker(bind=self.engine)
#         self.session = db.Session_webscrapping_latest_news()
#         self.Base = db.Base_webscraping_latest_news
#         # 2 - generate database schema
#         self.Base.metadata.create_all(self.engine)
#
#     def update(self,df, letter_name, section=None):
#         temp = []
#         for i in range(len(df)):
#             row = df.iloc[i, :]
#             title = row['title']
#             link = row['link']
#             image = row['image']
#             full_text = row['Text']
#             date = row['Dates']
#             summary = row['summary']
#             key_words = row['keywords']
#             author = row['Author']
#             news = self.session.query(Financial_News).filter_by(Title=title).first()
#             if news != None:
#                 print('esta noticia ya esta en la base de datos ')
#                 # if news.media_source.section == None and section != None:
#                 #     newsletter_media = self.session.query(Media_Source).filter_by(
#                 #         name=letter_name).filter_by(section=section).first()
#                 #     if newsletter_media == None:
#                 #         newsletter_media = Media_Source(name=letter_name, section=section)
#                 #         self.session.add(newsletter_media)
#                 #     news.media_source = newsletter_media
#                 #     self.session.commit()
#                 #     #news.repr()
#                 continue
#             newsletter_media = self.session.query(Media_Source).filter_by(name=letter_name).filter_by(
#                     section=section).first()
#             if newsletter_media == None:
#                 newsletter_media = Media_Source(name=letter_name, section=section)
#                 self.session.add(newsletter_media)
#             new = Financial_News(media_source=newsletter_media)
#             new.add_value(link=link, title=title, image=image, date=date, summary=summary, text=full_text)
#             new.add_key_words_value(keywords=key_words)
#             new.add_author(autor=author)
#             #new.repr()
#             self.session.add(new)
#             self.session.commit()
#             temp.append(i)
#             print('Updating...')
#
#         if temp:
#             output = df.loc[[i for i in temp]]
#             output.reset_index(inplace=True, drop=True)
#             add_value = True
#         else:
#             output = pd.DataFrame()
#             output['title'] = ['No updates yet.']
#             output['link'] = output['image'] = output['Text'] = output['Dates'] = output['summary'] = ['']
#             add_value = False
#
#         return output, add_value
#
#
#
#
#     def finish_connection(self):
#         self.session.close()


from .drawable import *
import pygame
import itertools

class Group(Drawable):
    DEFAULTATTRS = dict(
        Drawable.DEFAULTATTRS,
        children=[]
    )

    def add(self, *args):
        for child in args:
            self.children.append(child)
            if hasattr(self, 'parent'):
                child.setup(self.parent)
        if hasattr(self, 'parent'):
            self.set_position()
            self.create_bg_surf()

    def clear(self):
        self.children = []

    def setup(self, parent):
        self.parent = parent
        for child in self.children:
            child.setup(parent)
        self.set_position()
        self.create_bg_surf()

    def render_image(self):
        for child in self.children:
            child.render_image()

    def set_position(self):
        for child in self.children:
            child.set_position()
        self.bg_rect = self.children[0].bg_rect.unionall([child.bg_rect for child in self.children[1:]])
        print(self.bg_rect)

    def draw(self, surface):
        if hasattr(self, 'bg_surf') and self.bg_surf:
            surface.blit(self.bg_surf, self.bg_rect)
        for child in self.children:
            child.draw(surface)

class List(Group):
    DEFAULTATTRS = dict(
        Drawable.DEFAULTATTRS,
        direction='down',
        alignment='midtop',
        children=[],
        spacing=0
    )

    def update(self, **kwargs):
        if kwargs:
            self.set_attrs(kwargs)
        if hasattr(self, 'bg_color') and self.bg_color:
            if len(self.bg_color) == 3:
                self.color += 255
                self.create_bg_surf()
        self.render_image()
        self.set_position()
        self.create_bg_surf()

    def get_standalone_rect(self):
        if self.fixed_size:
            return pygame.Rect(0, 0, self.fixed_size[0], self.fixed_size[1])
        rect = self.get_standalone_fg_rect()
        if self.padding:
            return rect.inflate(self.padding[0], self.padding[1])
        return rect

    def get_standalone_fg_rect(self):
        if not self.children:
            return pygame.Rect(0, 0, 0, 0)
        try:
            child_rects = [child.get_standalone_rect() for child in self.children]
        except AttributeError:
            print(self, self.children)
            raise AttributeError
        if self.direction in ['left', 'right']:
            rect = pygame.Rect(0, 0,
                               sum(rect.width for rect in child_rects)
                               + (len(self.children) - 1) * self.spacing,
                               max(rect.height for rect in child_rects)
                               )
        elif self.direction in ['up', 'down']:
            rect = pygame.Rect(0, 0,
                               max(rect.width for rect in child_rects),
                               sum(rect.height for rect in child_rects)
                               + (len(self.children) - 1) * self.spacing
                               )
        return rect

    def set_position(self):
        self.fg_rect = self.get_standalone_fg_rect()
        self.parent_rect = self.parent.get_rect()
        if type(self.position) is str:
            setattr(self.fg_rect, self.position, getattr(self.parent_rect, self.position))
        elif type(self.position) is tuple:
            setattr(self.fg_rect, self.position[0], getattr(self.parent_rect, self.position[0]))
            self.fg_rect.move_ip(self.position[1], self.position[2])
        else:
            raise AttributeError

        if self.padding and self.fixed_size:
            raise AttributeError("Drawable " + str(type(self)) + " can't have both padding and fixed_size attributes")
        elif self.padding:
            self.bg_rect = self.fg_rect.inflate(self.padding[0], self.padding[1])
        elif self.fixed_size:
            self.bg_rect = pygame.Rect(0, 0, self.fixed_size[0], self.fixed_size[1])
            setattr(self.bg_rect, self.alignment, getattr(self.fg_rect, self.alignment))
        else:
            self.bg_rect = self.fg_rect
        self.create_bg_surf()
        self.set_pos_from_rect_children()

    def set_pos_from_rect_children(self):
        if self.direction == 'right':
            offset = self.fg_rect.left
            for child in self.children:
                child_rect = child.get_standalone_rect()
                child.set_pos_from_rect(
                    pygame.Rect(offset, self.fg_rect.top, child_rect.width, self.fg_rect.height), self.alignment)
                offset += child_rect.width + self.spacing
        elif self.direction == 'down':
            offset = self.fg_rect.top
            for child in self.children:
                child_rect = child.get_standalone_rect()
                child.set_pos_from_rect(
                    pygame.Rect(self.fg_rect.left, offset, self.fg_rect.width, child_rect.height), self.alignment)
                offset += child_rect.height + self.spacing

    def set_pos_from_rect(self, rect, alignment):
        self.fg_rect = self.get_standalone_fg_rect()
        setattr(self.fg_rect, alignment, getattr(rect, alignment))
        self.set_pos_from_rect_children()

        if self.padding and self.fixed_size:
            raise AttributeError("Drawable " + str(type(self)) + " can't have both padding and fixed_size attributes")
        if self.padding:
            self.bg_rect = self.fg_rect.inflate(self.padding[0], self.padding[1])
        elif self.fixed_size:
            self.bg_rect = pygame.Rect(0, 0, self.fixed_size[0], self.fixed_size[1])
            setattr(self.bg_rect, self.alignment, getattr(self.fg_rect, self.alignment))
        else:
            self.bg_rect = self.fg_rect

class Grid(List):
    def add(self, *args):
        for row in args:
            self.children.append(row)
            for item in row:
                if hasattr(self, 'parent'):
                    item.setup(self.parent)
        if hasattr(self, 'parent'):
            self.set_position()
            self.create_bg_surf()

    def setup(self, parent):
        self.parent = parent
        for row in self.children:
            for item in row:
                item.setup(parent)
        self.set_position()
        self.create_bg_surf()

    def draw(self, surface):
        if self.bg_color:
            surface.blit(self.bg_surf, self.bg_rect)
        for row in self.children:
            for item in row:
                item.draw(surface)

    def render_image(self):
        for row in self.children:
            for item in row:
                item.render_image()

    def get_standalone_rect(self):
        if not self.children:
            return pygame.Rect(0, 0, 0, 0)
        if self.direction == 'down':
            row_widths = []
            height = 0
            for row in self.children:
                rects = [item.get_standalone_rect() for item in row]
                row_widths.append(max(rect.width for rect in rects) * len(rects) + self.spacing * len(rects[:-1]))
                height += max(rect.height for rect in rects)
            height += self.spacing * len(self.children[:-1])
            width = max(row_widths)
        return pygame.Rect(0, 0, width, height)

    def get_standalone_fg_rect(self):
        return self.get_standalone_rect()

    def set_pos_from_rect_children(self):
        if self.children:
            child_rects = [item.get_standalone_rect() for row in self.children for item in row]
            item_height = max(rect.height for rect in child_rects)
            item_width = max(rect.width for rect in child_rects)
            init_offset_x = self.fg_rect.left
            init_offset_y = self.fg_rect.top
            item_offset_x = item_width + self.spacing
            item_offset_y = item_height + self.spacing
            if self.direction == 'down':
                for y_index, row in enumerate(self.children):
                    for x_index, item in enumerate(row):
                        _x = init_offset_x + x_index * item_offset_x
                        _y = init_offset_y + y_index * item_offset_y
                        item.set_pos_from_rect(
                            pygame.Rect(_x, _y, item_width, item_height),
                            self.alignment
                        )

    @property
    def flat_children(self):
        return itertools.chain.from_iterable(self.children)


import sys

## Rooms
from rooms.dungeon import Dungeon
from rooms.tunnel import Tunnel
from rooms.entranceway import Entranceway
from rooms.deadEnd import DeadEnd
from rooms.hallway import Hallway
from rooms.leftTunnel import LeftTunnel
from rooms.pit import Pit
from rooms.dragonRoom import  DragonRoom
from rooms.library import Library
from rooms.diningRoom import DiningRoom
from rooms.ballroom import Ballroom
from rooms.outside import Outside
from rooms.windyHallway import WindyHallway
from rooms.kitchen import Kitchen
from rooms.keyRoom import KeyRoom

## Objects
from objects.backpack import Backpack
from objects.map import Map

class Escape(object):

    def __init__(self):
        self.createRooms()
        self.backpack = Backpack()
        self.map = Map()

    def createRooms(self):
        self.rooms = {
            "Dungeon": Dungeon(),
            "Tunnel": Tunnel(),
            "Entranceway": Entranceway(),
            "Dead end": DeadEnd(),
            "Hallway": Hallway(),
            "Left Tunnel": LeftTunnel(),
            "Dragon Room": DragonRoom(),
            "Library": Library(),
            "Ballroom": Ballroom(),
            "Dining Room": DiningRoom(),
            "Outside": Outside(),
            "Pit": Pit(),
            "Windy Hallway": WindyHallway(),
            "Kitchen": Kitchen(),
            "Key Room": KeyRoom()
        }

    def start(self):
        print("")
        print("")
        print("Welcome to ESCAPE!!!!")
        print("Enjoy your stay.")
        print("Type in the commands in the square brackets to play.")
        print("Type 'Backpack' at any time to view what items you have.")
        print("")
        self.enterRoom("Dungeon")

    def end(self):
        print("Thank you for playing!")
        print("Would you like to play again (y/n)?")
        playAgain = input(">>> ")

        if playAgain.lower() == "y" or playAgain.lower() == "yes":
            self.start()
        elif playAgain.lower() == "n" or playAgain.lower() == "no":
            sys.exit()
        else:
            self.end()

    def openDoor(self, door, room):
        if door.requiredItem == None:
            self.enterRoom(door.leadsTo)
        else:
            item = self.backpack.getItem(door.requiredItem)

            if item != None:
                print("Used " + item.name)
                self.enterRoom(door.leadsTo)
            else:
                print("It looks like you need a " + door.requiredItem)
                self.askWhatToDo(room)

    def enterRoom(self, roomName):
        room = self.rooms[roomName]
        print(room.name)
        print(room.description)
        if room.name == "Outside" or room.name == "Pit":
            self.end()
        else:
            self.askWhatToDo(room)

    def askWhatToDo(self, room):
        # print out the options
        print("")
        print("Options:")

        # list the door options
        for door in room.doors:
            print ("[" + door.button + "] " + door.name)

        for item in room.items:
            print("[" + item.button + "] " + item.name)

        # ask the user to make a choice
        print("")
        buttonPressed = input(">>> ")
        print("")
        if buttonPressed.lower() == "backpack":
            if len(self.backpack.items) == 0:
                print("You currently have no items in your backpack")
            else:
                print("Items in your backpack:")
                for item in self.backpack.items:
                    print("    " + item.name)
            print("")
            self.askWhatToDo(room)
        if buttonPressed.lower() == "map":
            for item in self.backpack.items:
                if item.name == "Map":
                    Map.displayMap(self)
                else:
                    print("You do not currently have a map.")
            self.askWhatToDo(room)
        else:
            # try to find a door associated with that button
            chosenDoor = room.getDoorByButton(buttonPressed)
            # find item associated with that button
            chosenItem = room.getItemByButton(buttonPressed)

            if chosenDoor == None and chosenItem == None:
                print("This is not a valid input!")
                print("Please try again")
                self.askWhatToDo(room)
            else:
                if chosenDoor != None:
                    self.openDoor(chosenDoor, room)
                elif chosenItem != None:
                    self.takeItem(chosenItem, room)
                    self.askWhatToDo(room)

    def takeItem(self, item, room):
        room.removeItem(item)
        self.backpack.items.append(item)
        print("Added " + item.name + " to backpack.")
        if item.name == "Black key":
            print("The key is covered with poison which kills you instantly.")
            self.end()


import re
import pandas as pd

def check_company_country(data_frame, sheet_name, writer):
    if 'COUNTRY' in data_frame.columns:
        data_frame['country']=data_frame.COUNTRY
    if 'Company' in data_frame.columns:
        data_frame['company'] = data_frame.Company
    if 'Global_ID' in data_frame.columns:
        data_frame['global_id'] = data_frame.Global_ID
    else:
        data_frame['global_id'] = data_frame.index
    gb = data_frame.groupby(['country','company'],as_index=False)['global_id'].count()
    gb.to_excel(writer, sheet_name=sheet_name)
    if 'Q320_1' in data_frame.columns:
        data_frame['q320_1'] = data_frame.Q320_1
    gb = data_frame.groupby(['country', 'company','q320_1'], as_index=False)['global_id'].count()
    gb.to_excel(writer, sheet_name=sheet_name+'q')

def get_spss_integers(spss_meta):
    integer_cols = []
    for var_name, variables in spss_meta.variable_value_labels.items():
        for value, value_label in variables.items():
            if type(int(value)) == int and var_name not in integer_cols:
                integer_cols.append(var_name)
    return(integer_cols)

def get_customQ_renaming_dic(data_frame):
    rename_custom_dic = {}
    customs = re.compile('Q_.*A_.*') # cute
    col_names = data_frame.columns.tolist()

    for col in col_names:
        if customs.findall(col):
            rename_this = col.replace('A', '') # this pulls (Q_N_123_001 from Q_N_123A_001)
            rename_custom_dic[col] = rename_this
    return(rename_custom_dic)

def get_customQ_renaming_dic_Q_A(data_frame):
    rename_custom_dic = {}
    customs = re.compile('Q_.*A$') # cute
    col_names = data_frame.columns.tolist()

    for col in col_names:
        if customs.findall(col):
            rename_this = col[:-1] # this pulls (Q_N_123_001 from Q_N_123A_001)
            rename_custom_dic[col] = rename_this
    return(rename_custom_dic)

def get_customQ_renaming_dic_aus(data_frame):
    rename_custom_dic = {}
    customs = re.compile('AP_.*A_.*') # cute
    col_names = data_frame.columns.tolist()

    for col in col_names:
        if customs.findall(col):
            rename_this = col.replace('A_', '_') # this pulls (Q_N_123_001 from Q_N_123A_001)
            rename_custom_dic[col] = rename_this
    return(rename_custom_dic)

def get_customQ_renaming_dic_endingA(data_frame):
    rename_custom_dic = {}
    customs = re.compile('AUS_.*_A') # cute
    col_names = data_frame.columns.tolist()

    for col in col_names:
        if customs.findall(col):
            rename_this = col.replace('_A', '') # this pulls (Q_N_123_001 from Q_N_123A_001)
            rename_custom_dic[col] = rename_this
    return(rename_custom_dic)

def get_customQ_renaming_dic_aus_A(data_frame):
    rename_custom_dic = {}
    customs = re.compile('A*_.*A$') # cute
    col_names = data_frame.columns.tolist()

    for col in col_names:
        if customs.findall(col):
            rename_this = col[:-1]
            rename_custom_dic[col] = rename_this
    return(rename_custom_dic)

def stack_aus_data(data_frame, spss_metadata):
    '''
    Order of Operation
    1. Duplicate frame (DF1, DF2)
    2. DF2 - Drop first questions rating
    3. DF2 - Drop first stakeholders
    4. DF2 - Assign RATINGORDER
    5. DF2 - Rename
    6. DF1 - Drop second questions, ratings
    7. DF1 - Assign RATINGORDER
    8. Merge and return
    ''';

    # 1
    print("---\nSPSS: Original Shape of dataframe: {}".format(data_frame.shape))
    print("SPSS: Getting valid integers from metadata")
    valid_integers = get_spss_integers(spss_metadata)
    #print(f"SPSS: Converting all columns to Int64: {valid_integers}")
    data_frame_cols = [col for col in valid_integers if
                       ('OE' not in col and col in data_frame.columns)]
    print(f"SPSS: Identified {len(data_frame_cols)} that need to be converted to integers")
    i = 0
    for col in data_frame_cols:
        i += 1
        if i % 1000 == 0:
            print(f"SPSS: Data conversion complete on {i}th column, {col}")
        data_frame[col] = pd.to_numeric(data_frame[col].values, errors='ignore')
        data_frame[col] = data_frame[col].astype('Int64')

    df1 = data_frame.copy()
    df2 = data_frame.copy()

    rename_custom_q = get_customQ_renaming_dic(data_frame)
    rename_custom_aus_q = get_customQ_renaming_dic_aus(data_frame)
    aus_custom_questions = get_customQ_renaming_dic_endingA(data_frame)
    ending_a2 = get_customQ_renaming_dic_aus_A(data_frame)
    more_garbage = get_customQ_renaming_dic_Q_A(data_frame)

    for k, v in ending_a2.items():
        if k not in aus_custom_questions.keys():
            aus_custom_questions[k] = v

    for k, v in rename_custom_aus_q.items():
        if k not in aus_custom_questions.keys():
            aus_custom_questions[k] = v

    for k, v in rename_custom_q.items():
        if k not in aus_custom_questions.keys():
            aus_custom_questions[k] = v

    for k, v in more_garbage.items():
        if k not in aus_custom_questions.keys():
            aus_custom_questions[k] = v

    manual_renaming = {'CODERESP_1': 'CODERESP',
                       'CRT2': 'CRT1',
                       'Company_Type2': 'Company_Type1',
                       'Counter_306': 'Counter_305',
                       'EXPERIMENTAL_CO2': 'EXPERIMENTAL_CO1',
                       'GroupBenchmark2': 'GroupBenchmark1',
                       'GroupBenchmarkGroup2': 'GroupBenchmarkGroup1',
                       'GroupBenchmarkGroup_COMPANY2': 'GroupBenchmarkGroup_COMPANY1',
                       'Q306CEO_1': 'Q305CEO_1',
                       'Q331x1_99': 'Q330x1_99',
                       'Rated_306': 'Rated_305',
                       'Q601A': 'Q600A',
                       'S106CEO': 'S105CEO',
                       'Q306CEO': 'Q305CEO',
                       'Q316CEO': 'Q315CEO',
                       'AP_AW1_2': 'AP_AW1_1',
                       'AP_AW2_2_1': 'AP_AW2_1_1',
                       'AP_AW2_2x99_99': 'AP_AW2_1x99_99',
                       'AP_AW3_2': 'AP_AW3_1'
                       }

    for k, v in manual_renaming.items():
        if k not in aus_custom_questions.keys():
            aus_custom_questions[k] = v

    ceo = dict(zip([x for x in data_frame.columns if x.startswith('Q310CEO')],
                   [x for x in data_frame.columns if x.startswith('Q311CEO')]))

    for k, v in ceo.items():
        if k not in aus_custom_questions.keys():
            aus_custom_questions[k] = v

    # 2, 3
    batteries = ['Q305_' ,'Q320_','Q215_' ,'Q410_' ,'Q420_' ,'Q600_', 'Q605' ,'Q800_']
    drop_these_questions = []
    df1_batteries = []
    for battery in batteries:
        df1_batteries += [x for x in df2.columns.tolist() if x.startswith(battery)]

    stakeholders1 = [x for x in data_frame.columns if x.startswith('Stakeholder') &
                     x.endswith('1')]
    stakeholders1 += ['Company1', 'GlobalCompany1', 'Industry1','GroupBenchmark_COMPANY1',
                      'INDUSTRY_INDEX1', 'KYLIE_COMPANY1' ,'NASDAQ_COMPANY1',
                      'GRT100_COMPANY1', 'NEW_INDUSTRY1' ,'NL_30_1' ,'APAC_TOP1']
    drop_these_questions += df1_batteries
    drop_these_questions += stakeholders1
    drop_these_questions = [x for x in drop_these_questions if x in df2.columns]
    df2.drop(drop_these_questions, axis=1, inplace=True)
    # Check if drop was sucessful
    for col_name in drop_these_questions:
        if col_name in df2:
            print(f"JSON: {col_name} was found in df2 after dropping")

    # 4
    df2['RATINGORDER'] = 2

    # 5
    # Renaming variables in DF2 - we have a problem with Company2, etc.
    df2_batteries = []
    for question in df1_batteries:
        temp = list(question)
        temp[3] = str(int(temp[3]) + 1)
        question2 = "".join(temp)
        df2_batteries.append(question2)
    stakeholders2 = []
    for stakeholder in stakeholders1:
        stakeholders2.append(stakeholder[:-1] + "2")
    rename_battery_dic = dict(zip(df2_batteries, df1_batteries))
    rename_stakeho_dic = dict(zip(stakeholders2, stakeholders1))

    df2.rename(rename_battery_dic ,axis=1 ,inplace=True)
    df2.rename(rename_stakeho_dic ,axis=1 ,inplace=True)

    # Custom Question renaming/dropping
    print("SPSS: Dropping/Renaming custom questions from df2 ({} total)".format(len(aus_custom_questions)))
    drop_from_df2 = list(aus_custom_questions.values())  # NOTE: using values, which are the non _A values
    drop_from_df2 = [x for x in drop_from_df2 if x in df2.columns.tolist()]
    df2.drop(drop_from_df2, axis=1, inplace=True)
    df2.rename(aus_custom_questions, axis=1, inplace=True)

    # 6
    stakeholders2 = [x for x in data_frame.columns if x.startswith('Stakeholder') &
                     x.endswith('2')]
    stakeholders2 += ['Company2', 'GlobalCompany2', 'Industry2','GroupBenchmark_COMPANY2',
                      'INDUSTRY_INDEX2', 'KYLIE_COMPANY2' ,'NASDAQ_COMPANY2',
                      'GRT100_COMPANY2', 'NEW_INDUSTRY2' ,'NL_30_2' ,'APAC_TOP2']
    #df1.drop(stakeholders2, axis=1, inplace=True)
    df1.drop(df2_batteries, axis=1, inplace=True)
    drop_from_df1 = list(rename_custom_q.keys()) + list(rename_custom_aus_q.keys()) + stakeholders2  # drop all the _A questions
    drop_from_df1 = [x for x in drop_from_df1 if x in df1.columns.tolist()]
    df1.drop(drop_from_df1 ,axis=1 ,inplace=True)

    # 7
    df1['RATINGORDER'] = 1

    # 8
    df1 = df1.loc[:, ~df1.columns.duplicated()].copy()
    df2 = df2.loc[:, ~df2.columns.duplicated()].copy()
    print(f"SPSS: Shape of dataframes pre-merging:\n1: {df1.shape}, 2: {df2.shape}")
    final_data = pd.concat([df1 ,df2], ignore_index=True)
    final_data.reset_index(inplace=True, drop=True)
    final_data.rename({'GlobalCompany1': 'Company',
                       'Company1': 'Rating',
                       'Industry1':'NEW_INDUSTRY1'}, axis=1, inplace=True)
    del df1
    del df2
    print(f"SPSS: Shape of final dataframe: {final_data.shape}\n---\n")
    return(final_data)

def get_spss(dataframe, spss_metadata):
    # 0 Remove trash columns
    bad_s105s = ['S105_','S105x_','S105a_', 'S105Fielded_',
                 'S105_Qualified_','S105_DummyCompany_',
                 'S105_Eligible', 'S105_Shown_',  'S105_Qualified_',
                  'S105_SECOND_','S105a_SECOND_',
                  'XS105_Companies_',
                 'hidQ320_Company1', 'hidQ320_Company2',
                 'Q320x', 'Q320x1', 'XQ320', 'Q320x2',
                 'Q321x', 'Q321x1', 'Q321x2', 'Q320x',
                 'IndRepSecLF',
                 'AP_IND_REP1', 'AP_IND_REP2',
                 'INDREP', 'FLAGLATAM', 'ZEROQUOTA',
                 'hidMediaSplit',
                 'AP_MEDIA', 'tplVariables',
                 'Companies_','Companies2_']

    # Need to clean df_spss ---
    drop_all = []
    for bad_var in bad_s105s:
        drop_these = [x for x in dataframe.columns if x.startswith(bad_var)]
        drop_all += drop_these
    print(f"DI-SPSS: Identified {len(drop_all)} columns to be removed from SPSS data.")
    print(f"DI-SPSS: Size of dataframe before removing junk: {dataframe.shape}")  # from approx 8662
    dataframe.drop(drop_all, axis=1, inplace=True)
    print(f"DI-SPSS: Size of dataframe after removing junk: {dataframe.shape}")  # down to approx 3315

    # Rename Q320s
    rename_q320s = dict([[x, x.replace('_Selected_', '_')] for x in dataframe.columns
                         if (x.startswith('Q320_Selected') or x.startswith('Q321_Selected'))])
    dataframe.rename(rename_q320s, axis=1, inplace=True)

    # Rename S105s
    rename_s105s = dict([[x, x.replace('S105Fieldeda_', 'S105Fielded_')] for x in dataframe.columns
                         if x.startswith('S105Fieldeda_')])
    dataframe.rename(rename_s105s, axis=1, inplace=True)

    # Rename essential variables
    dataframe.rename({'Country': 'COUNTRY',
                      'STATUS': 'respStatus',
                      'AGE':'Age',
                      'AGE_ORI_1':'AGE_ORI'},
                     axis=1, inplace=True)
    dataframe['country_temp'] = pd.to_numeric(dataframe['COUNTRY'])
    dataframe.loc[dataframe.country_temp.isna(), 'country_temp'] = 99
    dataframe['country_temp'] = dataframe.country_temp.astype(int)
    dataframe['CODERESP'] = dataframe.country_temp.astype(str) + dataframe.index.astype(str)
    dataframe.drop('country_temp',axis=1,inplace=True)
    print(f"DI-SPSS: Stacking SPSS Data!")
    df_aus = stack_aus_data(dataframe, spss_metadata)
    df_aus['INDUSTRY_INDEX1'] = 0
    return(df_aus)


import random

# Welcome user and obtain user name and number guess

print("Hello! Welcome to the Guess a Number game. You'll have to guess my secret number. Good luck!")
user_name = input("What is your name? ")
random_number = random.randint(1, 20)
print(f"Well, {user_name}, I'm thinking of a number between 1 and 20.")

# Prompt user for number guesses

for user_tries in range(1, 7):
    print("Enter a guess: ")
    user_guess = int(input())
    if user_guess < random_number:
        print("Your guess is too low.")
    elif user_guess > random_number:
        print("Your guess is too high.")
    else:
        break

# User won or lost

if user_guess == random_number:
    print(f"Congratulations, {user_name}! You guessed my number in {user_tries} tries.")
else:
    print(f"Actually, the number I was thinking of was {random_number}.")


import os
from werkzeug import secure_filename

import sys
from sgb import db
import datetime
from datetime import timedelta

root = os.path.dirname((os.path.abspath(__file__)))
root += '/static/images'

ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'])

def upload_file(file):
    if file.filename == '':
        raise Exception('Arquivo sem nome!')

    if file:
        filename = secure_filename(file.filename)
        file.save(os.path.join(root, filename))

    return file

def add_punicao(data, id_socio):
    db.insert_bd('insert into punicao values(default, %s, "%s")' % (id_socio, data))
    db.insert_bd('update socio set status = "SUSPENSO" where id = %d' % id_socio)

def atualiza_punicao_socio(id_emprestimo):
    emprestimos = db.query_bd('select * from emprestimo where id = %s' % id_emprestimo)
    data_atual = datetime.date.today()

    for emprestimo in emprestimos:
        data_devolucao = emprestimo['devolucao']

        if data_atual >= data_devolucao + timedelta(days=3) and data_atual < data_devolucao + timedelta(days=29):
            add_punicao(data_atual + timedelta(days=30), emprestimo['id_socio'])

        elif data_atual >= data_devolucao + timedelta(days=29) and data_atual < data_devolucao + timedelta(days=60):
            add_punicao(data_atual + timedelta(days=90), emprestimo['id_socio'])

        elif data_atual >= data_devolucao + timedelta(days=60) and data_atual < data_devolucao + timedelta(days=120):
            add_punicao(data_atual + timedelta(days=120), emprestimo['id_socio'])

        elif data_atual >= data_devolucao + timedelta(days=120) and data_atual < data_devolucao + timedelta(days=180):
            add_punicao(data_atual + timedelta(days=210), emprestimo['id_socio'])

        elif data_devolucao >= data_atual + timedelta(days=180):
            add_punicao(data_atual + timedelta(days=365), emprestimo['id_socio'])

def atualizao_status_socio():
    punicoes = db.query_bd('select * from punicao')
    for punicao in punicoes:
        data_punicao = punicao['data_punicao']
        if datetime.date.today() > data_punicao:
            db.insert_bd('delete from punicao where id = %d' % punicao['id'])
            db.insert_bd('update socio set status = "ATIVO" where id = %d' % punicao['id_socio'])


#!/usr/bin/env python2

import sys

BOOT_PATH = "bootloader.obx"
BOOT_HS_PATH = "bootloader_hs.obx"
XEX_PATH = sys.argv[1]

data = open(BOOT_PATH, "rb").read()
if len(data) > 512:
    sys.stderr.write("ERROR: bootloader.obx too long (%d)\n" % len(data))
    sys.exit(1)

data_hs = open(BOOT_HS_PATH, "rb").read()
if len(data_hs) > 8*128:
    sys.stderr.write("ERROR: bootloader_hs.obx too long (%d)\n" % len(data_hs))
    sys.exit(1)

lines = open(XEX_PATH, "rt").readlines()

with open(XEX_PATH, "wt") as f:
    copy = True
    for line in lines:
        if "#bootloader_end#" in line or "#bootloader_hs_end#" in line:
            copy = True
        if copy:
            f.write(line)
        if "#bootloader_start#" in line or "#bootloader_hs_start#" in line:
            if "#bootloader_start#" in line:
                src = data
            else:
                src = data_hs
            copy = False
            i = 0
            for x in src:
                if i == 0:
                    f.write(" " * 8)
                f.write("(byte) 0x%02x, " % ord(x))
                i = i + 1
                if i == 8:
                    f.write("\n")
                    i = 0
            f.write("\n")


def read(filename='books_data'):
  file = open(filename, 'r')
  data_list = file.readlines()
  file.close()
  authors_books = []
  for book_info in data_list:
    author_books = {}
    book_info = book_info.strip('\n')
    book_list = book_info.split(',')
    i = 0
    books = []
    for item in book_list:
      if i == 0:
          author = item
      else:
          books.append(item.strip())
      i += 1
    author_books['author'] = author
    author_books['books'] = books
    authors_books.append(author_books)

  return authors_books

def add(author, filename='books_data.txt'):
  file = open(filename, "a")
  author_and_books = author["author"]
  books = author['books']
  for book in books:
    author_and_books += ", " + book
  file.write(author_and_books + '\n')
  file.close()


# -*- coding:utf-8 -*-
#
# Copyright © Spyder Project Contributors
# Licensed under the terms of the MIT License
# (see spyder/__init__.py for details)

"""Pylint Code Analysis Plugin."""

# pylint: disable=C0103
# pylint: disable=R0903
# pylint: disable=R0911
# pylint: disable=R0201

# Standard library imports
import os.path as osp

# Third party imports
from qtpy.QtCore import Qt, Signal, Slot
from qtpy.QtWidgets import QGroupBox, QInputDialog, QLabel, QVBoxLayout

# Local imports
from spyder.config.base import get_translation
from spyder.plugins import SpyderPluginMixin
from spyder.plugins.configdialog import PluginConfigPage
from spyder.utils import icon_manager as ima
from spyder.utils.qthelpers import create_action, MENU_SEPARATOR
from .widgets.pylintgui import (PYLINT_PATH, PylintWidget)

_ = get_translation("pylint", "spyder_pylint")

class PylintConfigPage(PluginConfigPage):
    def setup_page(self):
        settings_group = QGroupBox(_("Settings"))
        save_box = self.create_checkbox(_("Save file before analyzing it"),
                                        'save_before', default=True)

        hist_group = QGroupBox(_("History"))
        hist_label1 = QLabel(_("The following option will be applied at next "
                               "startup."))
        hist_label1.setWordWrap(True)
        hist_spin = self.create_spinbox(_("History: "),
                            _(" results"), 'max_entries', default=50,
                            min_=10, max_=1000000, step=10)

        results_group = QGroupBox(_("Results"))
        results_label1 = QLabel(_("Results are stored here:"))
        results_label1.setWordWrap(True)

        # Warning: do not try to regroup the following QLabel contents with
        # widgets above -- this string was isolated here in a single QLabel
        # on purpose: to fix Issue 863
        results_label2 = QLabel(PylintWidget.DATAPATH)

        results_label2.setTextInteractionFlags(Qt.TextSelectableByMouse)
        results_label2.setWordWrap(True)

        settings_layout = QVBoxLayout()
        settings_layout.addWidget(save_box)
        settings_group.setLayout(settings_layout)

        hist_layout = QVBoxLayout()
        hist_layout.addWidget(hist_label1)
        hist_layout.addWidget(hist_spin)
        hist_group.setLayout(hist_layout)

        results_layout = QVBoxLayout()
        results_layout.addWidget(results_label1)
        results_layout.addWidget(results_label2)
        results_group.setLayout(results_layout)

        vlayout = QVBoxLayout()
        vlayout.addWidget(settings_group)
        vlayout.addWidget(hist_group)
        vlayout.addWidget(results_group)
        vlayout.addStretch(1)
        self.setLayout(vlayout)

class Pylint(PylintWidget, SpyderPluginMixin):
    """Python source code analysis based on pylint"""
    CONF_SECTION = 'pylint'
    CONFIGWIDGET_CLASS = PylintConfigPage
    edit_goto = Signal(str, int, str)

    def __init__(self, parent=None):
        PylintWidget.__init__(self, parent=parent,
                              max_entries=self.get_option('max_entries', 50))
        SpyderPluginMixin.__init__(self, parent)

        # Initialize plugin
        self.initialize_plugin()

    #------ SpyderPluginWidget API --------------------------------------------
    def get_plugin_title(self):
        """Return widget title"""
        return _("Static code analysis")

    def get_plugin_icon(self):
        """Return widget icon"""
        path = osp.join(self.PLUGIN_PATH, self.IMG_PATH)
        return ima.icon('pylint', icon_path=path)

    def get_focus_widget(self):
        """
        Return the widget to give focus to when
        this plugin's dockwidget is raised on top-level
        """
        return self.treewidget

    def get_plugin_actions(self):
        """Return a list of actions related to plugin"""
        # Font
        history_action = create_action(self, _("History..."),
                                       None, ima.icon('history'),
                                       _("Set history maximum entries"),
                                       triggered=self.change_history_depth)
        self.treewidget.common_actions += (None, history_action)
        return []

    def on_first_registration(self):
        """Action to be performed on first plugin registration"""
        self.main.tabify_plugins(self.main.help, self)
        self.dockwidget.hide()

    def register_plugin(self):
        """Register plugin in Spyder's main window"""
        self.edit_goto.connect(self.main.editor.load)
        self.redirect_stdio.connect(self.main.redirect_internalshell_stdio)
        self.main.add_dockwidget(self)

        pylint_act = create_action(self, _("Run static code analysis"),
                                   triggered=self.run_pylint)
        pylint_act.setEnabled(PYLINT_PATH is not None)
        self.register_shortcut(pylint_act, context="Pylint",
                               name="Run analysis")

        self.main.source_menu_actions += [MENU_SEPARATOR, pylint_act]
        self.main.editor.pythonfile_dependent_actions += [pylint_act]

    def refresh_plugin(self):
        """Refresh pylint widget"""
        self.remove_obsolete_items()

    def closing_plugin(self, cancelable=False):
        """Perform actions before parent main window is closed"""
        return True

    def apply_plugin_settings(self, options):
        """Apply configuration file's plugin settings"""
        # The history depth option will be applied at
        # next Spyder startup, which is soon enough
        pass

    #------ Public API --------------------------------------------------------
    @Slot()
    def change_history_depth(self):
        "Change history max entries"""
        depth, valid = QInputDialog.getInt(self, _('History'),
                                       _('Maximum entries'),
                                       self.get_option('max_entries'),
                                       10, 10000)
        if valid:
            self.set_option('max_entries', depth)

    @Slot()
    def run_pylint(self):
        """Run pylint code analysis"""
        if self.get_option('save_before', True)\
           and not self.main.editor.save():
            return
        self.analyze( self.main.editor.get_current_filename() )

    def analyze(self, filename):
        """Reimplement analyze method"""
        if self.dockwidget and not self.ismaximized:
            self.dockwidget.setVisible(True)
            self.dockwidget.setFocus()
            self.dockwidget.raise_()
        PylintWidget.analyze(self, filename)


#!/usr/bin/env python
# -*- coding:utf-8 -*-
#
# @Version : 1.0
# @Time    : 2019-10-08
# @Author  : bading
# @File    : io_util.py

def write2file(data, file, is_first_time=False):
    if is_first_time:
        with open(file, "w") as fout:
            for ele in data:
                fout.write(ele)
                fout.write("\n")
    else:
        with open(file, "a") as fout:
            for ele in data:
                fout.write(ele)
                fout.write("\n")

def read_file(file, types=None, skip_head=True):
    res = list()
    with open(file, "r") as fin:
        for line in fin:
            if skip_head:
                skip_head = False
                continue
            fields = line.split(",")
            if types:
                assert len(types) == len(fields)
                fields = [types[i](fields[i]) for i in range(len(fields))]
            res.append(fields)
    return res


# -*- coding: utf-8 -*-
"""
Created on Tue Feb 28 10:43:02 2023

@author: Mario
"""

from models import dlib_model_68, mediapipe_model
from utils import find_face
import os
import numpy as np
import cv2

path_images = './Helen_testset/y_pred/'
path_pts = './Helen_testset/y_true/'
images = os.listdir(path_images)
images.sort()
pts = os.listdir(path_pts)
pts.sort()
print(len(images), len(pts))

a = "./img/a.jpg"
#dlib_model_68.dlib_model(a)
y_pred = []
#mediapipe_model.MediapipeModel(a, y_pred)
i = 315
image = path_images+images[i]
g = pts[i]
grand_truth = find_face.read_pts(g)

print(g)

find_face.grabcut(image, grand_truth)