# -*- coding: utf-8 -*-
"""

data engineering functions

"""

import sqlite3
import sys

# -*- coding: utf-8 -*-
import numpy as np
import pandas as pd
import pylab as plt
from scipy.sparse import coo_matrix
from sklearn.model_selection import train_test_split

from auxiliary import *

def list_channels_to_df(portfolio, channel_types=['web', 'email', 'mobile', 'social'], debug=False):
    """
    extracts information from the dictionary located in the channels column of the portfolio database to separate columns

    :param portfolio:
    :param channel_types:
    :param debug:
    :return:
    """
    #

    # initialize
    for channel in channel_types:
        portfolio[channel] = False

    # assign values
    for ind in portfolio.index:
        for channel in channel_types:
            if debug:
                print(portfolio.loc[ind, 'channels'])
            if channel in portfolio.loc[ind, 'channels']:
                portfolio.loc[ind, channel] = True
                if debug:
                    print(channel, portfolio[channel])
    return (portfolio)

def load_portfolio(debug=False, location='colab'):
    """
    loads and processes the portfolio database

    :param debug: print debug information
    :param location: portfolio.json location (Google Colab or local)
    :return:
    """

    if location == 'colab':
        portfolio = pd.read_json(r'/content/gdrive/My Drive/UD/Sbux/data/portfolio.json', orient='records', lines=True)
    else:
        portfolio = pd.read_json(r'portfolio.json', orient='records', lines=True)

    # extracts channel (web, email, ...) information
    portfolio = list_channels_to_df(portfolio);
    portfolio = portfolio.drop(columns=['channels'])
    if debug:
        print('portfolio.shape:', portfolio.shape)

    assert portfolio.shape[0] == 10, 'wrong portfolio size'
    n_offers = np.unique(portfolio['id']).shape[0]
    assert n_offers == 10, 'wrong portfolio size'
    portfolio.rename(columns={'id': 'offer', 'reward': 'offer_reward'}, inplace=True)
    portfolio['duration_hours'] = 24. * portfolio['duration']

    return (portfolio)

def load_transcript(debug=False, location='colab'):
    """
    loads and processes the transcript database

    :param debug: print debug information
    :param location: portfolio.json location (Google Colab or local)
    :return:
    """

    if location == 'colab':
        transcript = pd.read_json(r'/content/gdrive/My Drive/UD/Sbux/data/transcript.json', orient='records',
                                  lines=True)
    else:
        transcript = pd.read_json(r'transcript.json', orient='records', lines=True)

    # extract offers + transactions
    if debug:
        print('transcript.shape:', transcript.shape)

    # extract if/when the offer was viewed etc
    values = (pd.DataFrame(transcript['value'].values.tolist()))  # .stack()
    values.columns = ['transaction', 'offer', 'offer viewed', 'reward']

    transcript = pd.concat([transcript, values], axis=1)
    transcript['offer'] = transcript['offer'].fillna(transcript['offer viewed'])
    transcript = transcript.drop(columns=['offer viewed'])

    if debug:
        print('transcript.shape:', transcript.shape)

    n_persons_transcript = np.unique(transcript['person']).shape[0]

    if debug:
        print('n_persons_transcript:', n_persons_transcript)

    # separate into two different DF to handle better time splits
    t_offers = transcript[transcript.event.str.contains('offer')]
    t_purchases = transcript[transcript.event.str.contains('transaction')]

    return (t_offers, t_purchases)

def load_profile(debug=False, location='colab'):
    """
    loads and processes the customer profile db

    :param debug: print debug information
    :param location: portfolio.json location (Google Colab or local)
    :return:
    """

    if location == 'colab':
        profile = pd.read_json(r'/content/gdrive/My Drive/UD/Sbux/data/profile.json', orient='records', lines=True)
    else:
        profile = pd.read_json(r'profile.json', orient='records', lines=True)

    profile['became_member_on'] = pd.to_datetime(profile['became_member_on'].astype(str), format='%Y%m%d').astype(
        "datetime64")
    profile['joined_year'] = profile['became_member_on'].dt.year
    profile['joined_month'] = profile['became_member_on'].dt.month
    profile['joined_week'] = profile['became_member_on'].dt.week

    profile['gender_num'] = profile.gender.astype("category", categories=['F', 'M', 'O']).cat.codes

    time_start = profile.became_member_on.min()
    time_end = profile.became_member_on.max()
    time_delta = time_end - time_start

    if debug:
        print()
        print('time span in profile DB:', time_start, time_end)
        print('time_delta in profile DB:', time_delta.days, ' days ~ %d years' % np.round(time_delta.days / 365))

    profile['user_time'] = (time_end - profile['became_member_on']).dt.days
    profile = profile.rename(columns={'id': 'person'})

    return (profile)

def create_person_offer_table(transcript_offers, transcript_transactions, portfolio, add_missing=False, debug=False):
    """
    combines the transcript_offers, transcript_transactions, portfolio DBs into a single combined one that can be
    indexed by (person, offer, time)

    :param transcript_offers:
    :param transcript_transactions:
    :param portfolio:
    :param add_missing: adds back the persons (6) that did not receive an offer in the timeframe of transcript_offers
    :param debug:
    :return: s
    """
    # generates a main pivot table with customer and offer information (received, viewed, completed)

    # time of offer received / viewed / completed / reward
    combined_pot = extract_offer_times(transcript_offers, portfolio, debug=True);

    # adding the persons that don't receive any offer
    missing_persons = set(transcript_transactions.person) - set(combined_pot.person)
    print('persons with no offers received in timeframe: ', len(missing_persons))

    if add_missing:
        # adding back the persons that did not receive an offer in the timeframe of transcript_offers

        tmp1 = np.array(list(missing_persons))
        tmp = np.vstack((tmp1, np.empty(tmp1.shape) * np.nan)).T

        piv_missing = pd.DataFrame(tmp, columns=['person', 'offer'])
        # display(piv_missing)

        combined = pd.concat((combined_pot, piv_missing), sort=False)
    else:
        combined = combined_pot

    return combined

def add_offers_df(combined, portfolio, debug=False):
    """
    adds the offer details to the main pivot table

    :param combined:
    :param portfolio: portfolio of offers
    :param debug:
    :return:
    """

    #

    # merge the offers on (person, offer)
    if debug:
        print('combined.shape:', combined.shape)
    combined = combined.merge(portfolio[['difficulty', 'duration', 'offer', 'offer_type', 'web',
                                         'email', 'mobile', 'social', 'offer_reward', 'duration_hours']], on='offer',
                              how='left')
    combined.rename(columns={'offer completed': 'offer_completed', 'offer received': 'offer_received',
                             'offer viewed': 'offer_viewed'}, inplace=True)

    combined['offer_end'] = combined['offer_received'] + combined['duration_hours']
    combined['duration_viewed'] = combined['offer_end'] - combined['offer_viewed']
    combined['duration_viewed'] = combined['duration_viewed'].fillna(0)

    combined['duration_effective'] = combined[['duration_viewed', 'duration_hours']].min(axis=1, skipna=False)

    combined['viewed'] = np.invert(combined['offer_viewed'].isnull())
    combined['rewarded'] = np.invert(combined['reward'].isnull())
    combined['received'] = np.invert(combined['offer_received'].isnull())

    combined['comp_not_viewed'] = ((-combined['viewed']) & combined['rewarded'])
    # if an offer was viewed after being completed it also should not be rewarded
    combined['comp_not_viewed'] = combined.offer_viewed > combined.offer_completed

    combined['completed'] = combined['rewarded'] & (-combined['comp_not_viewed'])

    if debug:
        print('combined.shape:', combined.shape)

    return (combined)

def extract_offer_times(transcript_offers, portfolio, debug=True, ov_same_keep_earliest_viewed=True):
    """
    extracts the offer received, viewed, completed information from the transcript dataframe

    :param transcript_offers:
    :param portfolio:
    :param debug:
    :param ov_same_keep_earliest_viewed:
    :return: combined database by person, offer --> times (received, viewed, completed,...)
    """

    if debug:
        events_type = transcript_offers.event.value_counts()
        print(events_type)

    er = transcript_offers.loc[transcript_offers.event == 'offer received', ['person', 'offer', 'time']].rename(
        columns={'time': 'offer_received'})
    ec = transcript_offers.loc[
        transcript_offers.event == 'offer completed', ['person', 'offer', 'time', 'reward']].rename(
        columns={'time': 'offer_completed'})
    ev = transcript_offers.loc[transcript_offers.event == 'offer viewed', ['person', 'offer', 'time']].rename(
        columns={'time': 'offer_viewed'})

    if debug:
        print('er,ec,ev')
        print(er.shape, ec.shape, ev.shape)

    er = er.merge(portfolio[['offer', 'duration_hours']], on='offer')
    er['offer_end'] = er['offer_received'] + er['duration_hours'] - 1

    er.index.name = 'idx'
    er = er.reset_index()

    if debug:
        print('er ', er.shape)
    conn = sqlite3.connect(':memory:')

    er.to_sql('er', conn, index=False, if_exists='replace')  # ,index_label='idx')
    ec.to_sql('ec', conn, index=False, if_exists='replace')
    ev.to_sql('ev', conn, index=False, if_exists='replace')

    # SQL query for speed
    qry = '''
      SELECT
          er.person, er.offer, er.offer_received, ev.offer_viewed, ec.offer_completed, ec.reward, er.idx, er.offer_end
      FROM
          er
          left join  ec on
          er.person = ec.person and er.offer = ec.offer and ec.offer_completed <= er.offer_end and ec.offer_completed >= er.offer_received

          left join  ev on
          er.person = ev.person and er.offer = ev.offer and ev.offer_viewed <= er.offer_end and ev.offer_viewed >= er.offer_received

      '''
    combined = pd.read_sql_query(qry, conn)  # .drop(columns=['idx'])

    # remove duplicates from SQL
    combined.drop_duplicates(subset=['idx'], inplace=True)
    combined = combined.drop(columns=['idx'])

    #   # dealing with overlapping offers of the same type

    #   if ov_same_keep_earliest_viewed != 'skip':
    #     piv['distance']=piv['offer_viewed']-piv['offer_received']

    #     ordered=piv.loc[:,['person','offer','offer_viewed','distance']].sort_values(by='distance',ascending=True)
    #     ordered=ordered.drop(columns=['distance'])

    #     if ov_same_keep_earliest_viewed:
    #       keep='first'
    #     else:
    #       keep='last'
    #     duplicated=ordered.loc[ordered.duplicated(keep=keep)].index

    #     piv=piv.drop(index=duplicated)
    #     piv=piv.drop(columns=['distance'])

    #     piv=piv.reindex()

    #   if debug:
    #     print(piv.shape)
    return (combined)

def extract_transactions(combined, transcript, add_transactions=False, skip_overlap=True, debug=False):
    """
    extract_transactions
    separates purchases from transcript_offers in those during/outside offers

    :param combined:
    :param transcript:
    :param add_transactions:
    :param skip_overlap: if offers are overlapping the transaction is assigned to the first offer received
    :return:
    """

    # filter the transactions for purchases
    trans = transcript[transcript['event'] == 'transaction'][['person', 'time', 'transaction']].sort_values(
        by=['person', 'time'])
    trans.rename(columns={'transaction': 'payments'}, inplace=True)
    trans.reset_index(inplace=True)

    if debug:
        print('trans.shape:', trans.shape)

    # extracts the transactions during an offer period, uses group by to avoid repeated rows in the SQL query
    conn = sqlite3.connect(':memory:')

    combined.to_sql('piv', conn, index=False, if_exists='replace')

    # (tr_id) index is the real index in the transcript DataFrame, idx is the reindexed index
    trans = trans.rename(columns={'index': 'tr_id'})

    # to check for duplicate lines after SQL query
    trans.index.name = 'idx'
    trans.reset_index().to_sql('trans', conn, index=False, if_exists='replace', index_label='idx')

    if debug:
        print(trans.columns)

    # https://stackoverflow.com/questions/30627968/merge-pandas-dataframes-where-one-value-is-between-two-others
    if skip_overlap:
        qry = '''
        SELECT
            trans.person, trans.time, trans.payments, trans.tr_id, piv.offer, idx , piv.offer_viewed
        FROM
            trans inner join  piv on
            trans.person = piv.person and trans.time between offer_viewed and offer_end+1
        WHERE
            viewed = 1
        GROUP BY
            idx
        '''
    else:
        qry = '''
    SELECT
        trans.person, trans.time, trans.payments, trans.tr_id, piv.offer, idx , piv.offer_viewed
    FROM
        trans inner join  piv on
        trans.person = piv.person and trans.time between offer_viewed and offer_end+1
    WHERE
        viewed = 1
    '''

    transactions_during_offer = pd.read_sql_query(qry, conn)

    # check for duplicates and remove them. there should be none
    print('transactions_during_offer.shape:', transactions_during_offer.shape)

    if skip_overlap:
        print('skip_overlap: True')

        print(transactions_during_offer.columns)

        # each transaction is attributed to the first offer received
        #transactions_during_offer.set_index('idx', inplace=True)
        #transactions_during_offer = transactions_during_offer.groupby(by='idx').first()
        transactions_during_offer = transactions_during_offer.drop_duplicates(subset=['idx'])
        transactions_during_offer['overlaps'] = 0

        print(transactions_during_offer.columns)

    else:
        print('skip_overlap: False')
        print(transactions_during_offer.columns)

        # fix the issue that some transactions are counted multiple times on the same offer
        transactions_during_offer = transactions_during_offer.drop_duplicates(
            subset=['person', 'time', 'payments', 'tr_id', 'offer'])
        # however this does not resolve the issue that the same transactions may be assigned to two different offers.

        # each transaction is attributed to all the valid offers
        vc = transactions_during_offer['tr_id'].value_counts().rename('overlaps')
        transactions_during_offer = transactions_during_offer.merge(vc, left_on='tr_id', right_index=True)
        # overlaps = value_counts -1
        transactions_during_offer['overlaps'] -= 1

        print(transactions_during_offer.columns)

    if debug:
        print('combined.shape:', combined.shape)
        print('trans.shape:', trans.shape)

        print('transactions_during_offer.shape:', transactions_during_offer.shape)
        print('transactions_during_offer index unique :', len(set(transactions_during_offer.index)))
    n_trans_do = transactions_during_offer.shape[0]

    if add_transactions:
        combined = combined.merge(transactions_during_offer, on=['person', 'offer'], how='left')
        print('piv.shape:', combined.shape)

    #  get the remaining transactions
    ind = trans.index.difference(transactions_during_offer.index)
    transactions_outside_offer = trans.loc[ind, :]

    # check validity
    if debug:
        print()
        #   print('trans.shape:',trans.shape)
        print('transactions_outside_offer.shape:', transactions_outside_offer.shape)
        print('transactions_during_offer.shape:', transactions_during_offer.shape)
        print('total transactions: ', transactions_outside_offer.shape[0] + transactions_during_offer.shape[0])
        print('difference: ', transactions_outside_offer.shape[0] + transactions_during_offer.shape[0] - trans.shape[0])

    assert trans.shape[0] == transactions_outside_offer.shape[0] + transactions_during_offer.shape[
        0], "transactions not matching "

    # trans is the full transaction database
    return (combined, trans, transactions_during_offer, transactions_outside_offer)

def assign_overlapping_purchases(transactions_during_offer, assign_to='first'):
    """
    assign_overlapping_purchases
    resolve case of overlapping offers by the criteria assign_to

    :param transactions_during_offer:
    :param assign_to: first, last, ignore
    :return:
    """

    # resolve case of overlapping offers by
    conn = sqlite3.connect(':memory:')

    # find purchases with overlapping offers
    tdo_ov = transactions_during_offer.loc[(transactions_during_offer.overlaps > 0)]

    tdo_ov.to_sql('tdo', conn, index=False, if_exists='replace')

    # assign to first viewed offer
    if assign_to == 'first':
        qry = '''
    SELECT
         person,  time,  payments, tr_id,  offer,   idx, MIN(offer_viewed) AS offer_viewed
    FROM
        tdo
    GROUP BY
        tr_id
    '''

    # assign to last viewed offer
    elif assign_to == 'last':
        qry = '''
    SELECT
         person,  time,  payments, tr_id,  offer,   idx, MAX(offer_viewed) AS offer_viewed
    FROM
        tdo
    GROUP BY
        tr_id
    '''
    elif assign_to == 'ignore':
        return transactions_during_offer
    else:
        print('method not selected!')
        return (None)

    tdo_nov = pd.read_sql_query(qry, conn)
    if len(tdo_nov) > 0:
        tdo_nov['overlaps'] = 0
    else:
        print('tdo_nov:', tdo_nov)

    tdo_others = transactions_during_offer.loc[(transactions_during_offer.overlaps == 0)]

    return pd.concat([tdo_nov, tdo_others], ignore_index=True)

def add_stats_by_person(combined, transactions_outside_offer, transactions_during_offer, calc_net_offer_time=False):
    """
    add_stats_by_person

    :param combined:
    :param transactions_outside_offer:
    :param transactions_during_offer:
    :param calc_net_offer_time:
    :return:
    """

    # customers who made purchases but in times when no offer is received  (total $ sum and number of purchases)
    total_purchase_outside = transactions_outside_offer.groupby(by=['person'])['payments'].agg('sum').to_frame()
    total_purchase_outside.rename(columns={'payments': 'Tpay_out'}, inplace=True)

    total_purchase_outside_num = transactions_outside_offer.groupby(by=['person'])['payments'].agg('count').to_frame()
    total_purchase_outside_num.rename(columns={'payments': 'Npay_out'}, inplace=True)

    t_offers = transactions_during_offer.groupby(by=['person'])['payments'].agg(['sum', 'count', 'max', 'min'])
    t_offers.rename(columns={'sum': 'Tpay_offers_tot', 'count': 'Npay_offers_tot', 'max': 'Maxpay_offers_tot',
                             'min': 'Minpay_offers_tot'}, inplace=True)
    t_offers = t_offers.fillna(0)

    t_rewards = combined.groupby(by=['person'])['reward'].agg(['sum', 'count'])
    t_rewards.rename(columns={'sum': 'Trewards_tot', 'count': 'Nrewards_tot'}, inplace=True)
    t_rewards = t_rewards.fillna(0)

    ## SPECIFIC FOR A CUSTOMER
    # total offers time (gross, ie counting multiple times overlapping time intervals only once)
    offer_times = combined.groupby(by='person')['duration_effective'].agg('sum').to_frame()
    offer_times.rename(columns={'duration_effective': 'tot_gross_offer_time'}, inplace=True)

    # total offers time (gross, ie counting multiple times overlapping offer intervals)
    offer_timesN = combined.groupby(by=['person', 'offer'])[['offer_viewed', 'offer_end']].agg(np.mean)
    offer_timesN = offer_timesN[pd.notnull(offer_timesN['offer_viewed'])]

    if calc_net_offer_time:
        tot_net_offers_time = overlapping_offer_intevals(combined, final_time=744, return_person_occupacy=False)

    # completed offers
    p_offers_completed = combined.groupby(by='person')['completed', 'comp_not_viewed', 'received','viewed'].agg('sum')
    p_offers_completed['p_c_r_ratio'] = p_offers_completed['completed'] / p_offers_completed['received']
    p_offers_completed['p_c_v_ratio'] = p_offers_completed['completed'] / p_offers_completed['viewed']

    p_offers_completed['p_cnv_r_ratio'] = p_offers_completed['comp_not_viewed'] / p_offers_completed['received']
    p_offers_completed=p_offers_completed.fillna(0)

    print('add_stats_by_person:')
    print(combined.shape)
    #   print('transactions_during_offer',transactions_during_offer.shape)
    # add overlaps
    combined_with_person_stats = combined.merge(total_purchase_outside, on=['person'], how='left')

    combined_with_person_stats = combined_with_person_stats.merge(total_purchase_outside_num, on=['person'], how='left')
    combined_with_person_stats = combined_with_person_stats.merge(p_offers_completed[['p_c_r_ratio', 'p_cnv_r_ratio','p_c_v_ratio']],
                                                                  on=['person'], how='left')

    overlaps = transactions_during_offer[['person', 'offer', 'offer_viewed', 'overlaps']].groupby(
        by=['person', 'offer', 'offer_viewed']).agg('sum')
    combined_with_person_stats = combined_with_person_stats.merge(overlaps, on=['person', 'offer', 'offer_viewed'],
                                                                  how='left')
    #   piv2=piv2.merge( transactions_during_offer[['person', 'offer',  'offer_viewed', 'overlaps']].drop_duplicates() ,on=['person','offer','offer_viewed'],how='left')

    print(combined_with_person_stats.shape)

    combined_with_person_stats = combined_with_person_stats.merge(t_offers, on=['person'], how='left')
    combined_with_person_stats = combined_with_person_stats.merge(t_rewards, on=['person'], how='left')
    #   piv2=piv2.merge(t_offers_num,on=['person'],how='left')
    print(combined_with_person_stats.shape)

    combined_with_person_stats = combined_with_person_stats.merge(offer_times, on='person', how='left')
    print(combined_with_person_stats.shape)

    # piv2[['tot_net_offers_time'],'tot_net_offers_time','reward','Tpay_offer','Tpay_out','Npay_out','Npay_offer'
    if calc_net_offer_time:
        combined_with_person_stats = combined_with_person_stats.merge(tot_net_offers_time.reset_index(), on=['person'],
                                                                      how='left')
        combined_with_person_stats['tot_net_offers_time'] = combined_with_person_stats['tot_net_offers_time'].fillna(0)
        combined_with_person_stats['tot_not_offers_time'] = 744. - combined_with_person_stats['tot_net_offers_time']

        combined_with_person_stats['Avg_pay_offers'] = combined_with_person_stats['Tpay_offers_tot'] / \
                                                       combined_with_person_stats['tot_net_offers_time']
        combined_with_person_stats['Net_pay_offers'] = combined_with_person_stats['Tpay_offers_tot'] - \
                                                       combined_with_person_stats['Trewards_tot']
        combined_with_person_stats['Avg_net_pay_offers'] = (combined_with_person_stats['Tpay_offers_tot'] -
                                                            combined_with_person_stats['Trewards_tot']) / \
                                                           combined_with_person_stats['tot_net_offers_time']

        combined_with_person_stats['Avg_pay_outside'] = combined_with_person_stats['Tpay_out'] / \
                                                        combined_with_person_stats['tot_not_offers_time']
        combined_with_person_stats['Avg_pay_offers'] = combined_with_person_stats['Avg_pay_offers'].fillna(0)
        combined_with_person_stats['Avg_pay_outside'] = combined_with_person_stats['Avg_pay_outside'].fillna(0)

        combined_with_person_stats['Avg_D_O'] = combined_with_person_stats['Avg_pay_offer'] - \
                                                combined_with_person_stats['Avg_pay_outside']
        combined_with_person_stats['Avg_D_OS'] = combined_with_person_stats['Avg_pay_offers'] - \
                                                 combined_with_person_stats['Avg_pay_outside']
        combined_with_person_stats['Avg_D_nOS'] = combined_with_person_stats['Avg_net_pay_offers'] - \
                                                 combined_with_person_stats['Avg_pay_outside']

        combined_with_person_stats['Avg_D_nOS']=combined_with_person_stats['Avg_D_nOS'].fillna(0)

    combined_with_person_stats['reward'] = combined_with_person_stats['reward'].fillna(0)
    combined_with_person_stats['Tpay_offers_tot'] = combined_with_person_stats['Tpay_offers_tot'].fillna(0)
    combined_with_person_stats['Tpay_out'] = combined_with_person_stats['Tpay_out'].fillna(0)
    combined_with_person_stats['Npay_out'] = combined_with_person_stats['Npay_out'].fillna(0)
    combined_with_person_stats['Npay_offers_tot'] = combined_with_person_stats['Npay_offers_tot'].fillna(0)
    combined_with_person_stats['Avg_net_pay_offers'] = combined_with_person_stats['Avg_net_pay_offers'].fillna(0)

    combined_with_person_stats['Maxpay_offer'] = combined_with_person_stats['Maxpay_offer'].fillna(0)
    combined_with_person_stats['Minpay_offer'] = combined_with_person_stats['Minpay_offer'].fillna(0)
    combined_with_person_stats['Netpay_offer'] = combined_with_person_stats['Netpay_offer'].fillna(0)
    combined_with_person_stats['Maxpay_offers_tot'] = combined_with_person_stats['Maxpay_offers_tot'].fillna(0)
    combined_with_person_stats['Minpay_offers_tot'] = combined_with_person_stats['Minpay_offers_tot'].fillna(0)
    combined_with_person_stats['Net_pay_offers'] = combined_with_person_stats['Net_pay_offers'].fillna(0)
    combined_with_person_stats['overlaps'] = combined_with_person_stats['overlaps'].fillna(0)

    print(combined_with_person_stats.shape)

    return (combined_with_person_stats)

def add_purchases(combined, profile, transactions_outside_offer, tdo, assign_to='first', calc_net_offer_time=False,
                  add_person_stats=True, debug=False,skip_overlap=False):
    """
    adds the purchase information (during and outside offers) to the main pivot table.
    the information of average spending during and outside of offers is calculated

    :param combined:
    :param profile:
    :param transactions_outside_offer:
    :param tdo:
    :param assign_to:
    :param calc_net_offer_time:
    :param add_person_stats:
    :return:
    """

    if skip_overlap==False:
        # deal with multiple offers
        print('assign_overlapping_purchases:')
        transactions_during_offer = assign_overlapping_purchases(tdo, assign_to=assign_to)

        # this should be done in a separate function
        transactions_during_offer['payments_original'] = 1.0 * transactions_during_offer['payments']
        if assign_to == 'ignore':
            # a payment is received during multiple offers are received, the amount is split equally between the offers
            transactions_during_offer['payments'] = transactions_during_offer['payments_original'] / (
                    transactions_during_offer['overlaps'] + 1.)
    else:
        transactions_during_offer=tdo

    t_spec_offer = transactions_during_offer.groupby(by=['person', 'offer', 'offer_viewed'])['payments'].agg(
        ['sum', 'count', 'max', 'min'])
    t_spec_offer.rename(
        columns={'sum': 'Tpay_offer', 'count': 'Npay_offer', 'max': 'Maxpay_offer', 'min': 'Minpay_offer', },
        inplace=True)

    # adding the total purchases outside of offers and during offer (specific and total)
    if debug:
        print('add_purchases:')
        print(combined.shape)

    combined_with_purchases = combined.merge(t_spec_offer, on=['person', 'offer', 'offer_viewed'], how='left')
    print(combined_with_purchases.shape)

    combined_with_purchases['Tpay_offer'] = combined_with_purchases['Tpay_offer'].fillna(0)
    combined_with_purchases['Npay_offer'] = combined_with_purchases['Npay_offer'].fillna(0)

    combined_with_purchases['Netpay_offer'] = combined_with_purchases['Tpay_offer'] - combined_with_purchases['reward']

    # if the offer is not seen, it should not be considered. It also prevents the division by zero
    # prevents division by zero if there's a payment
    combined_with_purchases['Avg_pay_offer'] = combined_with_purchases['Tpay_offer'] / np.maximum(
        combined_with_purchases['duration_effective'],
        1.0 + 0.0 * combined_with_purchases['duration_effective'])
    combined_with_purchases.loc[(combined_with_purchases['duration_effective'] == 0), 'Avg_pay_offer'] = 0.0
    combined_with_purchases['Avg_pay_offer'] = combined_with_purchases['Avg_pay_offer'].fillna(0)

    combined_with_purchases = combined_with_purchases.merge(profile, on='person', how='left')

    if add_person_stats:
        combined_with_purchases = add_stats_by_person(combined_with_purchases, transactions_outside_offer,
                                                      transactions_during_offer,
                                                      calc_net_offer_time=calc_net_offer_time)

    return (combined_with_purchases)

def create_pivot_table(portfolio, profile_all, offers_all, transactions_all, add_transactions=False,
                       skip_overlap=False,
                       skip_add_purchases=False, calc_net_offer_time=True, assign_to='first'):
    """
    creates the pivot table with all the derived information
    # portfolio, profile_all,offers_all,  transactions_all

    :param portfolio:
    :param profile_all:
    :param offers_all:
    :param transactions_all:
    :param add_transactions:
    :param skip_overlap:
    :param skip_add_purchases:
    :param calc_net_offer_time:
    :param assign_to:
    :return:
    """

    combined = create_person_offer_table(offers_all, transactions_all, portfolio)
    combined = add_offers_df(combined, portfolio);
    combined, trans, transactions_during_offer, transactions_outside_offer = extract_transactions(combined,
                                                                                transactions_all,
                                                                                add_transactions=add_transactions,
                                                                                skip_overlap=skip_overlap);
    if not skip_add_purchases:
        combined = add_purchases(combined, profile_all, transactions_outside_offer, transactions_during_offer,
                                 calc_net_offer_time=calc_net_offer_time, assign_to=assign_to, skip_overlap=skip_overlap)

    return (combined, transactions_during_offer, transactions_outside_offer, trans)

def basic_load_data_cv(  rename_offers=True, location='local'):
    # for debugging

    rename_offers_dict = {'0b1e1539f2cc45b7b9fa7c272da2e1d7': 'a',
                          '2298d6c36e964ae4a3e7e9706d1fb8c2': 'b',
                          '2906b810c7d4411798c6938adc9daaa5': 'c',
                          '3f207df678b143eea3cee63160fa8bed': 'd',
                          '4d5c57ea9a6940dd891ad53e9dbe8da0': 'e',
                          '5a8bc65990b245e5a138643cd4eb9837': 'f',
                          '9b98b8c7a33c4b65b9aebfe6a799e6d9': 'g',
                          'ae264e3637204a6fb9bb56bc8210ddfd': 'h',
                          'f19421c1d4aa40978ebb69ca19b0e20d': 'i',
                          'fafdcd668e3743c1bb461111dcafc2a4': 'j'}

    portfolio = load_portfolio(location=location)
    offers_all, transactions_all = load_transcript(location=location)
    profile_all = load_profile(location=location)

    if rename_offers:
        portfolio['offer'] = portfolio['offer'].replace(rename_offers_dict)
        offers_all['offer'] = offers_all['offer'].replace(rename_offers_dict)
        transactions_all['offer'] = transactions_all['offer'].replace(rename_offers_dict)

    print(""" combined, transactions_during_offer, transactions_outside_offer, trans = create_pivot_table(portfolio, profile_all,offers_all,
        transactions_all,add_transactions=False, skip_overlap=False, skip_add_purchases=False, calc_net_offer_time=False, assign_to='first') """)

    return(portfolio,offers_all,profile_all ,transactions_all)

def load_data_cv(person_split=None, rename_offers=False, time_split_min=None, time_split_max=None,
                 add_transactions=False, skip_overlap=False,
                 skip_add_purchases=False, calc_net_offer_time=False, assign_to='first',location='colab'):
    """
    creates the main table and transaction tables
    can also generate CV splits by persons (by fraction) or by time (before a time in hours)

    :param person_split:
    :param rename_offers:
    :param time_split_min:
    :param time_split_max:
    :param add_transactions:
    :param skip_overlap:
    :param skip_add_purchases:
    :param calc_net_offer_time:
    :param assign_to:
    :return:
    """

    # print(person_split)

    rename_offers_dict = {'0b1e1539f2cc45b7b9fa7c272da2e1d7': 'a',
                          '2298d6c36e964ae4a3e7e9706d1fb8c2': 'b',
                          '2906b810c7d4411798c6938adc9daaa5': 'c',
                          '3f207df678b143eea3cee63160fa8bed': 'd',
                          '4d5c57ea9a6940dd891ad53e9dbe8da0': 'e',
                          '5a8bc65990b245e5a138643cd4eb9837': 'f',
                          '9b98b8c7a33c4b65b9aebfe6a799e6d9': 'g',
                          'ae264e3637204a6fb9bb56bc8210ddfd': 'h',
                          'f19421c1d4aa40978ebb69ca19b0e20d': 'i',
                          'fafdcd668e3743c1bb461111dcafc2a4': 'j'}

    if ((not time_split_min is None) & (not person_split is None)):
        print('CV split must be either by person or by time')
        sys.exit(1)

    portfolio = load_portfolio(location=location)
    offers_all, transactions_all = load_transcript(location=location)
    profile_all = load_profile(location=location)

    if rename_offers:
        portfolio['offer'] = portfolio['offer'].replace(rename_offers_dict)
        offers_all['offer'] = offers_all['offer'].replace(rename_offers_dict)
        transactions_all['offer'] = transactions_all['offer'].replace(rename_offers_dict)

    if ((person_split is None) & (time_split_min is None) & (time_split_max is None)):
        combined, transactions_during_offer, transactions_outside_offer, trans = create_pivot_table(portfolio,
                                                                                profile_all,
                                                                                offers_all,
                                                                                transactions_all,
                                                                                add_transactions=add_transactions,
                                                                                skip_overlap=skip_overlap,
                                                                                skip_add_purchases=skip_add_purchases,
                                                                                calc_net_offer_time=calc_net_offer_time,
                                                                                assign_to=assign_to)

        return (
        combined, profile_all, portfolio, offers_all, trans, transactions_during_offer, transactions_outside_offer)

        # creating CV splitting by persons:
    print()
    if person_split is None:
        print('using full person list')

        if not time_split_min is None:
            print()
            print()
            print('min time split:')
            print("time: ", offers_all.time.min(), offers_all.time.max())

            offers = offers_all.loc[offers_all.time < time_split_min, :]
            transactions = transactions_all.loc[transactions_all.time < time_split_min, :]

            if time_split_max is None:
                print('time splitting without max')
                offers_test = offers_all.loc[offers_all.time >= time_split_min, :]
                transactions_test = transactions_all.loc[transactions_all.time >= time_split_min, :]

            else:
                print('time splitting with max')
                offers_test = offers_all.loc[offers_all.time >= time_split_max, :]
                transactions_test = transactions_all.loc[transactions_all.time >= time_split_max, :]
                print("offers.shape: ", offers.shape)
                print("offers_test.shape: ", offers_test.shape)

    else:
        # person split
        Lprofile, Lprofile_test = train_test_split(profile_all.person, test_size=person_split)
        print('train set %d persons' % Lprofile.shape[0])

        profile = profile_all.loc[profile_all.person.isin(list(Lprofile))]
        profile_test = profile_all.loc[profile_all.person.isin(list(Lprofile_test))]

        offers = offers_all.loc[offers_all.person.isin(list(Lprofile))]
        offers_test = offers_all.loc[offers_all.person.isin(list(Lprofile_test))]

        transactions = transactions_all.loc[transactions_all.person.isin(list(Lprofile))]
        transactions_test = transactions_all.loc[transactions_all.person.isin(list(Lprofile_test))]

    combined, transactions_during_offer, transactions_outside_offer, trans = create_pivot_table(portfolio, profile_all,
                                                                                offers, transactions,
                                                                                add_transactions=add_transactions,
                                                                                skip_overlap=skip_overlap,
                                                                                skip_add_purchases=skip_add_purchases,
                                                                                calc_net_offer_time=calc_net_offer_time,
                                                                                assign_to=assign_to)
    out = {}
    out['train'] = combined
    out['train_tdo'] = transactions_during_offer
    out['train_too'] = transactions_outside_offer
    out['train_offers'] = offers
    out['train_transactions'] = trans

    if person_split is None:
        if not time_split_min is None:
            #       return(out ,profile_all,portfolio,offers_all,transactions_all )
            #     else:
            # time split
            combined_test, transactions_during_offer_test, transactions_outside_offer_test, trans_test = create_pivot_table(
                portfolio, profile_all, offers_test, transactions_test,
                add_transactions=add_transactions, skip_overlap=skip_overlap,
                skip_add_purchases=skip_add_purchases,
                calc_net_offer_time=calc_net_offer_time, assign_to=assign_to)
            out['test'] = combined_test
            out['test_tdo'] = transactions_during_offer_test
            out['test_too'] = transactions_outside_offer_test
            out['test_offers'] = offers_test
            out['test_transactions'] = trans_test

            return (out, profile_all, portfolio, offers_all, transactions_all)

    else:
        # person split
        print('test data')
        combined_test, transactions_during_offer_test, transactions_outside_offer_test, trans_test = create_pivot_table(
            portfolio, profile_test, offers_test, transactions_test,
            add_transactions=add_transactions, skip_overlap=skip_overlap,
            skip_add_purchases=skip_add_purchases,
            calc_net_offer_time=calc_net_offer_time, assign_to=assign_to)
        out['test'] = combined_test
        out['test_tdo'] = transactions_during_offer_test
        out['test_too'] = transactions_outside_offer_test
        out['test_offers'] = offers_test
        out['test_transactions'] = trans_test

        return (out, profile_all, portfolio, offers_all, transactions_all)

def overlapping_offer_intevals(pot_df, final_time=744, return_person_occupacy=True):
    """

    :param pot_df:
    :param final_time:
    :param return_person_occupacy:
    :return:
    """

    # derive the net offer time for a person, counting overlapping offers only once

    # work on subset pot (person, offer, time)
    pot = pot_df[['person', 'offer', 'offer_viewed', 'offer_end']].copy().dropna(subset=['offer_viewed'])
    pot['offer_viewed'] = pot['offer_viewed'].astype(int)
    pot['offer_end'] = pot['offer_end'].astype(int)

    start_time = pot['offer_viewed'].values
    end_time = np.minimum(pot['offer_end'].values, final_time + 0 * pot['offer_end'].values)
    ind = np.arange(0, len(pot))

    # Repeat start position index length times and concatenate
    lengths = end_time - start_time + 1
    cat_start = np.repeat(start_time, lengths)
    y_range = np.repeat(ind, lengths)

    # Create group counter that resets for each start/length
    cat_counter = np.arange(lengths.sum()) - np.repeat(lengths.cumsum() - lengths, lengths)

    # Add group counter to group specific starts
    x_range = cat_start + cat_counter

    # sparse matrix to vectorize computation
    occupancy = coo_matrix((1 + 0 * x_range, (y_range, x_range)))

    occupancy_df = pd.DataFrame(occupancy.toarray(), index=pot.index, columns=range(0, final_time + 1))

    person_occupacy = pd.concat((pot.drop(columns=['offer_viewed', 'offer_end']), occupancy_df), axis=1)

    # aggregate by person
    person_occupacy = person_occupacy.groupby(by='person').agg('sum')
    person_occupacy = person_occupacy.clip(upper=1)

    effective_duration = person_occupacy.sum(axis=1).to_frame().rename(columns={0: 'tot_net_offers_time'})
    if return_person_occupacy:
        return (effective_duration, person_occupacy)
    else:
        return (effective_duration)

def plot_offers_of_customer_time(piv, trans, tdo, too, customer, yc=0, step=0.1, plot_transactions=True,
                                 plot_offer_name=True, overlay_spacing=0.2):
    """
    plots the offers and the transactions for a customer versus time

    :param piv:
    :param trans:
    :param tdo:
    :param too:
    :param customer:
    :param yc:
    :param step:
    :param plot_transactions:
    :param plot_offer_name:
    :param overlay_spacing:
    :return:
    """

    gg = piv.loc[(piv.person == customer), 'offer_received'].drop_duplicates()
    ind = gg.index.values

    viewed_offers = []

    v_off_times = np.zeros((ind.shape[0], 2))
    v_off_set = set([])

    c = 0

    offer_colors = []

    for n in ind:
        hh = plt.plot([piv.loc[n, 'offer_received'], piv.loc[n, 'offer_end']], [yc, yc], 'o-', alpha=0.6)
        color = hh[0].get_color()
        offer_colors.append(color)

        if plot_offer_name:
            plt.text((piv.loc[n, 'offer_received'] + piv.loc[n, 'offer_end']) / 2, yc - .2, piv.loc[n, 'offer']);

        if not np.isnan(piv.loc[n, 'offer_viewed']):
            plt.plot([piv.loc[n, 'offer_viewed'], piv.loc[n, 'offer_end']], [yc, yc], '.-', alpha=1.0, color=color)
            viewed_offers.append(piv.loc[n, 'offer'])
            v_off_times[c, 0] = piv.loc[n, 'offer_viewed']
            v_off_times[c, 1] = piv.loc[n, 'offer_end']
            v_off_set = v_off_set.union(set(range(int(piv.loc[n, 'offer_viewed']), int(piv.loc[n, 'offer_end']) + 0)))
            c += 1

        yc += step
    v_off_times = v_off_times[0:c, :]

    diff = np.diff(np.array(list(v_off_set)))
    diff[diff > 1] = 0
    overlapping_offers_time = diff.sum()

    if plot_transactions:
        df = trans.loc[(trans.person == customer)]
        plt.plot(df['time'], df['payments'], 'o', color='k', alpha=0.3)

        # show transactions attributed to several offers
        n_offer = 0

        for n in ind:
            y_offer = n_offer + 1
            tdoc = tdo.loc[(tdo.person == customer) & (tdo.offer == piv.loc[n, 'offer']) & (
                    tdo.offer_viewed == piv.loc[n, 'offer_viewed'])]
            dy = (tdoc['overlaps']) * y_offer
            if (tdoc['overlaps'] > 0).any():
                y_offer += overlay_spacing

            if n_offer < len(offer_colors):
                #         print(n_offer)

                #         print(offer_colors[n_offer])
                plt.plot(tdoc['time'], tdoc['payments'] + dy, '.', color=offer_colors[n_offer], alpha=0.8)
            else:
                plt.plot(tdoc['time'], tdoc['payments'] + dy, '.', alpha=0.8)
            n_offer += 1

    plt.xlabel('hours');
    plt.ylabel('purchases');

    return (viewed_offers, v_off_times, overlapping_offers_time)

if __name__ == "__main__":
    out, profile_all, portfolio, offers_all, transactions_all, transactions_during_offer, transactions_outside_offer = load_data_cv(
        person_split=None, rename_offers=True, time_split_min=None, time_split_max=None,
        add_transactions=False, skip_overlap=False, skip_add_purchases=False, calc_net_offer_time=True,
        assign_to='ignore', location='local')


# created data bundle for each position of the testBench
def bundle(df):
    angles = df["Angle"].unique()
    dataBundle ={}
    for a in angles:
        subdf = df.loc[df['Angle'] == a]
        flat = flatten(subdf,df.columns.values[[2,3,4, 6,7,8, 10,11,12]])
        dataBundle[a*375/360] = flat

    return dataBundle,angles*375/360

def remAnormalRx(df):
    cdf = df
    rxTimestampsCol = df.columns.values[[2, 6, 10]]
    val0 = df[rxTimestampsCol[0]]
    val1 = df[rxTimestampsCol[1]]
    val2= df[rxTimestampsCol[2]]
    tdoaErrVal = 500
    numDrop = 0
    for i in range(0,len(val0)):
        if val0[i]-val1[i] not in range(-tdoaErrVal,tdoaErrVal) or val0[i]-val2[i] not in range(-tdoaErrVal,tdoaErrVal) or val1[i]-val2[i] not in range(-tdoaErrVal,tdoaErrVal):
            cdf = cdf.drop(i-numDrop)
    return cdf

def flatten (df,columns,x=None):
    flat_dic = {}
    if x is not None:
        for col in columns:
            flat_dic[col]=[]
            for itr in x:
                for item in df[itr][col]:
                        flat_dic[col].append(item)
    else:
        for col in columns:
            flat_dic[col]=[]
            for item in df[col]:
                    flat_dic[col].append(item)
    return flat_dic


"""Given a char array representing tasks CPU need to do. It contains capital letters A to Z where different letters represent different tasks.Tasks could be done without original order. Each task could be done in one interval. For each interval, CPU could finish one task or just be idle.

However, there is a non-negative cooling interval n that means between two same tasks, there must be at least n intervals that CPU are doing different tasks or just be idle.

You need to return the least number of intervals the CPU will take to finish all the given tasks.

Example 1:
Input: tasks = ["A","A","A","B","B","B"], n = 2
Output: 8
Explanation: A -> B -> idle -> A -> B -> idle -> A -> B.
Note:
The number of tasks is in the range [1, 10000].
The integer n is in the range [0, 100].

"""
class Solution:
    def leastInterval(self, tasks, n):
        """
        :type tasks: List[str]
        :type n: int
        :rtype: int
        """
        from queue import PriorityQueue
        from collections import Counter
        q = PriorityQueue()
        for k,v in Counter(tasks).items():
            q.put(tuple((-v,k)))

        count = 0
        while(not q.empty()):
            length = n + 1
            tmp_list = []
            while ((not q.empty()) and length>0):
                value = q.get()
                tmp_list.append(tuple((value[0]+1,value[1])))
                length-=1
                count+=1
            for i in tmp_list:
                if i[0] < 0:
                    q.put(i)

            if length and (not q.empty()) > 0:
                count+=length
        return count

if __name__ == '__main__':

    s = Solution()
    r = s.leastInterval(["A","A","A","A","A","A","B","C","D","E","F","G"],2)
    print(r)


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import pandas as pd
import numpy as np

# In[2]:

from tensorflow.keras.datasets import mnist

(x_train, y_train), (x_test, y_test) = mnist.load_data()

# ##  Visualizing the Image Data

# In[3]:

import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')

# In[4]:

x_train.shape

# In[5]:

single_image = x_train[0]

# In[6]:

single_image

# In[7]:

single_image.shape

# In[8]:

plt.imshow(single_image)

# # PreProcessing Data

# ## Labels

# In[9]:

y_train

# In[10]:

y_test

# In[11]:

from tensorflow.keras.utils import to_categorical

# In[12]:

y_train.shape

# In[13]:

y_example = to_categorical(y_train)

# In[14]:

y_example

# In[15]:

y_example.shape

# In[16]:

y_example[0]

# In[17]:

y_cat_test = to_categorical(y_test,10)

# In[18]:

y_cat_train = to_categorical(y_train,10)

# In[19]:

y_cat_test.shape

# ### Processing X Data
#
# We should normalize the X data

# In[21]:

single_image.max()

# In[22]:

single_image.min()

# In[23]:

x_train = x_train/255
x_test = x_test/255

# In[24]:

scaled_single = x_train[0]

# In[25]:

scaled_single.max()

# In[26]:

plt.imshow(scaled_single)

# ## Reshaping the Data

# In[27]:

x_train.shape

# In[28]:

x_test.shape

# In[29]:

x_train = x_train.reshape(60000, 28, 28, 1)

# In[30]:

x_train.shape

# In[31]:

x_test = x_test.reshape(10000,28,28,1)

# In[32]:

x_test.shape

# # Training the Model

# In[33]:

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, MaxPool2D, Flatten

# In[34]:

model = Sequential()

# CONVOLUTIONAL LAYER
model.add(Conv2D(filters=32, kernel_size=(4,4),input_shape=(28, 28, 1), activation='relu',))
# POOLING LAYER
model.add(MaxPool2D(pool_size=(2, 2)))

# FLATTEN IMAGES FROM 28 by 28 to 764 BEFORE FINAL LAYER
model.add(Flatten())

# 128 NEURONS IN DENSE HIDDEN LAYER (YOU CAN CHANGE THIS NUMBER OF NEURONS)
model.add(Dense(128, activation='relu'))

# LAST LAYER IS THE CLASSIFIER, THUS 10 POSSIBLE CLASSES
model.add(Dense(10, activation='softmax'))

# https://keras.io/metrics/
model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy']) # we can add in additional metrics https://keras.io/metrics/

# In[35]:

model.summary()

# In[36]:

from tensorflow.keras.callbacks import EarlyStopping

# In[37]:

early_stop = EarlyStopping(monitor='val_loss',patience=2)

# ## Train the Model

# In[38]:

model.fit(x_train,y_cat_train,epochs=10,validation_data=(x_test,y_cat_test),callbacks=[early_stop])

# ## Evaluate the Model

# In[42]:

model.metrics_names

# In[43]:

losses = pd.DataFrame(model.history.history)

# In[44]:

losses.head()

# In[45]:

losses[['accuracy','val_accuracy']].plot()

# In[46]:

losses[['loss','val_loss']].plot()

# In[50]:

print(model.metrics_names)
print(model.evaluate(x_test,y_cat_test,verbose=0))

# In[52]:

from sklearn.metrics import classification_report,confusion_matrix

# In[53]:

predictions = model.predict_classes(x_test)

# In[54]:

y_cat_test.shape

# In[55]:

y_cat_test[0]

# In[56]:

predictions[0]

# In[57]:

y_test

# In[59]:

print(classification_report(y_test,predictions))

# In[60]:

confusion_matrix(y_test,predictions)


def pyHelp(comando):
    from time import sleep
    def formataTexto(mgn, corTexto=37, corFunto=48):
        tm = len(mgn) + 4
        print('\033['+str(corTexto)+';'+str(corFunto)+'m'+'~' * tm)
        print(f'  {mgn}', flush=False)
        print('~' * tm)
        print('\033[m')
    while True:
        formataTexto('Sistema de Ajuda PyHelp', 32, 46)
        cm = str(input(comando))
        print()
        if cm.upper() == 'FIM':
            formataTexto('Até Logo!', corFunto=41)
            break
        formataTexto(f'Acessando o manual do comando  \'{cm}\'', 36, 42)
        sleep(3)
        print('\033[7;30m', end='')
        help(cm)
        print('\033[m')

pyHelp('Função ou Biblioteca: ')


from django.urls import path
from . import views, api

app_name = 'sales'

urlpatterns = [
    path('', views.index, name='site_index'),
    path('add', views.create_view, name='site_index'),
    path('register', views.register, name='site_index'),
    path('login', views.login, name='login'),
    path('logout', views.logout, name='logout'),
    path('list', views.list_view, name='Cars'),
    path('particular/<id>', views.detail_view, name='particular'),
    path('update/<id>', views.update_view, name='particular'),
    path('delete/<id>', views.delete_view, name='delete'),
    ]


import cv2
import numpy as np
import scipy

def extractKeypoints(diff_of_gaussians, contrast_threshold):
    num_octaves = len(diff_of_gaussians)
    keypoint_locations = []

    for oct_idx, dog in enumerate(diff_of_gaussians):
        dog_max = scipy.ndimage.maximum_filter(dog, [3,3,3])
        is_keypoint = (dog == dog_max) & (dog >= contrast_threshold)
        is_keypoint[:, :, 0] = False
        is_keypoint[:, :, -1] = False
        keypoint_locations.append( np.array(is_keypoint.nonzero()).T )

    return keypoint_locations


import sqlite3
import random

conn = sqlite3.connect('airConditioners.db')
conn2 = sqlite3.connect('boilers.db')
c=conn.cursor()
c2=conn2.cursor()

c.execute("""CREATE TABLE airConditionersData (
          airConditionerNumber integer,
          open_close text
           )""")

c2.execute("""CREATE TABLE boilersData (
           boilersNumber integer,
           open_close text
           )""")
conn.commit()
conn2.commit()

def insert_airConditioners(airConditionerNumber, open_close):
    with conn:
        c.execute("INSERT INTO airConditionersData VALUES (:airConditionerNumber, :open_close)",{'airConditionerNumber':airConditionerNumber, 'open_close':open_close})

def update_airConditioners(airConditionerNumber, open_close):
    with conn:
        c.execute("""UPDATE from airConditionersData SET open_close = :open_close WHERE airConditionerNumber =:airConditionerNumber""",{'airConditionerNumber':airConditionerNumber, 'open_close':open_close})

def get_status_by_number(airConditionerNumber):
    c.execute("SELECT * FROM airConditionersData WHERE airConditionerNumber=:airConditionerNumber", {'airConditionerNumber': airConditionerNumber})
    return c.fetchall()

def insert_boilers(boilersNumber, open_close):
    with conn2:
        c2.execute("INSERT INTO boilersData VALUES (:boilersNumber, :open_close)",{'boilersNumber':boilersNumber, 'open_close':open_close})

def update_window(boilersNumber, open_close):
    with conn2:
        c2.execute("""UPDATE from boilersData SET open_close = :open_close WHERE boilersNumber =:boilersNumber""",{'boilersNumber':boilersNumber, 'open_close':open_close})

def get_status_by_number(boilersNumber):
    c.execute("SELECT * FROM boilersData WHERE boilersNumber=:boilersNumber", {'boilersNumber': boilersNumber})
    return c.fetchall()

for x in range(21):

    mylist1 = ['open', 'close']
    insert_airConditioners(x+1,random.choice(mylist1))
    insert_boilers(x+1,random.choice(mylist1))

conn.close()
conn2.close()


from tfbldr.datasets import tbptt_file_list_iterator
import os
import numpy as np

files = os.listdir("/Tmp/kastner/lj_speech_hybrid_speakers/numpy_features/")
files = ["/Tmp/kastner/lj_speech_hybrid_speakers/numpy_features/" + f for f in files]
ljspeech_hybridset = [' ', '!', ',', '-', '.', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z']
hybrid_lookup = {v: k for k, v in enumerate(sorted(ljspeech_hybridset))}
hybrid_inverse_lookup = {v: k for k, v in hybrid_lookup.items()}

def file_access(f):
    d = np.load(f)
    text = d["text"]
    inds = [hybrid_lookup[t] for t in text.ravel()[0]]
    audio = d["audio_features"]
    return (audio, inds)

random_state = np.random.RandomState(1442)
batch_size = 8
truncation_length = 256
itr = tbptt_file_list_iterator(files, file_access,
                               batch_size,
                               truncation_length,
                               other_one_hot_size=[len(ljspeech_hybridset)],
                               random_state=random_state)
for i in range(100000):
    print(i)
    r = itr.next_masked_batch()


"""initial migrate

Revision ID: 53bde6239219
Revises:
Create Date: 2021-07-15 11:18:33.376830

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '53bde6239219'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('cryptozaken',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('zaaknaam', sa.String(length=64), nullable=True),
    sa.Column('verdachte', sa.String(length=64), nullable=True),
    sa.Column('currency', sa.String(length=64), nullable=True),
    sa.Column('aantal', sa.Float(), nullable=True),
    sa.Column('totale_waarde', sa.Float(), nullable=True),
    sa.Column('datum_tijd', sa.String(length=64), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('cryptozaken')
    # ### end Alembic commands ###


import sys

import src.Globals as gb
from src.Utils import *
from src.menus.MenuItem import *
from src.scenes.GameScene import *

# MenuScene is based on Nebelhom's tutorial: "Create a simple game menu with pygame" at:
# https://nebelprog.wordpress.com/2013/09/02/create-a-simple-game-menu-with-pygame-pt-4-connecting-it-to-functions/
class MenuScene(SceneBase):
    def __init__(self, funcs, font=None, font_size=30, font_color=Color('white')):
        SceneBase.__init__(self)
        self.bg_color = Color('black')

        self.funcs = funcs
        funcs['Start'] = self.next_scene
        funcs['Quit'] = sys.exit
        items = funcs.keys()
        self.items = []
        for index, item in enumerate(items):
            menu_item = MenuItem(item, font, font_size, font_color)

            t_h = len(items) * menu_item.height  # total height of text block
            pos_x = (gb.WINDOW_WIDTH / 2) - (menu_item.width / 2)
            pos_y = (gb.WINDOW_HEIGHT / 2) - (t_h / 2) + ((index * 2) + index * menu_item.height)

            menu_item.set_position(pos_x, pos_y)
            self.items.append(menu_item)

        self.mouse_is_visible = True
        self.cur_item = None

        self.mpos = None

    def next_scene(self):
        self.SwitchToScene(GameScene())

    def set_keyboard_selection(self, key):
        # Mark the MenuItem chosen via up and down keys
        for item in self.items:
            # Return all to neutral
            item.set_italic(False)
            item.set_font_color(Color('white'))

        if self.cur_item is None:
            self.cur_item = 0
        else:
            # Find the chosen item
            if key == pygame.K_UP and self.cur_item > 0:
                self.cur_item -= 1
            elif key == pygame.K_UP and self.cur_item == 0:
                self.cur_item = len(self.items) - 1
            elif key == pygame.K_DOWN and self.cur_item < len(self.items) - 1:
                self.cur_item += 1
            elif key == pygame.K_DOWN and self.cur_item == len(self.items) - 1:
                self.cur_item = 0

        self.items[self.cur_item].set_italic(True)
        self.items[self.cur_item].set_font_color(Color('red'))

        # Finally check if Enter or Space is pressed
        if key == pygame.K_SPACE or key == pygame.K_RETURN:
            text = self.items[self.cur_item].text
            self.funcs[text]()

    def ProcessInput(self, events, pressed_keys, mpos):
        self.mpos = mpos

        for event in events:
            if event.type == pygame.KEYDOWN:
                self.mouse_is_visible = False
                self.set_keyboard_selection(event.key)
            if event.type == pygame.MOUSEBUTTONDOWN:
                for item in self.items:
                    if item.is_mouse_selection(mpos):
                        self.funcs[item.text]()

        if pygame.mouse.get_rel() != (0, 0):
            self.mouse_is_visible = True
            self.cur_item = None

        pygame.mouse.set_visible(self.mouse_is_visible)

    def Update(self):
        pass

    def Render(self, screen):
        screen.fill(self.bg_color)

        if self.mouse_is_visible:
            for item in self.items:
                set_mouse_selection(item, self.mpos)
                screen.blit(item.label, item.position)


#  This file is part of Recent changes Goat compatible Discord webhook (RcGcDw).
#
#  RcGcDw is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  RcGcDw is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with RcGcDw.  If not, see <http://www.gnu.org/licenses/>.

import argparse

parser = argparse.ArgumentParser(description="Start RcGcDw")
parser.add_argument("--test", action='store_true', help="mode used for testing, sends only 5 entries of both rc and discussion changes and sends daily overview")
parser.add_argument("--settings", default="settings.json", type=argparse.FileType(encoding='utf8'), help="provides a path to settings file (default ./settings.json)")
command_args, unknown = parser.parse_known_args()


def main():

    file = open("animal rank.txt", "r")
    lstRank = []

    for strLine in file:
        lstAnimal = strLine.split()
        lstAnimal[1] = int(lstAnimal[1])

        # If the list is empty or this breed has the lowest rank so far, add
        # this breed to the end.
        if len(lstRank) == 0 or lstAnimal[1] > lstRank[-1][1]:
            lstRank.append(lstAnimal)
        else:

            # Create a sorted list of breeds by comparing this breed's rank to
            # the other breeds in the list, and adding it in the right place.
            for i in range(len(lstRank)):
                if lstAnimal[1] < lstRank[i][1]:
                    lstRank.insert(i, lstAnimal)
                    break

    print("Highest ranking animal: {} (#{})\n".format(lstRank[0][0],
                                                     lstRank[0][1]))

    print("All animals (in order of ranking):")
    for lstAnimal in lstRank:
        print("#{}. {}".format(lstAnimal[1], lstAnimal[0]))

main()


'''
If Statements
'''

cash_balance = 99.43
price_of_headphones = 150.99

# IF
if cash_balance >= price_of_headphones:
    print("You can afford the headphones.")
else:
    print("You cannot afford the headphones.")

if cash_balance >= price_of_headphones:
    print("You can afford the headphones.")
print("You cannot afford the headphones.")

# Tabs used to tell the Computer what code is inside of what
if True and False:
    print("In the if statement.")

if True or False:
    print('something')
print("Not in the if.")
if True:
    print("hey")

# Else-if
dad_is_home_for_dinner = False
mom_is_home_for_dinner = True
if dad_is_home_for_dinner:
    result = "McDonalds"
elif mom_is_home_for_dinner:
    result = "Harveys"
elif False:
    dosomething()
elif False:
    dosomething()
else:
    result = "Baskin Robbins"

## Nested Statements
if mom_is_home_for_dinner:
    if cash_balance >= price_of_headphones:
        print("BLAH")
    else:
        print("WOOHOO")
elif dad_is_home_for_dinner:
    print("Something else")
else:
    print("Another thing!")


from __future__ import (absolute_import, division, print_function)

import matplotlib as mpl
import matplotlib.pyplot as plt
# The two statemens below are used mainly to set up a plotting
# default style that's better than the default from Matplotlib 1.x
# Matplotlib 2.0 supposedly has better default styles.
import seaborn as sns
plt.style.use('bmh')
from pathlib import Path
data_folder = Path('D:/OneDrive/AAMOSM2018/0828mapdata')

#data_folder = Path('C:/Users/langzx/OneDrive/AAMOSM2018/0828mapdata')
from shapely.geometry import Point, Polygon
from matplotlib.lines import Line2D
import pandas as pd
import geopandas as gpd
from geopandas import GeoSeries, GeoDataFrame
#import pyepsg
mpl.__version__, pd.__version__, gpd.__version__

from shapely.wkt import loads
from fiona.crs import from_epsg
from pyproj import Proj
 #Proj('init=epsg:4326', preserve_flags=True)
# Determine the CRS of the GeoDataFrame

GeoSeries([loads('POINT(1 2)'), loads('POINT(1.5 2.5)'), loads('POINT(2 3)')])

subbasin = gpd.read_file(data_folder/"subbasins.shp")
subbasin_p4 =  subbasin.to_crs(epsg=4326)
subbasin.crs
subbasin.crs = from_epsg(4326)
subbasin_p4 =  subbasin.to_crs(epsg = 4326)
subbasin.crs = {'init' :'epsg:4326'}
subbasin.crs
subbasin['geometry']

#subbasin.plot( color='red', markersize=100, figsize=(4, 4))
gage = gpd.read_file(data_folder/"gage2.shp")
gs = GeoSeries([Point(-120, 45), Point(-121.2, 46), Point(-122.9, 47.5),Point(-122.9, 47.5),Point(-122.9, 47.5)])
#subbasin['Watershed_Zone'] = subbasin.watershed.map(str) + " " + subbasin.Zone

#subbasin = gpd.read_file("C:\\Users\langzx\\Onedrive\\AAMOSM2018\\newmapdata\\subs1.shp")
stream =  gpd.read_file(data_folder/ "LeSueur_Streams.shp")
## project
subbasin.crs = {'init' :'epsg:4326'}
subbasin.crs
#subbasin_p4 =  subbasin.to_crs(epsg=4326)
gage.crs = {'init' :'epsg:4326'}

#gage_p4.plot(marker='*', color='red', markersize=100, figsize=(4, 4))

#gage2_p4 =  gage2.to_crs
stream.crs= {'init' :'epsg:4326'}

len(subbasin.columns.values)
subbasin.columns.values
type(subbasin['geometry'])

subbasin['coords'] = subbasin['geometry'].apply(lambda x: x.representative_point().coords[:])
subbasin['coords'] = [coords[0] for coords in subbasin['coords']]

f, ax = plt.subplots(1, figsize=(20, 20))
ax.set_title('')

# Other nice categorical color maps (cmap) include 'Set2' and 'Set3'
subbasin.plot(ax = ax, column = 'watershed', linewidth=0.8, cmap='summer_r',edgecolor='#B3B3B3', legend = True)
stream.plot(ax = ax, edgecolor='blue')
gage.plot(ax = ax, marker='*', color='red', markersize=400)
ax.set_ylim([43.6, 44.3])
plt.axis('equal');

##Annotation
for idx, row in subbasin.iterrows():
    ax.annotate(s=row['Zone'], xy=row['coords'],
                 verticalalignment='center')

#plt.axis('equal');
custom_lines = [Line2D([0], [0], color=cmap(0.), lw=20),
                Line2D([0], [0], color=cmap(.5), lw=20),
                Line2D([0], [0], color=cmap(1.), lw=20),
                Line2D([0], [0], marker='*', color='w',markersize=20, markerfacecolor='r'),
                Line2D([0], [0],  color='w',markersize=20, markerfacecolor='w'),
                Line2D([0], [0],  color='w',markersize=20, markerfacecolor='w'),
                Line2D([0], [0],  color='w',markersize=20, markerfacecolor='w')]
legend = ax.legend(custom_lines, ['COB', 'LES', 'MAP','Gages','Zone 1 = Upland','Zone 2 = Transitional','Zone 3 = Incised'], fontsize=20)
legend.get_frame().set_facecolor('white')

type(subbasin.ix[23, 'geometry'])
subbasin['coords'] = subbasin['geometry'].apply(lambda x: x.representative_point().coords[:])
subbasin['coords'] = [coords[0] for coords in subbasin['coords']]

subbasin.plot( cmap='summer_r', legend=True, k=3, figsize=(12, 15))
for idx, row in subbasin.iterrows():
    plt.annotate(s=row['Zone'], xy=row['coords'],
                 horizontalalignment='center')
gage.plot( marker='*', color='red', markersize=200)
stream.plot(ax=ax, edgecolor='blue')

subbasin.plot(column='watershed', cmap='summer_r',  k=3, figsize=(8, 10));

subbasin.plot(color='white', edgecolor='black', figsize=(8, 8));

# Visualize
ax = subbasin.plot()

gdf.columns.values
gdf.plot(cmap='Set2', figsize=(10, 10));

ax = subbasin.plot(color='white', edgecolor='black')
ax.set_axis_off()
gdf.plot(ax = ax, edgecolor='white')
plt.show()

##################

plt.figure(figsize=(20,5))
cmap = plt.cm.summer_r
t = ("Zone 1 = Upland\n"
     "Zone 2 = Transitional\n"
     "Zone 3 = Incised")

f, ax = plt.subplots(1, figsize=(20,20))
ax.set_facecolor('white')
ax.set_ylim([43.6, 44.3])
ax.grid(False)
# Other nice categorical color maps (cmap) include 'Set2' and 'Set3'

type(subbasin.ix[23, 'geometry'])
subbasin['coords'] = subbasin['geometry'].apply(lambda x: x.representative_point().coords[:])
subbasin['coords'] = [coords[0] for coords in subbasin['coords']]

subbasin.plot(ax=ax,column='watershed',linewidth=0.8, cmap='summer_r',edgecolor='#B3B3B3', legend = True)
gage.plot(ax = ax, marker='*', color='red', markersize=200)
stream.plot(ax = ax, edgecolor='blue')

for idx, row in subbasin.iterrows():
    ax.annotate(s=row['Zone'], xy=row['coords'],
                 verticalalignment='center')

#plt.axis('equal');
custom_lines = [Line2D([0], [0], color=cmap(0.), lw=20),
                Line2D([0], [0], color=cmap(.5), lw=20),
                Line2D([0], [0], color=cmap(1.), lw=20),
                Line2D([0], [0], marker='*', color='w',markersize=20, markerfacecolor='r'),
                Line2D([0], [0],  color='w',markersize=20, markerfacecolor='w'),
                Line2D([0], [0],  color='w',markersize=20, markerfacecolor='w'),
                Line2D([0], [0],  color='w',markersize=20, markerfacecolor='w')]
legend = ax.legend(custom_lines, ['COB', 'LES', 'MAP','Gages','Zone 1 = Upland','Zone 2 = Transitional','Zone 3 = Incised'], fontsize=20)
legend.get_frame().set_facecolor('white')

#ax.text(0.97, 0.87, t,
#       verticalalignment='top', horizontalalignment='right',
#        transform=ax.transAxes,
#        fontsize=15,wrap=True)
#ax.text(0.5,0.5,u'\u25B2 \nN ', ha='center', fontsize=20, family='Arial')
plt.savefig('mosmmap.pdf', bbox_inches='tight')

subbasin.crs
# Visualize
ax = subbasin.plot()

gdf.columns.values
gdf.plot(cmap='Set2', figsize=(10, 10));

ax = subbasin.plot(color='white', edgecolor='black')
ax.set_axis_off()
gdf.plot(ax = ax, edgecolor='white')
plt.show()


from sys import argv;

import os;
import subprocess;

def run(command, exitcode = 0, expected_stderr = None):
  print("\033[33m" f" $ {' '.join(command)}" "\033[0m");
  result = subprocess.run(command, \
    capture_output = True, text = True, \
    env = os.environ.update({"print_stacktrace": "1", "halt_on_error": "1"}))
  if result.returncode != exitcode:
    print("\033[31m subcommand failed!");
    exit(1);
  elif expected_stderr is not None and result.stderr != expected_stderr:
    print("\033[31m output does not match expectations!");
    exit(1);

executable = argv[1];

run( \
  [executable, "-i", "/tmp/missing.guard", "-o", "/tmp/missing.c"], \
  expected_stderr = "guardian: open(\"missing.guard\"): No such file or directory\n", \
  exitcode = 1);


import unittest
import tempfile
import os

from staroid import Staroid

def integration_test_ready():
    return "STAROID_ACCESS_TOKEN" in os.environ and "STAROID_ACCOUNT" in os.environ

class TestCluster(unittest.TestCase):
    @unittest.skipUnless(integration_test_ready(), "Integration test environment is not configured")
    def test_initialize(self):
        s = Staroid(access_token=os.environ["STAROID_ACCESS_TOKEN"], account=os.environ["STAROID_ACCOUNT"])
        s.get_all_accounts()

    @unittest.skipUnless(integration_test_ready(), "Integration test environment is not configured")
    def test_crud(self):
        # given
        s = Staroid(access_token=os.environ["STAROID_ACCESS_TOKEN"], account=os.environ["STAROID_ACCOUNT"])
        all_accounts = s.get_all_accounts()
        clusters_before = s.cluster().get_all()

        # when create
        c = s.cluster().create("staroid-python it-test")

        # then
        self.assertEqual("staroid-python it-test", c.name())

        # when already exists on create
        c = s.cluster().create("staroid-python it-test")

        # then
        self.assertEqual("staroid-python it-test", c.name())

        # when list
        clusters_after = s.cluster().get_all()

        # then
        self.assertEqual(len(clusters_after), len(clusters_before) + 1)

        # when delete
        s.cluster().delete("staroid-python it-test")

        # then
        clusters_after_deleted = s.cluster().get_all()
        self.assertEqual(len(clusters_after_deleted), len(clusters_before))


from django.db.models.deletion import ProtectedError
from rest_framework import mixins, serializers
from rest_framework.viewsets import GenericViewSet

class DestroyModelMixin(mixins.DestroyModelMixin):
    """
    Destroy a model instance.  Raise a validation error if a ProtectedError is raised by a database constraint.
    """

    def perform_destroy(self, instance):
        try:
            instance.delete()
        except ProtectedError as e:
            msg = 'Referenced by protected {} {}'.format(
                e.protected_objects.model._meta.label,
                ', '.join([str(x) for x in e.protected_objects.values_list('id', flat=True)])
            )
            raise serializers.ValidationError(msg)

class ModelViewSet(mixins.CreateModelMixin,
                   mixins.RetrieveModelMixin,
                   mixins.UpdateModelMixin,
                   DestroyModelMixin,
                   mixins.ListModelMixin,
                   GenericViewSet):
    """
    A viewset that provides default `create()`, `retrieve()`, `update()`, `partial_update()`, `destroy()` and `list()`
    actions.  This class incorporates system-wide customizations of the basic DRF viewet behavior; in this case, uses
    our DestroyModelMixin.
    """
    pass


import os.path
import glob
from .listdataset import ListDataset
from .util import split2list

def make_dataset(dir, phase='train'):
    '''Will search for triplets that go by the pattern '[name]_img1.ppm  [name]_img2.ppm    [name]_flow.flo' '''
    images = []
    for flow_map in sorted(glob.glob(os.path.join(dir, phase+'/*/flow/*.flo'))):
        flow_map = os.path.relpath(flow_map, dir)
        img1 = flow_map.replace('/flow/', '/composition/')
        img1 = img1.replace('.flo', '.png')
        img2 = img1[:-9] + str(int(img1.split('/')[-1][:-4])+1).zfill(5) + '.png'

        if int(img1.split('/')[-1][:-4]) % 10 == 9:
            continue

        if not (os.path.isfile(os.path.join(dir,img1)) and os.path.isfile(os.path.join(dir,img2))):
            continue

        images.append([[img1,img2],flow_map])

    return images

def humanflow(root, transform=None, target_transform=None,
                  co_transform=None, split=None):
    train_list = make_dataset(root, 'train')
    test_list = make_dataset(root, 'test')
    train_dataset = ListDataset(root, train_list, transform, target_transform, co_transform)
    test_dataset = ListDataset(root, test_list, transform, target_transform)

    return train_dataset, test_dataset


# Define a dictionary representing a person
person1 = {
    "first_name": "Saif",
    "last_name": "Ur Rehman",
    "age": 21,
    "city": "Bahawalpur"
}

# Define a dictionary representing another person
person2 = {
    "first_name": "Maqsood",
    "last_name": "Ahmad",
    "age": 19,
    "city": "Isl"
}

person3 = {
    "first_name": "Ali",
    "last_name": "akber",
    "age": 21,
    "city": "Lahore"
}

people_list = [person1, person2, person3]

for person in people_list:
    print("First Name:", person["first_name"])
    print("Last Name:", person["last_name"])
    print("Age:", person["age"])
    print("City:", person["city"])
    print("next one")


class Solution:
    def groupAnagrams(self, strs: List[str]) -> List[List[str]]:
        dict1 = {}
        for word in strs:
            sortedwords = "".join(sorted(word))
            if sortedwords in dict1:
                dict1[sortedwords].append(word)
            else:
                dict1[sortedwords] = [word]

        return list(dict1.values())


from matplotlib import pyplot as plt
from torch.utils.data import Dataset, DataLoader
import torch
import os
import numpy as np

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

class EVRPDataset(Dataset):
    def __init__(self, train_size, num_nodes, t_limit, capacity, num_afs=3, seed=520):
        super().__init__()
        self.size = train_size

        torch.manual_seed(seed)
        afs = torch.rand(2,num_afs+1)
        customers = torch.rand(train_size, 2, num_nodes, device=device)

        self.static = torch.cat((afs.unsqueeze(0).repeat(train_size, 1, 1), customers), dim=2).to(device)  # (train_size, 2, num_nodes+4)

        self.dynamic = torch.ones(train_size, 3, 1+num_afs+num_nodes, device=device)   # time duration, capacity, demands
        self.dynamic[:, 0, :] *= t_limit
        self.dynamic[:, 1, :] *= capacity
        self.dynamic[:, 2, :num_afs+1] = 0

        seq_len = self.static.size(2)
        self.distances = torch.zeros(train_size, seq_len, seq_len, device=device)
        for i in range(seq_len):
            self.distances[:, i] = cal_dis(self.static[:, 0, :], self.static[:, 1, :], self.static[:, 0, i:i+1], self.static[:, 1, i:i+1])

    def __len__(self):
        return self.size

    def __getitem__(self, idx):
        return self.static[idx], self.dynamic[idx], self.distances[idx]    # dynamic: None

    def update_mask(self, what):
        pass

    def update_dynamic(self, old_idx, idx, mask, dynamic, distances, dis_by_afs, capacity, velocity, cons_rate, t_limit, num_afs):
        """
        :param old_idx: (batch*beam, 1)
        :param idx: ditto
        :param mask: (batch*beam, seq_len)
        :param dynamic: (batch*beam, dynamic_features, seq_len)
        :param distances: (batch*beam, seq_len, seq_len)
        :param dis_by_afs: (batch*beam, seq_len)
        :param capacity, velocity, cons_rate, t_limit, num_afs: scalar
        :return: updated dynamic AFS = merh pou kanoume recharge
        """

        #TODO: kane th maska apo 1:paw, 0 den paw, se 0 paw, -inf den paw
        # mask[mask == 1] = 0# mporeis na pas
        # mask[mask == 0] =  float('-inf')
        dis = distances[torch.arange(distances.size(0)), old_idx.squeeze(1), idx.squeeze(1)].unsqueeze(1)
        depot = idx.eq(0).squeeze(1)
        afs = (idx.gt(0) & idx.le(num_afs)).squeeze(1)
        fs = idx.le(num_afs).squeeze(1)
        customer = idx.gt(num_afs).squeeze(1)  # TODO: introduce num_afs
        time = dynamic[:, 0, :].clone()
        fuel = dynamic[:, 1, :].clone()
        demands = dynamic[:, 2, :].clone()

        time -= dis / velocity
        time[depot] = t_limit #exeis full diathesimh wra
        time[afs] -= 0.25 # katanalwneis ligh wra gia na forthseis
        time[customer] -= 0.5  # katanalwneis ligh wra gia na afhseis to proion

        fuel -= cons_rate * dis
        fuel[fs] = capacity
        demands.scatter_(1, idx, 0)

        dynamic = torch.cat((time.unsqueeze(1), fuel.unsqueeze(1), demands.unsqueeze(1)), dim=1).to(device)
        #print(f'Here are some demands left{((dynamic[:, 2, :] == 0)==False).nonzero()}')
        mask.scatter_(1, idx, float('-inf'))
        # forbid passing by afs if leaving depot, allow if returning to depot; forbid from afs to afs, not necessary but convenient
        mask[fs, 1:num_afs + 1] = float('-inf')
        mask[afs, 0] = 0
        # kapou edw
        mask[customer, :num_afs + 1] = 0  # phgaine
        mask[fs] = torch.where(demands[fs] > 0, torch.zeros(mask[fs].size(), device=device), mask[fs])
        #print(f'non inf mask {torch.nonzero(mask != float('-inf')).squeeze()}')
        # path1: ->Node->Depot
        dis1 = distances[torch.arange(distances.size(0)), idx.squeeze(1)].clone()
        fuel_pd0 = cons_rate * dis1 #poso fuel xreiazetai na paei apo node sto depot
        time_pd0 = dis1 / velocity # poso time xreiazetai
        dis1[:, num_afs + 1:] += distances[:, 0, num_afs + 1:]
        fuel_pd1 = cons_rate * dis1
        time_pd1 = (distances[torch.arange(distances.size(0)), idx.squeeze(1)] + distances[:, 0, :]) / velocity
        time_pd1[:, 1:num_afs + 1] += 0.25
        time_pd1[:, num_afs + 1:] += 0.5

        # path2: ->Node-> Station-> Depot(choose the station making the total distance shortest)
        dis2 = distances[:, 1:num_afs + 1, :].gather(1, dis_by_afs[1].unsqueeze(1)).squeeze(1)
        dis2[:, 0] = 0
        dis2 += distances[torch.arange(distances.size(0)), idx.squeeze(1)]
        fuel_pd2 = cons_rate * dis2
        time_pd2 = (distances[torch.arange(distances.size(0)), idx.squeeze(1)] + dis_by_afs[0]) / velocity
        time_pd2[:, 1:num_afs + 1] += 0.25
        time_pd2[:, num_afs + 1:] += 0.5

        # path3: ->Node-> Station-> Depot(choose the closest station to the node), ignore this path temporarily
        # the next node should be able to return to depot with at least one way; otherwise, mask it
        mask[~((fuel >= fuel_pd1) & (time >= time_pd1) | (fuel >= fuel_pd2) & (time >= time_pd2))] = float('-inf')
        # edw ginetai h malakia
        mask[(fuel < fuel_pd0) | (time < time_pd0)] = float('-inf')

        all_masked = mask[:, num_afs + 1:].eq(0).sum(1).le(0)
        mask[all_masked, 0] = 0  # unmask the depot if all nodes are masked

        return dynamic

    def update_dynamic2(self, old_idx, idx, mask, dynamic, distances, dis_by_afs, capacity, velocity, cons_rate, t_limit,
                       num_afs):
        """
        1: mporeis na pas
        0: den mporeis na pas
        :param old_idx: (batch*beam, 1)
        :param idx: ditto
        :param mask: (batch*beam, seq_len)
        :param dynamic: (batch*beam, dynamic_features, seq_len)
        :param distances: (batch*beam, seq_len, seq_len)
        :param dis_by_afs: (batch*beam, seq_len)
        :param capacity, velocity, cons_rate, t_limit, num_afs: scalar
        :return: updated dynamic
        """
        dis = distances[torch.arange(distances.size(0)), old_idx.squeeze(1), idx.squeeze(1)].unsqueeze(1)
        episkepthkes_depot = idx.eq(0).squeeze(1)
        afs = (idx.gt(0) & idx.le(num_afs)).squeeze(1) #episkepthkes fulling station
        fs = idx.le(num_afs).squeeze(1) ##episkepthkes fulling station
        customer = idx.gt(num_afs).squeeze(1)  # TODO: introduce num_afs
        time = dynamic[:, 0, :].clone()
        fuel = dynamic[:, 1, :].clone()
        demands = dynamic[:, 2, :].clone()

        time -= dis / velocity
        time[episkepthkes_depot] = t_limit
        time[afs] -= 0.25
        time[customer] -= 0.5

        fuel -= cons_rate * dis # poso kausimo xalases?
        fuel[fs] = capacity # edw episkfthkes fuel station
        #demands.scatter_(1, idx, 1) # mhpws 1?

        print(f'Chose indexes:{idx.squeeze()}')
        demands.scatter_(1, idx, 0) # satisfy the demands of the customer you visit

        # dynamic: [time, fuel, demand]
        dynamic = torch.cat((time.unsqueeze(1), fuel.unsqueeze(1), demands.unsqueeze(1)), dim=1).to(device)

        # mask.scatter_(1, idx, float('-inf'))
        mask.scatter_(1, idx, 0) # MHN XANAPAS STO IDIO

        # forbid passing by afs if leaving depot, allow if returning to depot; forbid from afs to afs, not necessary but convenient
        mask[fs, 1:num_afs + 1] = 0#float('-inf')
        mask[afs, 0] = 1 #0

        mask[customer, :num_afs + 1] = 1 # phgaine ok # 0
        # edw ta kanei mhden enw emeis theloume 1!
        mask[fs] = torch.where(demands[fs] > 0, torch.zeros(mask[fs].size(), device=device), mask[fs])

        # path1: ->Node->Depot
        dis1 = distances[torch.arange(distances.size(0)), idx.squeeze(1)].clone()
        fuel_pd0 = cons_rate * dis1
        time_pd0 = dis1 / velocity
        dis1[:, num_afs + 1:] += distances[:, 0, num_afs + 1:]
        fuel_pd1 = cons_rate * dis1
        time_pd1 = (distances[torch.arange(distances.size(0)), idx.squeeze(1)] + distances[:, 0, :]) / velocity
        time_pd1[:, 1:num_afs + 1] += 0.25
        time_pd1[:, num_afs + 1:] += 0.5

        # path2: ->Node-> Station-> Depot(choose the station making the total distance shortest)
        dis2 = distances[:, 1:num_afs + 1, :].gather(1, dis_by_afs[1].unsqueeze(1)).squeeze(1)
        dis2[:, 0] = 1 #0
        dis2 += distances[torch.arange(distances.size(0)), idx.squeeze(1)]
        fuel_pd2 = cons_rate * dis2
        time_pd2 = (distances[torch.arange(distances.size(0)), idx.squeeze(1)] + dis_by_afs[0]) / velocity
        time_pd2[:, 1:num_afs + 1] += 0.25
        time_pd2[:, num_afs + 1:] += 0.5

        # path3: ->Node-> Station-> Depot(choose the closest station to the node), ignore this path temporarily
        # the next node should be able to return to depot with at least one way; otherwise, mask it
        mask[~((fuel >= fuel_pd1) & (time >= time_pd1) | (fuel >= fuel_pd2) & (time >= time_pd2))] = 0#float('-inf')

        mask[(fuel < fuel_pd0) | (time < time_pd0)] = 0# float('-inf')

        #all_masked = mask[:, num_afs + 1:].eq(0).sum(1).le(0)
        all_masked = mask[:,num_afs + 1:].eq(0).all(dim=1)
        mask[all_masked, 0] = 1#0  # unmask the depot if all nodes are masked

        mask[mask == 0] = 1
        mask[mask == float('-inf')] = 0
        return dynamic

def cal_dis(lon1, lat1, lon2, lat2):
    """
    Calculate the great circle distance between two points
    on the earth (specified in decimal degrees)
    """
    # degrees to radians
    lon1, lat1, lon2, lat2 = map(lambda x: x/180*np.pi, [lon1, lat1, lon2, lat2])
    dlon = lon2 - lon1
    dlat = lat2 - lat1
    a = torch.pow(torch.sin(dlat / 2), 2) + torch.cos(lat1) * torch.cos(lat2) * torch.pow(torch.sin(dlon / 2), 2)
    c = 2 * 4182.44949 * torch.asin(torch.sqrt(a))    # miles
    # c = 2 * atan2(sqrt(a), sqrt(1 - a)) * 4182.44949
    return c

def reward_func2(tours, static, distances, beam_width=1):
    """
    :param tours: LongTensor, (batch*beam, seq_len)
    :param static: (batch, 2, num_nodes)
    :param distances: (batch, num_nodes, num_nodes)
    :param beam_width: set beam_width=1 when training
    :return: reward: Euclidean distance between each consecutive point， (batch)
    :return: locs: (batch, 2, seq_len)
    """
    bb_size, seq_len = tours.size()
    batch_size = static.size(0)
    depot = torch.zeros(bb_size, 1, dtype=torch.long, device=device)
    tours = torch.cat((depot, tours, depot), dim=1)         # start from depot, end at depot(although some have ended at depot)
    id0 = torch.arange(bb_size).unsqueeze(1).repeat(1, seq_len+1)
    reward = distances.repeat(beam_width, 1, 1)[id0, tours[:, :-1], tours[:, 1:]].sum(1)    # (batch*beam)
    # (batch*beam) -> (batch), choose the best reward
    reward, id_best = torch.cat(torch.chunk(reward.unsqueeze(1), beam_width, dim=0), dim=1).min(1)  # (batch)
    bb_idx = torch.arange(batch_size, device=device) + id_best * batch_size
    tours = tours[bb_idx]
    # print(tours)
    tours = tours.unsqueeze(1).repeat(1, static.size(1), 1)
    locs = torch.gather(static, dim=2, index=tours)  # (batch, 2, seq_len+)
    return reward, locs

def reward_func( static,tours, distances, beam_width=1):
    """
    :param tours: LongTensor, (batch*beam, seq_len)
    :param static: (batch, 2, num_nodes)
    :param distances: (batch, num_nodes, num_nodes)
    :param beam_width: set beam_width=1 when training
    :return: reward: Euclidean distance between each consecutive point， (batch)
    :return: locs: (batch, 2, seq_len)
    """
    bb_size, seq_len = tours.size()
    batch_size = static.size(0)
    depot = torch.zeros(bb_size, 1, dtype=torch.long, device=device)
    tours = torch.cat((depot, tours, depot), dim=1)         # start from depot, end at depot(although some have ended at depot)
    id0 = torch.arange(bb_size).unsqueeze(1).repeat(1, seq_len+1)
    reward = distances.repeat(beam_width, 1, 1)[id0, tours[:, :-1], tours[:, 1:]].sum(1)    # (batch*beam)

    # (batch*beam) -> (batch), choose the best reward
    reward, id_best = torch.cat(torch.chunk(reward.unsqueeze(1), beam_width, dim=0), dim=1).min(1)  # (batch)
    bb_idx = torch.arange(batch_size, device=device) + id_best * batch_size
    # print(tours)
    tours = tours[bb_idx]
    #print(tours)
    tours = tours.unsqueeze(1).repeat(1, static.size(1), 1)
    locs = torch.gather(static, dim=2, index=tours)  # (batch, 2, seq_len+)
    return reward#, locs

def render_func(locs, static, num_afs, save_dir="."):
    """
    :param locs: (batch, 2, sel_len)
    :param static: (batch, 2, num_nodes+1)
    :param num_afs: scalar
    :param save_dir: path to save figure
    :return: None
    """
    plt.close('all')
    data = locs[-1].cpu().numpy()   # (2, num_nodes+1), just plot the last one
    # demands = dynamic[-1, 1, 1:].cpu().numpy()*capacity   # (num_nodes), depot excluded
    # coords = static[-1][:, 1:].cpu().numpy().T     # (num_nodes, 2)
    plt.plot(data[0], data[1], zorder=1)
    origin_locs = static[-1, :, :].cpu().numpy()
    plt.scatter(origin_locs[0], origin_locs[1], s=4, c='r', zorder=2)
    plt.scatter(origin_locs[0, 0], origin_locs[1, 0], s=20, c='k', marker='*', zorder=3)  # depot
    plt.scatter(origin_locs[0, 1:num_afs+1], origin_locs[1, 1:num_afs+1], s=20, c='b', marker='+', zorder=4)    # afs
    for i, coords in enumerate(origin_locs.T[1:]):
        plt.annotate('%d' % (i+1), xy=coords, xytext=(2, 2), textcoords='offset points')  # mark numbers
    if os.path.isfile(save_dir):
        os.remove(save_dir)
    plt.axis('equal')
    plt.savefig(save_dir, dpi=400)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#@created: <data>
#@author: <name>
#@contact: <email>

import sys
import argparse
import os
import os.path

def main(input_dir, outdirectory):
    ''' Function description.
    '''

    def fasta_reader(file):
        fasta = {}
        header = None
        with open(file) as fh:
            for i, line in enumerate(fh):
                line = line.strip()
                if line.startswith(">"):
                    if header:
                        fasta[header]="".join(seq)
                    header = line
                    seq = []
                else:
                    seq.append(line)
        if header:
            fasta[header] = "".join(seq)
        return fasta

    def revcomp(seq):
        new_seq = ""
        for i in seq[::-1]:
            if i == "A":
                new_seq += "T"
            elif i == "T":
                new_seq += "A"
            elif i == "G":
                new_seq += "C"
            else:
                new_seq += "G"
        return new_seq

    path = os.path.abspath(input_dir)
    inp_dir_list = os.listdir(path)
    for i in inp_dir_list:
        if ".fna" in i:
            input_fasta = path + "/" + i
            print("INPUT FASTA FILE IS %s" % input_fasta)
        if ".gff" in i:
            input_gff = path + "/" + i
            print("INPUT GFF FILE IS %s" % input_gff)

    prefix = input_fasta.split("/")[-1].split(".")[0]
    outdir = outdirectory + "/" + prefix
    print("OUTPUT DIRECTORY IS %s" % outdir)
    out_file = outdir + "/" + prefix + "_genes_seq.txt"
    print("OUTPUT FILE IS %s" % out_file)

    if os.path.exists(outdir):
        pass
    else:
        os.mkdir(outdir)

    fasta = fasta_reader(input_fasta)

    #gff_reader
    i = 0
    with open(out_file, "w") as fw:
        with open(input_gff) as fh:
            for line in fh:
                if line.startswith("#"):
                    continue
                if line.startswith(">"): # to read until fasta part starts
                    break
                line = line.strip().split("\t")
                if not "gene" in line[8]:
                    continue
                gene = line[8].split("gene")[1].split(";")[0][1:]
                contig = line[0]
                start = int(line[3])
                stop = int(line[4])
                strain = line[6]
                product = line[8].split(";")[-1]
                new_header = ">%s:%s:%s:%s:%s:%s" % (gene, prefix, contig, start, stop, product)
                i+=1
                for header, seq in fasta.items():
                    if header == ">" + contig:
                        fw.write(new_header + "\n")
                        if strain == "+":
                            fw.write(seq[start-1:stop] + "\n")
                        elif strain == "-":
                            fw.write(revcomp(seq[start-1:stop]) + "\n") #add reverce compliment
    fw.close()

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Prokka runner')
    parser.add_argument('-i','--input', help='path to prokka dir', required=True)
    parser.add_argument('-o','--outdir', help='output directory', required=True)
    parser.add_argument('-d','--dry', help='dry run', action='store_true')
    args = vars(parser.parse_args())

    input_dir = os.path.abspath(args["input"])
    outdirectory = os.path.abspath(args["outdir"])
    dry = args["dry"]

    if os.path.exists(outdirectory):
        pass
    else:
        os.mkdir(outdirectory)

    main(input_dir, outdirectory)
    print("DONE for path %s" % input_dir)


def VideoKartaold(num):
    old = []
    for n in range(num):
        print(n + 1, "-", "VideoKartani kiriting: ", end='')
        num_vk = int(input())
        old.append(num_vk)
    return old

num = int(input("Videkartalar sonini kiriting: "))
old_vk = VideoKartaold(num)
print(f"Videokartalarning eski ro'yxati: {old_vk}")
maximum = max(old_vk)
def VideoKartanew(old_vk):
    new = []
    for n in old_vk:
        if n != maximum:
           new.append(n)
    return new
new_vk = VideoKartanew(old_vk)
print(f"Videokartalarning yangi ro'yxati: {new_vk}")


"""
This file is part of the everest project.
See LICENSE.txt for licensing, CONTRIBUTORS.txt for contributor information.

Package setup file.

Created on Nov 3, 2011.
"""

import os

from distribute_setup import use_setuptools
use_setuptools()

from setuptools import setup
from setuptools import find_packages

here = os.path.abspath(os.path.dirname(__file__))
README = open(os.path.join(here, 'README.txt')).read()

setup_requirements = []

install_requirements = [
    ]

tests_requirements = install_requirements + [
    'nose>=1.1.0,<=1.1.99',
    'nosexcover>=1.0.4,<=1.0.99',
    'coverage==3.4',
    ]

setup(name='tractor',
      version='0.1',
      description=
        'A Python library to manipulate Trac tickets programmatically.',
      long_description=README,
      classifiers=[
        "Development Status :: 4 - Beta",
        "Environment :: Web Environment",
        "Intended Audience :: Developers",
        "Intended Audience :: Information Technology",
        "License :: OSI Approved :: MIT License",
        "Programming Language :: Python",
        "Topic :: Internet :: WWW/HTTP",
        ],
      author='Anna-Antonia Berger',
      author_email='berger@cenix.com',
      license="MIT",
      url='https://github.com/cenix/tractor',
      keywords='web trac',
      packages=find_packages(),
      include_package_data=True,
      zip_safe=False,
      setup_requires=setup_requirements,
      install_requires=install_requirements,
      tests_require=tests_requirements,
      test_suite="tractor/tests",
      entry_points="""
      """
      )


class Node():

    def __init__(self, key, right = None, left = None, parent = None, height = 1):
        self.key = key
        self.right = right
        self.left = left
        self.parent = parent
        self.height = height

class BinaryTree():

    def __init__(self, key = None):
        aux = Node(key)
        if key != None:
            self.root = aux

    def search(self, k):
        aux = self.root

        while aux:
            if aux.key == k:
                return aux

            elif aux.key>k:
                if aux.left != None:
                    aux = aux.left
                else:
                    return aux

            elif aux.key<k:
                if aux.right != None:
                    aux = aux.right
                else:
                    return aux

    def insert(self, k):
        aux = search(k)
        if aux.key>k:
            aux.left = Node(k, parent=aux)
        else:
            aux.right = Node(k, parent=aux)

class AVLtree():

    def __init__(self, nodo = None):
        self.nodo = nodo

    def search(self, key, R):
        return

    def insert(self, key, R):
        return
    def Rebalance(N):
        P = N.parent
        if N.left.height > N.right.height+1:
                return

    def delete(self, value, R):
        return


"""add columns to user

Revision ID: cbf708203191
Revises: 240d251d908b
Create Date: 2017-12-21 12:34:05.816547

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'cbf708203191'
down_revision = '240d251d908b'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('users', sa.Column('image_filename', sa.String(), nullable=True))
    op.add_column('users', sa.Column('image_url', sa.String(), nullable=True))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('users', 'image_url')
    op.drop_column('users', 'image_filename')
    # ### end Alembic commands ###