import numpy as np
from collections import Counter
import pandas as pd
from math import log10
import matplotlib.pyplot as plt

from HMM_errors import *

class SequenceReader:
    """
    This class builds HMM from  protein or DNA sequence alignment.

    """

    def __init__(self, path_to_file, amino_acids=True, deletion_sign="-"):

        alignment_list = []

        with open(path_to_file) as seq_file:
            for line in seq_file:
                alignment_list.append(list(line.strip()))

        if not alignment_list:
            raise EmptyFile("File {} is empty!".format(path_to_file))

        self.alignment_list = alignment_list
        self.alignment_len = len(alignment_list)

        self.position_list = SequenceReader.transpose(alignment_list)
        self.position_len = len(self.position_list)

        self.deletion_sign = deletion_sign

        self.alignment_alphabet = (
            "A", "R", "N", "D", "C", "E", "Q", "G", "H", "I", "L", "K", "M", "F", "P", "S", "T", "W", "Y",
            "V") if amino_acids else ("A", "C", "G", "T")
        self.amino_acids = amino_acids

        self.true_seq, self.match_emission, self.insert_emission = self.calculate_emissions()

        self.match_emission = pd.DataFrame(SequenceReader.transpose(self.match_emission))
        self.match_emission.index = self.alignment_alphabet
        self.insert_emission = pd.DataFrame(SequenceReader.transpose(self.insert_emission))
        self.insert_emission.index = self.alignment_alphabet

        self.L = len(self.true_seq)
        self.trans = self.calculate_transitions()

    def calculate_transitions(self):
        match_transition = {"M": [0] * (self.L + 1), "D": [0] * (self.L + 1), "I": [0] * (self.L + 1)}
        insert_transition = {"M": [0] * (self.L + 1), "D": [0] * (self.L + 1), "I": [0] * (self.L + 1)}
        # delete_transition = {"M": [float('nan')] + ([0] * self.L), "D": [float('nan')] + ([0] * self.L),
        #                      "I": [float('nan')] + ([0] * self.L)}
        delete_transition = {"M": [float(0)] + ([0] * self.L), "D": [float(0)] + ([0] * self.L),
                             "I": [float(0)] + ([0] * self.L)}

        trans = {"M": match_transition, "D": delete_transition, "I": insert_transition}

        for alignment_i, sequence in enumerate(self.alignment_list):
            last_seq_position = 0
            first_transition_index = "M"

            for true_seq_i, x in enumerate(self.true_seq):
                self.process_transitions(sequence, trans, last_seq_position, x, true_seq_i, first_transition_index)
                first_transition_index = "D" if sequence[x] == self.deletion_sign else "M"
                last_seq_position = x + 1
            else:
                if last_seq_position != 0:
                    true_seq_i += 1
                    self.process_transitions(sequence, trans, last_seq_position, len(sequence), true_seq_i,
                                             first_transition_index)

        return SequenceReader.divide_dict(trans)

    def process_insertion(self, insertion, insert_transition, sequence_sign, true_seq_i):
        last_transition_index = "D" if sequence_sign == self.deletion_sign else "M"
        if insertion:
            insert_transition["I"][true_seq_i] += len(insertion) - 1
            insert_transition[last_transition_index][true_seq_i] += 1
            last_transition_index = "I"
        return last_transition_index

    def process_transitions(self, sequence, trans, last_seq_position, x, true_seq_i, first_transition_index):
        insertion = SequenceReader.clear_list(sequence[last_seq_position:x])
        sign_in_sequence = sequence[x] if x < len(sequence) else True
        last_transition_index = self.process_insertion(insertion, trans["I"], sign_in_sequence, true_seq_i)
        trans[first_transition_index][last_transition_index][true_seq_i] += 1

    def calculate_emissions(self, deletion_sign="-", treshold=0.5):
        true_seq = []

        match_emission = []
        insert_emission = []

        full_occurrence_count = {}

        for i, position_i in enumerate(self.position_list):

            current_occurrence_count = Counter(position_i)  # UPDATE HERE!

            deletion_count = current_occurrence_count.get(deletion_sign, 0)

            if deletion_count / self.alignment_len < treshold:
                match_emission.append(SequenceReader.divide_list(self.build_count_column(current_occurrence_count)))
                insert_emission.append(SequenceReader.divide_list(self.build_count_column(full_occurrence_count)))
                true_seq.append(i)
                full_occurrence_count = {}
            else:
                full_occurrence_count = Counter(full_occurrence_count) + Counter(current_occurrence_count)
        else:
            insert_emission.append(SequenceReader.divide_list(self.build_count_column(full_occurrence_count)))

        # match_emission.insert(0,[float("NaN")]*len(self.alignment_alphabet))
        match_emission.insert(0, [float(0)] * len(self.alignment_alphabet))

        return true_seq, match_emission, insert_emission

    def build_count_column(self, occurrence_count):
        match_column = []
        for letter in self.alignment_alphabet:
            match_column.append(occurrence_count.get(letter, 0))
        return match_column

    def forward(self, sequence):

        FM = np.zeros(shape=(len(sequence) + 1, self.L + 1), dtype=np.float64)
        FI = np.zeros(shape=(len(sequence) + 1, self.L + 1), dtype=np.float64)
        FD = np.zeros(shape=(len(sequence) + 1, self.L + 1), dtype=np.float64)

        # row_len = seqeunce len = i
        # column_len = model len = j

        FI[0, 0] = np.log(self.insert_emission.loc[sequence[0]][0])
        # FM[1, 1] = np.log(self.match_emission.loc[sequence[1]][1]) + np.log(
        #     self.trans["M"]["M"][0] * np.exp(FM[0, 0]) + self.trans["I"]["M"][0] * np.exp(FI[0, 0])
        # )
        #
        # FI[1, 1] = np.log(self.insert_emission.loc[sequence[1]][1]) + np.log(
        #     self.trans["M"]["I"][0] * np.exp(FM[0, 1]) + self.trans["I"]["I"][0] * np.exp(FI[0, 1])
        # )
        #
        # FD[1, 1] = np.log(self.trans["M"]["D"][0] * np.exp(FM[1, 0]) + self.trans["I"]["D"][0] * np.exp(FI[1, 0]))

        for i, sign in enumerate(sequence):
            i += 1
            for j in range(1, self.L + 1):
                FM[i, j] = np.log(self.match_emission.loc[sign][j] / (1 / len(self.alignment_alphabet))) + np.log(
                    (self.trans["M"]["M"][j - 1] * np.exp(FM[i - 1][j - 1])) +
                    (self.trans["I"]["M"][j - 1] * np.exp(FI[i - 1][j - 1])) +
                    (self.trans["D"]["M"][j - 1] * np.exp(FD[i - 1][j - 1]))
                )
                FI[i, j] = np.log(self.insert_emission.loc[sign][j] / (1 / len(self.alignment_alphabet))) + np.log(
                    (self.trans["M"]["I"][j - 1] * np.exp(FM[i - 1][j])) +
                    (self.trans["I"]["I"][j - 1] * np.exp(FI[i - 1][j])) +
                    (self.trans["D"]["I"][j - 1] * np.exp(FD[i - 1][j]))
                )
                FD[i, j] = np.log(
                    (self.trans["M"]["D"][j - 1] * np.exp(FM[i][j - 1])) +
                    (self.trans["I"]["D"][j - 1] * np.exp(FI[i][j - 1])) +
                    (self.trans["D"]["D"][j - 1] * np.exp(FD[i][j - 1]))
                )
        return FM[i][j]

    @staticmethod
    def compare(sequence, HMM_model_1, HMM_model_2=False):
        if not HMM_model_2:
            HMM_model_2 = SequenceReader.random_model(HMM_model_1.position_len, HMM_model_1.alignment_len,
                                                      HMM_model_1.amino_acids, HMM_model_1.deletion_sign)
        return log10(HMM_model_1.forward(sequence) / HMM_model_2.forward(sequence))

    @staticmethod
    def pairwise(iterable):
        "s -> (s0,s1), (s1,s2), (s2, s3), ..."

        from itertools import tee

        a, b = tee(iterable)
        next(b, None)
        return zip(a, b)

    @staticmethod
    def transpose(iterable):
        return list(map(list, zip(*iterable)))

    @staticmethod
    def clear_list(list_to_clear, element_to_delete="-"):
        return [x for x in list_to_clear if x != element_to_delete]

    @staticmethod
    def divide_list(list_to_divide):
        list_to_divide = [x + 1 for x in list_to_divide]
        list_sum = sum(list_to_divide)
        return [x / list_sum for x in list_to_divide]

    @staticmethod
    def divide_dict(dict_to_divide):
        for first, second in dict_to_divide.items():
            fraction = [sum(x) + 1 * 3 for x in zip(*second.values())]
            for key_of_list, list_to_divide in second.items():
                list_to_divide = [(x + 1) / y for x, y in zip(list_to_divide, fraction)]
                dict_to_divide[first][key_of_list] = list_to_divide
        return dict_to_divide

def plot_matrix(title, data):
    fig = plt.figure(figsize=(10, 6))
    plt.plot(data, linestyle="", marker="o")
    plt.title(title)

    fig = plt.gcf()
    fig.savefig('plots/{}.png'.format(title.replace(" ", "_")))
    plt.show()

def plot_model(model, name, position):
    print("{} model len: {}".format(name, model.L))
    print("{} T, mE, iE shapes: {},{},{}".format(name, (9, len(model.trans["M"]["M"])), model.match_emission.shape,
                                                 model.insert_emission.shape))

    print("----------Position {} of {} HMM model.------------".format(position, name))
    plot_matrix("Position {} of {} HMM model: Match emission".format(position, name),
                model.match_emission.loc[:, position])
    plot_matrix("Position {} of {} HMM model: Insert emission".format(position, name),
                model.insert_emission.loc[:, position])

    trans_table = []
    trans_names = []
    for x1, y1 in model.trans.items():
        for x2, y2 in y1.items():
            print("{}-{}: {}".format(x1, x2, y2[50]))
            trans_table.append([y2[50]])
            trans_names.append("{}-{}".format(x1, x2))

    trans_df =  pd.DataFrame(trans_table,index=trans_names)
    print(trans_df)
    plot_matrix("Position {} of {} HMM model: Transitions".format(position, name), trans_df)

def main():
    atp = SequenceReader("ATPases.txt")
    gtp = SequenceReader("GTP_binding_proteins.txt")

    for position in [49, 50]:
        for model, name in [(atp, "ATPases"), (gtp, "GTP binding proteins")]:
            plot_model(model, name, position)

    y = []
    with open("Unclassified_proteins.txt") as file:
        for line in file:
            atp_score = atp.forward(line.strip())
            gtp_score = gtp.forward(line.strip())
            print()
            print(line.strip())
            print("ATP score: {}".format(atp_score))
            print("GTP score: {}".format(gtp_score))
            y.append(atp_score - gtp_score)

    fig = plt.figure(figsize=(10, 6))

    y_plus = [x if x >= 0 else 0 for x in y]
    y_minus = [x if x < 0 else 0 for x in y]

    x = [x for x in range(len(y))]
    plt.bar(x, y, tick_label=x, width=0.5)

    fig = plt.figure(figsize=(10, 6))
    ax = fig.add_subplot(111)
    ax.bar(x, y_minus, tick_label=x, width=0.5, color='r', label='GTP binding proteins')
    ax.bar(x, y_plus, tick_label=x, width=0.5, color='b', label='ATPases')
    plt.title('Family membership')
    ax.legend()
    plt.show()

if __name__ == "__main__":
    main()


from django.core.paginator import Paginator
import plotly.graph_objects as go
from django.shortcuts import render
from django.contrib import messages
import matplotlib.dates as mdates
from .api_calls import (
    fetch_coins,
    fetch_market_charts,
    fetch_coin_details,
    fetch_order_book,
    fetch_coin_history,
    API_COINCAP_URL,
    API_BASE_URL,
    API_BASE_URL_CHARTS,
    API_BASE_URL_DETAILS,
    API_ORDER_BOOK_URL,
    params,
    chart_params,
    order_params
)
import matplotlib
matplotlib.use('Agg')
from dateutil.parser import parse as parse_date
import matplotlib.pyplot as plt
from io import BytesIO
import base64
from datetime import datetime
import re
import datetime

def crypto(request):
    # Fetch data
    coin_data = fetch_coins(API_BASE_URL, params)

    # Check if data was successfully fetch
    if coin_data is None:
        messages.error(request, 'The limit for fetch requests has been reached. Please try again later.')
        return render(request, 'crypto_pulse/crypto.html')

    if 'error' in coin_data and coin_data['error']['code'] == 429:
        messages.error(request, 'The limit for fetch requests has been reached. Please try again later.')
        return render(request, 'crypto_pulse/crypto.html')

    # Split data into 10 coins per page
    paginator = Paginator(coin_data, 10)
    page_number = request.GET.get('page')
    page_obj = paginator.get_page(page_number)

    # Store coin data in the session
    request.session['coin_data'] = coin_data

    # List to store coin data as dictionaries
    coins = []
    for coin in page_obj:
        coin_info = {
            'name': coin.get('name'),
            'symbol': coin.get('symbol'),
            'image': coin.get('image'),
            'market_cap': coin.get('market_cap'),
            'daily_price_change': coin.get('price_change_percentage_24h'),
            'daily_volume': coin.get('total_volume'),
            'current_price': coin.get('current_price'),
        }
        coins.append(coin_info)

    return render(request, 'crypto_pulse/crypto.html', {'page_obj': page_obj, 'coins': coins})

def coin_chart(request, coin_symbol):
    # Format symbol
    coin_symbol = coin_symbol.upper() + 'USDT'
    if coin_symbol == 'USDTUSDT':
        coin_symbol = 'BUSDUSDT'
    title_coin_symbol = coin_symbol.replace('USDT', '/USDT')

    # Fetch data
    candles_data = fetch_market_charts(API_BASE_URL_CHARTS, coin_symbol=coin_symbol, params=chart_params)
    orderbook_data = fetch_order_book(API_ORDER_BOOK_URL, coin_symbol=coin_symbol, params=order_params)

    # Check if data was successfully fetched
    if candles_data is None and orderbook_data is None:
        messages.error(request, 'Failed to fetch candlestick data. Please try again.')
        return render(request, 'crypto_pulse/coin_chart.html')

    # Parse the orderbook data from Binance API
    bids_data = orderbook_data['bids']
    asks_data = orderbook_data['asks']

    # Parse the data from the Binance API to create the candlestick chart
    timestamps = [candle[0] for candle in candles_data]
    open_prices = [float(candle[1]) for candle in candles_data]
    high_prices = [float(candle[2]) for candle in candles_data]
    low_prices = [float(candle[3]) for candle in candles_data]
    close_prices = [float(candle[4]) for candle in candles_data]

    # Calculate the date range for the last 3 months
    today = datetime.date.today()
    three_months_ago = today - datetime.timedelta(days=90)

    # Filter the data to show only the last 3 months
    filtered_data = [(timestamp, open_price, high_price, low_price, close_price)
                     for timestamp, open_price, high_price, low_price, close_price in zip(timestamps, open_prices, high_prices, low_prices, close_prices)
                     if datetime.datetime.fromtimestamp(timestamp / 1000).date() >= three_months_ago]

    # Create the candlestick chart
    fig = go.Figure(data=[go.Candlestick(x=[candle[0] for candle in filtered_data],
                                        open=[candle[1] for candle in filtered_data],
                                        high=[candle[2] for candle in filtered_data],
                                        low=[candle[3] for candle in filtered_data],
                                        close=[candle[4] for candle in filtered_data])])

    # Customize the chart layout
    fig.update_layout(
        xaxis=dict(
            type='date',  # Set the x-axis type to 'date'
            range=[three_months_ago, today + datetime.timedelta(days=1)],  # Set the x-axis range to include an extra day
            gridcolor='rgba(255, 255, 255, 0.2)',  # Set the grid color with transparency
            showgrid=True,  # Show the grid lines
        ),
        yaxis=dict(
            gridcolor='rgba(255, 255, 255, 0.2)',  # Set the grid color with transparency for the y-axis
            showgrid=True,  # Show the grid lines for the y-axis
        ),
        plot_bgcolor='rgba(0, 0, 0, 0)',  # Set the plot area background to transparent
        paper_bgcolor='rgba(0, 0, 0, 0)',  # Set the entire chart background to transparent
        font=dict(color='white'),  # Set the font color for the labels and legend to white
        title=dict(text=f'{title_coin_symbol}',  # Set the title of the chart
                   x=0.5,  # Set the title position to the center of the chart
                   y=0.95,  # Set the title position relative to the y-axis
                   font=dict(size=24)  # Set the font size of the title
                   ),
        xaxis_rangeslider_visible=False
    )

    # Convert the chart to JSON to pass it to the template
    chart_json = fig.to_json()

    # Get coin data from the session
    coin_data = request.session.get('coin_data', [])
    coin_info = {}
    for coin in coin_data:
        real_coin_symbol = coin_symbol.replace('USDT', '').lower()
        if coin['symbol'] == real_coin_symbol:
            coin_info = {
                'name': coin.get('name'),
                'symbol': real_coin_symbol,
                'current_price': coin.get('current_price'),
                'daily_price_change': coin.get('price_change_percentage_24h'),
                'daily_volume': coin.get('total_volume'),
                'market_cap': coin.get('market_cap'),
            }
            break

    # Pass the selected coin's data and chart JSON to the template
    return render(request, 'crypto_pulse/coin_chart.html', {'chart_json': chart_json,
                                                            'coin_info': coin_info,
                                                            'bids': bids_data,
                                                            'asks': asks_data})

def coin_details(request, coin_name):
    # Fetch data for description
    if coin_name.lower() == 'xrp':
        coin_name = 'ripple'
    if coin_name.lower() == 'binance usd':
        coin_name = 'binance-usd'
    if coin_name.lower() == 'bnb':
        coin_name = 'busd'
    if coin_name.lower() == 'usd coin' or coin_name.lower() == 'usd%20coin':
        coin_name = 'usd-coin'
    if coin_name.lower() == 'wrapped bitcoin' or coin_name.lower() == 'wrapped%20bitcoin':
        coin_name = 'wrapped-bitcoin'
    if coin_name.lower() == 'shiba inu' or coin_name.lower() == 'shiba%20inu':
        coin_name = 'shiba-inu'
    if coin_name.lower() == 'bitcoin cash' or coin_name.lower() == 'bitcoin%20cash':
        coin_name = 'bitcoin-cash'
    if coin_name.lower() == 'ethereum classic' or coin_name.lower() == 'ethereum%20classic':
        coin_name = 'ethereum-classic'
    if coin_name.lower() == 'cosmos hub' or coin_name.lower() == 'cosmos%20hub':
        coin_name = 'cosmos-hub'

    coin_details = fetch_coin_details(API_BASE_URL_DETAILS, coin_name=coin_name)

    if coin_name.lower() == 'ripple':
        coin_name = 'xrp'

    coin_history = fetch_coin_history(API_COINCAP_URL, coin_name=coin_name)

    if coin_history is None:
        messages.error(request, 'Failed to fetch data. Please try again.')
        return render(request, 'crypto_pulse/coin_chart.html')

    data = coin_history['data']

    # Parse coin history
    data = coin_history['data']

    prices = []
    dates = []

    for data_point in data:
        price = data_point['priceUsd']
        date = data_point['date']

        prices.append(float(price))
        dates.append(date)

    # Convert formatted_dates to datetime objects
    formatted_dates = [parse_date(date_string) for date_string in dates]

    # Clear plots
    plt.clf()
    plt.figure(figsize=(11, 4))  # Adjust the height to 4

    # Creating line chart
    plt.plot(formatted_dates, prices, linewidth=2)

    # Plot info and style
    plt.style.use('_mpl-gallery')
    plt.xlabel('', color='white')  # Increase font size to 12
    plt.ylabel('', color='white')  # Increase font size to 12
    plt.title(f'{coin_name} price', color='white')  # Increase font size to 16
    plt.tight_layout()

    # Remove the grid from both axes
    plt.grid(False)

    # Style tick labels on both axes to have a white font color and increase font size to 10
    plt.tick_params(axis='x', colors='white')
    plt.tick_params(axis='y', colors='white', labelsize=8)

    # Configure x-axis tick marks to display fewer dates
    locator = mdates.AutoDateLocator(minticks=5, maxticks=9)
    formatter = mdates.ConciseDateFormatter(locator)
    plt.gca().xaxis.set_major_locator(locator)
    plt.gca().xaxis.set_major_formatter(formatter)

    # Convert the plot to an image
    buffer = BytesIO()
    plt.savefig(buffer, format='png', transparent=True)
    buffer.seek(0)
    line_chart = base64.b64encode(buffer.getvalue()).decode('utf-8')
    buffer.close()

    # Check if data was successfully fetched
    if coin_details is None:
        messages.error(request, 'Failed to fetch data. Please try again.')
        return render(request, 'crypto_pulse/coin_chart.html')

    # Create a dictionary to store descriptions
    description = coin_details['description']['en']
    name = coin_details['name']
    symbol = coin_details['symbol'].upper()

    # Clean the description by removing URLs and links
    cleaned_description = description.replace('<a href="', '')
    url_pattern = re.compile(r'http[s]?://\S+|www\.\S+')
    cleaned_description = re.sub(url_pattern, '', cleaned_description)

    context = {
        'description': cleaned_description,
        'name': name,
        'symbol': symbol,
        'line_chart': line_chart
    }

    return render(request, 'crypto_pulse/coin_details.html', context)


import sys

if __name__=="__main__":
   """ Solution for challenge 1 """
   # Read the first line
   sys.stdin.readline()

   # Iterate the remaining lines
   for case in sys.stdin:
      print (int(case) + 1)/2


from sklearn.decomposition import FastICA
import numpy
import matplotlib.pyplot as plt
import time
from scipy.io import wavfile
from Processor import Processor
from sklearn import preprocessing

class ICA:
    def __init__(self, mixed_sig, no_of_components):
        self.mixed_signal = mixed_sig
        self.num_components = no_of_components

    def train(self, opt, proc):
        print("Training ICA...")
        time_start = time.time()
        # whitening necessary

        if opt == 2:
            ica = FastICA(n_components=self.num_components, whiten=False)
        elif opt == 3:
            ica = FastICA(n_components=self.num_components, fun='exp')
        elif opt == 4:
            ica = FastICA(n_components=self.num_components, fun='cube')
        elif opt == 5:
            self.mixed_signal = preprocessing.scale(self.mixed_signal)
            ica = FastICA(n_components=self.num_components)
        else:
            ica = FastICA(n_components=self.num_components)

        # self.mixed_signal = preprocessing.scale(self.mixed_signal)
        reconstruct_signal = ica.fit_transform(self.mixed_signal)
        mixing_matrix = ica.mixing_

        time_stop = time.time() - time_start
        print("Training Complete under {} seconds".format(time_stop))

        if opt == 2:
            assert numpy.allclose(self.mixed_signal, numpy.dot(reconstruct_signal, mixing_matrix.T))
        else:
            assert numpy.allclose(self.mixed_signal, numpy.dot(reconstruct_signal, mixing_matrix.T) + ica.mean_)

        if opt == 2:
            remixed_mat = numpy.dot(reconstruct_signal, mixing_matrix.T)
        else:
            remixed_mat = numpy.dot(reconstruct_signal, mixing_matrix.T) + ica.mean_

        for q in range(len(remixed_mat.T)):
            y = remixed_mat.T[q] - self.mixed_signal.T[q]
            q = sum([i**2 for i in y])
            print("Residual value: {}".format(q))

        # for i in range(len(remixed_mat.T)):
        #     wavfile.write('aaa.wav', proc[0], numpy.asarray(remixed_mat.T[0], dtype=numpy.int16))
        return reconstruct_signal, mixing_matrix, remixed_mat

    def create_audio(self, reconstructed_mat, s_hat_names, remixed_mat, recon_names, rates):
        print("Creating recovered audio...")
        for count in range(len(s_hat)):
            wavfile.write(s_hat[count], rates[count], reconstructed_mat.T[count])
            wavfile.write(recon_names[count], rates[count], numpy.asarray(remixed_mat.T[count], dtype=numpy.int16))

    def plot_audio(self, mixed_sig, recovered_sig):
        print("Plotting graphs...")
        models = [mixed_sig, recovered_sig]
        names = ['Observations (mixed signal)',
                 'ICA recovered signals']
        colors = ['red', 'steelblue', 'orange', 'green', 'brown']

        plt.figure()

        for ii, (model, name) in enumerate(zip(models, names), 1):
            plt.subplot(2, 1, ii)
            plt.title(name)
            for sig, color in zip(model.T, colors):
                plt.plot(sig, color=color)

        plt.subplots_adjust(0.09, 0.04, 0.94, 0.94, 0.26, 0.46)
        plt.show()

if __name__ == '__main__':
    audios = ['mic1.wav', 'mic2.wav', 'mic3.wav', 'mic4.wav', 'mic5.wav']
    s_hat = ['shat1.wav', 'shat2.wav', 'shat3.wav', 'shat4.wav', 'shat5.wav']
    recon = ['recon1.wav', 'recon2.wav', 'recon3.wav', 'recon4.wav', 'recon5.wav']

    processor = Processor(audios)
    processor.generate_wav_data()

    # Options: 1: default, 2: whitening off, 3: exp function, 4: cube function, 5: centering
    option = 1
    ica_ = ICA(processor.mixed_signal, 5)
    recon_matrix, mixing_matrix, remixed = ica_.train(option, processor.rates)
    ica_.create_audio(recon_matrix, s_hat, remixed, recon, processor.rates)
    ica_.plot_audio(processor.mixed_signal, recon_matrix)


import pandas as pd

# WordsExtracting - extracts the toCutNumnber-fromCutNumber words that appear the most often
# gets path csv and saves csv to savePath
# xendcolumn not included -> if you want the last piece as well =None
def WordsExtracting(path="../preprocessing/ProcessedData.csv", savePath = "SelectedData.csv", fromCutNumber=0, toCutNumber=1000,wordCountColumn=-2, xStartColumn =0, xEndColumn=-2, pathColumn=-1):

   print("Reading data")
   data = pd.read_csv(path)
   df= data

   if xEndColumn is not None:
      data = data.iloc[:, xStartColumn:xEndColumn]
   else:
      data = data.iloc[:, xStartColumn:]

   print("sorting")

   data = data.reindex(data.sum().sort_values(ascending=False).index, axis=1)

   print("cutting the words")
   data = data.iloc[:, fromCutNumber:toCutNumber]

   data['Word_Count'] = df[(list(df.columns))[wordCountColumn]]
   data['path_from_file'] = df[(list(df.columns))[pathColumn]]

   print("saving to csv file")
   data.to_csv(savePath, index=False)

# parameters for column indeces
# wordCountColumnNew - where in pathNewCSV is the Word_Count column
# pathColumnNew - where is the path column
# xStartColumnNew - where the words start in the new file
# xEndColumnNew - where the words end in the new file + 1 (not included) if you want the last piece as well =None )
# Old by analogy

def TakeMatchingWords(pathNewCSV = "ProcessedData.csv", savePath = "SelectedData.csv", trainedDataPath = "classifier/SelectedData.csv",wordCountColumnNew=-2, xStartColumnNew =0, xEndColumnNew=-2, pathColumnNew=-1, xStartColumnOld=3, xEndColumnOld=-1):
   print("Reading new data")
   dataNew = pd.read_csv(pathNewCSV)
   dfNew= dataNew

   print("reading old data")
   dataOld= pd.read_csv(trainedDataPath)

   print("droping path and word count")
   if xEndColumnOld is not None:
      dataOld = dataOld.iloc[:, xStartColumnOld:xEndColumnOld]
   else:
      dataOld = dataOld.iloc[:, xStartColumnOld:]

   if xEndColumnNew is not None:
      dataNew = dataNew.iloc[:, xStartColumnNew:xEndColumnNew]
   else:
      dataNew = dataNew.iloc[:, xStartColumnNew:]

   print("take the matching words")

   dataNewToSave = pd.DataFrame(columns=dataOld.keys())
   columnLength=dataNew.shape[0]

   for key in dataOld.keys():
      if key in dataNew.keys():
         dataNewToSave[key]=dataNew[key]
      else:
         dataNewToSave[key] = [0 for i in range(columnLength)]

   dataNewToSave['Word_Count'] = dfNew[(list(dfNew.columns))[wordCountColumnNew]]
   dataNewToSave['path_from_file'] = dfNew[(list(dfNew.columns))[pathColumnNew]]

   print("saving to csv file")
   dataNewToSave.to_csv(savePath, index=False)

if __name__ == '__main__':
   WordsExtracting(path="../preprocessing/ProcessedDataLem.csv",savePath="SelectedDataLem.csv")

   WordsExtracting(path="../preprocessing/ProcessedData.csv",savePath="SelectedData.csv")


"""GLOBIO InVEST Model"""

import os
import logging

import gdal
import osr
import numpy
import pygeoprocessing

logging.basicConfig(format='%(asctime)s %(name)-20s %(levelname)-8s \
%(message)s', level=logging.DEBUG, datefmt='%m/%d/%Y %H:%M:%S ')

LOGGER = logging.getLogger('invest_natcap.globio.globio')

def execute(args):
    """main execute entry point"""

    #append a _ to the suffix if it's not empty and doens't already have one
    try:
        file_suffix = args['results_suffix']
        if file_suffix != "" and not file_suffix.startswith('_'):
            file_suffix = '_' + file_suffix
    except KeyError:
        file_suffix = ''

    pygeoprocessing.geoprocessing.create_directories([args['workspace_dir']])

    if not args['predefined_globio']:
        out_pixel_size = pygeoprocessing.geoprocessing.get_cell_size_from_uri(
            args['lulc_uri'])
    else:
        out_pixel_size = pygeoprocessing.geoprocessing.get_cell_size_from_uri(
            args['globio_lulc_uri'])

    if not args['predefined_globio']:
        #reclassify the landcover map
        lulc_to_globio_table = pygeoprocessing.geoprocessing.get_lookup_from_table(
            args['lulc_to_globio_table_uri'], 'lucode')

        lulc_to_globio = dict(
            [(lulc_code, int(table['globio_lucode'])) for
             (lulc_code, table) in lulc_to_globio_table.items()])

        intermediate_globio_lulc_uri = os.path.join(
            args['workspace_dir'], 'intermediate_globio_lulc%s.tif' % file_suffix)
        globio_nodata = -1
        pygeoprocessing.geoprocessing.reclassify_dataset_uri(
            args['lulc_uri'], lulc_to_globio, intermediate_globio_lulc_uri,
            gdal.GDT_Int32, globio_nodata, exception_flag='values_required')

        globio_lulc_uri = os.path.join(
            args['workspace_dir'], 'globio_lulc%s.tif' % file_suffix)

        sum_yieldgap_uri = args['sum_yieldgap_uri']
        potential_vegetation_uri = args['potential_vegetation_uri']
        pasture_uri = args['pasture_uri']

        #smoothed natural areas are natural areas run through a gaussian filter
        natural_areas_uri = os.path.join(
            args['workspace_dir'], 'natural_areas%s.tif' % file_suffix)
        natural_areas_nodata = -1

        def natural_area_mask_op(lulc_array):
            """masking out natural areas"""
            nodata_mask = lulc_array == globio_nodata
            result = (
                (lulc_array == 130) | (lulc_array == 1))
            return numpy.where(nodata_mask, natural_areas_nodata, result)

        LOGGER.info("create mask of natural areas")
        pygeoprocessing.geoprocessing.vectorize_datasets(
            [intermediate_globio_lulc_uri], natural_area_mask_op,
            natural_areas_uri, gdal.GDT_Int32, natural_areas_nodata,
            out_pixel_size, "intersection", dataset_to_align_index=0,
            assert_datasets_projected=False, vectorize_op=False)

        LOGGER.info('gaussian filter natural areas')
        sigma = 9.0
        gaussian_kernel_uri = os.path.join(
            args['workspace_dir'], 'gaussian_kernel%s.tif' % file_suffix)
        make_gaussian_kernel_uri(sigma, gaussian_kernel_uri)
        smoothed_natural_areas_uri = os.path.join(
            args['workspace_dir'], 'smoothed_natural_areas%s.tif' % file_suffix)
        pygeoprocessing.geoprocessing.convolve_2d_uri(
            natural_areas_uri, gaussian_kernel_uri, smoothed_natural_areas_uri)

        ffqi_uri = os.path.join(
            args['workspace_dir'], 'ffqi%s.tif' % file_suffix)

        def ffqi_op(natural_areas_array, smoothed_natural_areas):
            """mask out ffqi only where there's an ffqi"""
            return numpy.where(
                natural_areas_array != natural_areas_nodata,
                natural_areas_array * smoothed_natural_areas,
                natural_areas_nodata)

        LOGGER.info('calculate ffqi')
        pygeoprocessing.geoprocessing.vectorize_datasets(
            [natural_areas_uri, smoothed_natural_areas_uri], ffqi_op,
            ffqi_uri, gdal.GDT_Float32, natural_areas_nodata,
            out_pixel_size, "intersection", dataset_to_align_index=0,
            assert_datasets_projected=False, vectorize_op=False)

        #remap globio lulc to an internal lulc based on ag and yield gaps
        #these came from the 'expansion_scenarios.py' script as numbers Justin
        #provided way back on the unilever project.
        high_intensity_agriculture_threshold = float(args['high_intensity_agriculture_threshold'])
        pasture_threshold = float(args['pasture_threshold'])
        yieldgap_threshold = float(args['yieldgap_threshold'])
        primary_threshold = float(args['primary_threshold'])

        sum_yieldgap_nodata = pygeoprocessing.geoprocessing.get_nodata_from_uri(
            args['sum_yieldgap_uri'])

        potential_vegetation_nodata = (
            pygeoprocessing.geoprocessing.get_nodata_from_uri(
                args['potential_vegetation_uri']))
        pasture_nodata = pygeoprocessing.geoprocessing.get_nodata_from_uri(
            args['pasture_uri'])

        def create_globio_lulc(
                lulc_array, sum_yieldgap, potential_vegetation_array, pasture_array,
                ffqi):

            #Step 1.2b: Assign high/low according to threshold based on yieldgap.
            nodata_mask = lulc_array == globio_nodata
            high_low_intensity_agriculture = numpy.where(
                sum_yieldgap < yieldgap_threshold *
                high_intensity_agriculture_threshold, 9.0, 8.0)

            #Step 1.2c: Stamp ag_split classes onto input LULC
            lulc_ag_split = numpy.where(
                lulc_array == 132.0, high_low_intensity_agriculture, lulc_array)
            nodata_mask = nodata_mask | (lulc_array == globio_nodata)

            #Step 1.3a: Split Scrublands and grasslands into pristine
            #vegetations, livestock grazing areas, and man-made pastures.
            three_types_of_scrubland = numpy.where(
                (potential_vegetation_array <= 8) & (lulc_ag_split == 131), 6.0,
                5.0)

            three_types_of_scrubland = numpy.where(
                (three_types_of_scrubland == 5.0) &
                (pasture_array < pasture_threshold), 1.0,
                three_types_of_scrubland)

            #Step 1.3b: Stamp ag_split classes onto input LULC
            broad_lulc_shrub_split = numpy.where(
                lulc_ag_split == 131, three_types_of_scrubland, lulc_ag_split)

            #Step 1.4a: Split Forests into Primary, Secondary
            four_types_of_forest = numpy.empty(lulc_array.shape)
            #1.0 is primary forest
            four_types_of_forest[(ffqi >= primary_threshold)] = 1
            #3 is secondary forest
            four_types_of_forest[(ffqi < primary_threshold)] = 3

            #Step 1.4b: Stamp ag_split classes onto input LULC
            globio_lulc = numpy.where(
                broad_lulc_shrub_split == 130, four_types_of_forest,
                broad_lulc_shrub_split) #stamp primary vegetation

            return numpy.where(nodata_mask, globio_nodata, globio_lulc)

        LOGGER.info('create the globio lulc')
        pygeoprocessing.geoprocessing.vectorize_datasets(
            [intermediate_globio_lulc_uri, sum_yieldgap_uri,
             potential_vegetation_uri, pasture_uri, ffqi_uri],
            create_globio_lulc, globio_lulc_uri, gdal.GDT_Int32, globio_nodata,
            out_pixel_size, "intersection", dataset_to_align_index=0,
            assert_datasets_projected=False, vectorize_op=False)
    else:
        LOGGER.info('no need to calcualte GLOBIO LULC because it is passed in')
        globio_lulc_uri = args['globio_lulc_uri']
        globio_nodata = pygeoprocessing.get_nodata_from_uri(globio_lulc_uri)

    """This is from Justin's old code:
    #Step 1.2b: Assign high/low according to threshold based on yieldgap.
    #high_low_intensity_agriculture_uri = args["export_folder"]+"high_low_intensity_agriculture_"+args['run_id']+".tif"
    high_intensity_agriculture_threshold = 1 #hardcode for now until UI is determined. Eventually this is a user input. Do I bring it into the ARGS dict?
    high_low_intensity_agriculture = numpy.where(sum_yieldgap < float(args['yieldgap_threshold']*high_intensity_agriculture_threshold), 9.0, 8.0) #45. = average yieldgap on global cells with nonzero yieldgap.

    #Step 1.2c: Stamp ag_split classes onto input LULC
    broad_lulc_ag_split = numpy.where(broad_lulc_array==132.0, high_low_intensity_agriculture, broad_lulc_array)

    #Step 1.3a: Split Scrublands and grasslands into pristine vegetations,
    #livestock grazing areas, and man-made pastures.
    three_types_of_scrubland = numpy.zeros(scenario_lulc_array.shape)
    potential_vegetation_array = geotiff_to_array(aligned_agriculture_uris[0])
    three_types_of_scrubland = numpy.where((potential_vegetation_array <= 8) & (broad_lulc_ag_split== 131), 6.0, 5.0) # < 8 min potential veg means should have been forest, 131 in broad  is grass, so 1.0 implies man made pasture
    pasture_array = geotiff_to_array(aligned_agriculture_uris[1])
    three_types_of_scrubland = numpy.where((three_types_of_scrubland == 5.0) & (pasture_array < args['pasture_threshold']), 1.0, three_types_of_scrubland)

    #Step 1.3b: Stamp ag_split classes onto input LULC
    broad_lulc_shrub_split = numpy.where(broad_lulc_ag_split==131, three_types_of_scrubland, broad_lulc_ag_split)

    #Step 1.4a: Split Forests into Primary, Secondary, Lightly Used and Plantation.
    sigma = 9
    primary_threshold = args['primary_threshold']
    secondary_threshold = args['secondary_threshold']
    is_natural = (broad_lulc_shrub_split == 130) | (broad_lulc_shrub_split == 1)
    blurred = scipy.ndimage.filters.gaussian_filter(is_natural.astype(float), sigma, mode='constant', cval=0.0)
    ffqi = blurred * is_natural

    four_types_of_forest = numpy.empty(scenario_lulc_array.shape)
    four_types_of_forest[(ffqi >= primary_threshold)] = 1.0
    four_types_of_forest[(ffqi < primary_threshold) & (ffqi >= secondary_threshold)] = 3.0
    four_types_of_forest[(ffqi < secondary_threshold)] = 4.0

    #Step 1.4b: Stamp ag_split classes onto input LULC
    globio_lulc = numpy.where(broad_lulc_shrub_split == 130 ,four_types_of_forest, broad_lulc_shrub_split) #stamp primary vegetation

    return globio_lulc"""

    #load the infrastructure layers from disk
    infrastructure_filenames = []
    infrastructure_nodata_list = []
    for root_directory, _, filename_list in os.walk(
            args['infrastructure_dir']):

        for filename in filename_list:
            LOGGER.debug(filename)
            if filename.lower().endswith(".tif"):
                LOGGER.debug("tiff added %s", filename)
                infrastructure_filenames.append(
                    os.path.join(root_directory, filename))
                infrastructure_nodata_list.append(
                    pygeoprocessing.geoprocessing.get_nodata_from_uri(
                        infrastructure_filenames[-1]))
            if filename.lower().endswith(".shp"):
                LOGGER.debug("shape added %s", filename)
                infrastructure_tmp_raster = (
                   os.path.join(args['workspace_dir'], os.path.basename(filename.lower() + ".tif")))
                pygeoprocessing.geoprocessing.new_raster_from_base_uri(
                    globio_lulc_uri, infrastructure_tmp_raster,
                    'GTiff', -1.0, gdal.GDT_Int32, fill_value=0)
                pygeoprocessing.geoprocessing.rasterize_layer_uri(
                    infrastructure_tmp_raster,
                    os.path.join(root_directory, filename), burn_values=[1],
                    option_list=["ALL_TOUCHED=TRUE"])
                infrastructure_filenames.append(infrastructure_tmp_raster)
                infrastructure_nodata_list.append(
                    pygeoprocessing.geoprocessing.get_nodata_from_uri(
                        infrastructure_filenames[-1]))

    if len(infrastructure_filenames) == 0:
        raise ValueError(
            "infrastructure directory didn't have any GeoTIFFS or "
            "Shapefiles at %s", args['infrastructure_dir'])

    infrastructure_nodata = -1
    infrastructure_uri = os.path.join(
        args['workspace_dir'], 'combined_infrastructure%s.tif' % file_suffix)

    def collapse_infrastructure_op(*infrastructure_array_list):
        """Combines all input infrastructure into a single map where if any
            pixel on the stack is 1 gets passed through, any nodata pixel
            masks out all of them"""
        nodata_mask = (
            infrastructure_array_list[0] == infrastructure_nodata_list[0])
        infrastructure_result = infrastructure_array_list[0] > 0
        for index in range(1, len(infrastructure_array_list)):
            current_nodata = (
                infrastructure_array_list[index] ==
                infrastructure_nodata_list[index])

            infrastructure_result = (
                infrastructure_result |
                ((infrastructure_array_list[index] > 0) & ~current_nodata))

            nodata_mask = (
                nodata_mask & current_nodata)

        return numpy.where(
            nodata_mask, infrastructure_nodata, infrastructure_result)

    LOGGER.info('collapse infrastructure into one raster')
    pygeoprocessing.geoprocessing.vectorize_datasets(
        infrastructure_filenames, collapse_infrastructure_op,
        infrastructure_uri, gdal.GDT_Byte, infrastructure_nodata,
        out_pixel_size, "intersection", dataset_to_align_index=0,
        assert_datasets_projected=False, vectorize_op=False)

    #calc_msa_f
    primary_veg_mask_uri = os.path.join(
        args['workspace_dir'], 'primary_veg_mask%s.tif' % file_suffix)
    primary_veg_mask_nodata = -1

    def primary_veg_mask_op(lulc_array):
        """masking out natural areas"""
        nodata_mask = lulc_array == globio_nodata
        result = (lulc_array == 1)
        return numpy.where(nodata_mask, primary_veg_mask_nodata, result)

    LOGGER.info("create mask of primary veg areas")
    pygeoprocessing.geoprocessing.vectorize_datasets(
        [globio_lulc_uri], primary_veg_mask_op,
        primary_veg_mask_uri, gdal.GDT_Int32, primary_veg_mask_nodata,
        out_pixel_size, "intersection", dataset_to_align_index=0,
        assert_datasets_projected=False, vectorize_op=False)

    LOGGER.info('gaussian filter primary veg')
    sigma = 9.0
    gaussian_kernel_uri = os.path.join(
        args['workspace_dir'], 'gaussian_kernel%s.tif' % file_suffix)
    make_gaussian_kernel_uri(sigma, gaussian_kernel_uri)
    smoothed_primary_veg_mask_uri = os.path.join(
        args['workspace_dir'], 'smoothed_primary_veg_mask%s.tif' % file_suffix)
    pygeoprocessing.geoprocessing.convolve_2d_uri(
        primary_veg_mask_uri, gaussian_kernel_uri, smoothed_primary_veg_mask_uri)

    primary_veg_smooth_uri = os.path.join(
        args['workspace_dir'], 'ffqi%s.tif' % file_suffix)

    def primary_veg_smooth_op(primary_veg_mask_array, smoothed_primary_veg_mask):
        """mask out ffqi only where there's an ffqi"""
        return numpy.where(
            primary_veg_mask_array != primary_veg_mask_nodata,
            primary_veg_mask_array * smoothed_primary_veg_mask,
            primary_veg_mask_nodata)

    LOGGER.info('calculate primary_veg_smooth')
    pygeoprocessing.geoprocessing.vectorize_datasets(
        [primary_veg_mask_uri, smoothed_primary_veg_mask_uri],
        primary_veg_smooth_op, primary_veg_smooth_uri, gdal.GDT_Float32,
        primary_veg_mask_nodata, out_pixel_size, "intersection",
        dataset_to_align_index=0, assert_datasets_projected=False,
        vectorize_op=False)

    msa_nodata = -1
    def msa_f_op(primary_veg_smooth):
        """calcualte msa fragmentation"""
        nodata_mask = primary_veg_mask_nodata == primary_veg_smooth

        msa_f = numpy.empty(primary_veg_smooth.shape)
        msa_f[:] = 1.0
        #These thresholds come from FFQI from Justin's code; I don't
        #know where they otherwise came from.
        msa_f[(primary_veg_smooth > .9825) & (primary_veg_smooth <= .9984)] = 0.95
        msa_f[(primary_veg_smooth > .89771) & (primary_veg_smooth <= .9825)] = 0.90
        msa_f[(primary_veg_smooth > .578512) & (primary_veg_smooth <= .89771)] = 0.7
        msa_f[(primary_veg_smooth > .42877) & (primary_veg_smooth <= .578512)] = 0.6
        msa_f[(primary_veg_smooth <= .42877)] = 0.3
        msa_f[nodata_mask] = msa_nodata

        return msa_f

    LOGGER.info('calculate msa_f')
    msa_f_uri = os.path.join(args['workspace_dir'], 'msa_f%s.tif' % file_suffix)
    pygeoprocessing.geoprocessing.vectorize_datasets(
        [primary_veg_smooth_uri], msa_f_op, msa_f_uri, gdal.GDT_Float32,
        msa_nodata, out_pixel_size, "intersection", dataset_to_align_index=0,
        assert_datasets_projected=False, vectorize_op=False)

    #calc_msa_i
    infrastructure_impact_zones = {
        'no impact': 1.0,
        'low impact': 0.9,
        'medium impact': 0.8,
        'high impact': 0.4
    }

    def msa_i_op(lulc_array, distance_to_infrastructure):
        """calculate msa infrastructure"""
        msa_i_tropical_forest = numpy.empty(lulc_array.shape)
        distance_to_infrastructure *= out_pixel_size #convert to meters
        msa_i_tropical_forest[:] = infrastructure_impact_zones['no impact']
        msa_i_tropical_forest[(distance_to_infrastructure > 4000.0) & (distance_to_infrastructure <= 14000.0)] = infrastructure_impact_zones['low impact']
        msa_i_tropical_forest[(distance_to_infrastructure > 1000.0) & (distance_to_infrastructure <= 4000.0)] = infrastructure_impact_zones['medium impact']
        msa_i_tropical_forest[(distance_to_infrastructure <= 1000.0)] = infrastructure_impact_zones['high impact']

        msa_i_temperate_and_boreal_forest = numpy.empty(lulc_array.shape)
        msa_i_temperate_and_boreal_forest[:] = infrastructure_impact_zones['no impact']
        msa_i_temperate_and_boreal_forest[(distance_to_infrastructure > 1200.0) & (distance_to_infrastructure <= 4200.0)] = infrastructure_impact_zones['low impact']
        msa_i_temperate_and_boreal_forest[(distance_to_infrastructure > 300.0) & (distance_to_infrastructure <= 1200.0)] = infrastructure_impact_zones['medium impact']
        msa_i_temperate_and_boreal_forest[(distance_to_infrastructure <= 300.0)] = infrastructure_impact_zones['high impact']

        msa_i_cropland_and_grassland = numpy.empty(lulc_array.shape)
        msa_i_cropland_and_grassland[:] = infrastructure_impact_zones['no impact']
        msa_i_cropland_and_grassland[(distance_to_infrastructure > 2000.0) & (distance_to_infrastructure <= 7000.0)] = infrastructure_impact_zones['low impact']
        msa_i_cropland_and_grassland[(distance_to_infrastructure > 500.0) & (distance_to_infrastructure <= 2000.0)] = infrastructure_impact_zones['medium impact']
        msa_i_cropland_and_grassland[(distance_to_infrastructure <= 500.0)] = infrastructure_impact_zones['high impact']

        msa_i = numpy.where((lulc_array >= 1) & (lulc_array <= 5), msa_i_temperate_and_boreal_forest, infrastructure_impact_zones['no impact'])
        msa_i = numpy.where((lulc_array >= 6) & (lulc_array <= 12), msa_i_cropland_and_grassland, msa_i)

        return msa_i

    LOGGER.info('calculate msa_i')
    distance_to_infrastructure_uri = os.path.join(
        args['workspace_dir'], 'distance_to_infrastructure%s.tif' % file_suffix)
    pygeoprocessing.geoprocessing.distance_transform_edt(
        infrastructure_uri, distance_to_infrastructure_uri)
    msa_i_uri = os.path.join(args['workspace_dir'], 'msa_i%s.tif' % file_suffix)
    pygeoprocessing.geoprocessing.vectorize_datasets(
        [globio_lulc_uri, distance_to_infrastructure_uri], msa_i_op, msa_i_uri,
        gdal.GDT_Float32, msa_nodata, out_pixel_size, "intersection",
        dataset_to_align_index=0, assert_datasets_projected=False,
        vectorize_op=False)

    #calc_msa_lu
    lu_msa_lookup = {
        0.0: 0.0, #map 0 to 0
        1.0: 1.0, #primary veg
        2.0: 0.7, #lightly used natural forest
        3.0: 0.5, #secondary forest
        4.0: 0.2, #forest plantation
        5.0: 0.7, #livestock grazing
        6.0: 0.1, #man-made pastures
        7.0: 0.5, #agroforesty
        8.0: 0.3, #low-input agriculture
        9.0: 0.1, #intenstive agriculture
        10.0: 0.05, #built-up areas
    }
    msa_lu_uri = os.path.join(
        args['workspace_dir'], 'msa_lu%s.tif' % file_suffix)
    LOGGER.info('calculate msa_lu')
    pygeoprocessing.geoprocessing.reclassify_dataset_uri(
        globio_lulc_uri, lu_msa_lookup, msa_lu_uri,
        gdal.GDT_Float32, globio_nodata, exception_flag='values_required')

    LOGGER.info('calculate msa')
    msa_uri = os.path.join(
        args['workspace_dir'], 'msa%s.tif' % file_suffix)
    def msa_op(msa_f, msa_lu, msa_i):
        return numpy.where(
            msa_f != globio_nodata, msa_f* msa_lu * msa_i, globio_nodata)
    pygeoprocessing.geoprocessing.vectorize_datasets(
        [msa_f_uri, msa_lu_uri, msa_i_uri], msa_op, msa_uri,
        gdal.GDT_Float32, msa_nodata, out_pixel_size, "intersection",
        dataset_to_align_index=0, assert_datasets_projected=False,
        vectorize_op=False)
    #calc msa msa = msa_f[tail_type] * msa_lu[tail_type] * msa_i[tail_type]

def make_gaussian_kernel_uri(sigma, kernel_uri):
    """create a gaussian kernel raster"""
    max_distance = sigma * 5
    kernel_size = int(numpy.round(max_distance * 2 + 1))

    driver = gdal.GetDriverByName('GTiff')
    kernel_dataset = driver.Create(
        kernel_uri.encode('utf-8'), kernel_size, kernel_size, 1,
        gdal.GDT_Float32, options=['BIGTIFF=IF_SAFER'])

    #Make some kind of geotransform, it doesn't matter what but
    #will make GIS libraries behave better if it's all defined
    kernel_dataset.SetGeoTransform([444720, 30, 0, 3751320, 0, -30])
    srs = osr.SpatialReference()
    srs.SetUTM(11, 1)
    srs.SetWellKnownGeogCS('NAD27')
    kernel_dataset.SetProjection(srs.ExportToWkt())

    kernel_band = kernel_dataset.GetRasterBand(1)
    kernel_band.SetNoDataValue(-9999)

    col_index = numpy.array(xrange(kernel_size))
    integration = 0.0
    for row_index in xrange(kernel_size):
        kernel = numpy.exp(
            -((row_index - max_distance)**2 +
                (col_index - max_distance) ** 2)/(2.0*sigma**2)).reshape(
                    1, kernel_size)

        integration += numpy.sum(kernel)
        kernel_band.WriteArray(kernel, xoff=0, yoff=row_index)

    for row_index in xrange(kernel_size):
        kernel_row = kernel_band.ReadAsArray(
            xoff=0, yoff=row_index, win_xsize=kernel_size, win_ysize=1)
        kernel_row /= integration
        kernel_band.WriteArray(kernel_row, 0, row_index)


import logging
import os

from django.conf import settings
from django.db.models import Q
from django.http import HttpResponse
from django.utils.safestring import mark_safe

from cvAppMain.helpers import get_cv_url
from cvAppMain.models import GeneratedPDF

logger = logging.getLogger('django')

def make_context(process, language):
    """Makes a context for CV generating"""
    context = {"process": process}
    for each in process.texts.filter(
            Q(language=language) |
            Q(language__isnull=True)):
        context[each.text_type.codename] = mark_safe(each.text)
    return context

def get_pdf(process, language):
    """Gets a cached PDF from DB or calls generating a new one"""
    saved_file = process.generatedpdf_set.filter(language=language).first()
    if saved_file:
        return saved_file.as_response()
    else:
        return render_to_pdf(process, language)

def render_to_pdf(process, language):
    """Uses WKHTMLTOPDF to generate a PDF from URL.
    This function requires PROTOCOL and HOST settings to be set."""
    temp_pdf = f'pdfs/{process.codename}_{language.lang}.pdf'
    wkhtmltopdf_command = f'wkhtmltopdf "{settings.PROTOCOL}{settings.HOST}{get_cv_url(process.codename, language, "html")}" {temp_pdf}'

    try:
        logger.info(f"Rendering PDF to:\n{temp_pdf}\nwith command:\n{wkhtmltopdf_command}\n")
        os.system(wkhtmltopdf_command)
    except Exception as e:
        logger.error("Error while generating PDF: " + str(e))

    try:
        saved_pdf = GeneratedPDF.objects.create(process=process, language=language, pdf=temp_pdf)
        return saved_pdf.as_response()
    except Exception as e:
        logger.error("Error while saving and returning PDF: " + str(e))
        return HttpResponse('Something went wrong while generating PDF! Sorry!')


from core.base_classes.entity import BaseEntityProps
from core.value_objects import DateVO, ID
from abc import ABC, abstractmethod
from typing import Any, Generic, TypeVar, get_args

from infrastructure.database.base_classes.mongodb import OrmEntityBase

Entity = TypeVar('Entity', bound=BaseEntityProps)
OrmEntity = TypeVar('OrmEntity')
Props = TypeVar('Props')

class OrmMapperBase(ABC, Generic[Entity, OrmEntity]):

    def __init__(self) -> None:
        super().__init__()

    @property
    @abstractmethod
    def entity_klass(self):
        # return get_args(self.__orig_bases__[0])[0]
        raise NotImplementedError()

    @property
    @abstractmethod
    def orm_entity_klass(self):
        # return get_args(self.__orig_bases__[0])[1]
        raise NotImplementedError()

    @abstractmethod
    def to_domain_props(self, orm_entity: OrmEntity) -> Any:
        return

    @abstractmethod
    def to_orm_props(self, entity: Entity) -> OrmEntity:
        return

    def to_domain_entity(self, orm_entity: OrmEntity) -> Entity:

        props = self.to_domain_props(orm_entity)

        return self.assign_props_to_entity(props, orm_entity)

    def to_orm_entity(self, entity: Entity) -> OrmEntity:

        props = self.to_orm_props(entity)

        return self.orm_entity_klass(**{
            **props,
            'id': entity.id.value,
            'created_at': entity.created_at.value,
            'updated_at': entity.updated_at.value
        })

    def assign_props_to_entity(
        self,
        entity_props: Any,
        orm_entity: OrmEntity
    ) -> Entity:

        return self.entity_klass.from_orm({
            **entity_props,
            "id": ID(str(orm_entity.id)),
            "created_at": DateVO(orm_entity.created_at),
            "updated_at": DateVO(orm_entity.updated_at)
        })


# 3. Your monthly expense list (from Jan to May) looks like this,
# ```
# expense_list = [2340, 2500, 2100, 3100, 2980]
# ```
# Write a program that asks you to enter an expense amount and program
# should tell you in which month that expense occurred. If expense is not
# found then it should print that as well.
print("\nExercise 3\n")
month_list = ["January", "February", "March", "April", "May"]
expense_list = [2340, 2500, 2100, 3100, 2980]
e = input("Enter expense amount: ")
e = int(e)

month = -1
for i in range(len(expense_list)):
    if e == expense_list[i]:
        month = i
        break

if month != -1:
    print(f'You spent {e} in {month_list[month]}')
else:
    print(f'You didn\'t spend {e} in any month')


from rest_framework import serializers
from rest_framework.renderers import JSONRenderer
from rest_framework.response import Response
from rest_framework.views import APIView

from .models import Group, Facultie, Student, HistoryOfRating

class FacultiesSerializers(serializers.ModelSerializer):
    class Meta:
        model = Facultie
        fields = ('id', 'name', 'dean', 'phone_number')

class GroupsSerializers(serializers.ModelSerializer):
    faculty_name = serializers.SerializerMethodField('get_faculty_name')

    class Meta:
        model = Group
        fields = ('id', 'name', 'faculty_name', 'year')

    def get_faculty_name(self, Group):
        faculty_name = Group.faculty.name
        return faculty_name

class StudentsSerializers(serializers.ModelSerializer):
    class Meta:
        model = Student
        fields = ('id', 'name', 'email', 'rating', 'description', 'group')

class StudentProfileSerializers(serializers.ModelSerializer):
    group_name = serializers.SerializerMethodField('get_group_name')
    class Meta:
        model = Student
        fields = ('id', 'name', 'email', 'rating', 'description', 'group_name')

    def get_group_name(self, Student):
        group_name = Student.group.name
        return group_name

# class HistoryOfRatingSerializers(serializers.ModelSerializer):
#     class Meta:
#         model = HistoryOfRating
#         fields = ('date_of_change', 'reason', 'rating_value', 'id_student')

class HistoryOfRatingSerializers(serializers.ModelSerializer):
    class Meta:
        model = HistoryOfRating
        fields = ('date_of_change', 'reason', 'rating_value', 'id_student')

    def update(self, instance, validated_data):
        instance.rating_value = validated_data.get('rating_value', instance.rating_value)
        return instance

class ReasonsSerializer(APIView):
    REASONS = {'reasons': [{'reason': 'Пользование мобильным устройством во время пары', 'rating': '-20'},
                           {'reason': 'Опоздание более чем на 0 милисекунд', 'rating': '-50'},
                           {'reason': 'Недостаточная вежливость по отношению к преподавателю и студентам',
                            'rating': '-10'},
                           {'reason': 'Невнимательное поведение на лекции', 'rating': '-20'},
                           {'reason': 'Некрасивый почерк у доски', 'rating': '-10'},
                           {'reason': 'Недостаточная активность на семинаре или практике', 'rating': '-30'},
                           {'reason': 'Плохое настроение', 'rating': '-5'},
                           {'reason': 'Оскорбление администрации', 'rating': '-100'},
                           {'reason': 'Немытая голова', 'rating': '-10'},
                           {'reason': 'Задержка в сдаче дз', 'rating': '-50'},

                           {'reason': 'Хорошие взаимоотношения с администратором', 'rating': '+100'},
                           {'reason': 'Своевременное выполнение дз', 'rating': '+10'},
                           {'reason': 'Активное поведение на семинаре', 'rating': '+20'},
                           {'reason': 'Подарок преподавателю', 'rating': '+50'},
                           {'reason': 'Участие в мероприятии', 'rating': '+30'},
                           {'reason': 'Ответ у доски', 'rating': '+5'},
                           {'reason': 'Отсутствие пропусков за месяц', 'rating': '+10'},
                           {'reason': 'Участие в олимпиаде', 'rating': '+20'},
                           {'reason': 'Отсутствие долгов на протяжении года', 'rating': '+50'},
                           {'reason': 'Отсутствие опозданий за неделю', 'rating': '+5'}]}

    def get(self, request):
        return Response(self.REASONS)


import torch;
import torch.nn as nn;
import torch.nn.functional as F;
import torchvision;
import torchvision.transforms as transforms;
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor;
from torchvision.models.detection.mask_rcnn import MaskRCNNPredictor;

from engine import train_one_epoch, evaluate;
import utils;

from pycocotools.coco import COCO;

import numpy as np;
from PIL import Image;

import matplotlib.pyplot as plt;

def barycenterWeight(an,w,h):
    box = an['bbox'];
    x = w/2 - (box[0] + box[2]/2);
    y = h/2 - (box[1] + box[3]/2);
    if (y == 0 and x == 0):
        return np.Inf;
    return an['area']/(x*x + y*y)**2;

def bestAnn(coco, imgData):
    annIds = coco.getAnnIds(imgIds=imgData['id'], catIds=[1]);
    anns = coco.loadAnns(annIds);
    bst = 0;
    bstAnn = None;
    for a in anns:
        if a['iscrowd'] == 1:
            continue;
        scr = barycenterWeight(a,imgData['width'],imgData['height']);
        if scr > bst:
            bst = scr;
            bstAnn = a;
    return bstAnn;

class MyCocoDataset(torch.utils.data.Dataset):
    def __init__(self, root, annFile, transform):
        self.root = root;
        self.transform = transform;
        self.coco = COCO(annFile);
        self.ds = [];
        imgIds = self.coco.getImgIds();
        for i in imgIds:
            imgData = self.coco.loadImgs(i)[0];
            ann = bestAnn(self.coco,imgData);
            if ann is not None:
                if (ann['area']/(imgData['width']*imgData['height']) > 0.40):
                    self.ds.append((imgData,ann));

    def __getitem__(self, idx):
        imgData, ann = self.ds[idx];
        img = Image.open(self.root + imgData['file_name']);
        if self.transform is not None:
            img = self.transform(img);
        target = {};
        target["masks"] = torch.as_tensor([self.coco.annToMask(ann)], dtype=torch.uint8);
        target['image_id'] =  torch.tensor([ann['image_id']]);
        x,y,dx,dy = ann['bbox'];
        target["boxes"] = torch.as_tensor([[x,y,x+dx,y+dy]], dtype=torch.float32);
        target['area'] = torch.as_tensor([ann['area']]);
        target['labels'] = torch.as_tensor([ann['category_id']]);
        target['iscrowd'] = torch.as_tensor([ann['iscrowd']]);
        return img, target;

    def __len__(self):
        return len(self.ds);

testset = MyCocoDataset(root=img_path_val, annFile=ann_path_val, transform=transforms.ToTensor());
testloader = torch.utils.data.DataLoader(testset,shuffle=False);
evaluate(model, testloader, device=device);

path = './data/';
ann_path = path + 'annotations_train_val2014/instances_train2014.json';
img_path = path + 'train2014/';
ann_path_val = path + 'annotations_train_val2014/instances_val2014.json';
img_path_val = path + 'val2014/';

trainset = MyCocoDataset(root=img_path, annFile=ann_path, transform=transforms.ToTensor());
testset = MyCocoDataset(root=img_path_val, annFile=ann_path_val, transform=transforms.ToTensor());
trainloader = torch.utils.data.DataLoader(trainset,shuffle=True);
testloader = torch.utils.data.DataLoader(testset,shuffle=False);
#dataiter = iter(testloader);
#img, trg = dataiter.next();

num_classes = 2;
model = torchvision.models.detection.maskrcnn_resnet50_fpn(pretrained=True);
for param in model.parameters():
    param.requires_grad = False;

in_features = model.roi_heads.box_predictor.cls_score.in_features;
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes);
in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels;
hidden_layer = 256;
model.roi_heads.mask_predictor = MaskRCNNPredictor(in_features_mask,hidden_layer,num_classes);

device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu');
model = model.to(device);

params = [p for p in model.parameters() if p.requires_grad];
optimizer = torch.optim.SGD(params, lr=0.005,momentum=0.9, weight_decay=0.0005);
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,step_size=3,gamma=0.1);

train_one_epoch(model, optimizer, trainloader, device, i, print_freq=10);
evaluate(model, testloader, device=device);

#train
#train_one_epoch(model, optimizer, trainloader, device, i, print_freq=10);
#lr_scheduler.step();
#test


# Sample code for the ChoiceMC implementation
from ChoiceMC import ChoiceMC
PIMC = ChoiceMC(m_max=5, P=9, g=1.0, MC_steps=50000, N=2, PIGS=True, Nskip=100, Nequilibrate=100, V0=5.0, potentialField='transverse', T=0.25)
# Creating the probability density matrix for each rotor
PIMC.createFreeRhoMarx()
# Creating the probability density matrix for nearest neighbour interactions
PIMC.createRhoVij()
# Performing MC integration
PIMC.runMC(averagePotential = True, averageEnergy = True, orientationalCorrelations = True, initialState = 'random')
# Saving plots of the histograms
PIMC.plotHisto_total('left', 'middle', 'right', 'initial')
PIMC.plotHisto_N('PIMC', 'middle')
# Example of how to access class attributes
print('<E> = ', PIMC.E_MC)

# Calculating the properties by exact diagonalization, this can only be done for N=2
PIMC.runExactDiagonalization()

# Calculating the properties using the NMM method, this can only be done for N=2
PIMC.runNMM()


# -*- coding: utf-8 -*-
"""
Jianyu Chen

This is a project for optimal control.
Python code for 1-dim double well system
"""
#%%
import torch
from torch import nn
from torch.nn import functional as F
import math
import numpy as np
import matplotlib.pyplot as plt
#%%
t = np.arange(100) * 0.01
print(t)
sigma = 1
#%%
u= np.random.normal(0, 0.008, 100)
#u = np.linspace(0, 1, 100)
print(u)
sigma = 1
def runge_kutta(x, t, dt, f, u, i):
    """   x is the initial value for x
          t is the initial value for t
          dt is the time step in t
          f is derivative of function x(t)

    """

    k1 = dt * (f(x, t)+u[i])
    k2 = dt * (f(x + k1, t + dt)+u[i+1])
    return x + (k1 + k2) / 2.

def get_x(u):
    t = 0.
    x = -1.
    T = 1.0
    dt = 0.01
    xs, ts = [x], [0.0]
    def func(x, t):
        return x-x**3
    i = 0
    while i < 99:
        #print (i, t)
        x = runge_kutta(x, t, dt, func, u, i)
        i += 1
        t += dt
        xs.append(x)
        ts.append(t)
    return np.array([np.float32(xx) for xx in xs])
#%%
import math
import numpy as np
import matplotlib.pyplot as plt
def runge_kutta(p, t, dt, f, xs, i):
    """   p is the terminal value for p
          t is the initial value for t
          dt is the time step in t
          f is derivative of function p(t)

    """

    k1 = -dt * (f(p,t)*(3*xs[i]**2-1)-3*xs[i])
    k2 = -dt * (f(p + k1, t + dt)*(3*xs[i-1]**2-1)-3*xs[i-1])
    return p + (k1 + k2) / 2.

def get_p(x):
    t = 1.0
    p = -2*(x[99]-1)/((x[99]-1)**2+1)**2
    dt = 0.01
    ps, ts = [p], [0.0]
    def func(p, t):
        return p
    i = 99
    while i > 0:
        #print (i, t)
        p = runge_kutta(p, t, dt, func, x, i)
        i -= 1
        t -= dt
        ps.append(p)
        ts.append(t)

    return np.array([np.float32(pp) for pp in ps])
#%%
class MLP(nn.Module):
     def __init__(self):#call the constractor class"Module" to perform necessary initialization.
        super().__init__()
        self.hidden1 = nn.Linear(1, 8)#hidden
        self.hidden2 = nn.Linear(8, 16)#hidden
        self.out = nn.Linear(16,1)#out put layer
 #define the forward propagation of the model,how to return the required model output
 #based on the input "t"（input t is "u"）
     def forward(self, u):
        u = self.hidden1(u)
        u = F.relu(u)
        u = self.hidden2(u)
        u = F.relu(u)
        out = self.out(u)
        return out
net = MLP()
#%%
t = torch.tensor(t)
t = t.to(torch.float32)
x = get_x(u)
p = get_p(x)
reversed_p = []
for i in reversed(p):
    reversed_p.append(i)

print(reversed_p)
p = reversed_p
x = torch.tensor(x).to(torch.float32)
p = torch.tensor(p).to(torch.float32)
#%%
print(x)
print(p)
#%%
import numpy as np
import matplotlib.pylab as plt
#from gradient import numerical_gradient
Loss = []
lr =0.05
epoch = 1000
N = 20#iteration times
Q = 2*(x[99]-1)**2
print(Q)
dt = 0.01
dx= (x[1:]-x[0:-1])/dt
dp = (p[1:]-p[0:-1])/dt
rho = 1 #修正系数
updater = torch.optim.SGD(net.parameters(), lr)  # change to adaptive
print(dx.shape)
#%%
def train():

    for ii in range(epoch):
        H = 0
        for j in range(99):
            t_j = t[j].reshape(1)
            u_hat_j = net(torch.tensor(t_j))
            x_j = x[j].reshape(1)
            p_j = p[j].reshape(1)
            H = H + p_j* (x_j-x_j**3+sigma*u_hat_j) - 0.5*(u_hat_j**2+1-3*x_j**2)-0.5*rho*(abs(dx[j]-x_j+x_j**3-sigma*u_hat_j)**2)+abs((dp[j]+p_j*(1-3*x_j**2)+3*x_j)**2)

        l = -(H-Q)/100 # l>0

        updater.zero_grad()
        l.backward()
        updater.step()
        Loss.append(l.detach().item())

    return
#%%
for k in range(N):
    print(k)
    U = [net(torch.tensor(t_j.reshape(1))) for t_j in t]
    u_op = [u.detach().numpy() for u in U]
    #print(u_op)
    flatten_u_op = [element for sublist in u_op for element in sublist]
    print(flatten_u_op)
    x = get_x(flatten_u_op)
    p = get_p(x)

    reversed_p = []
    for i in reversed(p):
        reversed_p.append(i)

    p = reversed_p

    x = torch.tensor(x).to(torch.float32)
    p = torch.tensor(p).to(torch.float32)
    train()
    print(x)
#%%
x = t
y = flatten_u_op
#y = list(map(lambda x:x.detach().numpy(), y))
plt.title("optimal control")
plt.xlabel("Time")
plt.ylabel("u")

plt.plot(x, y)
plt.show()
#%%
x = get_x(y)
print(x)
#%%
import scipy.io as scio
scio.savemat('pathway_data.mat',{'x':x})


"""
53. Maximum Subarray

Given an integer array nums, find the contiguous subarray (containing at least one number) which has the largest sum and return its sum.

Example 1:

Input: nums = [-2,1,-3,4,-1,2,1,-5,4]
Output: 6
Explanation: [4,-1,2,1] has the largest sum = 6.
"""

class Solution(object):
    def maxSubArray(self, nums):
        """
        :type nums: List[int]
        :rtype: int
        """
        max_sum = nums[0]
        for i in range(len(nums)):
            s = nums[i]
            for j in range(i, len(nums) - 1):
                if max_sum < s:
                    max_sum = s
                s += nums[j + 1]
            if max_sum < s:
                    max_sum = s
        return max_sum


import pandas as pd
import numpy
import os
import sys
import csv
import logging
import psycopg2

from psycopg2.extensions import register_adapter, AsIs
from cloudharness import applications

def addapt_numpy_int64(numpy_int64):
    return AsIs(numpy_int64)

register_adapter(numpy.int64, addapt_numpy_int64)

app = applications.get_configuration("pub-chem-index")
conn_string = f"postgres://{app.db_name}:{app.harness.database.postgres.ports[0]['port']}/asu?user={app.harness.database.user}&password={app.harness.database.get('pass')}"

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)

NUM_PROCESSES = 2
NUM_QUEUE_ITEMS = 20

#
# WARNING!!! use head command on files for debugging
# head -n 500000 CID-SMILES > CID-SMILES-head
#

# "https://ftp.ncbi.nlm.nih.gov/pubchem/Compound/Extras/CID-Synonym-unfiltered.gz"

added_col_dic = {
    "CID-InChI-Key": ["CID", "InChI", "Key"],
    "CID-Mass": ["CID", "Molecule", "Mass1", "Mass2"],
    "CID-PMID": ["CID", "PMID"],
    "CID-Parent": ["CID", "Parent"],
    "CID-Patent": ["CID", "Patent"],
    "CID-SID": ["CID", "SID"],
    "CID-MeSH": ["CID", "MeSH"],
    "CID-SMILES": ["CID", "SMILES"],
    "CID-Synonym-filtered": ["CID", "Synonym"],
    "CID-Synonym-unfiltered": ["CID", "Synonym"],
    "CID-Title": ["CID", "Title"],
    "CID-IUPAC": ["CID", "IUPAC"],
    "CID-Component": ["CID", "component"],
}

gin_indexes_tables = ['CID-Title', 'CID-MeSH', 'CID-IUPAC', 'CID-InChI-Key', 'CID-Synonym-filtered']

def change_permissions_recursive(path, mode):
    for root, dirs, files in os.walk(path, topdown=False):
        for file in [os.path.join(root, f) for f in files]:
            if file.startswith("export-"):
                os.chmod(file, mode)
    # os.chmod(path, mode)

def execute_sql(conn, command):
    with conn.cursor() as cur:
        logging.info(f"Execute {command}")
        cur.execute(command)
        conn.commit()

def create_table(conn, table_name):
    # Populate GIN indexed table, this will take about 30 minutes.
    column_names = added_col_dic[table_name]
    column_names = [x.upper() for x in column_names]
    table_name = table_name.replace("-", "_").upper()

    str_column_names = ""
    for i in column_names:
        if i == "CID":
            str_column_names += i + " INTEGER NOT NULL,"
        else:
            str_column_names += i + " VARCHAR,"

    str_column_names = str_column_names[: len(str_column_names) - 1]

    sql_drop_table = f"DROP TABLE IF EXISTS {table_name}"
    sql_create_table = f"CREATE TABLE {table_name} ({str_column_names})"

    execute_sql(conn, sql_drop_table)
    execute_sql(conn, sql_create_table)

    logging.info("Table created %s ", table_name)

def bulk_insert(conn, data, file_name):
    with conn.cursor() as cur:
        table_name = file_name.replace("-", "_").upper()
        columns = added_col_dic[file_name]
        column_list = ", ".join(columns)
        records_list_template = ",".join(["%s"] * len(data))
        insert_query = "insert into {table_name} ({columns}) values {};".format(
            records_list_template, table_name=table_name, columns=column_list
        )
        cur.execute(insert_query, data)
        conn.commit()

def create_indexes(conn, table_name, create_gin):
    column_names = added_col_dic[table_name]
    column_names = [x.upper() for x in column_names]
    main_column = column_names[1].lower()
    table_name = table_name.replace("-", "_").lower()

    if create_gin:
        logging.info("Start creating indexes")
        execute_sql(conn, "CREATE EXTENSION IF NOT EXISTS pg_trgm;")
        execute_sql(conn, f"CREATE INDEX IF NOT EXISTS idx_gin_{table_name} ON {table_name} USING gin ({main_column} gin_trgm_ops);")

    execute_sql(conn, f"CREATE INDEX IF NOT EXISTS cid_idx_{table_name} ON {table_name} (CID);")
    logging.info("Finish creating indexes")

def get_line(file_name):
    with open(file_name) as file:
        for i in file:
            yield i

def go():
    logging.info(f"Connecting with string: {conn_string}")
    conn = psycopg2.connect(conn_string)

    file = sys.argv[1]
    file_name = os.path.basename(file)
    logging.info(f"Populating table using file {file_name}")

    column_name = ["CID", file_name]
    types = {file_name: "string", "CID": "Int64"}
    column_names = added_col_dic[file_name]
    # column_names = [x.upper() for x in column_names]
    main_column = column_names[1]  # .upper()
    gin_indexed = file_name in gin_indexes_tables

    if file_name in added_col_dic:
        column_name = added_col_dic[file_name]
        types = {"CID": "string"}
        for c in column_name:
            if c != "CID":
                types[c] = "string"

    create_table(conn, file_name)

    encoding = "UTF-8"
    if file_name == "CID-Title":
        encoding = "Latin"
    chunksize = 200000
    column_slice_size = len(column_names)

    logging.info("Inserting...")
    record_counter = 0
    with open(file, "rb") as f:
        data = []
        for line in f:
            data.append(tuple(line.decode(encoding).replace("\n","").split("\t")[:column_slice_size]))
            if len(data) == chunksize:
                with conn:
                    bulk_insert(conn, data, file_name)
                    record_counter += chunksize
                    if record_counter%10000 == 0:
                        logging.info(f"Total number of records inserted: {record_counter}")
                data = []

    # insert the left over data (if there is any)
    if len(data) != chunksize:
        with conn:
            bulk_insert(conn, data, file_name)
            record_counter += len(data)
            logging.info(f"Total number of records inserted: {record_counter}")
    logging.info("Inserting done...")

    create_indexes(conn, file_name, gin_indexed)

go()


import os
import tarfile
from six.moves import urllib
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error
# sns.set()

DOWNLOAD_ROOT = "https://raw.githubusercontent.com/ageron/handson-ml2/master/"
HOUSING_PATH = os.path.join("data", "housing")
HOUSING_URL = DOWNLOAD_ROOT + "datasets/housing/housing.tgz"

def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):
    if not os.path.isdir(housing_path):
        os.makedirs(housing_path)
    tgz_path = os.path.join(housing_path, "housing.tgz")
    urllib.request.urlretrieve(housing_url, tgz_path)
    housing_tgz = tarfile.open(tgz_path)
    housing_tgz.extractall(path=housing_path)
    housing_tgz.close()

def load_housing_data(housing_path=HOUSING_PATH):
    csv_path = os.path.join(housing_path, "housing.csv")
    return pd.read_csv(csv_path)

def get_data_info():
    print(housing.info())
    data_describe = housing.describe()
    return data_describe

fetch_housing_data()
housing = load_housing_data()
data_describe = get_data_info()
housing = pd.get_dummies(housing)

plt.figure(1)
sns.histplot(data=housing)

plt.figure(2)
sns.scatterplot(data=housing, x='longitude', y='latitude', hue='median_house_value', s=1)

corr_matrix = housing.corr()

housing["rooms_per_household"] = housing["total_rooms"]/housing["households"]
housing["bedrooms_per_room"] = housing["total_bedrooms"]/housing["total_rooms"]
housing["population_per_household"]=housing["population"]/housing["households"]

target = housing.pop('median_house_value')
housing['median_house_value'] = target

housing = housing.dropna()
corr_matrix_2 = housing.corr()

X = housing.loc[:, 'longitude':'population_per_household']
y =  housing['median_house_value']
X_train, X_test, y_train, y_test = train_test_split(X, y)

def evaluate_model(estimator):
    model = make_pipeline(StandardScaler(), estimator)
    model.fit(X_train, y_train)
    y_predicted = model.predict(X_train)
    print('\nAccuracy on train set:', model.score(X_train, y_train))
    mse = np.sqrt(mean_squared_error(y_train, y_predicted))
    print('Mean squared error:', mse)

    print('Accuracy on test set:', model.score(X_test, y_test))

estimators = [LinearRegression(), RandomForestRegressor()]
for i in estimators:
    evaluate_model(i)

plt.show()


from Edge import Edge
from Vertex import Vertex

def parse_file(filename):
    data = []
    start = None
    end = None
    with open(filename) as f:
        for index, line in enumerate(f.readlines()):
            if index == 0:
                start = line.strip('\n')
            elif index == 1:
                end = line.strip('\n')
            else:
                origin, destination, weight = line.split(',')
                data.append((origin, destination, int(weight)))
    return data, start, end

def create_graph(filename):
    data, start, end = parse_file(filename)
    vertices = {}

    for line in data:
        from_name, to_name, weight = line

        vertex_from = vertices.get(from_name, Vertex(from_name, id=len(vertices)))
        vertices[from_name] = vertex_from

        vertex_to = vertices.get(to_name, Vertex(to_name, id=len(vertices)))
        vertices[to_name] = vertex_to

        # Create edge and residual edge
        edge = Edge(vertex_from, vertex_to, weight, residual=False)
        residual_edge = Edge(vertex_to, vertex_from, 0, residual=True)

        edge.inverse_edge = residual_edge
        residual_edge.inverse_edge = edge

        vertex_from.edges.append(edge)
        vertex_to.edges.append(residual_edge)

    return list(vertices.values()), vertices.get(start), vertices.get(end)


import pandas as pd

### Feature Encoding
### ----------------
### ****************

## One-Hot Encoding
## ****************
# Function to convert a column into one-hot encoding using get_dummies
def convert_col_one_hot_series(series, drop_first=False, method='get_dummies'):

    # If already a float or int, then simply return
    if series.dtype == float or series.dtype == int:
        return None, 0

    # If boolean, then return after converting the column to int16
    if series.dtype == bool:
        one_hot = series.astype('int16')
    else:
        one_hot = pd.get_dummies(series, prefix=(str(series.name)), drop_first=drop_first)
    return one_hot, 1

def replace_col_with_one_hot(df, col, drop_column=False, drop_first=False, method='get_dummies'):

    # Get one-hot encoding
    one_hot, conversion_signal = convert_col_one_hot_series(df[col])

    print('~~~~~~~~~~')

    if conversion_signal == 0:
        print('Not transforming column ' + str(col) + ' | already numerical')
        return df

    if df[col].dtype == bool:
        df[col] = one_hot
        print(one_hot.name)
    else:
        print(one_hot.columns)
        if drop_column:
            df = df.drop(col,axis = 1)

        # Join the encoded df
        df = df.join(one_hot)

    return df

## Freq and Mean Encoding
## ****************
# Stores the feature manipultation values from training set transformations that need to be placed on Test set
testFeatureTransforms = {}
# Frequency Encoding, Mean encoding of High cardinality categorical features (not the best compared to embedding etc. One hot would be too sparse)
def freqEncode(df, feature):
    encoding = df.groupby(feature).size()
    encoding = encoding/len(df)
    df[feature + 'FreqEnc'] = df[feature].map(encoding)
    testFeatureTransforms[(feature + 'FreqEnc')] = encoding

# Mean encoding
def meanEncode(df, feature, target):
    means = df.groupby(feature)[target].mean()
    df[feature + 'MeanEnc'] = df[feature].map(means)
    testFeatureTransforms[(feature + 'MeanEnc')] = means

### Clean DataFrames
### ----------------
### ****************

# Remove table alias from column names of a pandas dataframe
# **********************************************************
def remove_tbl_alias_cols(df):
    new_cols = []

    for col in df.columns:
        new_col = col[col.rfind('.') + 1:]
        new_cols.append(new_col)

    df.columns = new_cols

    return df

# Remove columns
# **************
def remove_unnamed_cols(df):
    num = 0
    col_list = df.columns[[col[0:7] == 'Unnamed' for col in df.columns]]
    for i in range(len(col_list)):
        col_name = col_list[i]
        df = df.drop(col_name, axis=1)
        num += 1
    print(' Removed Cols: ' + str(num))

    return df

# Clean column names to have only alphabetical/numerical or -,_
# *************************************************************
def cleanse_col_names(col_names):
    if ((type(col_names)) is not list) and ((type(col_names)) is not pd.core.indexes.base.Index):
        print('Handled Error: Input not a list')
        return 0

    # import dependencies
    import re

    i = 0
    new_names = []
    for elm in col_names:
        y = re.search("[a-z0-9_-]+", elm)
        if y is not None:
            new_names.append(y.group())
        else:
            new_names.append('NoName' + str(i))
            i += 1
    return new_names


import torch
from tqdm import tqdm, trange
from utils_squad_evaluate import EVAL_OPTS, main as evaluate_on_squad, plot_pr_curve
from utils_squad import (read_squad_examples, convert_examples_to_features,
                         RawResult, write_predictions,
                         RawResultExtended, write_predictions_extended)

def to_list(tensor):
    return tensor.detach().cpu().tolist()

def evaluate(model, validation_dataloader, features_val, examples_val, tokenizer, predict_file, len_val, batch_size, device):
  print("***** Running evaluation *****")
  print("  Num examples = %d" % len_val)
  print("  Batch size = %d" % batch_size)
  all_results = []
  for batch in tqdm(validation_dataloader, desc="Evaluating", miniters=100, mininterval=5.0):
    model.eval()
    batch = tuple(t.to(device) for t in batch)
    with torch.no_grad():
      inputs = {'input_ids':      batch[0],
                'attention_mask': batch[1],
                'token_type_ids': batch[2]
                }
      example_indices = batch[3]
      outputs = model(**inputs)

    for i, example_index in enumerate(example_indices):
      eval_feature = features_val[example_index.item()]
      unique_id = int(eval_feature.unique_id)

      result = RawResult(unique_id    = unique_id,
                         start_logits = to_list(outputs[0][i]),
                         end_logits   = to_list(outputs[1][i]))
      all_results.append(result)

  # Compute predictions
  output_prediction_file = "/predictions/predictions.json"
  output_nbest_file = "/predictions/nbest_predictions.json"
  output_null_log_odds_file = "/predictions/null_odds.json"
  output_dir = "/predictions/predict_results"

  write_predictions(examples_val, features_val, all_results, 10,
                  30, True, output_prediction_file,
                  output_nbest_file, output_null_log_odds_file, False,
                  True, 0.0)

  # Evaluate with the official SQuAD script
  evaluate_options = EVAL_OPTS(data_file=predict_file,
                               pred_file=output_prediction_file,
                               na_prob_file=output_null_log_odds_file,
                               out_image_dir=None)
  results = evaluate_on_squad(evaluate_options)
  return results


# core python programming p395

import os
import pickle

class FileDescr(object):
    saved = []

    def __init__(self, name=None):
        self.name = name

    def __get__(self, obj, typ=None):
        if self.name not in FileDescr.saved:
            raise AttributeError("%r used before assigment" % self.name)

        try:
            f = open(self.name, 'rb')
            val = pickle.load(f)
            f.close()
            return val
        except(pickle.UnpicklingError, IOError, EOFError, AttributeError, ImportError,
                IndexError) as e:
            raise AttributeError("could not read %r: %s" % self.name)

    def __set__(self, obj, val):
        f = open(self.name, 'wb')
        try:
            pickle.dump(val, f)
            FileDescr.saved.append(self.name)
        except(TypeError, pickle.PicklingError) as e:
            raise AttributeError("could not pickle %r" % self.name)
        finally:
            f.close()

    def __delete__(self, obj):
        try:
            os.unlink(self.name)
            FileDescr.saved.remove(self.name)
        except (OSError, ValueError) as e:
            pass

class MyFileVarClass(object):
    fo = FileDescr('fo')
    bar = FileDescr('bar')

if __name__ == '__main__':
    fvc = MyFileVarClass()
   # print(fvc.fo)
    fvc.fo = 1234
    fvc.bar = 'leanna'
    print (fvc.fo, fvc.bar)
    del fvc.fo
    print (fvc.fo, fvc.bar)

    # data1 = {'a': [1, 2.0, 3, 4+6j],
    #          'b': ('string', u'Unicode string'),
    #          'c': None}

    # selfref_list = [1, 2, 3]
    # selfref_list.append(selfref_list)

    # output = open('fo', 'wb')

    # # Pickle dictionary using protocol 0.
    # pickle.dump(data1, output)

    # # Pickle the list using the highest protocol available.
    # pickle.dump(selfref_list, output, -1)

    # output.close()


# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup

def main():
    #--- 取得URLの決定
    url = 'https://github.com/librariapj'
    #--- requestsでdom取得
    response = requests.get(url)
    #--- 取得結果の表示
    print(response)
    # print(response.text)
    #--- BS4に取得内容を格納
    soup = BeautifulSoup(response.content, 'html.parser')
    #--- BS4の出力
    # 全文
    #print(soup.prettify())
    # aタグ
    print(soup.find_all("a"))

if __name__ == "__main__":
    main()


import os
import numpy as np
from PIL import Image, ImageDraw, ImageFont
from scipy.ndimage import label

from config import cfg

import labels

def benchmark_vis(iou_ar, fn):
    pred_pgt_ar = np.zeros_like(iou_ar)
    structure = np.ones((3, 3), dtype=int)
    segment_ar = label(iou_ar, structure)[0]
    for idx in np.unique(segment_ar):
        if idx > 0:
            segment = np.where(segment_ar == idx)
            if len(segment[0]) >= cfg.MIN_ERROR_SIZE:
                pred_pgt_ar[segment] = iou_ar[segment]

    inf_mask_suffix = os.listdir(cfg.GT_MASKS_DIR)[0].split(".")[-1]
    inf_ar = np.asarray(Image.open(os.path.join(cfg.INFERENCE_OUTPUT_DIR, fn + "." + inf_mask_suffix)), dtype="uint8")

    result = np.zeros(pred_pgt_ar.shape + (3,), dtype='uint8')
    result[:,:,0] = pred_pgt_ar
    result[:,:,1] = inf_ar

    Image.fromarray(result).save(os.path.join(cfg.BENCHMARK_PROPOSAL_VIS_DIR, fn + "_proposals.png"))

def label_error_vis(fn):
    diff_mask = np.load(os.path.join(cfg.DIFF_DIR, fn + "_diff_map.npy"))
    img_suffix = os.listdir(cfg.GT_MASKS_DIR)[0].split(".")[-1]
    gt_mask = np.array(Image.open(os.path.join(cfg.GT_MASKS_DIR, fn + "." + img_suffix)))

    structure = np.ones((3, 3), dtype=int)
    error_segments = label(diff_mask, structure)[0]
    for idx in np.unique(error_segments):
        if idx > 0:
            segment = np.where(error_segments == idx)
            if len(segment[0]) < cfg.MIN_ERROR_SIZE:
                diff_mask[segment] = 0

    diff_mask[diff_mask==1] = gt_mask[diff_mask==1]
    Image.fromarray(diff_mask).save(os.path.join(cfg.ERROR_DIR, fn + "_label_errors." + img_suffix))

def proposal_vis(iou, fn, seg_id, cls_id):
    """Visualization of one potential label error in image fn"""
    inf_mask_suffix = os.listdir(cfg.GT_MASKS_DIR)[0].split(".")[-1]
    inf_ar = np.asarray(Image.open(os.path.join(cfg.INFERENCE_OUTPUT_DIR, fn + "." + inf_mask_suffix)), dtype="uint8")

    gt_mask_suffix = os.listdir(cfg.GT_MASKS_DIR)[0].split(".")[-1]
    net_input_suffix = os.listdir(cfg.NET_INPUT_DIR)[0].split(".")[-1]
    gt_mask = np.array(Image.open(os.path.join(cfg.GT_MASKS_DIR, fn + "." + gt_mask_suffix)))
    net_input = np.array(Image.open(os.path.join(cfg.NET_INPUT_DIR, fn + "." + net_input_suffix)))

    Dataset = getattr(labels, cfg.DATASET.capitalize())
    trainId2color = Dataset.trainId2color
    trainId2name = Dataset.trainId2name
    num_classes = Dataset.num_classes

    gt_rgb = np.zeros(inf_ar.shape + (3,), dtype="uint8")
    pred_rgb = np.zeros(inf_ar.shape + (3,), dtype="uint8")
    for id in range(num_classes):
        gt_rgb[gt_mask==id] = trainId2color[id]
        pred_rgb[inf_ar==id] = trainId2color[id]

    rgb_blend_img = np.copy(net_input)
    components = np.load(os.path.join(cfg.COMPONENTS_DIR, fn + "_components.npy"))
    seg_idx = np.where(np.abs(components)==seg_id)

    bbox_0 = (max(np.min(seg_idx[1])-15, 0), max(np.min(seg_idx[0])-15, 0))
    bbox_1 = (min(np.max(seg_idx[1])+15, rgb_blend_img.shape[1]), min(np.max(seg_idx[0])+15, rgb_blend_img.shape[1]))

    rgb_blend_img[seg_idx] = 0.4*rgb_blend_img[seg_idx] + 0.6*pred_rgb[seg_idx]

    img_shape = rgb_blend_img.shape[0]

    net_input = Image.fromarray(net_input)
    rgb_blend_img = Image.fromarray(rgb_blend_img)
    gt_rgb_img = Image.fromarray(gt_rgb)
    pred_rgb_img = Image.fromarray(pred_rgb)

    font = ImageFont.truetype("arial.ttf", size=int(0.05*img_shape))

    draw = ImageDraw.Draw(rgb_blend_img)
    draw.rectangle([bbox_0, bbox_1], outline='red', width=4)
    draw.text((0, 0), "Predicted Label: " + trainId2name[int(cls_id)], fill='red', font=font)

    draw = ImageDraw.Draw(net_input)
    draw.rectangle([bbox_0, bbox_1], outline='red', width=4)
    draw.text((0, 0), "Ground Truth", fill='red', font=font)

    draw = ImageDraw.Draw(gt_rgb_img)
    draw.rectangle([bbox_0, bbox_1], outline='white', width=4)

    draw = ImageDraw.Draw(pred_rgb_img)
    draw.rectangle([bbox_0, bbox_1], outline='white', width=4)

    out_img = Image.new('RGB', (2*rgb_blend_img.width, 2*rgb_blend_img.height))
    out_img.paste(net_input, (0, 0))
    out_img.paste(rgb_blend_img, (0, net_input.height))
    out_img.paste(gt_rgb_img, (net_input.width, 0))
    out_img.paste(pred_rgb_img, (net_input.width, net_input.height))

    saved = False
    i = 0
    while not saved:
        try:
            open(os.path.join(cfg.ERROR_PROPOSAL_DIR, fn + f"_proposal_{i}." + net_input_suffix))
            i += 1
        except IOError:
            out_img.save(os.path.join(cfg.ERROR_PROPOSAL_DIR, fn + f"_proposal_{i}." + net_input_suffix))
            saved = True


# FLASK: location API functions

from flask import Flask, jsonify, request
import pymysql

from __main__ import app
from helpers.location import *
from db_connect.db_connection import connect

# Open database connection
db = connect()
cursor = db.cursor()

# New payment or get payments to a tutor
@app.route("/location", methods=['GET','POST'])
def location():

    # Retrieve parameters
    dist = request.args.get('District')
    city = request.args.get('City')
    country = request.args.get('Country')

    # Find locations
    if request.method == 'GET':
        cursor.execute(get_location(dist, city, country))
        return jsonify(cursor.fetchall())

    # Create new location in database
    elif request.method == 'POST':
        pc = request.args.get('PostalCode')
        cursor.execute(new_location(pc, dist, city, country))
        db.commit()
        return jsonify("Success!")