import logging
import re
import time

import ldap
import pytest
from lib389 import Entry
from lib389._constants import *
from lib389.topologies import topology_m1c1
from lib389.utils import *

pytestmark = pytest.mark.tier2

logging.getLogger(__name__).setLevel(logging.DEBUG)
log = logging.getLogger(__name__)

TEST_REPL_DN = "cn=test_repl, %s" % SUFFIX
ENTRY_DN = "cn=test_entry, %s" % SUFFIX

MUST_OLD = "(postalAddress $ preferredLocale $ telexNumber)"
MAY_OLD = "(postalCode $ street)"

MUST_NEW = "(postalAddress $ preferredLocale)"
MAY_NEW = "(telexNumber $ postalCode $ street)"

def pattern_errorlog(file, log_pattern):
    try:
        pattern_errorlog.last_pos += 1
    except AttributeError:
        pattern_errorlog.last_pos = 0

    found = None
    log.debug("_pattern_errorlog: start at offset %d" % pattern_errorlog.last_pos)
    file.seek(pattern_errorlog.last_pos)

    # Use a while true iteration because 'for line in file: hit a
    # python bug that break file.tell()
    while True:
        line = file.readline()
        log.debug("_pattern_errorlog: [%d] %s" % (file.tell(), line))
        found = log_pattern.search(line)
        if ((line == '') or (found)):
            break

    log.debug("_pattern_errorlog: end at offset %d" % file.tell())
    pattern_errorlog.last_pos = file.tell()
    return found

def _oc_definition(oid_ext, name, must=None, may=None):
    oid = "1.2.3.4.5.6.7.8.9.10.%d" % oid_ext
    desc = 'To test ticket 47573'
    sup = 'person'
    if not must:
        must = MUST_OLD
    if not may:
        may = MAY_OLD

    new_oc = "( %s  NAME '%s' DESC '%s' SUP %s AUXILIARY MUST %s MAY %s )" % (oid, name, desc, sup, must, may)
    return ensure_bytes(new_oc)

def add_OC(instance, oid_ext, name):
    new_oc = _oc_definition(oid_ext, name)
    instance.schema.add_schema('objectClasses', new_oc)

def mod_OC(instance, oid_ext, name, old_must=None, old_may=None, new_must=None, new_may=None):
    old_oc = _oc_definition(oid_ext, name, old_must, old_may)
    new_oc = _oc_definition(oid_ext, name, new_must, new_may)
    instance.schema.del_schema('objectClasses', old_oc)
    instance.schema.add_schema('objectClasses', new_oc)

def trigger_schema_push(topology_m1c1):
    """
        It triggers an update on the supplier. This will start a replication
        session and a schema push
    """
    try:
        trigger_schema_push.value += 1
    except AttributeError:
        trigger_schema_push.value = 1
    replace = [(ldap.MOD_REPLACE, 'telephonenumber', ensure_bytes(str(trigger_schema_push.value)))]
    topology_m1c1.ms["supplier1"].modify_s(ENTRY_DN, replace)

    # wait 10 seconds that the update is replicated
    loop = 0
    while loop <= 10:
        try:
            ent = topology_m1c1.cs["consumer1"].getEntry(ENTRY_DN, ldap.SCOPE_BASE, "(objectclass=*)",
                                                         ['telephonenumber'])
            val = ent.telephonenumber or "0"
            if int(val) == trigger_schema_push.value:
                return
            # the expected value is not yet replicated. try again
            time.sleep(1)
            loop += 1
            log.debug("trigger_schema_push: receive %s (expected %d)" % (val, trigger_schema_push.value))
        except ldap.NO_SUCH_OBJECT:
            time.sleep(1)
            loop += 1

def test_ticket47573_init(topology_m1c1):
    """
        Initialize the test environment
    """
    log.debug("test_ticket47573_init topology_m1c1 %r (supplier %r, consumer %r" %
              (topology_m1c1, topology_m1c1.ms["supplier1"], topology_m1c1.cs["consumer1"]))
    # the test case will check if a warning message is logged in the
    # error log of the supplier
    topology_m1c1.ms["supplier1"].errorlog_file = open(topology_m1c1.ms["supplier1"].errlog, "r")

    # This entry will be used to trigger attempt of schema push
    topology_m1c1.ms["supplier1"].add_s(Entry((ENTRY_DN, {
        'objectclass': "top person".split(),
        'sn': 'test_entry',
        'cn': 'test_entry'})))

def test_ticket47573_one(topology_m1c1):
    """
        Summary: Add a custom OC with MUST and MAY
            MUST = postalAddress $ preferredLocale
            MAY  = telexNumber   $ postalCode      $ street

        Final state
            - supplier +OCwithMayAttr
            - consumer +OCwithMayAttr

    """
    log.debug("test_ticket47573_one topology_m1c1 %r (supplier %r, consumer %r" % (
    topology_m1c1, topology_m1c1.ms["supplier1"], topology_m1c1.cs["consumer1"]))
    # update the schema of the supplier so that it is a superset of
    # consumer. Schema should be pushed
    new_oc = _oc_definition(2, 'OCwithMayAttr',
                            must=MUST_OLD,
                            may=MAY_OLD)
    topology_m1c1.ms["supplier1"].schema.add_schema('objectClasses', new_oc)

    trigger_schema_push(topology_m1c1)
    supplier_schema_csn = topology_m1c1.ms["supplier1"].schema.get_schema_csn()
    consumer_schema_csn = topology_m1c1.cs["consumer1"].schema.get_schema_csn()

    # Check the schemaCSN was updated on the consumer
    log.debug("test_ticket47573_one supplier_schema_csn=%s", supplier_schema_csn)
    log.debug("ctest_ticket47573_one onsumer_schema_csn=%s", consumer_schema_csn)
    assert supplier_schema_csn == consumer_schema_csn

    # Check the error log of the supplier does not contain an error
    regex = re.compile("must not be overwritten \(set replication log for additional info\)")
    res = pattern_errorlog(topology_m1c1.ms["supplier1"].errorlog_file, regex)
    assert res is None

def test_ticket47573_two(topology_m1c1):
    """
        Summary: Change OCwithMayAttr to move a MAY attribute to a MUST attribute

        Final state
            - supplier OCwithMayAttr updated
            - consumer OCwithMayAttr updated

    """

    # Update the objectclass so that a MAY attribute is moved to MUST attribute
    mod_OC(topology_m1c1.ms["supplier1"], 2, 'OCwithMayAttr', old_must=MUST_OLD, new_must=MUST_NEW, old_may=MAY_OLD,
           new_may=MAY_NEW)

    # now push the scheam
    trigger_schema_push(topology_m1c1)
    supplier_schema_csn = topology_m1c1.ms["supplier1"].schema.get_schema_csn()
    consumer_schema_csn = topology_m1c1.cs["consumer1"].schema.get_schema_csn()

    # Check the schemaCSN was NOT updated on the consumer
    log.debug("test_ticket47573_two supplier_schema_csn=%s", supplier_schema_csn)
    log.debug("test_ticket47573_two consumer_schema_csn=%s", consumer_schema_csn)
    assert supplier_schema_csn == consumer_schema_csn

    # Check the error log of the supplier does not contain an error
    regex = re.compile("must not be overwritten \(set replication log for additional info\)")
    res = pattern_errorlog(topology_m1c1.ms["supplier1"].errorlog_file, regex)
    assert res is None

def test_ticket47573_three(topology_m1c1):
    '''
        Create a entry with OCwithMayAttr OC
    '''
    # Check replication is working fine
    dn = "cn=ticket47573, %s" % SUFFIX
    topology_m1c1.ms["supplier1"].add_s(Entry((dn,
                                             {'objectclass': "top person OCwithMayAttr".split(),
                                              'sn': 'test_repl',
                                              'cn': 'test_repl',
                                              'postalAddress': 'here',
                                              'preferredLocale': 'en',
                                              'telexNumber': '12$us$21',
                                              'postalCode': '54321'})))
    loop = 0
    ent = None
    while loop <= 10:
        try:
            ent = topology_m1c1.cs["consumer1"].getEntry(dn, ldap.SCOPE_BASE, "(objectclass=*)")
            break
        except ldap.NO_SUCH_OBJECT:
            time.sleep(1)
            loop += 1
    if ent is None:
        assert False

    log.info('Testcase PASSED')

if __name__ == '__main__':
    # Run isolated
    # -s for DEBUG mode
    CURRENT_FILE = os.path.realpath(__file__)
    pytest.main("-s %s" % CURRENT_FILE)


# -*- coding: utf-8 -*-
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Layout, Div, Fieldset, Row, Field, ButtonHolder, Submit, LayoutObject, HTML
from django import forms
from django.forms.models import inlineformset_factory
from django.template.loader import render_to_string
import re

from wastd.observations.models import Encounter, AnimalEncounter, TagObservation

class EncounterListFormHelper(FormHelper):
    """django-crispy-forms FormHelper for Encounter."""

    model = Encounter
    # form_class = 'form-horizontal'    # Adding a Filter Button
    form_show_labels = True             # form field labels
    layout = Layout(
        'name',
        'source_id',
        'when',
        ButtonHolder(Submit('submit', 'Filter', css_class='button white right')),
    )

class AnimalEncounterListFormHelper(EncounterListFormHelper):
    """django-crispy-forms FormHelper for AnimalEncounter."""

    model = AnimalEncounter

class Formset(LayoutObject):
    template = 'observations/formset.html'

    def __init__(self, formset_context_name, helper_context_name=None, template=None, label=None):
        self.formset_context_name = formset_context_name
        self.helper_context_name = helper_context_name
        self.fields = []

        # Overrides class variable with an instance level variable
        if template:
            self.template = template

    def render(self, form, form_style, context, **kwargs):
        formset = context.get(self.formset_context_name)
        helper = context.get(self.helper_context_name)
        if helper:
            helper.form_tag = False

        context.update({'formset': formset, 'helper': helper})
        return render_to_string(self.template, context.flatten())

class BaseForm(forms.ModelForm):
    """Base ModelForm class with a crispy_forms FormHelper.
    """
    save_button = Submit('save', 'Save', css_class='btn-lg')
    cancel_button = Submit('cancel', 'Cancel')

    def __init__(self, *args, **kwargs):
        self.helper = FormHelper()
        self.helper.form_method = 'POST'
        self.helper.form_class = 'form-horizontal'
        self.helper.label_class = 'col-xs-12 col-sm-4 col-md-3 col-lg-2'
        self.helper.field_class = 'col-xs-12 col-sm-8 col-md-6 col-lg-4'
        self.helper.help_text_inline = True
        super(BaseForm, self).__init__(*args, **kwargs)

class AnimalEncounterForm(BaseForm):

    class Meta:
        model = AnimalEncounter
        fields = ['where', 'when', 'taxon', 'species', 'health', 'sex', 'maturity', 'checked_for_flipper_tags']

    def __init__(self, *args, **kwargs):
        super(AnimalEncounterForm, self).__init__(*args, **kwargs)
        self.fields['when'].help_text = 'Date & time (AWST) of the observation.'
        self.helper.layout = Layout(
            Div(
                Fieldset(
                    'Marine animal encounter details',
                    Field('where'),
                    Field('when'),
                    Field('taxon'),
                    Field('species'),
                    Field('health'),
                    Field('sex'),
                    Field('maturity'),
                    Field('checked_for_flipper_tags'),
                ),
                Fieldset(
                    'Flipper tags',
                    HTML('
The ID of any tag must be unique within the tag type.
'),
                    Formset('flipper_tags', template='observations/formset_simple.html')
                ),
                HTML('
'),
                Div(
                    self.save_button,
                    self.cancel_button,
                    css_class='offset-sm-4 offset-md-3 offset-lg-2',
                )
            ),
        )

class TagObservationForm(forms.ModelForm):

    class Meta:
        model = TagObservation
        fields = ['name', 'tag_location']

    def __init__(self, *args, **kwargs):
        super(TagObservationForm, self).__init__(*args, **kwargs)
        formtag_prefix = re.sub('-[0-9]+$', '', kwargs.get('prefix', ''))
        self.helper = FormHelper()
        self.helper.form_tag = False
        self.fields['name'].help_text = None
        self.helper.layout = Layout(
            Row(
                Field('name'),
                Field('tag_location'),
                Field('DELETE', type='hidden'),
                css_class='formset_row-{}'.format(formtag_prefix)
            )
        )

FlipperTagObservationFormSet = inlineformset_factory(
    parent_model=AnimalEncounter,
    model=TagObservation,
    form=TagObservationForm,
    can_delete=True,
    extra=1,
)


import sys
import json
from yahoo_fin.stock_info import get_data
from yahoo_fin.stock_info import tickers_nifty50
import matplotlib.pyplot as plt
import pandas as pd
from datetime import datetime
from datetime import date
from datetime import timedelta
import requests
import matplotlib.pyplot as plt
import yfinance as yf  # Make sure to import the yfinance library

data_input = sys.argv[1]

#json_string = json.loads(data_input)
#data_output = json_string[0]["country"]

# start and end date
startDate='2023-03-01'
endDate='2023-05-31'

#############

startDate = date.today() - timedelta(days=365)
endDate = date.today()

# Factory Pattern for Trade Creation
class Trade:
    def __init__(self, id, country, stockName, price, amount, tradeDate, trade, created_at, updated_at, user_id):
        self.id = id
        self.country = country
        self.stockName = stockName
        self.price = price
        self.amount = amount
        self.tradeDate = tradeDate
        self.trade = trade
        self.created_at = created_at
        self.updated_at = updated_at
        self.user_id = user_id

    def __str__(self):
        return f"Trade(name={self.stockName}, price={self.price})"

    def lastPrice(self):
        if self.country == "India":
            tickerData = yf.Ticker(str(self.stockName) + ".NS")
        else:
            tickerData = yf.Ticker(str(self.stockName))
        priceData = tickerData.history(period='1d', start=startDate, end=endDate)
        lastPrice = priceData.tail(1)
        return lastPrice["Close"].to_numpy().tolist()[0]

    def totalValueNew(self):
        return self.amount * self.lastPrice()

class USTrade(Trade):
    def __init__(self, id, country, stockName, price, amount, tradeDate, trade, created_at, updated_at, user_id,
                 exchange_rate):
        super().__init__(id, country, stockName, price, amount, tradeDate, trade, created_at, updated_at, user_id)
        self.exchange_rate = exchange_rate

class IndianTrade(Trade):
    def __init__(self, id, country, stockName, price, amount, tradeDate, trade, created_at, updated_at, user_id,
                 exchange_rate):
        super().__init__(id, country, stockName, price, amount, tradeDate, trade, created_at, updated_at, user_id)
        self.exchange_rate = exchange_rate

class TradeFactory:
    def __init__(self):
        self.value_provider = PortfolioValueProvider()

    def create_trade(self, id, country, stockName, price, amount, tradeDate, trade, created_at, updated_at, user_id):
        exchange_rate = self.value_provider.get_exchange_rate(country)
        if country == "USA":
            return USTrade(id, country, stockName, price, amount, tradeDate, trade, created_at, updated_at, user_id,
                           exchange_rate)
        elif country == "India":
            return IndianTrade(id, country, stockName, price, amount, tradeDate, trade, created_at, updated_at, user_id,
                               exchange_rate)
        else:
            raise ValueError("Invalid country")

# Portfolio Class with Aggregation and API Interaction
class Portfolio:
    def __init__(self):
        self.trades = []
        self.value_provider = PortfolioValueProvider()
        self.net_values = None  # Store the calculated net values
        self.portfolio_status = {}  # Store the portfolio's status for each date

    def add_trade(self, trade):
        self.trades.append(trade)
        self.update_portfolio_status()

    def add_trades(self, trade_data):
        trade_factory = TradeFactory()
        for trade_info in trade_data:
            self.trades.extend([trade_factory.create_trade(**trade_info)])
        self.update_portfolio_status()

    def calculate_net(self):
        if self.net_values is None:
            net_value = {}
            total_quantity = {}
            average_price = {}
            current_value = {}
            for trade in self.trades:
                if trade.stockName in net_value:
                    net_value[trade.stockName] += trade.amount * trade.price
                    total_quantity[trade.stockName] += trade.amount
                    average_price[trade.stockName] = net_value[trade.stockName] / total_quantity[trade.stockName]
                    current_value[trade.stockName] = total_quantity[trade.stockName] * self.value_provider.get_asset_value(
                        trade.stockName, trade.country, startDate, endDate)
                else:
                    net_value[trade.stockName] = trade.amount * trade.price
                    total_quantity[trade.stockName] = trade.amount
                    average_price[trade.stockName] = net_value[trade.stockName] / total_quantity[trade.stockName]
                    current_value[trade.stockName] = total_quantity[trade.stockName] * self.value_provider.get_asset_value(
                        trade.stockName, trade.country, startDate, endDate)
            self.net_values = net_value, total_quantity, average_price, current_value

        return self.net_values

    def calculate_current_value(self, currency):
        current_value = {}
        for trade in self.trades:
            if trade.country == "USA":
                if currency == "USA":
                    currency_converter = 1
                else:
                    currency_converter = self.value_provider.get_exchange_rate("India")
            else:
                if currency == "USA":
                    currency_converter = self.value_provider.get_exchange_rate("USA")
                else:
                    currency_converter = 1
            asset_value = self.value_provider.get_asset_value(trade.stockName, trade.country, startDate,
                                                             trade.tradeDate)
            if asset_value is not None:
                current_value[trade.stockName] = asset_value * trade.amount * currency_converter
        return current_value

    def update_portfolio_status(self):
        for trade in self.trades:
            if trade.tradeDate not in self.portfolio_status:
                self.portfolio_status[trade.tradeDate] = {}

            if trade.stockName not in self.portfolio_status[trade.tradeDate]:
                self.portfolio_status[trade.tradeDate][trade.stockName] = trade.amount
            else:
                self.portfolio_status[trade.tradeDate][trade.stockName] += trade.amount

    def get_portfolio_status_by_date(self, date):
        return self.portfolio_status.get(date, {})

    def get_portfolio_status_for_dates(self, dates):
        portfolio_status_for_dates = []

        for date in dates:
            portfolio_status_for_dates.append({
                'date': date,
                'assets_quantity': self.get_portfolio_status_by_date(date)
            })

        return portfolio_status_for_dates

    def calculate_daily_values_oneyear(self, currency):
        daily_values = []
        trade_dates = []
        stock_quantity = {}
        start_dt = startDate
        end_dt = endDate
        delta = timedelta(days=1)

        # store the dates between two dates in a list
        dates_oneyear = []

        while start_dt <= end_dt:
            dates_oneyear.append(start_dt.isoformat())
            start_dt += delta

        sorted_trades = sorted(self.trades, key=lambda trade: trade.tradeDate)
        for date in dates_oneyear:
            total_value_for_date = 0
            for trade in sorted_trades:
                if trade.tradeDate == date:
                    total_value_for_date += trade.amount * self.value_provider.get_asset_value(trade.stockName,
                                                                                trade.country, date)
            daily_values.append(total_value_for_date)

        portfolio_status = self.get_portfolio_status_for_dates(dates_oneyear)

        return portfolio_status, dates_oneyear

    def assets_on_dates_oneyear(self):
        dates_oneyear = []
        start_dt = startDate
        end_dt = endDate
        delta = timedelta(days=1)

        while start_dt <= end_dt:
            dates_oneyear.append(start_dt.isoformat())
            start_dt += delta

        assets_by_date = {}

        trade_dates_set = set(trade.tradeDate for trade in self.trades)
        sorted_trades = sorted(self.trades, key=lambda trade: trade.tradeDate)

        for date1 in dates_oneyear:
            assets_on_date = {}
            for trade in self.trades:
                if trade.tradeDate <= date1:
                    if trade.country not in assets_on_date:
                        assets_on_date[trade.country] = {}
                    if trade.stockName in assets_on_date[trade.country]:
                        assets_on_date[trade.country][trade.stockName] += trade.amount
                    else:
                        assets_on_date[trade.country][trade.stockName] = trade.amount
            assets_by_date[date1] = assets_on_date

        return assets_by_date

    @staticmethod
    def portfolio_value_for_dates(portfolio_dict, currency):
        stock_data = {}

        for date, country_stocks in portfolio_dict.items():
            for country, stocks in country_stocks.items():
                for stock_name in stocks.keys():
                    if stock_name not in stock_data:
                        if country == 'India':
                            stock_data[stock_name] = yf.Ticker(str(stock_name) + ".NS").history(period='1d',
                                                                                start=startDate,
                                                                                end=endDate)
                        else:
                            stock_data[stock_name] = yf.Ticker(str(stock_name)).history(period='1d', start=startDate,
                                                                                end=endDate)

        portfolio_value_by_dates = {}

        for date, country_stocks in portfolio_dict.items():
            total_value = 0

            for country, stocks in country_stocks.items():
                for stock_name, quantity in stocks.items():
                    stock_history = stock_data[stock_name]

                    if date in stock_history.index:
                        stock_value_usd = stock_history.loc[date]["Close"] * quantity

                        if currency == 'USD':
                            total_value += stock_value_usd
                        elif currency == 'INR' and country != 'India':
                            exchange_rate = PortfolioValueProvider.get_exchange_rate(country)
                            stock_value_rupees = stock_value_usd * exchange_rate
                            total_value += stock_value_rupees

            if total_value > 0:
                portfolio_value_by_dates[date] = total_value
        return portfolio_value_by_dates

# Data Collection
class PortfolioValueProvider:
    def __init__(self):
        pass

    def get_asset_value(self, stockName, country, startDate, endDate):
        if country == "India":
            tickerData = yf.Ticker(str(stockName) + ".NS")
        else:
            tickerData = yf.Ticker(str(stockName))
        priceData = tickerData.history(period='1d', start=startDate, end=endDate)
        lastPrice = priceData.tail(1)
        return lastPrice["Close"].to_numpy().tolist()[0]

    def get_asset_values(self, stockName, country, startDate, endDate):
        if country == "India":
            tickerData = yf.Ticker(str(stockName) + ".NS")
        else:
            tickerData = yf.Ticker(str(stockName))
        priceData = tickerData.history(period='1d', start=startDate, end=endDate)
        return priceData["Close"].to_numpy().tolist()

    @staticmethod
    def get_exchange_rate(country, target_currency='USD'):
        if country in PortfolioValueProvider.exchange_rates and target_currency in PortfolioValueProvider.exchange_rates[
            country]:
            return PortfolioValueProvider.exchange_rates[country][target_currency]
        else:
            return 1.0

    # Sample exchange rates (USD to other currencies)
    exchange_rates = {
        'USA': {
            'USD': 1.0,  # 1 USD to USD
            'INR': 75.0,  # Sample exchange rate: 1 USD to 75 INR
            'EUR': 0.85,  # Sample exchange rate: 1 USD to 0.85 EUR
        },
        'India': {
            'INR': 1.0,  # 1 INR to INR
            'USD': 0.013,  # Sample exchange rate: 1 INR to 0.013 USD
            'EUR': 0.011,  # Sample exchange rate: 1 INR to 0.011 EUR
        }
    }

# Output
class PortfolioOutput:
    def __init__(self, portfolio, currency):
        self.portfolio = portfolio
        self.currency = currency

    def portfolio_daily_value(self):
        return self.portfolio.calculate_current_value(self.currency)

    def portfolio_net_cost(self):
        return self.portfolio.calculate_net()[0]

    def portfolio_total_quantity(self):
        return self.portfolio.calculate_net()[1]

    def portfolio_net_cost_per_unit(self):
        return self.portfolio.calculate_net()[2]

    def portfolio_net(self):
        return self.portfolio.calculate_net()

    def portfolio_daily_values_oneyear(self):
        return self.portfolio.assets_on_dates_oneyear()

    def portfolio_value_date_rs(self):
        input_dict = self.portfolio_daily_values_oneyear()
        values = self.portfolio.portfolio_value_for_dates(input_dict, "USD")
        return values

# Final Output
class OutputForPHP:
    def __init__(self, currency):
        self.output_list = []
        self.portfolio = Portfolio()
        self.currency = currency

    def output_portfolio(self, trade_data):
        self.portfolio.add_trades(trade_data)
        portfolioOutput = PortfolioOutput(self.portfolio, self.currency)
        output = portfolioOutput.portfolio_net()
        output_currentValue = portfolioOutput.portfolio_daily_value()
        output_dailyvalues2 = portfolioOutput.portfolio_value_date_rs()
        output_dailyvalues = portfolioOutput.portfolio_daily_values_oneyear()

        output_list = [
            [
                key,
                [list(output[1].values())[i], list(output[0].values())[i], list(output[2].values())[i],
                 list(output[3].values())[i]]
            ]
            for i, key in enumerate(output[0].keys())
        ]
        #output_list.append([list(output[0].values()), list(output[0].keys()), list(output[3].values())])

        return list(output_dailyvalues2.keys()), list(output_dailyvalues2.values()), output_list,list(output[0].values()), list(output[0].keys()), list(output[3].values())

# Get the portfolio output

def main():
    dataInput = json.loads(data_input)
    output = OutputForPHP("USA").output_portfolio(dataInput)
    #portfolio
    data_portfolio = output[2]

    #portfolio price chart data
    #data2_output= tickerEgData["close"]
    date_array= output[0]
    price_array = output[1]
    #final output array
    data_price= date_array,price_array,data_input,data_portfolio,output[3],output[4],output[5]
    # Date Array, Price Array, Input data, Portfolio Array

#removed print
    return(print(json.JSONEncoder().encode(data_price)))

if __name__=="__main__":
    main()


import numpy as np

from utils import mean_squared_error, d_mean_squared_error

class Model:
  def __init__(self):
    self.layers = []

  def add_layer(self, layer):
    self.layers.append(layer)

  def evaluate(self, inputs):
    # initialize "result" to inputs
    result = inputs
    # loop through layers and feed results forward
    for layer in self.layers:
      result = layer.forward_pass(result)

    # return result of network
    return result

  def gradient_descent(self, loss):
    running_gradient = loss
    for layer in reversed(self.layers):
      running_gradient = layer.back_propagate(running_gradient)
    for layer in self.layers:
      layer.update()

  def train(self, input_matrix, expected_results, validation_inputs, validation_expected):
    num_correct = 0
    for i, (_input, expected_result) in enumerate(zip(input_matrix, expected_results)):
      # add extra dimension to input to make matrix math work out
      input = np.expand_dims(_input, 0)

      # compute result of input
      actual_result = self.evaluate(input)
      print(np.argmax(actual_result))

      if np.argmax(expected_result) == np.argmax(actual_result):
        num_correct += 1

      # compute loss of result vs expected
      loss = mean_squared_error(expected_result, actual_result)
      d_loss = d_mean_squared_error(expected_result, actual_result)
      if i % 10000 == 0:
        print('Loss on sample ', i, ': ', np.linalg.norm(loss))
      # perform gradient descent
      self.gradient_descent(d_loss)

    print('num_correct: ', num_correct)
    print('Accuracy: ', round(num_correct / input_matrix.shape[0], 4) * 100, '%')

  def test(self, input_matrix, expected_results):
    num_correct = 0
    for _input, expected_result in zip(input_matrix, expected_results):
      # add extra dimension to input to make matrix math work out
      input = np.expand_dims(_input, 0)
      # compute result of input
      actual_result = self.evaluate(input)

      if np.argmax(expected_result) == np.argmax(actual_result):
        num_correct += 1

    print('Accuracy: ', round(num_correct / input_matrix.shape[0], 4) * 100, '%')


class Solution:
    def threeSum(self, nums: List[int]) -> List[List[int]]:
        if len(nums) < 3:
            return []

        """
        在固定第一个数v的状况，那当其中一个数为x，
        另一个数只能是-v-x(因为加总要等于0)。
        故每次将对应的互补数加入到dict中，
        我们只要从头到尾检查，即可得到所有的正确组合。
        """
        nums.sort()
        res = set()
        for i, v in enumerate(nums[:-2]): # 因为一次看三个
            if i >= 1 and v == nums[i-1]: # sort后，不用处理一样的数
                continue
            d = {}
            for x in nums[i+1:]:
                if x not in d:
                    d[-v-x] = 1
                else:
                    res.add((v, -v-x, x))

        return list(res)


# -*- coding: utf-8 -*-

# Standard imports
import argparse as ap
from pathlib import Path
import os
import pickle

# External imports
import pandas as pd
from sklearn import metrics
from sklearn.model_selection import cross_validate, train_test_split
from sklearn.model_selection import GridSearchCV, KFold
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import LinearRegression, Lasso, Ridge
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.ensemble import VotingRegressor
from sklearn.neighbors import KNeighborsClassifier
from matplotlib import pyplot as plt
from matplotlib import cm
from matplotlib.colors import Normalize
import numpy as np
from scipy.spatial.distance import cdist

import warnings
warnings.filterwarnings("ignore")

# Script information
__author__ = "Marti Municoy"
__license__ = "GPL"
__version__ = "1.0.1"
__maintainer__ = "Marti Municoy"
__email__ = "marti.municoy@bsc.es"

def parse_args():
    parser = ap.ArgumentParser()
    parser.add_argument("descriptors_path", metavar="PATH", type=str,
                        help="Path to descriptors")
    parser.add_argument("labels_path", metavar="PATH", type=str,
                        help="Path to labels")
    parser.add_argument("-o", "--output", metavar="PATH", type=str,
                        default="model_out", help="Output path")

    args = parser.parse_args()

    return args.descriptors_path, args.labels_path, args.output

def generate_logistic_model(X_train, y_train, X_test, y_test):
    print(' - Generating logistic regression model')
    log_reg = LogisticRegression(random_state=1993)
    cv = KFold(n_splits=5)
    results = cross_validate(log_reg, X_train, y_train, cv=cv,
                             return_estimator=True,
                             scoring='balanced_accuracy')

    best_model = None
    best_scorer = 0
    for m, s in zip(results['estimator'], results['test_score']):
        if (best_model is None or best_scorer < s):
            best_scorer = s
            best_model = m

    print('   - Cross-validation results:')
    print('     - Balanced accuracy:', np.mean(results['test_score']))

    y_pred = best_model.predict(X_test)

    print("   - Test set results:")
    print("     - Confusion matrix:")
    confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
    print("        | {} | {} |".format(*confusion_matrix[0]))
    print("        | {} | {} |".format(*confusion_matrix[1]))
    print("     - Accuracy:", metrics.accuracy_score(y_test, y_pred))
    print("     - Balanced accuracy:",
          metrics.balanced_accuracy_score(y_test, y_pred))
    print("     - Precision:", metrics.precision_score(y_test, y_pred))
    print("     - Recall:", metrics.recall_score(y_test, y_pred))

    return best_model

def generate_forest_model(X_train, y_train, X_test, y_test):
    print(' - Generating random forest model')

    for_cla = RandomForestClassifier(class_weight='balanced',
                                     random_state=1993)
    cv = KFold(n_splits=5)
    params_rf = {'n_estimators': [50, 100], 'max_depth': [2, 3, 4]}
    for_cla_gs = GridSearchCV(for_cla, params_rf, cv=cv,
                              scoring='balanced_accuracy')
    for_cla_gs.fit(X_train, y_train)

    print('   - Cross-validation results:')
    print('     - Balanced accuracy:', for_cla_gs.best_score_)

    best_model = for_cla_gs.best_estimator_

    y_pred = best_model.predict(X_test)

    print("   - Best parameters:")
    print("     - {}: {}".format(
        *[(i, j) for i, j in list(for_cla_gs.best_params_.items())][0]))
    print("     - {}: {}".format(
        *[(i, j) for i, j in list(for_cla_gs.best_params_.items())][1]))
    print("   - Test set results:")
    print("     - Confusion matrix:")
    confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
    print("        | {} | {} |".format(*confusion_matrix[0]))
    print("        | {} | {} |".format(*confusion_matrix[1]))
    print("     - Accuracy:", metrics.accuracy_score(y_test, y_pred))
    print("     - Balanced accuracy:",
          metrics.balanced_accuracy_score(y_test, y_pred))
    print("     - Precision:", metrics.precision_score(y_test, y_pred))
    print("     - Recall:", metrics.recall_score(y_test, y_pred))

    return best_model

def generate_neighbors_model(X_train, y_train, X_test, y_test):
    print(' - Generating k-nearest neighbors model')
    for_cla = KNeighborsClassifier()
    cv = KFold(n_splits=5)
    params_rf = {'n_neighbors': [1, 2, 3, 4, 5]}
    for_cla_gs = GridSearchCV(for_cla, params_rf, cv=cv,
                              scoring='balanced_accuracy')
    for_cla_gs.fit(X_train, y_train)

    print('   - Cross-validation results:')
    print('     - Balanced accuracy:', for_cla_gs.best_score_)

    best_model = for_cla_gs.best_estimator_

    y_pred = best_model.predict(X_test)

    print("   - Best parameters:")
    print("     - {}: {}".format(
        *[(i, j) for i, j in list(for_cla_gs.best_params_.items())][0]))
    print("   - Test set results:")
    print("     - Confusion matrix:")
    confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
    print("        | {} | {} |".format(*confusion_matrix[0]))
    print("        | {} | {} |".format(*confusion_matrix[1]))
    print("     - Accuracy:", metrics.accuracy_score(y_test, y_pred))
    print("     - Balanced accuracy:",
          metrics.balanced_accuracy_score(y_test, y_pred))
    print("     - Precision:", metrics.precision_score(y_test, y_pred))
    print("     - Recall:", metrics.recall_score(y_test, y_pred))

    return best_model

def generate_ensemble_classifier(models_to_combine,
                                 X_train, y_train, X_test, y_test):
    print(' - Generating ensemble model')

    ensemble = VotingClassifier(estimators=models_to_combine, voting='soft')
    cv = KFold(n_splits=5)
    results = cross_validate(ensemble, X_train, y_train, cv=cv,
                             return_estimator=True,
                             scoring='balanced_accuracy')

    best_model = None
    best_scorer = 0
    for m, s in zip(results['estimator'], results['test_score']):
        if (best_model is None or best_scorer < s):
            best_scorer = s
            best_model = m

    print('   - Cross-validation results:')
    print('     - Balanced accuracy:', np.mean(results['test_score']))

    y_pred = best_model.predict(X_test)

    print("   - Test set results:")
    print("     - Confusion matrix:")
    confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
    print("        | {} | {} |".format(*confusion_matrix[0]))
    print("        | {} | {} |".format(*confusion_matrix[1]))
    print("     - Accuracy:", metrics.accuracy_score(y_test, y_pred))
    print("     - Balanced accuracy:",
          metrics.balanced_accuracy_score(y_test, y_pred))
    print("     - Precision:", metrics.precision_score(y_test, y_pred))
    print("     - Recall:", metrics.recall_score(y_test, y_pred))

    return best_model

def generate_regression_model(X_train, y_train, X_test, y_test):
    print(' - Generating linear regression model')
    lin_reg = LinearRegression()
    cv = KFold(n_splits=5)
    results = cross_validate(lin_reg, X_train, y_train, cv=cv,
                             return_estimator=True,
                             scoring='r2')

    best_model = None
    best_scorer = 0

    for m, s in zip(results['estimator'], results['test_score']):
        if (best_model is None or best_scorer < s):
            best_scorer = s
            best_model = m

    print(X_test)

    y_pred = best_model.predict(X_test)

    print('   - Cross-validation results:')
    print('     - r2:', np.max(results['test_score']))
    print("   - Coefficients:")
    print("     - be / atoms : {}".format(best_model.coef_[0]))
    print("     - rotamers : {}".format(best_model.coef_[1]))
    print("     - internal : {}".format(best_model.coef_[2]))
    print("   - Intercept:")
    print("     - {}".format(best_model.intercept_))
    print("   - Test set results:")
    print("     - r2:", metrics.r2_score(y_test, y_pred))

    return best_model

def generate_lasso_model(X_train, y_train, X_test, y_test):
    print(' - Generating linear lasso model')
    lin_las = Lasso()
    cv = KFold(n_splits=5)
    params_rf = {'alpha': np.arange(0, 1, 0.05)}
    lin_las_gs = GridSearchCV(lin_las, params_rf, cv=cv,
                              scoring='r2')
    lin_las_gs.fit(X_train, y_train)

    best_model = lin_las_gs.best_estimator_

    y_pred = best_model.predict(X_test)

    print('   - Cross-validation results:')
    print('     - r2:', lin_las_gs.best_score_)
    print("   - Best parameters:")
    print("     - {}: {}".format(
        *[(i, j) for i, j in list(lin_las_gs.best_params_.items())][0]))
    print("   - Coefficients:")
    print("     - be / atoms : {}".format(best_model.coef_[0]))
    print("     - rotamers : {}".format(best_model.coef_[1]))
    print("     - internal : {}".format(best_model.coef_[2]))
    print("   - Intercept:")
    print("     - {}".format(best_model.intercept_))
    print("   - Test set results:")
    print("     - r2:", metrics.r2_score(y_test, y_pred))

    return best_model

def generate_ridge_model(X_train, y_train, X_test, y_test):
    print(' - Generating linear ridge model')
    lin_rid = Ridge()
    cv = KFold(n_splits=5)
    results = cross_validate(lin_rid, X_train, y_train, cv=cv,
                             return_estimator=True,
                             scoring='r2')

    best_model = None
    best_scorer = 0

    for m, s in zip(results['estimator'], results['test_score']):
        if (best_model is None or best_scorer < s):
            best_scorer = s
            best_model = m

    y_pred = best_model.predict(X_test)

    print('   - Cross-validation results:')
    print('     - r2:', np.max(results['test_score']))
    print("   - Coefficients:")
    print("     - be / atoms : {}".format(best_model.coef_[0]))
    print("     - rotamers : {}".format(best_model.coef_[1]))
    print("     - internal : {}".format(best_model.coef_[2]))
    print("   - Intercept:")
    print("     - {}".format(best_model.intercept_))
    print("   - Test set results:")
    print("     - r2:", metrics.r2_score(y_test, y_pred))

    return best_model

def generate_ensemble_regressor(models_to_combine,
                                X_train, y_train, X_test, y_test):
    print(' - Generating ensemble model')

    ensemble = VotingRegressor(estimators=models_to_combine)
    cv = KFold(n_splits=5)
    results = cross_validate(ensemble, X_train, y_train, cv=cv,
                             return_estimator=True,
                             scoring='r2')

    best_model = None
    best_scorer = 0

    for m, s in zip(results['estimator'], results['test_score']):
        if (best_model is None or best_scorer < s):
            best_scorer = s
            best_model = m

    y_pred = best_model.predict(X_test)

    print('   - Cross-validation results:')
    print('     - r2:', np.max(results['test_score']))
    print("   - Test set results:")
    print("     - r2:", metrics.r2_score(y_test, y_pred))

    return best_model

def plot_classification(activity_classifier_1, X_test, y_test,
                        title, output_path):
    disp = metrics. plot_confusion_matrix(activity_classifier_1,
                                          X_test, y_test,
                                          cmap=plt.cm.Blues)

    disp.ax_.set_title(title)

    plt.savefig(output_path)
    plt.clf()
    plt.cla()

def plot_regressor(model, X_test, y_test, X_train, y_train, title,
                   output_path, nn=None, min_nn=1):
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 6))
    plt.subplots_adjust(left=0.15, bottom=0.3, right=0.95, top=0.9)
    fig.suptitle(title, fontsize=16)

    y_pred = model.predict(X_train)
    ax1.plot(((min(y_train), max(y_train))), ((min(y_pred), max(y_pred))),
             'k--', linewidth=1)
    ax1.scatter(y_train, y_pred, color='red')
    ax1.set_title("Cross-validation results")
    ax1.set_ylabel("Predicted -pIC50")
    ax1.set_xlabel("Experimental -pIC50")

    y_pred = model.predict(X_test)
    ax2.plot(((min(y_test), max(y_test))), ((min(y_pred), max(y_pred))),
             'k--', linewidth=1)
    if (nn is None):
        ax2.scatter(y_test, y_pred, color='red', c=nn)
    else:
        colormap = cm.get_cmap('Wistia', max(nn) - (min_nn - 1))
        norm = Normalize(0, max(nn) - (min_nn - 1))
        colormap.set_under('grey')
        for x, y, n in zip(y_test, y_pred, nn):
            ax2.scatter(x, y, c=[colormap(n - min_nn)], label=n)
        fig.colorbar(cm.ScalarMappable(cmap=colormap, norm=norm), ax=ax2,
                     extend='min')
        X_test, y_test = filter_entries_by_nn(X_test, y_test,
                                              nn,
                                              min_nn=min_nn)
    ax2.set_title("Test set results")
    ax2.set_xlabel("Experimental -pIC50")

    try:
        props = dict(boxstyle='round', facecolor='orange', alpha=0.5)
        text = 'Coefficients:\n'
        text += ' • be / atoms = {:.2f}\n'.format(model.coef_[0])
        text += ' • rotamers = {:.2f}\n'.format(model.coef_[1])
        text += ' • internal = {:.2f}'.format(model.coef_[2])
        # place a text box in upper left in axes coords
        plt.text(0.1, 0.1, text, fontsize=12,
                 verticalalignment='center', bbox=props,
                 transform=fig.transFigure)

        text = 'Intercept:\n'
        text += ' • {:.2f}'.format(model.intercept_)
        plt.text(0.35, 0.1, text, fontsize=12,
                 verticalalignment='center', bbox=props,
                 transform=fig.transFigure)

        text = 'Cross-validation r2:\n'
        text += ' • {:.2f}'.format(model.score(X_train, y_train))
        plt.text(0.55, 0.1, text, fontsize=12,
                 verticalalignment='center', bbox=props,
                 transform=fig.transFigure)

        text = 'Test set r2:\n'
        text += ' • {:.2f}'.format(model.score(X_test, y_test))
        plt.text(0.8, 0.1, text, fontsize=12,
                 verticalalignment='center', bbox=props,
                 transform=fig.transFigure)

    except AttributeError:
        text = 'Cross-validation r2:\n'
        text += ' • {:.2f}'.format(model.score(X_train, y_train))
        plt.text(0.30, 0.1, text, fontsize=12,
                 verticalalignment='center', bbox=props,
                 transform=fig.transFigure)

        text = 'Test set r2:\n'
        text += ' • {:.2f}'.format(model.score(X_test, y_test))
        plt.text(0.75, 0.1, text, fontsize=12,
                 verticalalignment='center', bbox=props,
                 transform=fig.transFigure)

    plt.savefig(output_path)
    plt.clf()
    plt.cla()

def domain_analysis(train_domain):
    print(" - Computing applicability domains")

    # Matrix of descriptor distances between entries
    distances = np.array([cdist([x], train_domain) for x in train_domain])
    distances_sorted = [np.sort(d[0]) for d in distances]

    # Discard 0s, which are distances to the same entry (diagonal
    # elements in the matrix)
    distances_matrix = [d[1:] for d in distances_sorted]

    # Empirical calculation of the smoothing parameter k
    k = int(round(pow(len(train_domain), 1 / 3)))

    # Calculate the mean of first k distances for each sample
    d_means = [np.mean(d[:k][0]) for d in distances_matrix]

    # Calculate quartiles
    Q1 = np.quantile(d_means, .25)
    Q3 = np.quantile(d_means, .75)

    # Calculate reference value
    d_ref = Q3 + 1.5 * (Q3 - Q1)

    # Calculate thresholds
    Dijs = []
    Kis = []
    for distances in distances_matrix:
        allowed_distances = [d for d in distances if d <= d_ref]
        Dijs.append(allowed_distances)
        Kis.append(len(allowed_distances))

    thresholds = []
    for Dij, Ki in zip(Dijs, Kis):
        if (Ki == 0):
            thresholds.append(0)
        else:
            thresholds.append(np.sum(Dij) / Ki)

    thresholds = np.array(thresholds)

    # Convert 0 to minimum value not zero
    thresholds[thresholds == 0] = min(thresholds[thresholds != 0])

    return thresholds

def check_applicability_domain(train_domain, test_domain, thresholds):
    print(" - Checking applicability domains")

    # Matrix of descriptor distances between entries
    distances = np.array([cdist([x], train_domain) for x in test_domain])

    results = []
    for Dijs in distances:
        # get nearest neighbors
        NN = len([j for j, Dij in enumerate(Dijs[0]) if Dij <= thresholds[j]])
        results.append(NN)

    return results

def filter_entries_by_nn(X, y, nn, min_nn):
    f_X = X[[i for i, _ in enumerate(X) if nn[i] >= min_nn]]
    f_y = y[[i for i, _ in enumerate(y) if nn[i] >= min_nn]]

    return f_X, f_y

def main():
    descriptors_path, labels_path, output_path = parse_args()

    descriptors_path = Path(descriptors_path)
    labels_path = Path(labels_path)
    output_path = Path(output_path)

    if (not descriptors_path.is_file()):
        print(' - Error: invalid descriptors path \'{}\''.format(
            descriptors_path))

    if (not labels_path.is_file()):
        print(' - Error: invalid dataset path \'{}\''.format(
            labels_path))

    if (not output_path.is_dir()):
        os.mkdir(str(output_path))

    descriptors = pd.read_csv(str(descriptors_path))
    labels = pd.read_csv(str(labels_path))

    dataset = descriptors.merge(labels, left_on='path', right_on='path')

    # Remove unnamed columns
    dataset = dataset.loc[:, ~dataset.columns.str.contains('^Unnamed')]

    # Remove non numeric IC50 (exclamation mark!)
    f_dataset = dataset[pd.to_numeric(dataset.IC50,
                                      errors='coerce').notnull()]

    # Calculate BE / number of atoms
    f_dataset['be / atoms'] = f_dataset['be'] / f_dataset['atoms']

    # Drop unnecessary columns
    f_dataset.drop(['toten', 'weight', 'atoms', 'be', 'clusters'], axis=1, inplace=True)

    # Split by activity
    f_dataset.loc[:, 'IC50'] = f_dataset.loc[:, 'IC50'].apply(pd.to_numeric)
    f_dataset['pIC50'] = - np.log10(f_dataset.loc[:, 'IC50'] / 1000000)
    actives, inactives = [x for _, x in f_dataset.groupby(
        [f_dataset['IC50'] < 0])]

    # Convert activity in a boolean
    f_dataset['activity'] = f_dataset['IC50'] > 0
    f_dataset.drop(['IC50'], axis=1, inplace=True)

    """
    # Split data into training and test set
    X_train, X_test, y_train, y_test = train_test_split(
        f_dataset.loc[:, ['be / atoms', 'rotamers', 'internal']].to_numpy(),
        f_dataset['activity'].to_list(),
        train_size=0.75,
        random_state=123,
        stratify=f_dataset['activity'].to_list())

    activity_classifier_1 = generate_logistic_model(X_train, y_train,
                                                    X_test, y_test)

    with open(str(output_path.joinpath('activity_classifier_1.pkl')), 'wb') \
            as handle:
        pickle.dump(activity_classifier_1, handle)

    plot_classification(activity_classifier_1, X_test, y_test,
                        "Activity logistic regression classification",
                        str(output_path.joinpath('activity_classifier_1.png')))

    activity_classifier_2 = generate_forest_model(X_train, y_train,
                                                  X_test, y_test)

    with open(str(output_path.joinpath('activity_classifier_2.pkl')), 'wb') \
            as handle:
        pickle.dump(activity_classifier_2, handle)

    plot_classification(activity_classifier_2, X_test, y_test,
                        "Activity random forest classification",
                        str(output_path.joinpath('activity_classifier_2.png')))

    activity_classifier_3 = generate_neighbors_model(X_train, y_train,
                                                     X_test, y_test)

    with open(str(output_path.joinpath('activity_classifier_3.pkl')), 'wb') \
            as handle:
        pickle.dump(activity_classifier_3, handle)

    plot_classification(activity_classifier_3, X_test, y_test,
                        "Activity k-nearest neighbor classification",
                        str(output_path.joinpath('activity_classifier_3.png')))

    activity_classifier_4 = generate_ensemble_classifier(
        [('Logistic regression', activity_classifier_1),
         ('Random forest', activity_classifier_2,),
         ('k-nearest neighbors', activity_classifier_3)],
        X_train, y_train, X_test, y_test)

    with open(str(output_path.joinpath('activity_classifier_4.pkl')), 'wb') \
            as handle:
        pickle.dump(activity_classifier_4, handle)

    plot_classification(activity_classifier_4, X_test, y_test,
                        "Activity ensemble classification",
                        str(output_path.joinpath('activity_classifier_4.png')))
    """

    # Split data into training and test set
    X_train, X_test, y_train, y_test = train_test_split(
        actives.loc[:, ['be / atoms', 'rotamers', 'internal']].to_numpy(),
        actives['pIC50'].to_numpy(),
        train_size=0.75,
        random_state=123)

    thresholds = domain_analysis(X_train)

    nn_results = check_applicability_domain(X_train, X_test, thresholds)

    min_nn = 0
    f_X_test, f_y_test = filter_entries_by_nn(X_test, y_test,
                                              nn_results,
                                              min_nn=min_nn)
    affinity_predictor_1 = generate_regression_model(X_train, y_train,
                                                     f_X_test, f_y_test)

    with open(str(output_path.joinpath('affinity_predictor_1.pkl')),
              'wb') as handle:
        pickle.dump(affinity_predictor_1, handle)
        pickle.dump(thresholds, handle)

    plot_regressor(affinity_predictor_1, X_test, y_test, X_train, y_train,
                   'Linear regression',
                   str(output_path.joinpath('affinity_predictor_1.png')),
                   nn=nn_results, min_nn=min_nn)

    affinity_predictor_2 = generate_lasso_model(X_train, y_train,
                                                f_X_test, f_y_test)

    with open(str(output_path.joinpath('affinity_predictor_2.pkl')),
              'wb') as handle:
        pickle.dump(affinity_predictor_2, handle)
        pickle.dump(thresholds, handle)

    plot_regressor(affinity_predictor_2, X_test, y_test, X_train, y_train,
                   'Lasso regression',
                   str(output_path.joinpath('affinity_predictor_2.png')),
                   nn=nn_results, min_nn=min_nn)

    affinity_predictor_3 = generate_ridge_model(X_train, y_train,
                                                f_X_test, f_y_test)

    with open(str(output_path.joinpath('affinity_predictor_3.pkl')),
              'wb') as handle:
        pickle.dump(affinity_predictor_3, handle)
        pickle.dump(thresholds, handle)

    plot_regressor(affinity_predictor_3, X_test, y_test, X_train, y_train,
                   'Ridge regression',
                   str(output_path.joinpath('affinity_predictor_3.png')),
                   nn=nn_results, min_nn=min_nn)

    affinity_predictor_4 = generate_ensemble_regressor(
        [('Linear regression', affinity_predictor_1),
         ('Ridge regression', affinity_predictor_3)],
        X_train, y_train, f_X_test, f_y_test)

    with open(str(output_path.joinpath('affinity_predictor_4.pkl')),
              'wb') as handle:
        pickle.dump(affinity_predictor_4, handle)
        pickle.dump(thresholds, handle)

    plot_regressor(affinity_predictor_4, X_test, y_test, X_train, y_train,
                   'Ensemble regression',
                   str(output_path.joinpath('affinity_predictor_4.png')),
                   nn=nn_results, min_nn=min_nn)

if __name__ == "__main__":
    main()


from typing import Any, Dict, Iterable, List, Tuple, TypeVar, Union

import tensorflow as tf
import torch

from fastestimator.backend._huber import huber
from fastestimator.backend._l1_loss import l1_loss
from fastestimator.backend._reduce_mean import reduce_mean
from fastestimator.backend._smooth_l1_loss import smooth_l1_loss
from fastestimator.op.tensorop.loss.loss import LossOp
from fastestimator.util.traceability_util import traceable

Tensor = TypeVar('Tensor', tf.Tensor, torch.Tensor)

@traceable()
class L1_Loss(LossOp):
    """Calculate the L1 loss between two tensors.

    This LossOp can be used to Implement:
        L1 loss: Is a criterion that calculates Mean Absolute Error between the elements ([y_pred, y_true]).
        Smooth_L1 loss: Is a criterion that uses squared loss if absolute element wise subtraction (y_pred - y_true) is less than
                        'beta' and vanilla L1 loss otherwise.
        Huber loss: Is a criterion that uses squared loss if absolute element wise subtraction (y_pred - y_true) is less than
                    'beta' and a 'beta' scaled L1 loss otherwise.

    Args:
        inputs: A tuple or list like: [y_pred, y_true].
        outputs: String key under which to store the computed loss.
        mode: What mode(s) to execute this Op in. For example, "train", "eval", "test", or "infer". To execute
            regardless of mode, pass None. To execute in all modes except for a particular one, you can pass an argument
            like "!infer" or "!train".
        ds_id: What dataset id(s) to execute this Op in. To execute regardless of ds_id, pass None. To execute in all
            ds_ids except for a particular one, you can pass an argument like "!ds1".
        average_loss: Whether to average the element-wise loss after the Loss Op.
        loss_type: What type of L1 loss. Can either be 'L1' (L1 Loss), 'Smooth' (Smooth L1 Loss) or 'Huber' (Huber loss). Default:'L1'
        beta: A threshold at which to change between L1 and L2 loss. Needs to be a positive number. Default:1.0 . dtype: float16 or float32.
    """
    def __init__(self,
                 inputs: Union[Tuple[str, str], List[str]],
                 outputs: str,
                 mode: Union[None, str, Iterable[str]] = "!infer",
                 ds_id: Union[None, str, Iterable[str]] = None,
                 average_loss: bool = True,
                 loss_type: str = 'L1',
                 beta: Union[None, float] = 1.0):
        self.average_loss = average_loss
        self.loss_type = loss_type
        self.beta = beta
        super().__init__(inputs=inputs, outputs=outputs, mode=mode, ds_id=ds_id)

    def forward(self, data: List[Tensor], state: Dict[str, Any]) -> Tensor:
        y_pred, y_true = data
        if self.loss_type == 'L1':
            loss = l1_loss(y_true=y_true, y_pred=y_pred)
        elif self.loss_type == 'Smooth':
            loss = smooth_l1_loss(y_true=y_true, y_pred=y_pred, beta=self.beta)
        elif self.loss_type == 'Huber':
            loss = huber(y_true=y_true, y_pred=y_pred, beta=self.beta)
        else:
            raise ValueError(
                "Unrecognized Loss type. Can either be None (L1 Loss), 'Smooth' (Smooth L1 Loss) or 'Huber' (Huber loss)"
            )
        if self.average_loss:
            loss = reduce_mean(loss)
        return loss


# -*- coding: utf-8 -*-
import io
import os
import os.path
from hashlib import md5

from django.http import HttpResponse, Http404
from django.views.generic.create_update import create_object
from django.shortcuts import render_to_response, get_object_or_404, redirect
from django.template import loader, RequestContext
from django.utils import translation
from django.utils.translation import ugettext_lazy as _
from django.core.urlresolvers import reverse
from django.core.mail import mail_managers
from django.conf import settings

from apps.cpm2013.constants import APP_ROOT
from apps.cpm2013.models import Submission, NewsEntry, Page,\
     SubmissionFileUpload
from apps.cpm2013.forms import SubmissionForm, FileUploadForm
from apps.cpm2013.tasks import SendSubmissionEmail

def index(request):

    return render_to_response(
        'cpm2013/results_index.html',
        context_instance=RequestContext(request),
    )

def submit(request):
    return render_to_response(
        'cpm2013/submit_closed.html',
        context_instance=RequestContext(request),
    )

    form = SubmissionForm(request.POST or None)
    if request.method == 'POST' and form.is_valid():
        submission = form.save(commit=False)
        submission.submission_language = translation.get_language()
        submission.save()

        SendSubmissionEmail().apply_async(args=[submission])

        return render_to_response(
            'cpm2013/submit_done.html',
            {'email': submission.applicant_email},
            context_instance=RequestContext(request),
        )

    return render_to_response(
        'cpm2013/submit.html',
        {'form': form},
        context_instance=RequestContext(request),
    )

def page(request, slug):
    base_page = get_object_or_404(Page, slug=slug)
    pages = base_page._meta.translations_model.objects.filter(master=base_page)
    pages = dict((t.language_code, t) for t in pages)

    current_lang = translation.get_language()
    if current_lang in pages:
        page = pages[current_lang]
    else:
        for lang_code in ['en', 'ru', 'be']:
            if lang_code in pages:
                page = pages[lang_code]
                break

    return render_to_response(
        'cpm2013/page.html',
        {'page': page},
        context_instance=RequestContext(request),
    )

class Rules:
    LANGUAGES = {
        'be': 'rules_ru.md',
        'ru': 'rules_ru.md',
        'en': 'rules_en.md',

        'ar': 'rules_ar.md',
        'cn': 'rules_cn.md',
        'de': 'rules_de.md',
        'es': 'rules_es.md',
        'fr': 'rules_fr.md',
        'it': 'rules_it.md',
        'pl': 'rules_pl.md',
        'pt': 'rules_pt.md',
    }
    RTL = set(('ar',))

    PATH = os.path.join(APP_ROOT, 'docs')

    @classmethod
    def translation(cls, lang):
        if lang is None:
            lang = translation.get_language()

        lang = lang.lower()
        if lang not in cls.LANGUAGES:
            lang = 'en'

        rules = os.path.join(cls.PATH, cls.LANGUAGES[lang])
        rtl = lang in cls.RTL

        return lang, rules, rtl

    def __call__(self, request, lang=None):
        lang, rules_filename, rtl = self.translation(lang)
        rules = io.open(
            rules_filename,
            'r', encoding='utf-8'
        ).read()
        return render_to_response(
            'cpm2013/rules.html',
            {
                'rules': rules,
                'rtl': rtl,
                'current_lang': lang,
                'languages': sorted(self.LANGUAGES.iterkeys()),
            },
            context_instance=RequestContext(request),
        )
rules = Rules()

def partners(request):
    img_dir = '/static/cpm2013/banners/'
    main_banners = [
        (
            img_dir + 'where450.png',
            'http://www.spn.ru/publishing/whereminsk/',
            'Where Minsk'
        ),
        (
            img_dir + 'relax450.png',
            'http://relax.by/',
            'relax.by - развлечения в Минске, развлекательные центры столицы'
        ),

    ]
    banners = [
        (
            img_dir + 'lamora.png',
            'http://la-mora.info/',
            'Арт-пространство ДК «La мора»'
        ),        (
            img_dir + 'bvc.png',
            'http://www.belvc.by/',
            'Белорусский видеоцентр'
        ),
        (
            img_dir + 'minsk24dok.png',
            'http://mtis.tv/',
            'Минск 24 ДОК'
        ),
        (
            img_dir + 'paramonak.png',
            'http://paramonak.by/',
            'Креативное агенство Парамонак'
        ),
        (
            img_dir + 'kinolife.png',
            'http://kinolife.eu/',
            'Film & TV Online Platform'
        ),
        (
            img_dir + 'iysff.png',
            'http://www.makesilentfilm.com/',
            'International Youth Silent Film Festival'
        ),
        (
            img_dir + 'luma.png',
            'http://luma.net.my/',
            'LUMA – a creative hub for the media arts'
        ),
        (
            img_dir + 'euroradio.png',
            'http://euroradio.fm/',
            'Последние новости политики и культуры Беларуси - Euroradio'
        ),
        (
            img_dir + 'bolshoi.png',
            'http://bolshoi.by/',
            'Журнал "Большой"'
        ),
        (
            img_dir + 'minchanka.png',
            'http://www.minchanka.by/',
            'Минчанка: быть женщиной - это интересно!'
        ),
        (
            img_dir + 'open.png',
            'http://open.by/',
            'Интернет-портал OPEN.BY'
        ),
        (
            img_dir + 'mart.png',
            'http://mart.by/',
            'Современное белорусское искусство'
        ),
        (
            img_dir + 'vgomele.png',
            'http://vgomele.by/',
            'vGomele.by - гомельский информационно-развлекательный портал'
        ),
        (
            img_dir + 'belarusdigest.png',
            'http://belarusdigest.com/',
            'Беларусы за мяжой'
        ),
        (
            img_dir + 'ky.png',
            'http://kyky.org/',
            'KYKY.ORG - Культурный портал'
        ),
        (
            img_dir + 'nomadic.png',
            'http://vagrant.kinaklub.org/',
            'Nomadic Film Club'
        ),
    ]
    return render_to_response(
        'cpm2013/partners.html',
        {'main_banners': main_banners, 'banners': banners},
        context_instance=RequestContext(request),
    )

def submission_info(request, subm_id, subm_hash):
    submission = get_object_or_404(Submission, id=subm_id)

    real_hash = md5('%s%s' % (settings.SECRET_KEY, subm_id)).hexdigest()
    if real_hash != subm_hash:
        return HttpResponse('Forbidden', status=403)

    file_uploads = SubmissionFileUpload.objects.filter(submission=submission)

    return render_to_response(
        'cpm2013/submission_info.html',
        {
            'submission': submission,
            'files': list(file_uploads),
            'subm_hash': subm_hash
        },
        context_instance=RequestContext(request),
    )

def submission_info_upload(request, subm_id, subm_hash):
    submission = get_object_or_404(Submission, id=subm_id)

    real_hash = md5('%s%s' % (settings.SECRET_KEY, subm_id)).hexdigest()
    if real_hash != subm_hash:
        return HttpResponse('Forbidden', status=403)

    form = FileUploadForm(request.POST or None, request.FILES or None)
    error = ''
    if request.method == 'POST' and form.is_valid():
        uploaded_file = form.cleaned_data['file']

        name, ext = os.path.splitext(uploaded_file.name)
        real_name = md5(
            '%s%s' % (settings.SECRET_KEY, uploaded_file.name)
        ).hexdigest() + ext
        subm_file_upload = SubmissionFileUpload(
            submission=submission,
            display_name=name + ext,
            real_name=real_name
        )

        real_path = os.path.join(settings.MEDIA_ROOT, real_name)
        if not os.path.exists(real_path):
            with open(real_path, 'w') as f:
                buf = uploaded_file.read(4096)
                while buf:
                    f.write(buf)
                    buf = uploaded_file.read(4096)

            subm_file_upload.save()
            mail_managers(
                'CPM2013: new file upload',
                '%s at %s' % (
                    uploaded_file.name,
                    request.build_absolute_uri(reverse(
                        'cpm2013:submission_info',
                        args=[subm_id, subm_hash]
                    )),
                )
            )

            return redirect('cpm2013:submission_info', subm_id, subm_hash)

        error = _('File with such name exists')

    return render_to_response(
        'cpm2013/submission_info_upload.html',
        {'submission': submission, 'form': form, 'error': error},
        context_instance=RequestContext(request),
    )

def press_kit(request):
    programs
    return render_to_response(
        'cpm2013/press_kit.html', {},
        context_instance=RequestContext(request),
    )

def programs(request):
    from apps.cpm2013.programs import programs

    return render_to_response(
        'cpm2013/programs.html', {'programs': programs},
        context_instance=RequestContext(request),
    )

def program(request, program_id):
    from apps.cpm2013.programs import get_program

    program = get_program(program_id)
    if program is None:
        raise Http404

    return render_to_response(
        'cpm2013/program.html',
        {
            'title': program.title,
            'description': program.description,
            'parts': program.parts,
        },
        context_instance=RequestContext(request),
    )


# bfi - BrainFuck GNU interpreter, (C) 2002 Philippe Biondi, biondi@cartel-securite.fr
# Modified for ShedSkin, but can be used with Python/Psyco too

from sys import stdin, stdout, argv

def BF_interpreter(prog):
    CELL = 255 # Or 65535  default 255
    # clear program to speed up execution
    prog = "".join([c for c in prog if c in "><+-.,[]"])
    len_prog = len(prog)

    tape = [0] # This can be initialized to 30000 cells
    ip = 0
    p = 0
    level = 0

    while ip < len_prog:
        x = prog[ip]
        ip += 1

        if x == '+':
            tape[p] = (tape[p]+1) & CELL
        elif x == '-':
            tape[p] = (tape[p]-1) & CELL
        elif x == '>':
            p += 1
            if len(tape) <= p:
                tape.append(0)
        elif x == '<':
            if p:
                p -= 1
            else:
                #print "Warning: inserting one element at the begining"
                tape.insert(0, 0)
        elif x == '.':
            stdout.write( chr(tape[p]) )
        elif x == ',':
            tape[p] = ord(stdin.read(1))
        elif x == '[':
            if not tape[p]:
                while True:
                    if prog[ip] == '[':
                        level += 1
                    if prog[ip] == ']':
                        if level:
                            level -= 1
                        else:
                            break
                    ip += 1
                ip += 1
        elif x == ']':
            ip -= 2
            while True:
                if prog[ip] == ']':
                    level += 1
                if prog[ip] == '[':
                    if level:
                        level -= 1
                    else:
                        break
                ip -= 1

if __name__ == '__main__':
    program = open('testdata/99bottles.bf', 'r').read()
    BF_interpreter(program)


from qrcode import make
from discord.ext import commands
from discord import File
from io import BytesIO

class QRGen(commands.Cog):
    """
    Module for generating QR codes from attachments and URLs
    """

    def __init__(self, bot):
        self.bot = bot

    @commands.command()
    async def qr(self, ctx, url: str=None):
        """Generate QR Code"""

        if not url:
            async for m in ctx.channel.history():
                if len(m.attachments) == 1:  # Currently this only supports 1 attachment at most
                    img = make(m.attachments[0].url)
                    imgbuf = BytesIO()
                    img.save(imgbuf, 'png')
                    imgbuf.seek(0)
                    await ctx.send(file=File(imgbuf, "qr_code.png"))
                    return
        else:
            img = make(url)
            imgbuf = BytesIO()
            img.save(imgbuf, 'png')
            imgbuf.seek(0)
            await ctx.send(file=File(imgbuf, "qr_code.png"))

def setup(bot):
    bot.add_cog(QRGen(bot))


# 1: statements and expressions

# python expressions are pieces of python code that produce a value
# i.e., their value will eventually be substituted into the code where
# they're used. Arithmetic expressions, function calls, and method calls
# are examples of python expressions. Other common expressions are list
# literals, dictionary literals, string literals, etc.
1 + 2

"a string"

max(2, 3)

print("Hello!")

# statements are python code that contain a python keyword or an assignment operator.
# statements might produce a value just like an expression does,
# but can have other effects as well, like creating a variable. Statements
# are also used to introduce blocks (see below).

xs = [1, 2, 3]  # an assignment statement
xs += [4, 5]    # an augmented assignment operator
2 and 3         # a statement using the `and` keyword
letters = ''.join([chr(97 + i) for i in range(26)])

# 2: blocks

# blocks represent code whose execution is controlled by some python statement.
# the simplest example of a block is the block under an `if` statement, (it only
# runs when the `if` statement produces a value of `True`) but `for` and `while` loops
# also use blocks. A block is treated as all of the code after a colon that is indented
# one level more than the statement that introduced the block.

if len(xs) >= 3:
    print("xs is a long list.")

for letter in letters:
    letters +=  letter

with open("students.txt") as students:
    lines = students.read()
    print(lines)

# 3: types

# every value in python has a type. A value's type determines what operations it works with.
# integers work with addition, but not inside a `for` loop.

2 + 3  # fine!
# for i in 5: # an error!

# most common built-in types are int, float, str, list, tuple, and dict (examples of each below)
1

2.5

"2.5"

[2, 5]

(2, 5)

{ 2: 5, "will": "teacher" }
# 4: collections

# collections are groups of python objects. Collections can be thought of as 'things it makes
# sense to use within a for loop.'

setx = {1, 2, 3, 4}

# 5: function definitions

# function definitions are introduced with the `def` keyword and indicate
# the start of a function. A function takes variables, substitutes them in its
# bodies and produces a result (indicated by `return`).
def my_function():
    print ("Here is a function")
    return True

def add(a, b):
    return a + b

# 6: modules

# modules are units of python code outside the current Python file that can be used within the current Python file.
# modules can be included in the current file using the `import` keyword, or specific code can be imported using the
# `from` <module> `import` <thing> special syntax.

# some python standard library modules
import math
import random
import json
import os
import sys
import pathlib

from collections import Counter, deque
from statistics import mean

import functools as ft
import itertools as it
import operator as op

# 7: methods and attributes

# attributes are data that can be accessed by name using the `.` operator. Methods are functions defined inside the body
# of a `class` (we'll see more of this tomorrow). Methods are also called using the `.` operator (in a way, classes are objects,
# and methods are attributes of those objects). It's a good rule of thumb to allow object modification (adding or removing members)
# to take place only within methods (rather than within arbitrary functions).

print(wordlist)

f"{wordlist}".lower()
"HELLO HOW ARE YOU".lower()

xs.append(6)

xs.remove(3)

# 8: Comprehensions

# Comprehensions are a syntax for building collections using a variant of the for statement.

[x for x in range(10)]                        # a simple comprehension
[chr(x*2) for x in range(100)]                # a more complex expression
[chr(x*2) for x in range(100) if x % 4 == 0]  # with a filter

# Comprehensions can be used to build lists, dicts, sets and generators

{random.randint(0,1000) for _ in range(25)}
{i: True for i in range(25)}
(chr(random.randint(0, 20000)) for _ in range(25))


import numpy as np
from sklearn.impute import SimpleImputer
import pandas as pd
import sklearn
from scipy import stats
import statsmodels.api as sm
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2
import seaborn as sns
import matplotlib.pyplot as plt

#Feature_Selection
def univariate_feature_selection(data, target, features):
    best_feat = SelectKBest(score_func=chi2, k=5)
    fit = best_feat.fit(data, target)
    scores = pd.DataFrame(fit.scores_)
    cols = pd.DataFrame(data.columns)
    score_feat = pd.concat([cols, scores], axis=1)
    score_feat.columns = ["Col", "Score"]
    largest = score_feat.nlargest(features, "Score")
    largest = largest.index.values.tolist()
    return largest, score_feat

def corr_heatmap_pearson(dataframe):
    corrmat = dataframe.corr()
    corrmat.to_csv('pearson_corr.csv')
    top_corr_features = corrmat.index
    plt.figure(figsize=(50, 50))
    #g = sns.heatmap(dataframe[top_corr_features].corr(), annot=True, cmap="RdYlGn", fmt='.1g')
    sns.heatmap(dataframe[top_corr_features].corr(), annot=False, cmap="Blues", fmt='.1g', linewidths=1, linecolor='black', robust = True)
    plt.savefig('pearsonR.png')
    #plt.gcf().set_size_inches(15, 8)
    #plt.show()

def corr_heatmap_spearman(dataframe):
    corrmat = dataframe.corr()
    corrmat.to_csv('spearman_corr.csv')
    top_corr_features = corrmat.index
    plt.figure(figsize=(50, 50))
    #g = sns.heatmap(dataframe[top_corr_features].corr(method = 'spearman'), annot=True, cmap="RdYlGn", fmt='.1g')
    sns.heatmap(dataframe[top_corr_features].corr(method = 'spearman'), annot=False, cmap="Blues", fmt='.1g', linewidths=1, linecolor='black', robust = True)
    plt.savefig('spearmanR.png')
    #plt.gcf().set_size_inches(15, 8)
    #plt.show()

###ANALYSIS####

file = "/Users/ciaraconway/Desktop/all_NHANES_KEY.xlsx"
sheet = "Data_No_Comments"
df = pd.read_excel(file, sheet_name=sheet)
df = df[1:len(df)]

corr_heatmap_pearson(df)
corr_heatmap_spearman(df)


import copy
import constants as c

def extract_hits(event):
    """
    Extract photon and predefined massive particle hits with energy larger than 0 from an event.
    :param event: one event in the format defined by the store_event() method.
    :return:      A dictionary containing 3 dictionaries that describe the hits. The three
                  dictionaries contain the particle hits --which generate the photons--, and the
                  photon hits that deposited energy in the endplates. The first is added to the
                  output of the program, while the two other are used to generate the gruid hits.
                  The dictionaries' keys are the following:
                    * n: hit identifier. Unused by this program, but useful in reconstruction.
                    * x: x position of the hit in cm.
                    * y: y position of the hit in cm.
                    * t: time of the hit in ns (t=0 is defined as the moment when the event takes
                         place).
                    * Edep: energy deposited by the hit in eV.
                    * TrkE: energy of the track to which the hit belongs.
    """
    if event is None: return None
    # Define hits dictionaries (one per detecting surface).
    hitdict = {c.S_N:[], c.S_ID:[], c.S_PID:[], c.S_X:[], c.S_Y:[], c.S_Z:[], c.S_T:[],
               c.S_ED:[], c.S_TRKE:[]}
    pidlist = [c.S_MMPID, c.S_MPPID, c.S_EMPID, c.S_EPPID, c.S_NPID]
    hits = {c.S_MASSHITS:   copy.deepcopy(hitdict),
            c.S_PHOTONHITS: copy.deepcopy(hitdict),
            c.S_PHOTONH1:   copy.deepcopy(hitdict),
            c.S_PHOTONH2:   copy.deepcopy(hitdict),}

    for hi in range(len(event[c.IDBANK][c.S_HITN])):
        if event[c.IRBANK][c.S_EDEP][hi] == '0': continue # Ignore hits with no energy deposited.

        # Determine hit source.
        key = None
        if event[c.IRBANK][c.S_PID][hi] in pidlist:         key = c.S_MASSHITS
        elif event[c.IRBANK][c.S_PID][hi] == c.S_PHOTONPID: key = c.S_PHOTONHITS

        # Process photon hits
        if key == c.S_PHOTONHITS:
            volid = int(int(event[c.IDBANK][c.S_VOL][hi])/10**8)
            if volid == c.SENSOR1A_ID or volid == c.SENSOR1B_ID: key = c.S_PHOTONH1
            if volid == c.SENSOR2A_ID or volid == c.SENSOR2B_ID: key = c.S_PHOTONH2

        if key is None: continue # Ignore uninteresting hits.

        # Add hit to dictionary, converting data to appropiate units.
        hits[key][c.S_N]   .append(int  (event[c.IDBANK][c.S_HITN]  [hi]))     # hit id.
        hits[key][c.S_ID]  .append(int  (event[c.IRBANK][c.S_TID]   [hi]))     # track id.
        hits[key][c.S_PID] .append(int  (event[c.IRBANK][c.S_PID]   [hi]))     # particle id.
        hits[key][c.S_X]   .append(float(event[c.IRBANK][c.S_AVGX]  [hi])/10.) # x position (cm).
        hits[key][c.S_Y]   .append(float(event[c.IRBANK][c.S_AVGY]  [hi])/10.) # y position (cm).
        hits[key][c.S_Z]   .append(float(event[c.IRBANK][c.S_AVGZ]  [hi])/10.) # z position (cm).
        hits[key][c.S_T]   .append(float(event[c.IRBANK][c.S_AVGT]  [hi]))     # Time (ns).
        hits[key][c.S_ED]  .append(float(event[c.IRBANK][c.S_EDEP]  [hi]))     # EDep (MeV).
        hits[key][c.S_TRKE].append(float(event[c.IRBANK][c.S_TRACKE][hi]))     # TrkE (MeV).

    return hits


def FindSpecialPosition(P, start):
    needToFind = start + 1
    for i in range(start, len(P)):
        if P[i] == (('+') + str(needToFind)) or P[i] == (('-') + str(needToFind)):
            return i
    return 0

def GreedySorting(P):
    approxReversalDistance = 0
    for k in range(0, len(P)):
        if P[k] == '-' + str(k + 1):
            P[k] = '+' + str(k + 1)
            approxReversalDistance += 1
            print(*P)
        if P[k] != '+' + str(k + 1):
            endPosition = FindSpecialPosition(P, k)
            startPosition = k
            syntenyBlock = P[startPosition:endPosition+1]
            syntenyBlock.reverse()
            for j in range(0, len(syntenyBlock)):
                if syntenyBlock[j][0] == '+':
                    syntenyBlock[j] = '-' + syntenyBlock[j][1:len(syntenyBlock[j])]
                else:
                    syntenyBlock[j] = '+' + syntenyBlock[j][1:len(syntenyBlock[j])]
            P = P[0:startPosition] + syntenyBlock + P[endPosition+1:len(P)]
            approxReversalDistance += 1
            print(*P)
        if P[k] == '-' + str(k + 1):
            P[k] = '+' + str(k + 1)
            approxReversalDistance += 1
            print(*P)
#     print(approxReversalDistance)
    return

P = ["-3", "+4", "+1", "+5", "+2"]
GreedySorting(P)

f = open('task1.txt', 'rt')
P = f.read().rstrip('\n').split(" ")
GreedySorting(P)


from tool.runners.python import SubmissionPy

def find_common(part1, part2):
    for x in part1:
        if x in part2:
            return x

priorities = {x: i + 1 for i, x in enumerate('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ')}

class BebertSubmission(SubmissionPy):
    def run(self, s):
        """
        :param s: input in string format
        :return: solution flag
        """
        res = 0
        for line in s.splitlines():
            middle = len(line) // 2
            part1 = line[:middle]
            part2 = line[middle:]
            common = find_common(part1, part2)
            res += priorities[common]
        return res

def test_bebert():
    """
    Run `python -m pytest ./day-03/part-1/bebert.py` to test the submission.
    """
    assert (
            BebertSubmission().run(
                """vJrwpWtwJgWrhcsFMMfFFhFp
jqHRNqRjqzjGDLGLrsFMfFZSrLrFZsSL
PmmdzqPrVvPwwTWBwg
wMqvLMZHhHMvwLHjbvcjnnSBnvTQFn
ttgJtRGJQctTZtZT
CrZsJsPPZsGzwwsLwLmpwMDw
""".strip()
            )
            == 157
    )


#!/usr/bin/python3
"""

@@@@@@@@@@@@@@@@@@@@@@
@@@@@@@@@@@@@@@@@@@@@@             @@@.    @@@    @@@.     @@@@    @@@@        @@@@@@@@@        @@@@@@@@@@@@@.     .@@@@@@@@@@@@@        @@@@@@@@@
@@@@@%%@@@%%@@@%%@@@@@             @@@@    @@@    @@@@     @@@@    @@@@       @@@@@@@@@@        @@@@@@@@@@@@@@     @@@@@@@@@@@@@@        @@@@@@@@@
@@@@@  @@@  @@@  @@@@@             @@@@    @@@    @@@@     @@@@    @@@@       @@@@   @@@@        @@@@@   @@@@@       @@@@@   @@@@       @@@@   @@@@
@@@@@            @@@@@             @@@@   @@@@    @@@@     @@@@@@@@@@@@       @@@@   @@@@        @@@@@   @@@@@       @@@@@   @@@@       @@@@   @@@@
@@@@@   @    @   @@@@@             @@@@   @@@@    @@@@     @@@@@@@@@@@@       @@@@@@@@@@@        @@@@@   @@@@@       @@@@@   @@@@       @@@@@@@@@@@
@@@@@            @@@@@             @@@@###@@@@@##@@@@@     @@@@    @@@@      @@@@@@@@@@@@       #@@@@@###@@@@@     ##@@@@@###@@@@       @@@@@@@@@@@
@@@@@@@@@@@@@@@@@@@@@@             @@@@@@@@@@@@@@@@@@@     @@@@    @@@@      @@@@    @@@@@      @@@@@@@@@@@@@@     @@@@@@@@@@@@@@      @@@@     @@@@
@@@@@@@@@@@@@@@@@@@@@@

Project name: Whadda you see? (Raspberry pi edition)
Colour sensor Raspberry Pi Demo

Description:
  This project uses a colour sensor to determine the main colour of an object.
  The Hue color parameter (a number that represents the "pure colour") is calculated based on the RGB-values that are read from the sensor.
  From the Hue parameter it is possible to determine which basic color is presented to the sensor.
  If the button (connected to GPIO pin 20) is pressed, the corresponding colour is printed on the terminal and can be heard using the on-board audio (using a Text-To-Speech program)

  More information on the working principles can be found on whadda.com

Level of difficulty: Intermediate

Parts required:
  - Raspberry Pi 3B/B+ or 4B or equivalent (e.g. Whadda PI4SET)
  - 0.96" or 1.3" OLED screen (e.g. Whadda INSERT WHADDA CODE/VMA437 1.3 INCH OLED SCREEN FOR ARDUINO or INSERT WHADDA CODE/VMA438 0.96 INCH OLED SCREEN WITH I2C FOR ARDUINO)
  - TCS3200 color sensor module (e.g. Whadda INSERT WHADDA CODE/VMA325 COLOR SENSOR TCS3200 MODULE)

See illustrated wiring diagram for the necessary connections, available on whadda.com

For more information, check out whadda.com

Required Python modules:
    - RPi.GPIO (should be installed by default, if not use folowing command to install: sudo apt install python-rpi.gpio python3-rpi.gpio)
    - Pillow (command to install: pip3 install Pillow)
    - adafruit_ssd1306 (command to install: pip3 install adafruit-circuitpython-ssd1306)

Code inspired by how to Mechatronics (https://www.electronicshub.org/raspberry-pi-color-sensor-tutorial/)

Author: Midas Gossye
(c) 2020 Whadda, premium makers brand by Velleman
"""

# Import necessary modules
import RPi.GPIO as GPIO
from time import sleep
import time
import colorsys
import os
from PIL import Image, ImageDraw, ImageFont
import argparse

# Parse the command arguments
parser = argparse.ArgumentParser(prog='whadda_you_see', description='Determine and display colour from TCS3200 colour sensor')
parser.add_argument('--OLED', help='Add if OLED screen is installed', action='store_true', default=False)

args = parser.parse_args()
OLED_SCREEN_INSTALLED = args.OLED
###############################################################################################################################

# Import the OLED modules if an OLED screen is installed
if OLED_SCREEN_INSTALLED:
    import adafruit_ssd1306
    from board import SCL, SDA
    import busio
########################################################

GPIO.setmode(GPIO.BCM) # Use processor pin numbering system

NUM_CYCLES = 10 # number of measurements to perform to determine colour

# pin connections
S0 = 5      # S0 colour sensor
S1 = 6      # S1 colour sensor
S2 = 13     # 20 colour sensor
S3 = 19     # S3 colour sensor
Out = 26    # Out colour sensor
Button = 20 # Button
###############################

if OLED_SCREEN_INSTALLED:
    i2c = busio.I2C(SCL, SDA) # Initialize the I2C bus for the OLED screen

    oled = adafruit_ssd1306.SSD1306_I2C(128, 64, i2c) # Initialize OLED screen (128x64 pixels)
    oled.fill(0) # fill the screen with 0's (clear screen)

# setup GPIO pins (inputs/outputs)
GPIO.setup(S0, GPIO.OUT)
GPIO.setup(S1, GPIO.OUT)
GPIO.setup(S2, GPIO.OUT)
GPIO.setup(S3, GPIO.OUT)
GPIO.setup(Out, GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(Button, GPIO.IN, pull_up_down=GPIO.PUD_UP)
#####################################################

# pin reading sequence for Red, Green, Blue measurements [S2, S3]
pin_reading_seq = [[GPIO.LOW, GPIO.LOW], [GPIO.HIGH, GPIO.HIGH], [GPIO.LOW, GPIO.HIGH], [GPIO.HIGH, GPIO.LOW]]
##############################################################################################################

RGBW = [0, 0, 0, 0] # Initialize empty array to hold Red, Green, Blue & White colour sensor readings

# Set colour frequency scaling to 2% / 12 kHz
GPIO.output(S0, GPIO.LOW)
GPIO.output(S1, GPIO.HIGH)

colour_names = ['Red', 'Green', 'Blue', 'White']

if OLED_SCREEN_INSTALLED:
    # Show Raspberry Pi Logo on OLED screen for 1 sec
    image = Image.open("RPI.bmp")
    oled.image(image)
    oled.show()
    sleep(1)
    ##################################################

    oled.fill(0) # fill the screen with 0's (clear screen)

    # Show Whadda Logo on OLED screen for 2 secs
    image = Image.open("WHADDA.bmp")
    oled.image(image)
    oled.show()
    sleep(2)
    ##################################################

    # Open a new blank image and draw object to later draw tex and shapes on
    image = Image.new("1", (oled.width, oled.height))
    draw = ImageDraw.Draw(image)
    #########################################################################

    oled.fill(0) # fill the screen with 0's (clear screen)

    # Load a font in 2 different sizes.
    font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 10)
    font2 = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 22)

    # Draw the text that doesn't need updating
    draw.text((0, 0), "Red: ", font=font, fill=255, align="right")
    draw.text((0, 10), "Green: ", font=font, fill=255, align="right")
    draw.text((0, 20), "Blue: ", font=font, fill=255, align="right")

    draw.text((70, 0), "Hue: ", font=font, fill=255, align="right")
    draw.text((70, 10), "Sat.: ", font=font, fill=255, align="right")
    draw.text((70, 20), "Value: ", font=font, fill=255, align="right")
    ##################################################################

    draw.line((64, 0, 64, 32), width=1, fill=255)  # draw vertical seperation line
    draw.line((0, 32, 127, 32), width=1, fill=255) # draw horizontal seperation line

    # Update image on OLED
    oled.image(image)
    oled.show()
    ######################

"""
Function descrition:
    Reads the Red, Green, Blue & White colour components from the TCS3200 colour sensor,
    converts the RGB colour into the HSV colour space, determines the colour name from the Hue value,
    and displays all of the above information on the OLED screen.

Arguments:
    None

Returns:
    -color_str (String with the colour name)
    -HSV_Val (list with Hue, Saturation and Value components respectively)
"""
def read_color():
    for idx, pin_states in enumerate(pin_reading_seq):
        # Select colour filter
        GPIO.output(S2, pin_states[0])
        GPIO.output(S3, pin_states[1])
        ##############################
        time.sleep(0.01)

        start = time.time() # start timing measurement to time pulse length

        # Wait until the NUM_cycles pulses have been counted
        for impulse_count in range(NUM_CYCLES):
            GPIO.wait_for_edge(Out, GPIO.FALLING)
        ####################################################

        duration = time.time() - start # calculate duration of pulses
        RGBW[idx] = ((NUM_CYCLES / duration) / 10000) *255 # compute Red, Green and White colour components and scale to 0-255

    # Scale Red, Green & Blue components with respect to the White/Brightness reading
    RGBW[0] = (RGBW[0] /RGBW[3])
    RGBW[1] = (RGBW[1] / RGBW[3])
    RGBW[2] = (RGBW[2] / RGBW[3])
    #################################################################################

    HSV_val = colorsys.rgb_to_hsv(RGBW[0], RGBW[1], RGBW[2]) # Convert RGB colour readings into HSV colour space

    color_str = ""

    # Convert Red, Green & Blue values to rounded percentages
    R_perc = round(RGBW[0]*100)
    G_perc = round(RGBW[1]*100)
    B_perc = round(RGBW[2]*100)
    #########################################################

    hue_deg = round(HSV_val[0]*360) # Convert Hue to rounded degrees

    # Convert value and saturation to rounded percentages
    val_perc = round(HSV_val[1]*100)
    sat_perc = round(HSV_val[2]*100)
    #####################################################

    if OLED_SCREEN_INSTALLED:
        # draw "black" rectangle to clear screen space for updated numbers
        draw.rectangle((34, 0, 63, 30), fill=0)
        draw.rectangle((100, 0, 127, 30), fill=0)
        ###################################################################

        # Put updated Red, Green and Blue percentages on screen
        draw.text((30, 0), "{:3d} %".format(R_perc), font=font, fill=255, align="right")
        draw.text((30, 10), "{:3d} %".format(G_perc), font=font, fill=255, align="right")
        draw.text((30, 20), "{:3d} %".format(B_perc), font=font, fill=255, align="right")
        #################################################################################

        # Put updated Hue, value and saturation numbers on screen
        draw.text((100, 0), "{:3d} °".format(hue_deg), font=font, fill=255, align="right")
        draw.text((100, 10), "{:3d} %".format(val_perc), font=font, fill=255, align="right")
        draw.text((100, 20), "{:3d} %".format(sat_perc), font=font, fill=255, align="right")
        ####################################################################################

    # Determine colour based on hue parameter
    if sat_perc < 35:
        color_str = "Black/Dark"
    elif hue_deg > 357 or hue_deg < 10:
        color_str = "ORANGE"
    elif hue_deg > 10 and hue_deg < 40:
        color_str = "YELLOW"
    elif hue_deg > 45 and hue_deg < 160:
        color_str = "GREEN"
    elif hue_deg > 210 and hue_deg < 270:
        color_str = "BLUE"
    elif hue_deg > 270 and hue_deg < 330:
        color_str ="PURPLE"
    elif hue_deg > 340 and hue_deg < 356:
        color_str = "RED"
    else:
        color_str = "No valid color detected"
    #########################################

    if OLED_SCREEN_INSTALLED:
        draw.rectangle((0, 34, 128, 60), fill=0) # clear bottom part of screen by drawing "black" rectangle

        # if a valid color is detected...
        if color_str != "No valid color detected" and color_str != "Black/Dark":
            draw.rectangle((0, 34, 128, 60), fill=255)                  # Draw a filled rectangle as background
            w, h = draw.textsize(color_str, font=font2)                 # Calculate pixel width and height of color name
            draw.text(((128-w)/2, 33), color_str, font=font2, fill=0)   # Put color name on screen with inversed text

        # Update OLED screen with new image
        oled.image(image)
        oled.show()
        ###################################

    return color_str, HSV_val

print("Whadda you see?")
print("===============")
print("\nPush button to get colour info\n")

while True:
    color_str, HSV_val = read_color() # Read and display colour name and values

    # If button is pressed...
    if GPIO.input(Button) == GPIO.LOW:
        # Print colour details on terminal
        print("Detected color: {}".format(color_str))
        print("Hue:", HSV_val[0]*360)
        print("\n")
        ##################################

        # Using the Pico Text-To-Speech program SAY the colour
        os.system('pico2wave -w tts.wav "{}" && aplay tts.wav'.format(color_str))

        print("=================")
        print("\n")


# Import the library
try:
    # This is the statement you normally use.
    graph = ximport("graph")
except ImportError:
    # But since these examples are "inside" the library
    # we may need to try something different when
    # the library is not located in /Application Support
    graph = ximport("__init__")
    reload(graph)

size(500, 500)

g = graph.create()

# Create some relations.
g.add_edge("roof"        , "house")
g.add_edge("garden"      , "house")
g.add_edge("room"        , "house")
g.add_edge("kitchen"     , "room")
g.add_edge("bedroom"     , "room")
g.add_edge("bathroom"    , "room")
g.add_edge("living room" , "room")
g.add_edge("sofa"        , "living room")
g.add_edge("table"       , "living room")

# Calculate a good layout.
g.solve()

# Apply default rules for node colors and size,
# for example, important nodes become blue.
g.styles.apply()

# Draw the graph, indicating the direction of each relation
# and the two nodes that get the most traffic.
g.draw(
    directed=True,
    traffic=1
)

# You'll see that "house" is the key node,
# and that "room" gets the most traffic.


from google_images_download import google_images_download

def imageCrwaling(keyword, dir):
    response = google_images_download.googleimagesdownload()
    arguments = {"keywords": keyword,
                 "limit": 1000,
                 "chromedriver": "chromedriver",
                 "print_urls": False,
                 "no_directory": True,
                 "output_directory": dir}
    paths = response.download(arguments)
    print(paths)

# imageCrwaling('maltese', 'datasets/maltese')
imageCrwaling('말티즈', 'datasets/maltese')


"""
以数组 intervals 表示若干个区间的集合，其中单个区间为 intervals[i] = [starti, endi] 。请你合并所有重叠的区间，并返回一个不重叠的区间数组，该数组需恰好覆盖输入中的所有区间。

输入：intervals = [[1,3],[2,6],[8,10],[15,18]]
输出：[[1,6],[8,10],[15,18]]
解释：区间 [1,3] 和 [2,6] 重叠, 将它们合并为 [1,6].

解题思路如下：

先对输入数组按照区间左边的值进行升序排列
初始化一个变量 outputs，用于存储合并直接的区间结果
遍历排序后的所有区间，针对每个区间做如下的处理：
如果当前处理的区间是第一个区间的话，那么直接将区间加入到 outputs
比较当前处理区间左边的值 (currLeft) 和 outputs 中最后一个区间右边的值 (outputsLastRight) ：
如果 outputsLastRight < currLeft，说明没有重叠，那么直接将当前处理的区间加入到 outputs
否则，说明有重叠，那么将 outputs 中最后一个区间的右边的值更新为：当前处理区间右边值和 outputsLastRight 的最大值
将 outputs 转成数组，并返回

"""
intervals = [[2,6],[1,3],[8,10],[15,18]]

if not intervals:
    print(intervals)
intervals.sort(key=lambda x: x[0])
result = []
result.append(intervals[0])
for i in range(1, len(intervals)):
    last = result[-1]
    if last[1] >= intervals[i][0]:
        result[-1] = [last[0], max(last[1], intervals[i][1])]
    else:
        result.append(intervals[i])
print(result)


from pyspark.sql import SparkSession

# Local SparkSession
spark = (SparkSession
         .builder
         .master("local[*]")
         .appName("Rate-DataFrame_Nothing-StdOut")
         .getOrCreate())

# 1. Input data: test DataFrame with sequence and timestamp
df = (spark
      .readStream
      .format("rate")
      .option("rowsPerSecond", 1)
      .load())

# 2. Data processing: nothing

# 3. Output data: show results in the console
query = (df
         .writeStream
         .outputMode("append")
         .format("console")
         .start())

query.awaitTermination()