#!/usr/bin/env python3

from migen import *
from migen.fhdl import verilog
from migen.genlib.fifo import *
from migen.genlib.coding import *
import midiori_platform
import subprocess

base_addr = Constant(0xeafa00 >> 1)

describe = subprocess.check_output(["git", "describe", "--tags", "--long"]).strip().decode()
version_string = Array(("midiori "+describe+"\x00").encode('shift_jis'))

def _divisor(freq_in, freq_out, max_ppm=None):
    divisor = freq_in // freq_out
    if divisor <= 0:
        raise ArgumentError("output frequency is too high")

    ppm = 1000000 * ((freq_in / divisor) - freq_out) / freq_out
    if max_ppm is not None and ppm > max_ppm:
        raise ArgumentError("output frequency deviation is too high")

    return divisor

class UART(Module):
    def __init__(self, tx, clk_freq, baud_rate):
        self.tx_data = Signal(8)
        self.tx_ready = Signal()
        self.tx_ack = Signal()

        divisor = _divisor(freq_in=clk_freq, freq_out=baud_rate, max_ppm=50000)

        tx_counter = Signal(max=divisor)
        self.tx_strobe = tx_strobe = Signal()
        self.comb += tx_strobe.eq(tx_counter == 0)
        self.sync += \
            If(tx_counter == 0,
                tx_counter.eq(divisor - 1)
            ).Else(
                tx_counter.eq(tx_counter - 1)
            )

        self.tx_bitno = tx_bitno = Signal(3)
        self.tx_latch = tx_latch = Signal(8)
        self.submodules.tx_fsm = FSM(reset_state="IDLE")
        self.tx_fsm.act("IDLE",
            self.tx_ack.eq(1),
            If(self.tx_ready,
                NextValue(tx_counter, divisor - 1),
                NextValue(tx_latch, self.tx_data),
                NextState("START")
            ).Else(
                NextValue(tx, 1)
            )
        )
        self.tx_fsm.act("START",
            If(self.tx_strobe,
                NextValue(tx, 0),
                NextState("DATA")
            )
        )
        self.tx_fsm.act("DATA",
            If(self.tx_strobe,
                NextValue(tx, tx_latch[0]),
                NextValue(tx_latch, Cat(tx_latch[1:8], 0)),
                NextValue(tx_bitno, tx_bitno + 1),
                If(self.tx_bitno == 7,
                    NextState("STOP")
                )
            )
        )
        self.tx_fsm.act("STOP",
            If(self.tx_strobe,
                NextValue(tx, 1),
                NextState("IDLE")
            )
        )

class Midiori(Module):
    def __init__(self):
        # tx uart registers
        self.tx = Signal()
        self.uart_tx = Signal()
        self.uart = UART(self.uart_tx, 16000000, 31250)
        self.brke = Signal()
        self.txe = Signal()
        self.comb += If(self.brke,
            self.tx.eq(0),
        ).Else(
            self.tx.eq(self.uart_tx)
        )
        self.submodules += self.uart
        self.fifo = SyncFIFOBuffered(8, 16)
        self.submodules += self.fifo
        # the itx fifo is described as a 4 deep fifo,
        # but it's really just 4 different types of
        # messages that can be sent
        itx_fifo_fe = Signal()
        itx_fifo_in_progress = Signal()
        self.comb += self.uart.tx_ready.eq(self.fifo.readable | itx_fifo_fe)
        self.comb += If(itx_fifo_in_progress,
            self.fifo.re.eq(0)
        ).Else(
            self.fifo.re.eq(self.uart.tx_ack)
        )
        self.comb += If(itx_fifo_fe,
            self.uart.tx_data.eq(0xfe)
        ).Else(
            self.uart.tx_data.eq(self.fifo.dout)
        )
        self.sync += If(self.uart.tx_ack & itx_fifo_fe & ~itx_fifo_in_progress,
            itx_fifo_fe.eq(0),
            itx_fifo_in_progress.eq(1)
        ).Elif(self.uart.tx_ack & itx_fifo_in_progress,
            itx_fifo_in_progress.eq(0),
        )
        self.addr = Signal(23)
        self._irq = Signal(reset=1)
        self._iack = Signal()
        self._as = Signal()
        self._lds = Signal()
        self._dtready = Signal(reset=1)
        self._rw = Signal()
        self.exreset = Signal(reset=1)
        self.data = TSTriple(8)
        self.addr_num = Signal(3)
        self.register_num = Signal(8)
        self.version_index = Signal(5)

        # internal read-only registers

        self.txemp = Signal()
        self.comb += self.txemp.eq(self.fifo.level == 0)
        self.txrdy = Signal()
        self.comb += self.txrdy.eq(self.fifo.writable)
        self.txidl = Signal()
        self.ase = Signal()
        txidl_counter = Signal(21)
        self.sync += If(txidl_counter >= 1280000,
            txidl_counter.eq(0),
            self.txidl.eq(1),
            itx_fifo_fe.eq(self.ase)
        ).Elif(~self.txemp | ~self.txe,
            txidl_counter.eq(0)
        ).Else(
            txidl_counter.eq(txidl_counter + 1)
        )
        self.txbsy = Signal()
        self.comb += self.txbsy.eq(0)
        self.tsr = Signal(8)
        self.comb += self.tsr.eq(Cat(self.txbsy,0,self.txidl,0,0,0,self.txrdy,self.txemp))

        # internal read-write registers
        self.group_num = Signal(4)
        self.ier = Signal(8)
        self.ivo = Signal(3)
        self.ic = Signal()

        #8us clock divider
        midi_divider = Signal(7)
        #all games should set this to 1, but
        #simulate it for completeness
        clkm = Signal()
        self.sync += midi_divider.eq(midi_divider-1)
        midi_clk_en = Signal()
        self.comb += If(clkm,
                            midi_clk_en.eq(midi_divider == 0)
                        ).Else(
                            midi_clk_en.eq(midi_divider[0:6] == 0)
                        )

        # irq controller
        self.isr = Signal(8, reset=0x00)
        self.ivr = Signal(8)
        self.vec = Signal(4)
        self.isr_masked = Signal(8)
        self.comb += self.isr_masked.eq(self.isr & self.ier)
        isr_pe = PriorityEncoder(8)
        self.submodules += isr_pe
        self.comb += isr_pe.i.eq(self.isr_masked)
        self.comb += self.vec.eq(Cat(isr_pe.o, isr_pe.n))
        self.comb += self.ivr.eq(Cat(0,self.vec,self.ivo))
        self.comb += self._irq.eq(self.isr_masked == 0)

        # irq sets
        self.previous_empty = Signal()
        self.sync += If((self.previous_empty == 0) & (self.txemp == 1),
                        self.isr[6].eq(1)
        )
        self.sync += self.previous_empty.eq(self.txemp)

        #general purpose timer
        gpt_low_byte_cache = Signal(8)
        gpt_reset_value = Signal(16)
        gpt_counter = Signal(14)
        self.sync += If(midi_clk_en,
            If((gpt_counter == 0) & (gpt_reset_value > 1),
                gpt_counter.eq(gpt_reset_value),
                self.isr[7].eq(1)
            ).Else(
                gpt_counter.eq(gpt_counter-1)
            )
        )

        #midi clock timer
        clock_low_byte_cache = Signal(8)
        clock_reset_value = Signal(16)
        clock_counter = Signal(14)
        self.sync += If(midi_clk_en,
            If((clock_counter == 0) & (clock_reset_value > 1),
                clock_counter.eq(clock_reset_value),
                self.isr[1].eq(1)
            ).Else(
                clock_counter.eq(clock_counter-1)
            )
        )

        #midi click counter
        # todo: clock off midi clock and implement isr mux
        click_counter = Signal(7)
        click_reset_value = Signal(7)
        self.sync += If(0,
            If((click_counter == 0) & (click_reset_value > 0),
               click_counter.eq(click_reset_value),
               self.isr[1].eq(1)
            ).Else(
                click_counter.eq(click_counter-1)
            )
        )

        #register io state machine
        self.xltr_oe = Signal()
        self.comb += self.addr_num.eq(self.addr[0:3])
        self.comb += self.register_num.eq(Cat(self.addr_num, 0, self.group_num))
        fsm = FSM(reset_state="IDLE")
        self.submodules += fsm
        fsm.act("IDLE",
                self._dtready.eq(1),
                self.xltr_oe.eq(1),
                If((self._iack == 0) & (self._irq == 0),
                   #enable xltr early
                   self.xltr_oe.eq(0),
                   NextState("IACK")
                ).Elif((self._as == 0) &
                   (self.addr[3:24] == base_addr[3:24]),
                   # enable xltr early
                   self.xltr_oe.eq(0),
                   If(self._lds == 0,
                       If((self._rw == 1),
                           NextState("RDATA"),
                       ).Else(
                           NextState("WDATA"),
                       )
                   )
                )
        )
        fsm.act("IACK",
                self._dtready.eq(0),
                self.data.o.eq(self.ivr),
                If(self._iack == 1,
                   NextState("IDLE")
                )
        )
        fsm.act("RDATA",
                self._dtready.eq(0),
                If(self.addr_num == 0,
                   # irq vector register
                   self.data.o.eq(self.ivr)
                ).Elif(self.addr_num == 2,
                    self.data.o.eq(self.isr)
                ).Else(
                    If(self.register_num == 0x34,
                       self.data.o.eq(0x04)
                    ).Elif(self.register_num == 0x36,
                           self.data.o.eq(0x00)
                    ).Elif(self.register_num == 0x54,
                           self.data.o.eq(self.tsr)
                    ).Elif(self.register_num == 0x64,
                           self.data.o.eq(0xa0)
                    ).Elif(self.register_num == 0x74,
                           self.data.o.eq(0x00)
                    ).Elif(self.register_num == 0x96,
                           self.data.o.eq(0xFF)
                    ).Elif(self.register_num == 0xF5,
                           self.data.o.eq(version_string[self.version_index])
                    )
                ),
                If(self._as == 1,
                   NextState("IDLE")
                )
        )
        fsm.act("WDATA",
                self._dtready.eq(0),
                If(self.addr_num == 0x01,
                   NextValue(self.group_num, self.data.i[0:4]),
                   NextValue(self.ic, self.data.i[7])
                ).Elif(self.addr_num == 0x03,
                      NextValue(self.isr, self.isr & ~self.data.i)
                ).Else(
                    If(self.register_num == 0x04,
                       NextValue(self.ivo, self.data.i[5:8])
                    ).Elif(self.register_num == 0x06,
                           NextValue(self.ier, self.data.i)
                    ).Elif(self.register_num == 0x14,
                           NextValue(self.ase, self.data.i[5])
                    ).Elif(self.register_num == 0x55,
                           NextValue(self.txe, self.data.i[0]),
                           If(self.data.i[2],
                              NextValue(self.txidl, 0)
                           ),
                           NextValue(self.brke, self.data.i[3])
                    ).Elif(self.register_num == 0x56,
                       NextValue(self.fifo.we, 1),
                       NextValue(self.fifo.din, self.data.i),
                       # clear tx empty isr
                       NextValue(self.isr[6], 0)
                    ).Elif(self.register_num == 0x66,
                           NextValue(clkm, self.data.i[1])
                    ).Elif(self.register_num == 0x67,
                           NextValue(click_reset_value, self.data.i[0:7]),
                           If(self.data.i[7],
                              NextValue(click_counter, self.data.i[0:7]),
                           )
                    ).Elif(self.register_num == 0x84,
                           NextValue(gpt_low_byte_cache, self.data.i)
                    ).Elif(self.register_num == 0x85,
                           NextValue(gpt_reset_value, Cat(gpt_low_byte_cache, self.data.i[0:6])),
                           If(self.data.i[7],
                              NextValue(gpt_counter, Cat(gpt_low_byte_cache, self.data.i[0:6]))
                           )
                    ).Elif(self.register_num == 0x86,
                           NextValue(clock_low_byte_cache, self.data.i)
                    ).Elif(self.register_num == 0x87,
                           NextValue(clock_reset_value, Cat(clock_low_byte_cache, self.data.i[0:6])),
                           If(self.data.i[7],
                              NextValue(clock_counter, Cat(clock_low_byte_cache, self.data.i[0:6]))
                           )
                    ).Elif(self.register_num == 0xF4,
                           NextValue(self.version_index, self.data.i)
                    )
                ),
                #only spend one cycle in WDATA
                #so that writes only happen once
                NextState("WWAIT")
        )
        fsm.act("WWAIT",
                self._dtready.eq(0),
                NextValue(self.fifo.we, 0),
                If(self._as == 1,
                   NextState("IDLE")
                )
        )

        #manual resets - the bus FSM cannot be reset during an ic
        #todo: consider using clock domain reset instead
        self.sync += If(self.exreset == 0 | self.ic,
                        self.group_num.eq(0),
                        clkm.eq(0),
                        self.ivo.eq(0),
                        self.ier.eq(0),
                        self.isr.eq(0),
                        self.txe.eq(0),
                        self.brke.eq(0),
                        self.ase.eq(0),
                        gpt_counter.eq(0),
                        gpt_low_byte_cache.eq(0),
                        gpt_reset_value.eq(0),
                        clock_counter.eq(0),
                        clock_low_byte_cache.eq(0),
                        clock_reset_value.eq(0),
                        )

def midi_read(m, reg):
    yield m.addr.eq(base_addr+1)
    yield m._as.eq(0)
    yield m._lds.eq(0)
    yield m._rw.eq(0)
    yield m.data.i.eq(reg >> 4)
    while (yield m._dtready == 1):
        yield
    yield m._as.eq(1)
    yield m._lds.eq(1)
    while (yield m._dtready == 0):
        yield
    yield m._rw.eq(1)
    yield m.addr.eq(base_addr+(reg&0x0F))
    yield m._as.eq(0)
    yield m._lds.eq(0)
    while (yield m._dtready == 1):
        yield
    yield m._as.eq(1)
    yield m._lds.eq(1)
    while (yield m._dtready == 0):
        yield

def midi_write(m, reg, value):
    yield m.addr.eq(base_addr+1)
    yield m._as.eq(0)
    yield m._lds.eq(0)
    yield m._rw.eq(0)
    yield m.data.i.eq(reg >> 4)
    while (yield m._dtready == 1):
        yield
    yield m._as.eq(1)
    yield m._lds.eq(1)
    while (yield m._dtready == 0):
        yield
    yield m._rw.eq(0)
    yield m.data.i.eq(value)
    yield m.addr.eq(base_addr+(reg&0x0F))
    yield m._as.eq(0)
    yield m._lds.eq(0)
    while (yield m._dtready == 1):
        yield
    yield m._as.eq(1)
    yield m._lds.eq(1)
    while (yield m._dtready == 0):
        yield

def midi_wait_empty(m):
    while (yield m.fifo.level > 0):
        yield

def midi_iack(m):
    while (yield m._irq == 1):
        yield
    yield m._iack.eq(0)
    while (yield m._dtready == 1):
        yield
    yield m._iack.eq(1)
    while (yield m._dtready == 0):
        yield

def test(m):
    yield m.addr.eq(0)
    yield m._as.eq(1)
    yield m._lds.eq(1)
    yield m._rw.eq(1)
    yield m._iack.eq(1)
    yield
    assert(yield m._dtready == 1)
    yield m.addr.eq(base_addr)
    yield
    yield m._as.eq(0)
    yield m._lds.eq(0)
    yield
    yield m._as.eq(1)
    yield m._lds.eq(1)
    yield
    assert(yield m._dtready == 0)
    yield
    # test reset
    yield from midi_write(m, 0x06, 0xFF)
    yield
    yield from midi_write(m, 0x01, 0x80)
    yield
    yield from midi_write(m, 0x01, 0x00)
    yield
    assert(yield m.ier == 0x00)
    #configure
    yield from midi_read(m, 0x34)
    yield from midi_read(m, 0x16)
    yield from midi_write(m, 0x04, 0xE0)
    yield from midi_write(m, 0x06, 0x40) #tx irq only
    yield from midi_write(m, 0x55, 0x01)
    for i in range(0,2):
        yield from midi_write(m, 0x56, i)
    assert(yield m.isr == 0x00)
    assert(yield m.vec == 8)
    assert(yield m.fifo.dout == 0x00)
    yield from midi_iack(m)
    yield from midi_wait_empty(m)
    assert(yield m.isr == 0x40)
    assert(yield m.vec == 6)
    yield from midi_write(m, 0xf4, 0x00)
    yield from midi_read(m, 0xf5)
    for i in range(1, 10000):
        yield

if __name__ == "__main__":
    import sys
    if sys.argv[1] == "sim":
        m = Midiori()
        run_simulation(m, test(m), vcd_name="midiori.vcd")
    else:
        plat = midiori_platform.Platform()
        m = Midiori()
        m.comb += m.addr.eq(plat.request("addr"))
        m.comb += m._as.eq(plat.request("as"))
        m.comb += m._lds.eq(plat.request("lds"))
        m.comb += m._rw.eq(plat.request("rw"))
        m.comb += plat.request("dtack").eq(m._dtready)
        m.specials += m.data.get_tristate(plat.request("data"))
        m.comb += plat.request("xltr_oe").eq(m.xltr_oe)
        m.comb += m.data.oe.eq(~plat.request("iddir"))
        m.comb += plat.request("tx").eq(m.tx)
        m.comb += plat.request("irq2").eq(m._irq)
        m.comb += m._iack.eq(plat.request("iack2"))
        m.comb += m.exreset.eq(plat.request("exreset"))
        plat.build(m)


import turtle

def draw_nested_squares():
    number_of_squares = 10
    side_length = 25
    offset_value = 10
    side_gain = offset_value * 2

    turtle.shape('turtle')

    for i in range(number_of_squares):
        draw_square(side_length)
        perform_offset(offset_value)
        side_length += side_gain

    turtle.exitonclick()

def perform_offset(offset_value: int):
    turtle.penup()
    turtle.goto(turtle.xcor() - offset_value, turtle.ycor() - offset_value)
    turtle.pendown()

def draw_square(side_length: int):
    number_of_sides = 4
    angle_of_rotation = 90

    for i in range(number_of_sides):
        turtle.forward(side_length)
        turtle.left(angle_of_rotation)

if __name__ == '__main__':
    draw_nested_squares()


from django.shortcuts import render
from django.views.generic import ListView, TemplateView, CreateView, DeleteView, UpdateView, DetailView
from .models import Book
from django.urls import reverse_lazy

# Create your views here.
class HomePageView(ListView):
    template_name = 'home.html'
    model = Book
    context_object_name = 'books_list'

class AboutPageView(TemplateView):
    template_name = 'about.html'

class ContactsPageView(TemplateView):
    template_name = 'contacts.html'

class BookDetailView(DetailView):
    model = Book
    template_name = 'book_detail.html'
    context_object_name = 'book'

class BookCreateView(CreateView):
    model = Book
    template_name = 'book_new.html'
    fields = '__all__'

class BookUpdateView(UpdateView):
    model = Book
    template_name = 'book_edit.html'
    fields = ['year', 'isbn']

class BookDeleteView(DeleteView):
    model = Book
    template_name = 'book_delete.html'
    success_url = reverse_lazy('home')


import re
import sys

def get_lines(f=None):
    if not f:
        f = sys.stdin
    return [x.strip() for x in f]

pattern = re.compile('Sensor at x=(.*), y=(.*): closest beacon is at x=(.*), y=(.*)')

def get_coords(line: str):
    m = pattern.match(line)
    n = [int(m.group(i)) for i in range(1, 5)]
    return (n[0], n[1]), (n[2], n[3])

def get_all_coords(lines: list):
    return [get_coords(line) for line in lines]

def dist(p, q):
    x, y = p
    v, w = q
    return abs(x - v) + abs(y - w)

def add_interval(intervals: set, x1: int, x2: int):
    ints = set()
    for (x3, x4) in intervals:
        if x1 > x4 + 1 or x2 < x3 - 1:
            ints.add((x3, x4))
        else:
            x1 = min(x1, x3)
            x2 = max(x2, x4)
    ints.add((x1, x2))
    return ints

def cover(sensor, d, y):
    sx, sy = sensor
    dx = d - abs(sy - y)
    return sx - dx, sx + dx

if __name__ == '__main__':

    file_name = sys.argv[1]
    with open(file_name) as f:
        all_coords = get_all_coords(get_lines(f))

    sensors = dict()
    beacons = set()
    for p, q in all_coords:
        beacons.add(q)
        sensors[p] = dist(p, q)

    print("part 1")
    y = 10 if file_name.endswith('sample') else 2000000
    ints = set()
    beacon_or_sensor = set()
    if y % 1000 == 0:
        print("y =", y)
    for (sx, sy) in sensors:
        if sy  == y:
            ints = add_interval(ints, sx, sx)
            beacon_or_sensor.add(sx)
    for bx, by in beacons:
        if by == y:
            ints = add_interval(ints, bx, bx)
            beacon_or_sensor.add(bx)
    for s, d in sensors.items():
        x1, x2 = cover(s, d, y)
        if x1 <= x2:
            ints = add_interval(ints, x1, x2)
    nblocked = 0
    for (x1, x2) in ints:
        nblocked += 1 + x2 - x1
    nblocked -= len(beacon_or_sensor)
    print(nblocked)
#    blocked = set()
#    for s, d in sensors.items():
#        x1, x2 = cover(s, d, y)
#        if x1 > x2:
#            continue
#        for x in range(x1, x2 + 1):
#            if (x,y) not in sensors and (x, y) not in beacons:
#                blocked.add((x,y))
#    print(sorted(blocked))
#    print(len(blocked))

    print()
    print("part 2")
    xmin, ymin = 0, 0
    xmax, ymax = 20, 20 if file_name.endswith('sample') else 4000000
    for y in range(ymin, ymax + 1):
        ints = set()
        if y % 100000 == 0:
            print("y =", y)
        for (sx, sy) in sensors:
            if sy  == y:
                ints = add_interval(ints, sx, sx)
        for bx, by in beacons:
            if by == y:
                ints = add_interval(ints, bx, bx)
        for s, d in sensors.items():
            x1, x2 = cover(s, d, y)
            if x1 <= x2:
                ints = add_interval(ints, x1, x2)
        if len(ints) > 1:
            print(y, ints)
            [(x1, x2), (x3, x4)] = ints
            print((x2 + 1) * 4000000 + y)
            print("^^^^^")
        else:
            [(x1, x2)] = ints
            if x1 > xmin or x2 < xmax:
                print(y, ints)
                print("^^^^^")
                raise Exception("We have assumed this does not happen")


# -*- coding: utf-8 -*-
# Created by chenlu on May 2nd
import numpy as np
import pandas as pd
import time
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import NMF, LatentDirichletAllocation, TruncatedSVD
import gc

def clicks_by_ip(X_total):
    start_time = time.time()
    # Count the number of clicks by ip
    ip_count = X_total.groupby(['ip']).size().rename('clicks_by_ip', inplace=True).reset_index()
    X_total = X_total.merge(ip_count, on='ip', how='left', sort=False)
    X_total['clicks_by_ip'] = X_total['clicks_by_ip'].astype('uint16')

    print(f'[{time.time() - start_time}] Finished to extract clicks_by_ip')
    return X_total

# Aggregation function
def rate_calculation(x):
    """Calculate the attributed rate. Scale by confidence"""
    rate = x.sum() / float(x.count())
    log_group = np.log(100000) # 1000 views -> 60% confidence, 100 views -> 40% confidence
    conf = np.min([1, np.log(x.count()) / log_group])
    return rate * conf

def time_features(df):
    start_time = time.time()
    # Make some new features with click_time column
    df['day'] = df['click_time'].dt.day.astype('uint8')
    df['hour'] = df['click_time'].dt.hour.astype('uint8')
    df['minute'] = df['click_time'].dt.minute.astype('uint8')

    print(f'[{time.time() - start_time}] Finished to extract time_features')
    return df

def confidence_rate_feature(X_total, train):
    start_time = time.time()
    ATTRIBUTION_CATEGORIES = [
        # V1 Features #
        ###############
        ['ip'], ['app'], ['device'], ['os'], ['channel'],

        # V2 Features #
        ###############
        ['app', 'channel'],
        ['app', 'os'],
        ['app', 'device'],

        # V3 Features #
        ###############
        ['channel', 'os'],
        ['channel', 'device'],
        ['os', 'device']
    ]

    # Find frequency of is_attributed for each unique value in column
    freqs = {}
    for cols in ATTRIBUTION_CATEGORIES:

        # New feature name
        new_feature = '_'.join(cols)+'_confRate'

        # Perform the groupby
        group_object = train.groupby(cols)

        # Group sizes
        group_sizes = group_object.size()
        log_group = np.log(100000) # 1000 views -> 60% confidence, 100 views -> 40% confidence
        print(">> Calculating confidence-weighted rate for: {}.\n   Saving to: {}. Group Max /Mean / Median / Min: {} / {} / {} / {}"
              .format(
            cols, new_feature,
            group_sizes.max(),
            np.round(group_sizes.mean(), 2),
            np.round(group_sizes.median(), 2),
            group_sizes.min()
        ))

        # Perform the merge
        train = train.merge(
            group_object['is_attributed']. \
                apply(rate_calculation). \
                reset_index(). \
                rename(
                    index=str,
                    columns={'is_attributed': new_feature}
                )[cols + [new_feature]],
            on=cols, how='left'
        )
    print(f'[{time.time() - start_time}] Finished to extract confidence_rate_feature')
    return train

def group_by_feature(train):
    start_time = time.time()
    # Define all the groupby transformations
    GROUPBY_AGGREGATIONS = [

        # V1 - GroupBy Features #
        #########################
        # Variance in day, for ip-app-channel
        {'groupby': ['ip','app','channel'], 'select': 'day', 'agg': 'var'},
        # Variance in hour, for ip-app-os
        {'groupby': ['ip','app','os'], 'select': 'hour', 'agg': 'var'},
        # Variance in hour, for ip-day-channel
        {'groupby': ['ip','day','channel'], 'select': 'hour', 'agg': 'var'},
        # Count, for ip-day-hour
        {'groupby': ['ip','day','hour'], 'select': 'channel', 'agg': 'count'},
        # Count, for ip-app
        {'groupby': ['ip', 'app'], 'select': 'channel', 'agg': 'count'},
        # Count, for ip-app-os
        {'groupby': ['ip', 'app', 'os'], 'select': 'channel', 'agg': 'count'},
        # Count, for ip-app-day-hour
        {'groupby': ['ip','app','day','hour'], 'select': 'channel', 'agg': 'count'},
        # Mean hour, for ip-app-channel
        {'groupby': ['ip','app','channel'], 'select': 'hour', 'agg': 'mean'},

        # V2 - GroupBy Features #
        #########################
        # Average clicks on app by distinct users; is it an app they return to?
        {'groupby': ['app'],
         'select': 'ip',
         'agg': lambda x: float(len(x)) / len(x.unique()),
         'agg_name': 'AvgViewPerDistinct'
        },
        # How popular is the app or channel?
        {'groupby': ['app'], 'select': 'channel', 'agg': 'count'},
        {'groupby': ['channel'], 'select': 'app', 'agg': 'count'},

        # V3 - GroupBy Features                                              #
        # https://www.kaggle.com/bk0000/non-blending-lightgbm-model-lb-0-977 #
        ######################################################################
        {'groupby': ['ip'], 'select': 'channel', 'agg': 'nunique'},
        {'groupby': ['ip'], 'select': 'app', 'agg': 'nunique'},
        {'groupby': ['ip','day'], 'select': 'hour', 'agg': 'nunique'},
        {'groupby': ['ip','app'], 'select': 'os', 'agg': 'nunique'},
        {'groupby': ['ip'], 'select': 'device', 'agg': 'nunique'},
        {'groupby': ['app'], 'select': 'channel', 'agg': 'nunique'},
        {'groupby': ['ip', 'device', 'os'], 'select': 'app', 'agg': 'nunique'},
        {'groupby': ['ip','device','os'], 'select': 'app', 'agg': 'cumcount'},
        {'groupby': ['ip'], 'select': 'app', 'agg': 'cumcount'},
        {'groupby': ['ip'], 'select': 'os', 'agg': 'cumcount'},
        {'groupby': ['ip','day','channel'], 'select': 'hour', 'agg': 'var'}
    ]

    # Apply all the groupby transformations
    for spec in GROUPBY_AGGREGATIONS:

        # Name of the aggregation we're applying
        agg_name = spec['agg_name'] if 'agg_name' in spec else spec['agg']

        # Name of new feature
        new_feature = '{}_{}_{}'.format('_'.join(spec['groupby']), agg_name, spec['select'])

        # Info
        print("Grouping by {}, and aggregating {} with {}".format(
            spec['groupby'], spec['select'], agg_name
        ))

        # Unique list of features to select
        all_features = list(set(spec['groupby'] + [spec['select']]))

        # Perform the groupby
        gp = train[all_features]. \
            groupby(spec['groupby'])[spec['select']]. \
            agg(spec['agg']). \
            reset_index(). \
            rename(index=str, columns={spec['select']: new_feature})

        # Merge back to X_total
        if 'cumcount' == spec['agg']:
            train[new_feature] = gp[0].values
        else:
            train = train.merge(gp, on=spec['groupby'], how='left')

        # Clear memory
        del gp

    print(f'[{time.time() - start_time}] Finished to extract group_by_feature')
    return train

def next_click_feature(train):
    start_time = time.time()
    GROUP_BY_NEXT_CLICKS = [

        # V1
        {'groupby': ['ip']},
        {'groupby': ['ip', 'app']},
        {'groupby': ['ip', 'channel']},
        {'groupby': ['ip', 'os']},

        # V3
        {'groupby': ['ip', 'app', 'device', 'os', 'channel']},
        {'groupby': ['ip', 'os', 'device']},
        {'groupby': ['ip', 'os', 'device', 'app']}
    ]

    # Calculate the time to next click for each group
    for spec in GROUP_BY_NEXT_CLICKS:

        # Name of new feature
        new_feature = '{}_nextClick'.format('_'.join(spec['groupby']))

        # Unique list of features to select
        all_features = spec['groupby'] + ['click_time']

        # Run calculation
        print(f">> Grouping by {spec['groupby']}, and saving time to next click in: {new_feature}")
        train[new_feature] = train[all_features].groupby(spec['groupby']).click_time.transform(lambda x: x.diff().shift(-1)).dt.seconds

    print(f'[{time.time() - start_time}] Finished to extract next_click_feature')
    return train

def history_click_feature(train):
    start_time = time.time()
    HISTORY_CLICKS = {
        'identical_clicks': ['ip', 'app', 'device', 'os', 'channel'],
        'app_clicks': ['ip', 'app']
    }

    # Go through different group-by combinations
    for fname, fset in HISTORY_CLICKS.items():

        # Clicks in the past
        train['prev_'+fname] = train. \
            groupby(fset). \
            cumcount(). \
            rename('prev_'+fname)

        # Clicks in the future
        train['future_'+fname] = train.iloc[::-1]. \
            groupby(fset). \
            cumcount(). \
            rename('future_'+fname).iloc[::-1]

    # Count cumulative subsequent clicks
    print(f'[{time.time() - start_time}] Finished to extract history_click_feature')
    return train

def topic_feature(X_total):
    """
    categorical feature embedding by using LDA/NMF/LSA(PCA)
    """
    start_time = time.time()
    features = ['ip', 'app', 'device', 'os', 'channel']
    feat_len = len(features)
    n_components = 5

    for i in range(feat_len):
        for j in range(feat_len):
            if i == j:
                continue
            x = features[i]
            y = features[j]
            x_of_y = {}
            print(f">> generate topic embedding by {x}_of_{y}")
            for index, sample in X_total.iterrows():
                x_of_y.setdefault(sample[y], []).append(str(sample[x]))
            ys = list(x_of_y.keys())
            x_as_sentence = [' '.join(x_of_y[y]) for y in ys]
            x_as_matrix = CountVectorizer().fit_transform(x_as_sentence)
            print(f"[{time.time()-start_time}] generate x matrix done")
            # compute LDA topics of ys related to x
            lda = LatentDirichletAllocation(n_components=n_components,
                                            max_iter=3,
                                            learning_method='online',
                                            learning_offset=50.,
                                            random_state=0).fit_transform(x_as_matrix) # topics_of_ys
            lda_array = np.column_stack((np.array(ys), lda))
            print(f"[{time.time()-start_time}] LDA done")
            gc.collect()

            # computer NMF
            nmf = NMF(n_components=n_components,
                      random_state=1,
                      alpha=.1,
                      l1_ratio=.5).fit_transform(x_as_matrix)
            lda_nmf_array = np.column_stack((lda_array, nmf))
            print(f"[{time.time()-start_time}] NMF done")
            gc.collect()

            # compute LSA
            lsa = TruncatedSVD(n_components=n_components,
                               n_iter=3,
                               random_state=42).fit_transform(x_as_matrix)
            lda_nmf_lsa_array = np.column_stack((lda_nmf_array, lsa))
            print(f"[{time.time()-start_time}] LSA done")
            gc.collect()

            # add to X_total
            rename_dict = {0:y}
            t = ['LDA','NMF','LSA']
            for k in range(15):
                rename_dict[k+1] = f'{x}_of_{y}_{t[int(k/5)]}_{k%5+1}'
            temp = pd.DataFrame(data=lda_nmf_lsa_array).rename(index=str, columns=rename_dict)
            X_total = X_total.merge(temp, on=y, how='left', sort=False)
            print(f"[{time.time()-start_time}] merge done")
    X_total['ip'] = X_total['ip'].astype('uint32')
    X_total['app'] = X_total['app'].astype('uint16')
    X_total['device'] = X_total['device'].astype('uint16')
    X_total['os'] = X_total['os'].astype('uint16')
    X_total['channel'] = X_total['channel'].astype('uint16')

    print(f'[{time.time() - start_time}] Finished to extract topic_feature')
    return X_total


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jan 17 14:11:18 2018

@author: will
"""
import matplotlib.pyplot as plt
from tqdm import tqdm
from dendritic_backprop_classes import *

auto_set_optimal_weights = True
train_interneuron_weights = True
train_inter_pyrimidal_weights = True
train_backward_weights = False

hidden_1 = hidden_neuron_layer(30,20,10,10)
output  = output_layer(20,10)
if(auto_set_optimal_weights):
   hidden_1.int_to_pyrA = -hidden_1.above_to_pyrA
   hidden_1.pyr_to_int = output.input_to_pyrB
#test3 = hidden_neuron_layer(20,10,10,10)

vA_store = []
weight_store_int_pyr = []
weight_store_above_pyrA = []
uI_store = []
uP_store = []
soma_store = []

iters  = 10000
for i in tqdm(range(iters)):
   input_layer = np.zeros(30)
   if(i%1000== 0):
      x = np.random.uniform(-1,1,30)
      targets = target_matrix_2.dot(target_matrix_1.dot(x))
      x = non_linearity(x)
   input_layer  = low_pass_filter(input_layer,x,low_pass_filter_timescale_inputs,timestep)
   y = np.ones(10)
   hidden_1.update_membrane_potentials(x,output.uP,1)
   output.update_membrane_potentials(hidden_1.uP,targets)

   if(train_interneuron_weights):
      hidden_1.update_pyr_to_int_weights()
      hidden_1.update_int_to_pyr_weights()

   if(train_inter_pyrimidal_weights):
      hidden_1.update_input_pyr_weights(input_layer)
      output.update_input_pyr_weights(hidden_1.uP)

   if(i%1== 0):
      weight_store_int_pyr.append(np.copy(hidden_1.int_to_pyrA))
      weight_store_above_pyrA .append(np.copy(hidden_1.above_to_pyrA))
      uP_store.append(np.copy(output.uP))
      uI_store.append(np.copy(hidden_1.uI))
      vA_store.append(np.copy(hidden_1.vA))

vA_store = np.array(vA_store)
weight_store_int_pyr= np.array(weight_store_int_pyr)
weight_store_above_pyrA= np.array(weight_store_above_pyrA)
uP_store = np.array(uP_store)
uI_store = np.array(uI_store)
plt.plot(vA_store[:,:,0])
plt.title("Potential of Apical Dendrites")
plt.show()
plt.figure()
plt.plot(np.mean(np.abs(uI_store[:,:,0]- uP_store[:,:,0]),axis=1))
plt.title("Mean abs value of difference between pyr soma rate and interneuron rate")
plt.yscale('log')
plt.show()

diff_vector = []
for i in range(weight_store_int_pyr.shape[0]):
   diff_vector.append(tot_weight_sum(weight_store_int_pyr[i,:,:],weight_store_above_pyrA[i,:,:]))
diff_vector = np.array(diff_vector)
plt.figure()
plt.plot(diff_vector)


import json

from wfscript.constants.payload import PayloadKey
from wfscript.utils.json import preserialize

from django_wfscript.utils.payload import hash_request_body
from django_wfscript.wfscript_api.models import Payload, Response, Run
from django_wfscript.wfscript_api.views import core_domain

def be_idempotent(payload_hash):
    try:
        payload = Payload.objects.get(md5=payload_hash)
        return dict(payload.response.data, **{PayloadKey.DUPLICATE: True})
    except Payload.DoesNotExist:
        pass

def confirm_resume_info(payload_data, run):
    if not payload_data[PayloadKey.RESUME]:
        raise RuntimeError(f'If {PayloadKey.RUN_ID} is provided, {PayloadKey.RESUME} must be provided as well')
    else:
        provided_method = payload_data[PayloadKey.RESUME].get(PayloadKey.METHOD)
        if run.resume_method != provided_method:
            RuntimeError(f'Next method for Run {run.identity} is {run.resume_method}; '
                         f'Provided: {provided_method}')
        provided_step = payload_data[PayloadKey.RESUME].get(PayloadKey.STEP)
        if run.resume_step != provided_step:
            RuntimeError(f'Next step for Run {run.identity} is {run.resume_step}; '
                         f'Provided: {provided_step}')

def load_run_and_confirm_resume_info(payload_data):
    run = Run.objects.get(identity=payload_data[PayloadKey.RUN_ID])
    confirm_resume_info(payload_data, run)
    return run

def handle_resume_state(run, response_data):
    if response_data[PayloadKey.RESUME]:
        if PayloadKey.STATE in response_data[PayloadKey.RESUME]:
            run.state = response_data[PayloadKey.RESUME].pop(PayloadKey.STATE)
        run.resume_method = response_data[PayloadKey.RESUME][PayloadKey.METHOD]
        run.resume_step = response_data[PayloadKey.RESUME][PayloadKey.STEP]
        run.save()

def api_run_method(request_body):
    payload_hash = hash_request_body(request_body)
    duplicate_results = be_idempotent(payload_hash)
    if duplicate_results:
        return duplicate_results
    else:
        payload_data = json.loads(request_body)
        method_identity = payload_data[PayloadKey.METHOD]
        if payload_data.get(PayloadKey.RUN_ID):
            run = load_run_and_confirm_resume_info(payload_data)
        else:
            run = Run.objects.create(method=method_identity)
        payload = Payload.objects.create(md5=payload_hash, run=run, data=payload_data)
        method_executor = core_domain.get_method_executor(method_identity)
        response_data = preserialize(
            method_executor.run(input_data=payload_data[PayloadKey.INPUT],
                                state=run.state,
                                resume_info=payload_data.get(PayloadKey.RESUME, {})))
        response_data[PayloadKey.RUN_ID] = run.identity
        handle_resume_state(run, response_data)
        Response.objects.create(payload=payload, data=response_data)
        return dict(response_data, **{PayloadKey.DUPLICATE: False})


import pandas as pd

x = pd.read_pickle('bible.pkl')
y1 = pd.read_pickle('Finaltbl_bibleStandard.pkl')
y2 = pd.read_pickle('Finaltbl_bibleRevision.pkl')
y3 = pd.read_pickle('Finaltbl_bibleNIV.pkl')

shortBookList = ['창','출','레','민','신','수','삿','룻','삼상','삼하','왕상','왕하','대상','대하','스','느','에',
                '욥','시','잠','전','아','사','렘','애','겔','단','호','욜','암','옵','욘','미','나','합',
                '습','학','슥','말','마','막','눅','요','행','롬','고전','고후','갈','엡','빌','골','살전',
                '살후','딤전','딤후','딛','몬','히','약','벧전','벧후','요일','요이','요삼','유','계']

shortBookDict = {}

for index_1 in range(len(shortBookList)):
    index = str(index_1+1)
    shortBookDict[shortBookList[index_1]] = index

x['book']= x['book'].replace(shortBookDict)

xTitle = x[ x['verseTitle']!='' ]

y1['verseTitle']=''
y2['verseTitle']=''
y3['verseTitle']=''

numError1 = 0
numError2 = 0

yList = [y1,y2,y3]
yTitle = ['./FinalBibleStandard.pkl','./FinalBibleRevision.pkl','./FinalBibleNIV.pkl']

for yInd, y in enumerate(yList):
    for ind, row in xTitle.iterrows():
        title = row.verseTitle
        book = row.book
        chapter = row.chapter
        verse = row.verse
        try:
            dfFilter = (y['index']==book) & (y['chapter']==chapter) & (y['verse']==verse)
            y.loc[dfFilter, ['verseTitle']] = title
        except:
            numError2 += 1

    #print('Error1 occurred '+str(numError1)+' times')
    #print('Error2 occurred '+str(numError2)+' times')

    y = y[['index', 'book', 'chapter', 'verseTitle', 'verse', 'sentence', 'etc', 'key']]
    y = y.to_pickle(yTitle[yInd])


from random import randint
lst = ['3', '5', '6', '4', '9']

def summ(x):
    return x + 2
# lambda x: x + 2

def func(x): return x + 2

# DRY - DON'T REPEAT YOURSELF

res = map(int, lst)
print(res)
res2 = map(lambda x: x * 2, res)
print(tuple(res2))

def more_five(x):
    if x > 5:
        return True

res = filter(more_five, res)
res = filter(lambda x: x > 5, res)
print(list(res))

# for i in 'строка':
#     print(i)

print(more_five(10))
print(more_five)

res_range = []
for i in range(10):
    res_range.append(randint(0, 10))

print(res_range)

rand_list = [randint(0, 10) for i in range(10)]
print(rand_list)


# -*- coding: utf-8 -*-
# Scipy Sample.
# http://cs231n.github.io/python-numpy-tutorial/
# pip install --upgrade Pillow
"""
Note that the Python Imaging Library (PIL) is not a dependency of SciPy and therefore the pilutil module is not available on systems that don’t have PIL installed.

by http://docs.scipy.org/doc/scipy/reference/misc.html

http://pillow.readthedocs.org/en/latest/installation.html#mac-os-x-installation

pip3 uninstall scipy
pip3 install --upgrade scipy

brew install libtiff libjpeg webp little-cms2
pip3 install --upgrade Pillow
"""
import numpy as np
from scipy.misc import imread, imsave, imresize

# Load an image as an nparray.
img = imread('./dog.png')
print(img.dtype, img.shape)

# Resize.
img_resize = imresize(img, (100, 100))
imsave('./dist/dog_resize.png', img_resize)

# Modify.
img_changed = img * [0, 1, 0, 0.5]
imsave('./dist/dog_green.png', img_changed)

### Distance between points
from scipy.spatial.distance import euclidean, cosine
# Euclidean distance
d = euclidean([1,0], [2,5])
print(d)
# Cosine distance
d = cosine([1,1], [1,5])
print(d)


## Import Environment and FileSystemLoader so we can load in our template.
## Import yaml as our config is yaml.
from jinja2 import Environment, FileSystemLoader
import yaml

def get_template(env):

    ## Open up our yaml file which will map any services depending on the environment.
    with open('service-discovery.yaml', 'r') as file:
        svc_conf = yaml.safe_load(file)

    ## Pulling the values from the service discovery yaml and assigning them to variables.
    monitoringserver = svc_conf[env]['monitoring-server']
    keystore = svc_conf[env]['key-store']
    webservicefrontend = svc_conf[env]['web-service-frontend']
    dbservicebackend = svc_conf[env]['db-service-backend']

    ## Load the templates directory.
    file_loader = FileSystemLoader('templates')
    ## Create our Jinja environment
    jinja_env = Environment(loader=file_loader)
    ## Get our template config file.
    template = jinja_env.get_template("config.yaml")
    ## Create the config file using are variables from the service-discovery yaml.
    outputText = template.render(
        monitoringserver = monitoringserver,
        keystore = keystore,
        webservicefrontend = webservicefrontend,
        dbservicebackend = dbservicebackend
    )

    ## Write the new file
    f = open(f"output/{env}Config.yaml", "w")
    f.write(outputText)
    f.close()


import os

from django.conf import settings

from domain.Process import exonstodomain as exd
from domain.Process import process_data as pr
import pandas as pd
import numpy as np
import networkx as nx
from django.urls import reverse

PPI=exd.load_obj("PPI")
DDI=exd.load_obj("DDI")
g2d=exd.load_obj("g2d")

data = pd.read_csv( "domain/data/all_Proteins.csv")

# --- Create folder
# Global jobs table path
jobs_table_path = os.path.join(settings.MEDIA_ROOT, 'jobs', 'tables')
if not os.path.exists(jobs_table_path):
    os.makedirs(jobs_table_path)

# Global jobs networks path
jobs_networks_path = os.path.join(settings.MEDIA_ROOT, 'jobs', 'networks')
if not os.path.exists(jobs_networks_path):
    os.makedirs(jobs_networks_path)

def Construct_network(proteins_id, missing,job_ID):

      E=[]
      N=[]
      affected_nodes=[]
      DDI_nodes=[]
      p1=[]
      p2=[]
      p1_name=[]
      p2_name=[]
      DDIs=[]
      missing_DDIs=[]

      #DDIS with liknks for web display only:
      DDIs2=[]
      missing_DDIs2=[]

      #Score = (retained DDI/All interactions)
      score=[]
      source=[]
      home=reverse('home')
      # Get the subgraph:
      H = PPI.subgraph(proteins_id)
      #print('interactions:',H)
      if len(H.edges())==0: return 0
      for e in H.edges():
          if e[0] not in N: N.append(e[0])
          if e[1] not in N: N.append(e[1])

          gene1=entrez_to_ensembl(e[0])
          gene2=entrez_to_ensembl(e[1])

          #check if gene is in missing dictionary
          #missing{} have all proteins with domain

          #Draw The PPI edge for every case

          #one of the gene have a domain
          if (gene1 in missing) and (gene2 in missing):
              inter=False

              if e[0] in g2d:
                    domains1=g2d[e[0]]
              else:
                  domains1=[]
              if e[1] in g2d:
                  domains2=g2d[e[1]]
              else:
                  domains2=[]

              DDIs_tmp=[]
              missing_DDIs_tmp=[]

              #DDIS with liknks for web display only:

              DDIs_tmp2=[]
              missing_DDIs_tmp2=[]

              for d1 in domains1:
                  for d2 in domains2:
                      if DDI.has_edge(d1,d2):
                           inter=True

                           #Interacted domain is missing
                           if (d1 in missing[gene1]) or (d2 in missing[gene2]):
                                  E.append("{from: '"+e[0]+"', to: '"+e[1]+"', dashes:  true,title:' Lost "+d1+'-'+d2+"', color: 'red'},")
                                  #print('yeaaaaah')
                                  #print(d1,d2)
                                  #print(gene1,gene2)
                                  affected_nodes.append(e[0])
                                  affected_nodes.append(e[1])

                                  missing_DDIs_tmp.append(d1+'-'+d2)
                                  missing_DDIs_tmp2.append(link(d1)+'-'+link(d2))

                           else:
                                  #Interaction retained
                                  E.append("{from: '"+e[0]+"', to: '"+e[1]+"',title:'"+d1+'-'+d2+"',  color: 'red'},")
                                  DDI_nodes.append(e[0])
                                  DDI_nodes.append(e[1])

                                  DDIs_tmp.append(d1+'-'+d2)
                                  DDIs_tmp2.append(link(d1)+'-'+link(d2))

              if inter:
                   # inter=True: a Domain-Domain Interaction was found:
                   #for the table
                   p1.append(e[0])
                   p1_name.append(pr.entrez_to_name(e[0]))
                   p2.append(e[1])
                   p2_name.append(pr.entrez_to_name(e[1]))

                   if DDIs_tmp!=[]:
                      DDIs.append(' ; '.join(DDIs_tmp))
                      DDIs2.append(' ; '.join(DDIs_tmp2))

                   else:
                      DDIs.append('-')
                      DDIs2.append('-')

                   if missing_DDIs_tmp!=[]:
                      missing_DDIs.append(' ; '.join(missing_DDIs_tmp))
                      missing_DDIs2.append(' ; '.join(missing_DDIs_tmp2))

                   else:
                        missing_DDIs.append('-')
                        missing_DDIs2.append('-')

                   score.append(str(float("{0:.2f}".format(len(DDIs_tmp)/(len(DDIs_tmp)+len(missing_DDIs_tmp))))))
                   source.append('PPI-DDI')

              if not inter:
                   E.append("{from: '"+e[0]+"', to: '"+e[1]+"', title:'PPI',  color: CHOOSEN,       smooth: {type: 'continuous'}},")
                   p1.append(e[0])
                   p1_name.append(pr.entrez_to_name(e[0]))
                   p2.append(e[1])
                   p2_name.append(pr.entrez_to_name(e[1]))

                   DDIs.append('-')
                   DDIs2.append('-')

                   missing_DDIs.append('-')
                   missing_DDIs2.append('-')

                   score.append('1.0')
                   source.append('PPI')
          else : #Both genes have no domains
                   E.append("{from: '"+e[0]+"', to: '"+e[1]+"', title:'PPI evidence',  color: CHOOSEN,       smooth: {type: 'continuous'}},")
                   E.append("{from: '"+e[0]+"', to: '"+e[1]+"', title:'PPI',  color: CHOOSEN,       smooth: {type: 'continuous'}},")
                   p1.append(e[0])
                   p1_name.append(pr.entrez_to_name(e[0]))
                   p2.append(e[1])
                   p2_name.append(pr.entrez_to_name(e[1]))

                   DDIs.append('-')
                   DDIs2.append('-')
                   missing_DDIs.append('-')
                   missing_DDIs2.append('-')

                   score.append('1.0')
                   source.append('PPI')
      nodes=[]
      for n in N:
          ensembl=entrez_to_ensembl(n)
          if (n in affected_nodes) and (missing[ensembl]!=[]):
              nodes.append("{id: \""+n+"\",url:  '"+home+"ID/gene/"+ensembl+"' , color: CHOOSEN2, label:  '"+pr.entrez_to_name(n)+"'}, ")

          elif n in DDI_nodes:
              nodes.append("{id: \""+n+"\", url:  '"+home+"ID/gene/"+ensembl+"' , color: CHOOSEN3, label:  '"+pr.entrez_to_name(n)+"'},")

          else:
              nodes.append("{id: \""+n+"\", url:  '"+home+"ID/gene/"+ensembl+"' , label:  '"+pr.entrez_to_name(n)+"'},")

      # Table of interactions
      pd_interaction=pd.DataFrame( data=[p1,p1_name,p2,p2_name,DDIs2,  missing_DDIs2,DDIs,missing_DDIs, score,source ], index=['Protein 1', 'Protein 1 name','Protein 2', 'Protein 2 name', "Retained domain-domain interactions","Missing domain-domain interactions",'Retained DDIs', 'Lost DDIs','Score','Source of the interaction'] )
      pd_interaction=pd_interaction.transpose()
      pd.set_option('display.max_colwidth',1000)

      pd_interaction.drop(columns=["Retained domain-domain interactions","Missing domain-domain interactions"]).to_csv(f'{jobs_table_path}/{job_ID}.csv', index=False)

      pd_html=pd_interaction.drop(columns=['Protein 1', 'Protein 2','Retained DDIs', 'Lost DDIs'])

      pd_html.sort_values(by=['Source of the interaction'], ascending=[True])

      pd_html=pd_html.rename(columns={

          "Score": "Score*",

          })

      # Convert Table to HTML
      pd_html=pd_html.to_html(table_id='results', **settings.TO_HTML_RESPONSIVE_PARAMETERS)

      #Generate the network file (only edges and scores)
      pd_interaction=pd_interaction.rename(columns={
          "Score": "weight",
          })

      pd_interaction['weight']=pd_interaction['weight'].astype(float)
      edges =pd_interaction.drop(columns=['Protein 1 name', 'Protein 2 name','Retained DDIs','Lost DDIs','Source of the interaction',"Retained domain-domain interactions","Missing domain-domain interactions"])
      edges.to_csv(f'{jobs_networks_path}/{job_ID}.sif', index=False,sep='\t')

      edges=edges.rename(columns={"Protein 1": "source",
       "Protein 2": "target",

       })

      weighted_G=nx.from_pandas_edgelist(edges, edge_attr=True)

      #print(weighted_G.edges())
      nx.write_graphml(weighted_G, f'{jobs_networks_path}/{job_ID}.graphml')

      return nodes,E,pd_interaction,pd_html

def analysis_input_isoforms(Inputs):

        filtred=filter_proteins_list(Inputs)
        print('-----------filtred--------------')

        n=len(filtred)
        #Inputs are so many
        if n>2000:
              return False

        else:
                print('-----------yeaaaaaaah--------------')
                gene_domains={}
                #all genes and missing domains
                missing={}
                # all entrex ID of the genes
                proteins_id=[]

                for tr in filtred:
                          #print(tr)
                          gene=tr_to_gene(tr)
                          domains=tr_to_domain(tr)

                          if len(domains)==0:
                              #print(tr,domains)
                              gene_domains[gene]=[]

                          if len(domains)>0:
                              #Check for every domain
                              for domain in domains:

                                      if gene not in gene_domains:

                                          gene_domains[gene]=[domain]

                                      else :
                                          gene_domains[gene].append(domain)
                                          gene_domains[gene]=Remove(gene_domains[gene])

                for tr in filtred:
                    ensembl=tr_to_gene(tr)
                    entrez=str(ensembl_to_entrez(ensembl))
                    proteins_id.append(entrez)
                    if entrez in g2d:
                        domains=g2d[entrez]

                        #all domains are lost for this gene
                        if ensembl not in gene_domains :
                          missing[ensembl]=domains

                        #check if all domains are there
                        else:
                          missing[ensembl]=[]
                          for d in domains:
                              if d not in gene_domains[ensembl]:
                                  missing[ensembl].append(d)

                proteins_id=Remove(proteins_id)

                return proteins_id, missing,n

def analysis_input_genes(Inputs):

    protein_id=[]
    missing={}
    for i in Inputs:
        f=i.replace(" ", "")
        f=f.split(".")[0]

        if len(f)==15 and f[0:4]=='ENSG' :

              #check if gene is coding:
              if  len(data[data['Gene stable ID'].isin([f])])!=0:

                  #print(f)
                  #check PPI status:

                  entrez_id=str(ensembl_to_entrez(f))
                  #print(entrez_id)
                  if PPI.has_node(entrez_id):
                      print('yeaaaaaah')
                      protein_id.append(entrez_id)
                      missing[f]=[]
    print(len(protein_id))
    if len(protein_id)>2000 or len(protein_id)<1:
            return False
    else : return protein_id,missing,0

def pr_to_tr(pr):
    df_filter = data['Protein stable ID'].isin([pr])
    try: return data[df_filter]['Transcript stable ID'].unique()[0]
    except IndexError:
        return False

def tr_to_gene(tr):
    df_filter = data['Transcript stable ID'].isin([tr])
    try: return data[df_filter]['Gene stable ID'].unique()[0]
    except IndexError:
        return False

def ensembl_to_entrez(gene):
    df_filter = data['Gene stable ID'].isin([gene])
    try: return data[df_filter]['NCBI gene ID'].unique()[0]
    except IndexError:
        return False

def entrez_to_ensembl(gene):
    df_filter = data['NCBI gene ID'].isin([gene])
    try: return data[df_filter]['Gene stable ID'].unique()[0]
    except IndexError:
        return False

def tr_to_domain(tr):
    df_filter = data['Transcript stable ID'].isin([tr])
    tdata=data[df_filter]
    tdata=tdata[tdata["Pfam ID"].notna()].drop_duplicates()
    try: return tdata["Pfam ID"].unique()
    except IndexError:
        return False

#add to view
def check_PPI_status(tr):
    df_filter = data['Transcript stable ID'].isin([tr])
    return PPI.has_node(data[df_filter]['NCBI gene ID'].astype('str').unique()[0])

def tr_is_coding(tr):
    return  len(data[data['Transcript stable ID'].isin([tr])])!=0

def filter_proteins_list(List):
    filtred_list=[]
    for tr in List:
        #Correct spaces, dotsof version....
        ftr=tr=tr.replace(" ", "")
        ftr=ftr.split(".")[0]
        ftr=ftr.split("'\'")[0]
        ftr=ftr.split("+")[0]
        #print(ftr)
        #make sure Correct ID
        ftr=ftr[0:15]
        if ftr[0:3]=='ENS':

            #Check if protein coding

            #input is a protein and coverted successfully
            if ftr[3]=='P' :

                tmp= pr_to_tr(ftr)

                # The protein is converted to a transcript
                if tmp!=False:
                        # Check PPI status
                        if check_PPI_status(tmp):
                            filtred_list.append(tmp)

            #check if transcript is a coding protein
            elif ftr[3]=='T'and tr_is_coding(ftr) and check_PPI_status(ftr):
               filtred_list.append(ftr)

    return filtred_list

def Remove(duplicate):
    final_list = []
    for num in duplicate:
        if num not in final_list:
            final_list.append(num)
    return final_list

def link(id):
  return '<a href="http://pfam.xfam.org/family/'+id+'" target="_blank">'+id+' </a>'


n, m = map(int, input().split())
money = []
for i in range(n):
    money.append(int(input()))
dp = [10001] * (m + 1)
dp[0] = 0
for i in range(n):
    for j in range(money[i], m + 1):
        if dp[j - money[i]] != 10001:
            dp[j] = min(dp[j], dp[j - money[i]] + 1)
res = -1 if dp[m] == 10001 else dp[m]
print(res)
print(dp)


# -*- coding: utf-8 -*-
''' Data structures to describe a finite element mesh.'''

import os
import json
from import_export import mesh_entities as me
from import_export import block_topology_entities as bte

class GroupRecord(object):
    ''' Object defining a grop of entities (nodes, cells, points, or lines).

    :ivar name: group name.
    :ivar nodeIds: identifiers of the nodes.
    :ivar cellIds: identifiers of the cells.
    :ivar pointIds: identifiers of the points.
    :ivar lineIds: identifiers of the lines.
    '''

    def readFromDATFile(self,fName):
        self.name=  os.path.splitext(fName)[0]
        meshDesc= me.MeshData()
        meshDesc.readFromDATFile(fName)
        self.nodeIds= meshDesc.nodes.getTags()
        self.cellIds= meshDesc.cells.getTags()
        self.pointIds= []
        self.lineIds= []

    def setUp(self,name,points,lines):
        self.name= name
        self.nodeIds= []
        self.cellIds= []
        self.pointIds= points
        self.lineIds= lines

    def empty(self):
        return not(self.nodeIds or self.cellIds or self.pointIds or self.lineIds)

    def writeDxfFile(self, dummy):
        '''groups have not representation in dxf files.

        :param dummy: dummy argument (not used).
        '''
        return

    def writeToXCFile(self, xcImportExportData):
        ''' writes the XC commands to define the group in a file.

        :param xcImportExportData:Data used when importing and/or exporting XC models.
        '''
        if(not self.empty()):
            f= xcImportExportData.outputFile
            strCommand= self.name + '= ' + xcImportExportData.setHandlerName + '.defSet("' + self.name +'")'
            f.write(strCommand+'\n')
            for n in self.nodeIds:
                strCommand= self.name + '.getNodes.append(n' + str(n) + ')'
                f.write(strCommand+'\n')
            for e in self.cellIds:
                strCommand= self.name + '.elements.append(e' + str(e) + ')'
                f.write(strCommand+'\n')
            for p in self.pointIds:
                strCommand= self.name + '.getMultiBlockTopology.getPoints.append(' + str(p) + ')'
                f.write(strCommand+'\n')
            for l in self.lineIds:
                strCommand= self.name + '.getMultiBlockTopology.getLines.append(' + str(l) + ')'
                f.write(strCommand+'\n')

    def getDict(self):
        ''' Return the object members in a Python dictionary.'''
        return {'name':self.name, 'nodeIds':self.nodeIds, 'cellIds':self.cellIds, 'pointIds':self.pointIds, 'lineIds':self.lineIds}

    def setFromDict(self, dct):
        ''' Set the data values from the dictionary argument.

        :param dct: dictionary containing the values of the object members.
        '''
        self.name= dct['name']
        self.nodeIds= dct['nodeIds']
        self.cellIds= dct['cellIds']
        self.pointIds= dct['pointIds']
        self.lineIds= dct['lineIds']

class XCImportExportData(object):
    ''' Data used when importing and/or exporting XC
        models.

    :ivar outputFileName: name to use for the output file.
    :ivar problemName: name for the XC problem object.
    :ivar nodeHandlerName: name for the XC node handler.
    :ivar cellHandlerName: name for the XC element handler.
    :ivar setHandlerName: name for the XC group handler.
    :ivar pointHandlerName: name for the XC point handler.
    :ivar lineHandlerName: name for the XC line handler.
    :ivar surfaceHandlerName: name for the XC surface handler.
    :ivar cellConversion: dictionary for cell conversion.
    :ivar outputFile: Python file object used for output.
    :ivar meshDesc: string describing the mesh
    :ivar blockData: block data (points, lines, surfaces, bodies).
    '''

    def __init__(self):
        self.mainDATFile= ""
        self.groupDATFiles= []
        self.dxfLayers= []
        self.outputFileName= 'xc_mesh'
        self.problemName= None
        self.nodeHandlerName= "nodes"
        self.cellHandlerName= "elements"
        self.setHandlerName= "groups"
        self.pointHandlerName= "points"
        self.lineHandlerName= "lines"
        self.surfaceHandlerName= "surfaces"
        self.cellConversion= {}
        self.outputFile= None
        self.meshDesc= None
        self.blockData= None

    def getXCFileName(self):
        return self.outputFileName+'.py'

    def getJSONFileName(self):
        return self.outputFileName+'.json'

    def getDxfFileName(self):
        return self.outputFileName+'.dxf'

    def getBlockHandlerName(self,blockType):
        if(blockType=='line'):
            return 'lines'
        elif(blockType=='face'):
            return 'surfaces'
        elif(blockType=='solid'):
            return 'solids'

    def convertCellType(self,tp):
        if tp in self.cellConversion:
            return self.cellConversion[tp]
        else:
            return None

    def readDATFiles(self):
        self.meshDesc= me.MeshData()
        self.meshDesc.readFromDATFile(self.mainDATFile)
        for fName in self.groupDATFiles:
            grp= GroupRecord()
            grp.name= os.path.splitext(fName)[0]
            grp.readFromDATFile(fName)
            self.meshDesc.groups.append(grp)

    def readDxfFile(self, fName, preprocessor):
        ''' Read from DXF file.

        :param fName: file name.
        :param preprocessor: xc preprocessor for the FE problem.
        '''
        self.blockData= bte.BlockData()
        self.blockData.name= fName
        self.blockData.readFromDxfFile(fName,preprocessor,self.dxfLayers)
        for l in self.dxfLayers:
            grp= GroupRecord()
            grp.setUp(l,self.blockData.pointsInLayer[l],self.blockData.blocksInLayer[l])
            self.blockData.groups.append(grp)

    def readJSONFile(self, fName):
        ''' Read from JSON file.

        :param fName: file name.
        '''
        f= open(fName,'r') # Open JSON file
        dataDict= json.load(f) # returns JSON object as a dictionary.
        self.setFromDict(dataDict)
        f.close()

    def writeDxfFile(self,fileName):
        self.blockData.writeDxfFile(fileName)

    def getXCCommandString(self):
        ''' Return a string with the XC commands that define the model.'''
        strCommand= ''
        if(self.problemName!=None):
            strCommand+= 'problemName= \'' + self.problemName+'\'\n'
            strCommand+= self.problemName + '= xc.FEProblem()'+'\n'
            strCommand+= 'preprocessor= ' + self.problemName + '.getPreprocessor\n'
        strCommand+= self.nodeHandlerName + '= preprocessor.getNodeHandler\n'
        strCommand+= self.cellHandlerName + '= preprocessor.getElementHandler\n'
        strCommand+= self.pointHandlerName + '= preprocessor.getMultiBlockTopology.getPoints\n'
        strCommand+= self.lineHandlerName + '= preprocessor.getMultiBlockTopology.getLines\n'
        strCommand+= self.surfaceHandlerName + '= preprocessor.getMultiBlockTopology.getSurfaces\n'
        #strCommand+= self.lineHandlerName + '= preprocessor.getMultiBlockTopology.getLines\n'
        strCommand+= self.setHandlerName + '= preprocessor.getSets\n'
        if(self.blockData):
            strCommand+= self.blockData.getXCCommandString(self)
        if(self.meshDesc):
            strCommand+= self.meshDesc.getXCCommandString(self)
        return strCommand

    def dumpToXC(self, preprocessor):
        ''' Dump the objects in this container into the preprocessor argument.

        :param preprocessor: XC finite element problem preprocessor.
        '''
        blockTopology= None
        if(self.blockData):
            blockTopology= self.blockData.dumpToXC(preprocessor)
        mesh= None
        if(self.meshDesc):
            mesh= self.meshDesc.dumpToXC(preprocessor)
        return blockTopology, mesh

    def readFromXCSet(self, xcSet):
        ''' Read the objects from the XC set and create the
            corresponding entities here.

        :param preprocessor: XC finite element problem preprocessor.
        '''
        if(len(xcSet.points)>0):
            self.blockData= bte.BlockData()
            self.blockData.readFromXCSet(xcSet)
        if(len(xcSet.nodes)>0):
            self.meshDesc= me.MeshData()
            self.meshDesc.readFromXC(xcSet)

    def writeToXCFile(self):
        ''' Write the model to a XC file.'''
        self.outputFile= open(self.getXCFileName(),"w")
        xcCommandString= self.getXCCommandString()
        self.outputFile.write(xcCommandString)
        self.outputFile.close()

    def writeToJSON(self):
        '''Write the objects that define the mesh.

        :param xcImportExportData: import/export parameters.
        '''
        self.outputFile= open(self.getJSONFileName(),"w")
        blockDict= self.getDict()
        jsonString= json.dumps(blockDict)
        self.outputFile.write(jsonString)
        self.outputFile.close()

    def getDict(self):
        ''' Return the object members in a Python dictionary.'''
        retval= {'mainDATFile':self.mainDATFile, 'groupDATFiles':self.groupDATFiles, 'dxfLayers':self.dxfLayers, 'outputFileName':self.outputFileName, 'problemName':self.problemName, 'nodeHandlerName':self.nodeHandlerName, 'cellHandlerName':self.cellHandlerName, 'setHandlerName':self.setHandlerName, 'pointHandlerName':self.pointHandlerName, 'lineHandlerName':self.lineHandlerName, 'surfaceHandlerName':self.surfaceHandlerName, 'cellConversion':self.cellConversion}
        meshDict= None
        if(self.meshDesc):
            meshDict= self.meshDesc.getDict()
        blockDict= None
        if(self.blockData):
            blockDict= self.blockData.getDict()
        retval.update({'meshDesc':meshDict, 'blockData': blockDict})
        return retval

    def setFromDict(self, dct):
        ''' Set the data values from the dictionary argument.

        :param dct: dictionary containing the values of the object members.
        '''
        self.mainDATFile= dct['mainDATFile']
        self.groupDATFiles= dct['groupDATFiles']
        self.dxfLayers= dct['dxfLayers']
        self.outputFileName= dct['outputFileName']
        self.problemName= dct['problemName']
        self.nodeHandlerName= dct['nodeHandlerName']
        self.cellHandlerName= dct['cellHandlerName']
        self.setHandlerName= dct['setHandlerName']
        self.pointHandlerName= dct['pointHandlerName']
        self.lineHandlerName= dct['lineHandlerName']
        self.surfaceHandlerName= dct['surfaceHandlerName']
        self.cellConversion= dct['cellConversion']
        meshDict= dct['meshDesc']
        if(meshDict):
            self.meshDesc= me.MeshData()
            self.meshDesc.setFromDict(meshDict)
        blockDict= dct['blockData']
        if(blockDict):
            self.blockData= bte.BlockData()
            self.blockData.setFromDict(blockDict)

class MEDMeshData(me.MeshData):
    meshDimension= None
    spaceDimension= None

    def __init__(self,umesh):
        super(MEDMeshData, self).__init__()
        self.meshDimension= umesh.getMeshDimension()
        self.spaceDimension= umesh.getSpaceDimension()
        self.numberOfCells= umesh.getNumberOfCells()
        self.numberOfNodes= umesh.getNumberOfNodes()

        self.nodes.readFromUMesh(umesh)
        self.cells.readFromUMesh(umesh)

    def __str__(self):
        retval= "meshDimension= " + str(self.meshDimension) + '\n'
        retval+= "spaceDimension= " +' '+str(self.spaceDimension) + '\n'
        retval+= super(MEDMeshData, self).__str__()
        return retval

    def getDict(self):
        ''' Return the object members in a Python dictionary.'''
        retval= super().getDict()
        retval.update({'meshDimension':self.meshDimension, 'spaceDimension':self.spaceDimension, 'numberOfCells':self.numberOfCells, 'numberOfNodes':self.numberOfNodes})
        return retval

    def setFromDict(self, dct):
        ''' Set the data values from the dictionary argument.

        :param dct: dictionary containing the values of the object members.
        '''
        super().setFromDict(dct)
        self.meshDimension= dct['meshDimension']
        self.spaceDimension= dct['spaceDimension']
        self.numberOfCells= dct['numberOfCells']
        self.numberOfNodes= dct['numberOfNodes']

import pickle
def dumpMeshes(meshes,fName):
    with open(fName + '.pkl', 'wb') as f:
        pickle.dump(meshes, f, pickle.HIGHEST_PROTOCOL)

def loadMeshes(fName):
    with open(fName + '.pkl', 'r') as f:
        return pickle.load(f)


import os
import pytest
import time
from pathlib import Path
from indico.client import IndicoClient
from indico.queries import CreateDataset, CreateModelGroup
from indico.types import ModelGroup, Dataset

PUBLIC_URL = "https://github.com/IndicoDataSolutions/indico-client-python/raw/master/tests/integration/data/"

@pytest.fixture(scope="module")
def airlines_dataset(indico):
    client = IndicoClient()
    dataset_filepath = str(Path(__file__).parents[0]) + "/AirlineComplaints.csv"

    response = client.call(
        CreateDataset(
            name=f"AirlineComplaints-test-{int(time.time())}", files=[dataset_filepath]
        )
    )
    assert response.status == "COMPLETE"
    return response

@pytest.fixture(scope="module")
def too_small_dataset(indico):
    client = IndicoClient()
    dataset_filepath = str(Path(__file__).parents[0]) + "/TooSmall.csv"

    response = client.call(
        CreateDataset(
            name=f"TooSmall-test-{int(time.time())}", files=[dataset_filepath]
        )
    )
    assert response.status == "COMPLETE"
    return response

@pytest.fixture(scope="module")
def airlines_model_group(indico, airlines_dataset: Dataset) -> ModelGroup:
    client = IndicoClient()
    name = f"TestCreateModelGroup-{int(time.time())}"
    mg: ModelGroup = client.call(
        CreateModelGroup(
            name=name,
            dataset_id=airlines_dataset.id,
            source_column_id=airlines_dataset.datacolumn_by_name("Text").id,
            labelset_id=airlines_dataset.labelset_by_name("Target_1").id,
            wait=True,
        )
    )
    return mg

@pytest.fixture(scope="module")
def cats_dogs_image_dataset(indico):
    client = IndicoClient()
    dataset_filepath = str(Path(__file__).parents[0]) + "/dog_vs_cats_small.csv"

    response = client.call(
        CreateDataset(
            name=f"DogsAndCats-test-{int(time.time())}",
            files=dataset_filepath,
            from_local_images=True,
        )
    )
    assert response.status == "COMPLETE"
    return response

@pytest.fixture(scope="module")
def cats_dogs_modelgroup(indico, cats_dogs_image_dataset: Dataset) -> ModelGroup:
    client = IndicoClient()
    name = f"TestCreateObjectDetectionMg-{int(time.time())}"

    model_training_options = {
        "max_iter": 20,
        "lr": 0.1,
        "batch_size": 1,
        "filter_empty": False,
        "test_size": 0.2,
        "use_small_model": True,
    }

    mg: ModelGroup = client.call(
        CreateModelGroup(
            name=name,
            dataset_id=cats_dogs_image_dataset.id,
            source_column_id=cats_dogs_image_dataset.datacolumn_by_name("urls").id,
            labelset_id=cats_dogs_image_dataset.labelset_by_name("label").id,
            model_training_options=model_training_options,
            wait=True,
        )
    )
    return mg

@pytest.fixture(scope="module")
def org_annotate_dataset(indico):
    client = IndicoClient()
    dataset_filepath = str(Path(__file__).parents[0]) + "/org-annotate-labeled.csv"

    response = client.call(
        CreateDataset(
            name=f"OrgAnnotate-test-{int(time.time())}", files=[dataset_filepath]
        )
    )
    assert response.status == "COMPLETE"
    return response

@pytest.fixture(scope="module")
def org_annotate_model_group(indico, org_annotate_dataset: Dataset) -> ModelGroup:
    client = IndicoClient()
    name = f"TestFinetuneModelGroup-{int(time.time())}"
    mg: ModelGroup = client.call(
        CreateModelGroup(
            name=name,
            dataset_id=org_annotate_dataset.id,
            source_column_id=org_annotate_dataset.datacolumn_by_name(
                "News Headlines w/Company Names"
            ).id,
            labelset_id=org_annotate_dataset.labelset_by_name("question_825").id,
            wait=True,
        )
    )
    return mg


import time

list1 = ['nemo' for m in range(100000)]
list2 = ['nemo' for n in range(100000)]

def find_nemo(array1, array2):

    t0 = time.time()                # O(1)
    for i in range(0,len(array1)):  # O(m)
        if array1[i] == 'nemo':
            print("Found Nemo!!")
    t1 = time.time()                # O(1)
    print(f'The search took {t1-t0} seconds.')  # O(1)

    t0 = time.time()                 # O(1)
    for i in range(0, len(array2)):  # O(n)
        if array2[i] == 'nemo':
            print("Found Nemo!!")
    t1 = time.time()                 # O(1)
    print(f'The search took {t1 - t0} seconds.')  # O(1)

find_nemo(list1, list2)

# Here there are two different inputs array1 and array2, which could be of two different sizes m and n,
# hence you can't call it 2n instead it's O(m +n)


from datetime import datetime
from models.model_peer import Peer
from typing import Optional

class AddProviderQuery:
    cid: str
    target_peer: Peer
    started_at: datetime
    ended_at: Optional[datetime]
    success: Optional[bool]
    errors: Optional[str]

    def __init__(self, target_peer: Peer, cid: str, started_at: datetime) -> None:
        self.cid = cid
        self.started_at = started_at
        self.target_peer = target_peer
        self.ended_at = None
        self.success = None
        self.errors = None

    def succeeded(self, ended_at: datetime):
        if self.is_done:
            raise Exception(
                f"cid {self.cid} add provider query already done {self.target_peer}")
        self.ended_at = ended_at
        self.success = True

    def failed(self, ended_at: datetime, errors: str):
        if self.is_done:
            raise Exception(
                f"cid {self.cid} add provider query already done {self.target_peer}")
        self.ended_at = ended_at
        self.success = False
        self.errors = errors

    @property
    def is_done(self) -> bool:
        return self.success is not None


# Databricks notebook source
from pyspark.sql import SparkSession

# COMMAND ----------

spark = SparkSession.builder.appName("RS").getOrCreate()

# COMMAND ----------

from pyspark.ml.recommendation import ALS

# COMMAND ----------

from pyspark.ml.evaluation import RegressionEvaluator

# COMMAND ----------

data = spark.read.csv("/FileStore/tables/movielens_ratings.csv", inferSchema=True, header=True)

# COMMAND ----------

data.show()

# COMMAND ----------

data.describe().show()

# COMMAND ----------

training,test = data.randomSplit([0.7,0.3])

# COMMAND ----------

als = ALS(maxIter=5, regParam=0.01, userCol='userId', itemCol='movieId', ratingCol='rating')

# COMMAND ----------

model = als.fit(training)

# COMMAND ----------

predictions = model.transform(test)

# COMMAND ----------

predictions.show()

# COMMAND ----------

evaluator = RegressionEvaluator(metricName="rmse", labelCol="rating", predictionCol="prediction")

# COMMAND ----------

rmse = evaluator.evaluate(predictions)

# COMMAND ----------

print("RSME")
print(rmse)

# COMMAND ----------

single_user = test.filter(test["userId"]==11).select(["movieId", "userId"])

# COMMAND ----------

single_user.show()

# COMMAND ----------

recommendations = model.transform(single_user)

# COMMAND ----------

recommendations.orderBy('prediction', ascending=False).show()

# COMMAND ----------


# -*- coding: utf-8 -*-
# pylint: disable-msg=C0103,E1101
"""
BURPG Echo
See LICENSE for MIT/X11 license info.
See README.md for all other help.

Note: This class was adapted from Google's Drive API Python Quickstart
https://developers.google.com/drive/v3/web/quickstart/python
"""

#from __future__ import print_function
import os
import httplib2

from apiclient import discovery
from apiclient import http
import oauth2client
from oauth2client import client
from oauth2client import tools

class GoogleDrive:
    """
    GoogleDrive provides a way to connect to a user's Google Drive and upload
    files to it
    """

    SCOPES = 'https://www.googleapis.com/auth/drive.file'
    CLIENT_SECRET_FILE = 'client_secrets.json'
    CLIENT_CREDENTIAL_FILE = 'drive.credentials'
    APPLICATION_NAME = 'BURPG Echo'
    CREDENTIALS = None
    DRIVE_SERVICE = None
    logger = None

    def __init__(self, logger, secret_path='client_secrets.json',
                 credentials_path='drive.credentials', noauth_local_webserver=False):
        """
        Gets valid GDrive user credentials from storage.

        If nothing has been stored, or if the stored credentials are invalid,
        the OAuth2 flow is completed to obtain the new credentials.
        """
        self.CLIENT_SECRET_FILE = secret_path
        self.CLIENT_CREDENTIAL_FILE = credentials_path
        self.logger = logger

        credstore = oauth2client.file.Storage(self.CLIENT_CREDENTIAL_FILE)
        credentials = credstore.get()
        if not credentials or credentials.invalid:
            flow = client.flow_from_clientsecrets(self.CLIENT_SECRET_FILE, self.SCOPES)
            flow.user_agent = self.APPLICATION_NAME

            # Workaround to prevent tools.run_flow from trying to grab our args
            import argparse
            parser = argparse.ArgumentParser(description=__doc__,
                                             formatter_class=argparse.RawDescriptionHelpFormatter,
                                             parents=[tools.argparser])
            dummy_args = []
            if noauth_local_webserver:
                logger.log_verbose("Performing offline auth-flow")
                dummy_args = ["--noauth_local_webserver"]

            credentials = tools.run_flow(flow, credstore, flags=parser.parse_args(dummy_args))
            self.logger.log_verbose('Storing credentials to ' + self.CLIENT_CREDENTIAL_FILE)
        self.CREDENTIALS = credentials
        self.DRIVE_SERVICE = discovery.build(
            'drive', 'v3', http=credentials.authorize(httplib2.Http()))
        self.logger.log_verbose("Google Drive authentication complete")

    def upload_file(self, file_path, parent_folder_id=None):
        """
        Upload a file to Google Drive in a "chunked" manner

        :param file_path: the local path to the file being uploaded
        :param parent_folder_id: the id of the folder where this file should be
        placed
        :returns: the API's response object
        """
        self.logger.log_verbose("Attempting upload:" + file_path +" to " + str(parent_folder_id))
        media = http.MediaFileUpload(file_path, resumable=True)
        file_metadata = {'name': os.path.basename(file_path)}
        if parent_folder_id is not None:
            file_metadata['parents'] = [parent_folder_id]

        request = self.DRIVE_SERVICE.files().create(body=file_metadata, media_body=media)
        response = None
        while response is None:
            status, response = request.next_chunk()
            if status:
                self.logger.log_verbose("Uploaded {:d}%.".format(int(status.progress() * 100)))
        self.logger.log_verbose("Upload complete: " + file_path)
        return response

    def create_folder(self, folder_name):
        """
        Creates a folder in Google Drive and returns the ID

        Note: Google Drive treats folders just like files, except that they have
        the special MIME type "application/vnd.google-apps.folder". To place a
        file in a folder, create the folder using this method and then add the
        ID to the "parents" array property of the "body" argument in
        files().create()

        :param folder_name: the desired name of the created folder
        :returns: the ID of the newly-created folder
        """
        folder_metadata = {
            'name': folder_name,
            'mimeType': 'application/vnd.google-apps.folder'
        }

        folder = self.DRIVE_SERVICE.files().create(body=folder_metadata, fields='id').execute()
        self.logger.log_verbose("Created folder '" + folder_name + "' with ID " + folder.get('id'))
        return folder.get('id')


from Tool import app, db
import os
from hashids import Hashids
from flask import Flask, render_template, request, flash, redirect, url_for
from Tool.models import Urls

hashids = Hashids(min_length=4, salt=app.config['SECRET_KEY'])

@app.route('/', methods=['GET', 'POST'])
def index():
    if request.method == 'POST':
        url_data = Urls(og_url=request.form['url'])

        db.session.add(url_data)
        db.session.commit()

        if url_data is None:
            flash('URL is rquired!')
            return redirect(url_for('index'))

        url_id = url_data.id
        hashid = hashids.encode(url_id)
        short_url = request.host_url + hashid

        url_data.short_url = short_url
        db.session.commit()

        return render_template('index.html', short_url=short_url)

    return render_template('index.html')

@app.route('/<id>')
def url_redirect(id):

    original_id = hashids.decode(id)

    if original_id:
        original_id = original_id[0]
        url_data = Urls.query.filter_by(id=original_id).first()
        print(url_data)
        og_url = url_data.og_url
        return redirect(og_url)
    else:
        flash('INVALID URL')
        return redirect(url_for('index'))

if __name__ == '__main__':
    app.run(debug=True)


import requests

url = 'http://10.10.10.139/post.php'

getPara = {
    "wd": "林俊杰"
    }

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36 Edg/93.0.961.47"
}

res = requests.get(url = url,params=getPara,headers=headers)
print(res.text)
print(res.url)
input()


import cv2
import cv2.cv as cv
import numpy as np

global src_pts
global count
count = 0
src_pts = cv2.cv.CreateMat(4,1,cv2.CV_64FC2)

def getPoint(event,x,y,flags,param):
    if event == cv2.EVENT_LBUTTONDOWN:
        global src_pts
        global count
        src_pts[count,0] = (x,y)
        count = count + 1

baseIm = cv2.imread('P1010580.jpg')

cv2.namedWindow('Calibrate')

cv.SetMouseCallback('Calibrate',getPoint)

cv2.imshow('Calibrate',baseIm)

cv2.waitKey(0)

if count==4:
    cv2.destroyAllWindows()

dest_pts = cv2.cv.CreateMat(4,1,cv2.CV_64FC2)

#src_pts[0,0] = (392,144)
#src_pts[1,0] = (592,237)
#src_pts[2,0] = (272,322)
#src_pts[3,0] = (494,436)

dest_pts[0,0] = (0,0)
dest_pts[1,0] = (520,0)
dest_pts[2,0] = (0,520)
dest_pts[3,0] = (520,520)

src_pts = np.array(src_pts)
dest_pts = np.array(dest_pts)

M,mask = cv2.findHomography(src_pts, dest_pts, cv2.RANSAC, 5.0)

#dst = cv2.perspectiveTransform(baseIm,M)

output = cv2.warpPerspective(baseIm,M,(900,900))

cv2.imshow('base',baseIm)
cv2.imshow('output',output)

k = cv2.waitKey(0)


from flask import Flask, request, abort
import yaml
import json
from functions import *
import datetime

"""
VARIABLE ASSIGNEMENT
"""
# Open the config.yml file and load its contents into the 'config' variable
with open('config.yml', 'r') as file:
    config = yaml.safe_load(file)

# Webex card
with open('ressources/surveyCard.json', 'r') as json_file:
    surveyCard = json.load(json_file)
with open('ressources/resultCard.json', 'r') as json_file:
    resultCard = json.load(json_file)

"""
HANDLE WEBHOOKS
"""
deleteAllWebhooks(config["botToken"])
sendCardWebhookUrl = config["baseWebhookUrl"]+config["sendCardWebhookPath"]
getCardSubmissionUrl = config["baseWebhookUrl"]+config["getCardSubmissionPath"]
getDetailUrl = config["baseWebhookUrl"]+"details"

response = createWebhook("getDetail", getDetailUrl, "attachmentActions", "", config["botToken"])
status_code = 0
while status_code != 200:
    response, status_code = createWebhook("getCardSubmission", getCardSubmissionUrl, "attachmentActions", "", config["botToken"])

initaliseDB(config["botToken"])

print("\nWebhooks creation output: "+str(status_code)+"\n")

app = Flask(__name__)

"""
LISTENING ON /webhook/getcardsubmission RESSOURCE FOR HTTP POST (WEBHOOKS)
"""
@app.route(config["getCardSubmissionPath"], methods=['POST'])
def webhookGet():

    if request.method == 'POST':

        weekNum = getWeekNum()
        data = request.json
        attachement = getAttachement(config["botToken"], data["data"]["id"])

        # READ DATABASE
        with open('data/answers.json', 'r') as openfile:
            previousWeeksAllRooms = json.load(openfile)

        previousWeeks = previousWeeksAllRooms[data["data"]["roomId"]]["roomResults"]
        # REMOVE USERNAME FROM WEEK
        username = getUsernameFromUserid(config["botToken"], attachement["personId"])
        for day in previousWeeks[weekNum-1]["days"]:
            newLine = [s for s in previousWeeks[weekNum-1]["days"][day] if s != username]
            previousWeeks[weekNum-1]["days"][day] = newLine

        # ADD USERNAME IN WEEK
        for day in attachement["inputs"]:
            if attachement["inputs"][day] == "true":
                previousWeeks[weekNum-1]["days"][day].append(username)

        previousWeeksAllRooms[data["data"]["roomId"]]["roomResults"]=previousWeeks

        # UPDATE DATABASE
        with open("data/answers.json", "w") as outfile:
            json.dump(previousWeeksAllRooms, outfile, indent=4)

        # SEND RESULT BACK TO USER
        listDays = []
        listDays = getListDays()
        populateResultCard(resultCard, previousWeeks, weekNum, getRoomSize(config["botToken"], data["data"]["roomId"]), listDays)
        sendCardToPersonId(config["botToken"], data["data"]["personId"], resultCard)

        return 'success', 200

    # ELSE RETURN A HTTP 400 STATUS CODE
    else:
        abort(400)

"""
LISTENING ON LOCALHOST:42101 FOR HTTP POST (WEBHOOKS)
"""
if __name__ == '__main__':
    app.run(host="0.0.0.0", port=42101)


from urllib.parse import parse_qs

from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait

from court_scraper.base.selenium_helpers import SeleniumHelpers

from ..search_api import SearchApi

class ResultsLocators:

    SHOW_ALL_RESULTS = (
        By.XPATH,
        '//*[@id="caseSearchResults_length"]/label/select/option[5]',
    )
    RESULTS_TABLE_ROWS = (By.XPATH, '//*[@id="caseSearchResults"]/tbody/tr')
    RESULTS_TABLE = (By.XPATH, '//*[@id="caseSearchResults"]')
    YEAR_FILTER = (By.XPATH, '//*[@id="caseSearchResults"]/thead/tr[2]/td[1]/input')
    SINGLE_CASE_RETURN = (By.XPATH, '//*[@id="case-header-info"]/h4')
    EMPTY_RETURN = "No"
    SORT = (By.XPATH, '//*[@id="caseSearchResults"]/thead/tr[1]/th[1]')

class ResultRow:
    def __init__(self, county, cookies, row_obj, captcha_solution=None):
        self.county = county
        self.cookies = cookies
        self.captcha_solution = captcha_solution
        self._set_attrs_from_row_obj(row_obj)

    def _set_attrs_from_row_obj(self, row_obj):
        anchor = row_obj.find_element_by_tag_name("a")
        case_detail_url = anchor.get_attribute("href")
        base_url, query_string = case_detail_url.split("?")
        qs = parse_qs(query_string)
        self.case_num = anchor.text
        self.county_num = int(qs["countyNo"][0])

    def case_details(self, use_captcha_solution=False):
        try:
            return self._case_details
        except AttributeError:
            search_api = SearchApi(self.county)
            kwargs = {
                "county_num": self.county_num,
                "cookies": self.cookies,
            }
            if use_captcha_solution:
                kwargs["captcha_solution"] = self.captcha_solution
            case_info = search_api.case_details(self.case_num, **kwargs)
            self._case_details = case_info
            return case_info

class Results(SeleniumHelpers):

    locators = ResultsLocators

    def __init__(self, driver, county, captcha_api_key, captcha_solution=None):
        self.driver = driver
        self.county = county
        self.captcha_api_key = captcha_api_key
        self.captcha_solution = captcha_solution

    def get(self, use_captcha_solution=False):
        """Get case detail data"""
        WebDriverWait(self.driver, 10).until(
            EC.element_to_be_clickable(self.locators.RESULTS_TABLE)
        )
        # Maximize displayed results
        self.click(self.locators.SHOW_ALL_RESULTS)
        # Harvest links on page
        cookies = self.cookies_as_dict()
        rows = self._get_elements_by_locator(self.locators.RESULTS_TABLE_ROWS)
        # Use cookies and harvested Captcha key to make case_detail POST request
        # NOTE: Captcha key only appears to be required on the first request. Strange...
        results = []
        for idx, row in enumerate(rows):
            row_obj = ResultRow(self.county, cookies, row, self.captcha_solution)
            # Only use captcha solution on first case in first daily search result page
            if idx == 0 and use_captcha_solution:
                case_info = row_obj.case_details(use_captcha_solution=True)
            else:
                case_info = row_obj.case_details(use_captcha_solution=False)
            results.append(case_info)
        return results

class SearchResultsPage(SeleniumHelpers):

    locators = ResultsLocators

    def __init__(self, driver, county, captcha_api_key, captcha_solution):
        self.driver = driver
        self.county = county
        self.captcha_api_key = captcha_api_key
        self.captcha_solution = captcha_solution
        self.results = Results(driver, county, captcha_api_key, captcha_solution)


import os.path
import scipy.sparse

import numpy as np
import oct2py

from .utils import adjacency_matrix_to_tree
from .reconstruct_tree import ReconstructionMethod
from .raxml_reconstruction import SPECTRALTREE_LIB_PATH

SPECTRALTREE_CHOI_PATH = os.path.join(SPECTRALTREE_LIB_PATH, "ChoilatentTree")

class RG(ReconstructionMethod):
    def __init__(self, distance_metric):
        self.distance_metric = distance_metric

    def __call__(self, observations, taxa_metadata=None):
        oct2py.octave.addpath(SPECTRALTREE_CHOI_PATH)
        oc = oct2py.Oct2Py()
        num_taxa = observations.shape[0]

        D = self.distance_metric(observations)
        #adj_mat = oc.feval("./spectraltree/ChoilatentTree/toolbox/RGb.m",observations+1,0)
        adj_mat = oc.feval(os.path.join(SPECTRALTREE_CHOI_PATH, "toolbox", "RGb.m"),D,1,observations.shape[1], verbose=False)
        adj_mat = np.array(scipy.sparse.csr_matrix.todense(adj_mat))
        tree_RG = adjacency_matrix_to_tree(adj_mat,num_taxa,taxa_metadata)
        return tree_RG

    def __repr__(self):
        return "RG"

class CLRG(ReconstructionMethod):
    def __call__(self, observations, taxa_metadata=None):
        oct2py.octave.addpath('./spectraltree/ChoilatentTree/')
        oc = oct2py.Oct2Py()
        num_taxa = observations.shape[0]
        adj_mat = oc.feval(os.path.join(SPECTRALTREE_CHOI_PATH, "toolbox", "CLRGb.m"),observations+1,0, verbose=False)
        adj_mat = np.array(scipy.sparse.csr_matrix.todense(adj_mat))
        tree_CLRG = adjacency_matrix_to_tree(adj_mat,num_taxa,taxa_metadata)
        return tree_CLRG

    def __repr__(self):
        return "CLRG"


#!/usr/bin/env python3
"""
Alta3 Research | RZFeeser
StephanieCobble | Lab 45 - IPv4 Testing with if
Conditionals - strings test true
"""

ipchk = "192.168.0.1"

# a string tests as True
if ipchk:
   print("Looks like the IP address was set: " + ipchk)


# Databricks notebook source
# MAGIC %md
# MAGIC # Migrate Delta Table to New Location
# MAGIC ### What?
# MAGIC Title ^
# MAGIC ### Why?
# MAGIC Many users desire a way to re-organize their Delta Tables as they evolve with Databricks and see new patterns. We also want to help
# MAGIC     customers migrate away from DBFS managed tables and FUSE Mounts to prepare them for Unity Catalog.
# MAGIC ### Who?
# MAGIC Customers using DBFS Tables/FUSE Mounts
# MAGIC Customers who ask for a way to move a Delta Lake table to a new location
# MAGIC Persona: Data Engineers can run this interactively and get feedback from the cell output OR DE/Ops could script this out using Widget
# MAGIC     Paremeters to migrate many tables at once.
# MAGIC ### How?
# MAGIC Delta Lake includes a great feature called [Delta Clone](https://docs.databricks.com/spark/latest/spark-sql/language-manual/delta-clone.html).
# MAGIC This notebook implements Delta Clone and Delta Lake w/ the PySpark and SparkSQL APIs to facilitate a migration of your Delta Lake
# MAGIC     Table and associated Hive Metadata to a new location.
# MAGIC ### Caveats
# MAGIC - Currently Delta Clone does not support migrating table history
# MAGIC - I have not tested this with FUSE Mounts yet

# COMMAND ----------

# MAGIC %md
# MAGIC ## Initialize the environment

# COMMAND ----------

# MAGIC %md
# MAGIC EXAMPLE BASE PATH FOR AZURE:
# MAGIC ##### abfss://container-name@storage-account-name.dfs.core.windows.net/directory-name/my-metastore/gold-zone

# COMMAND ----------

dbutils.widgets.text("external_url", "abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<directory-name>", label = "New Base Url")
dbutils.widgets.text("table_name", "", label = "Hive Table Name")
dbutils.widgets.text("db_name", "", label = "Database Name")
dbutils.widgets.dropdown("managed", "Yes", ["Yes","No"],"Moving a Managed Table?")
table_name = dbutils.widgets.get("table_name")
spark.sql("USE {}".format(dbutils.widgets.get("db_name")))
new_url = "{0}{1}".format(dbutils.widgets.get("external_url"),table_name)
managed = dbutils.widgets.get("managed")

# COMMAND ----------

# MAGIC %md
# MAGIC ## Verify the correct target and source

# COMMAND ----------

location = spark.sql("DESCRIBE DETAIL {}".format(table_name)).collect()[0].location
print("{0} LOCATED AT {1} IS MOVING {2}".format(table_name, location, new_url))

# COMMAND ----------

# MAGIC %md
# MAGIC ## Clone the table

# COMMAND ----------

spark.sql("""
CREATE OR REPLACE TABLE delta.`{0}` CLONE `{1}`
""".format(new_url, table_name))

# COMMAND ----------

# MAGIC %md
# MAGIC ## Verify the data is all moved

# COMMAND ----------

old = spark.table(table_name)
new = spark.read.load(path=new_url)

# COMMAND ----------

# MAGIC %md
# MAGIC ### Ensure the old table matches the new table

# COMMAND ----------

display(old.exceptAll(new))

# COMMAND ----------

# MAGIC %md
# MAGIC ### Ensure the new table matches the old table

# COMMAND ----------

display(new.exceptAll(old))

# COMMAND ----------

# MAGIC %md
# MAGIC ## !!! DROP ORIGINAL TABLE & DATA !!!

# COMMAND ----------

spark.sql("""
DROP TABLE {}""".format(table_name))

# COMMAND ----------

# MAGIC %md
# MAGIC ### If table is not managed we have to also clean up the existing data

# COMMAND ----------

if managed == 'No':
  dbutils.fs.rm(location,True)

# COMMAND ----------

# MAGIC %md
# MAGIC ## Register new table with hive

# COMMAND ----------

spark.sql("""
CREATE TABLE {0}
USING DELTA
LOCATION "{1}"
""".format(table_name, new_url))

# COMMAND ----------

# MAGIC %md
# MAGIC ## Verify that we can query the cloned table by name

# COMMAND ----------

display(spark.sql("SELECT * FROM {}".format(table_name)))

# COMMAND ----------

# MAGIC %md
# MAGIC ## Verify the new location is correct

# COMMAND ----------

new_location = spark.sql("DESCRIBE DETAIL {}".format(table_name)).collect()[0].location
if new_location == new_url:
  print("The new table location matched the target.\nThe migration process is complete for {}".format(table_name))
else:
  print("The new table location does not match the target.")
  print("New Table Location: {}".format(new_location))
  print("Target Table Location: {}".format(new_url))
  print("Try running the 'Register new table with Hive' step")

# COMMAND ----------