import tensorflow as tf

from tf_agents.networks import network
from tf_agents.agents.dqn import dqn_agent
from tf_agents.environments import tf_py_environment
from tf_agents.policies import random_tf_policy
from tf_agents.replay_buffers import tf_uniform_replay_buffer
from tf_agents.trajectories import trajectory
from tf_agents.utils import common

from sudoku_env import SudokuEnvironment
from utils import load_dataset

if __name__ == "__main__":
    ds_train, ds_eval = load_dataset("data/sudoku.csv")

    train_py_env = SudokuEnvironment(ds_train)
    train_env = tf_py_environment.TFPyEnvironment(train_py_env)
    eval_py_env = SudokuEnvironment(ds_eval)
    eval_env = tf_py_environment.TFPyEnvironment(eval_py_env)

    num_iterations = 20000

    initial_collect_steps = 1000
    collect_steps_per_iteration = 1
    replay_buffer_max_length = 100000

    batch_size = 64
    learning_rate = 1e-3
    log_interval = 200

    num_eval_episodes = 30
    eval_interval = 1000

    class MyQNetwork(network.Network):
        def __init__(self, input_tensor_spec, action_spec, name="MyQnet"):
            super(MyQNetwork, self).__init__(
                input_tensor_spec=input_tensor_spec, state_spec=(), name=name
            )
            action_spec = tf.nest.flatten(action_spec)[0]
            num_actions = action_spec.maximum - action_spec.minimum + 1

            self._forward = tf.keras.Sequential(
                [
                    tf.keras.layers.Reshape((9, 9, 9)),
                    tf.keras.layers.Conv2D(
                        64,
                        kernel_size=(3, 3),
                        activation="relu",
                        padding="same",
                    ),
                    tf.keras.layers.BatchNormalization(),
                    # tf.keras.layers.Conv2D(
                    #     64,
                    #     kernel_size=(3, 3),
                    #     activation="relu",
                    #     padding="same",
                    # ),
                    # tf.keras.layers.BatchNormalization(),
                    tf.keras.layers.Conv2D(
                        128,
                        kernel_size=(1, 1),
                        activation="relu",
                        padding="same",
                    ),
                    tf.keras.layers.Flatten(),
                    tf.keras.layers.Dense(num_actions),
                ]
            )

        def call(self, observation, step_type=None, network_state=()):
            observation = tf.cast(observation, dtype=tf.float32)
            logits = self._forward(observation)
            return logits, network_state

    q_net = MyQNetwork(train_env.observation_spec(), train_env.action_spec())

    optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=learning_rate)

    train_step_counter = tf.Variable(0)

    agent = dqn_agent.DqnAgent(
        train_env.time_step_spec(),
        train_env.action_spec(),
        q_network=q_net,
        optimizer=optimizer,
        td_errors_loss_fn=common.element_wise_squared_loss,
        train_step_counter=train_step_counter,
    )

    agent.initialize()

    random_policy = random_tf_policy.RandomTFPolicy(
        train_env.time_step_spec(), train_env.action_spec()
    )

    def compute_avg_return(environment, policy, num_episodes=10):
        total_return = 0.0
        for _ in range(num_episodes):

            time_step = environment.reset()
            episode_return = 0.0

            while not time_step.is_last():
                action_step = policy.action(time_step)
                time_step = environment.step(action_step.action)
                episode_return += time_step.reward
            total_return += episode_return

        avg_return = total_return / num_episodes
        return avg_return.numpy()[0]

    replay_buffer = tf_uniform_replay_buffer.TFUniformReplayBuffer(
        data_spec=agent.collect_data_spec,
        batch_size=train_env.batch_size,
        max_length=replay_buffer_max_length,
    )

    def collect_step(environment, policy, buffer):
        time_step = environment.current_time_step()
        action_step = policy.action(time_step)
        next_time_step = environment.step(action_step.action)
        traj = trajectory.from_transition(
            time_step, action_step, next_time_step
        )

        # Add trajectory to the replay buffer
        buffer.add_batch(traj)

    def collect_data(env, policy, buffer, steps):
        for _ in range(steps):
            collect_step(env, policy, buffer)

    collect_data(train_env, random_policy, replay_buffer, steps=100)

    dataset = replay_buffer.as_dataset(
        num_parallel_calls=3, sample_batch_size=batch_size, num_steps=2
    ).prefetch(3)
    iterator = iter(dataset)

    agent.train = common.function(agent.train)

    # Reset the train step
    agent.train_step_counter.assign(0)

    # Evaluate the agent's policy once before training.
    avg_return = compute_avg_return(eval_env, agent.policy, num_eval_episodes)
    returns = [avg_return]

    for _ in range(num_iterations):

        # Collect a few steps using collect_policy and save to the replay buffer.
        for _ in range(collect_steps_per_iteration):
            collect_step(train_env, agent.collect_policy, replay_buffer)

        # Sample a batch of data from the buffer and update the agent's network.
        experience, unused_info = next(iterator)
        train_loss = agent.train(experience).loss

        step = agent.train_step_counter.numpy()

        if step % log_interval == 0:
            print("step = {0}: loss = {1}".format(step, train_loss))

        if step % eval_interval == 0:
            avg_return = compute_avg_return(
                eval_env, agent.policy, num_eval_episodes
            )
            print("step = {0}: Average Return = {1}".format(step, avg_return))
            returns.append(avg_return)


import os
import time

from flask import Flask, render_template, request
from flask_socketio import SocketIO, emit, disconnect

from collections import deque

app = Flask(__name__)

# socket-io configure
app.config["SECRET_KEY"] = os.getenv("SECRET_KEY")
socketio = SocketIO(app)

# in-memory data
USERS = {}
CHANNELS = {"general": deque([], maxlen=100)}

@app.route("/")
def index():
    return render_template("index.html")

@socketio.on('connect')
def connection():
    print("new user connected")

@socketio.on('userdata')
def user_data(data):
    if 'username' in data:
        USERS[data['username']] = request.sid

@socketio.on('new channel')
def new_channel(data):
    if data['name'] in CHANNELS:
        return False
    else:
        CHANNELS[data['name']] = deque(maxlen=100)
        emit('new channel', { "name" : data['name']}, broadcast=True)

@socketio.on('new msg')
def new_msg(data):

    if 'channel' in data:
        data['created_at'] = int(time.time())
        CHANNELS[data['channel']].append(data)
        emit('msg', data, broadcast=True)

@socketio.on('get channels')
def get_channels():
    emit('channels', list(CHANNELS.keys()))

@socketio.on('get msgs')
def get_msgs(data):
    if 'name' in data:
        emit('msgs', list(CHANNELS[data['name']]))

if __name__ == "__main__":
    socketio.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", 5000)))


import csv
import logging
import json
import math
import random
import re
import time
import urllib.request
from pathlib import Path
import sys
from bs4 import BeautifulSoup
import requests

import get_edgar.common.my_csv as mc
import get_edgar.common.utils as utils

logger = logging.getLogger(__name__)

EDGAR_PREFIX = "https://www.sec.gov/Archives/"
SEC_PREFIX = "https://www.sec.gov"

def save_file_info(csv_in,folder):
    """ from csv containing EDGAR records to csv containting EDGAR record and file information
        the new csv file has prefix "info_" to the original csv file name

    Arguments:
        csv_in {[Path]} -- [the csv file that contains EDGAR records]
        folder {[Path]} -- [the folder to save the new csv file]

    Returns:
        {[Path]} -- [the path for the csv created]
    """
    csv_save = folder / f'info_{csv_in.name[6:]}'
    if csv_save.exists() == False:
        logger.info(f"start save file info to {csv_in.name}")
        new_rows = get_file_info(csv_in)
        sorted_rows = mc.multikeysort_int(new_rows,'cik','filing_date')
        mc.save_dict_csv(sorted_rows,csv_save)
        logger.info(f"{csv_save.name} created")
    else:
        logger.info(f"{csv_save} already exists")
    return csv_save

## Add file information and htm link to the index downloaded

def get_file_info(csv_in):
    """ Get form and filer information for all EDGAR records in a csv file

    Arguments:
        csv_in {Path} -- the Path object for the csv file that contains EDGAR records,
                         the columns should include "cik", "filing_date", and "html_index"

    Returns:
        [list] -- [list of dictionaries containing original EDGAR record, form, and filer information]
    """
    logger.debug(f"start add file info to {csv_in.name}")
    new_rows = []
    with open(csv_in, 'r', newline='') as f:
        reader = csv.DictReader(f)
        for row in reader:
            time.sleep(1)
            isoup = get_isoup(row.get('html_index'))
            if isoup == None:
                continue
            links = get_htm_links(isoup, row.get('html_index'))
            for num,link in links:
                row[f'htm_link_{num}'] = link
            form_infos = get_form_info(isoup, row.get('html_index'))
            row.update(form_infos)
            filer_infos = get_filer_info(isoup, row)
            row.update(filer_infos)
            # row['year'] = int(f'{csv_in.name[-8:-4]}')
            new_rows.append(row)
    logger.debug(f"list of records with file info created for {csv_in.name}")
    return new_rows

def get_isoup(page):
    """[Get soup for the EDGAR index page for each EDGAR file]

    Arguments:
        page {str} -- [webpage address]

    Returns:
        [BeautifulSoup] -- [the soup parsed using BeautifulSoup]
    """
    i = 0
    while True:
        try:
            html_index = requests.get(page, headers=utils.headers)
            break
        except (requests.exceptions.HTTPError):
            logger.error(f'try wait a minute to reopen {page}')
            time.sleep(70)
        except Exception:
            logger.error(f"cannot download {page}", exc_info=True)
            i += 1
            if i > 10:
                return None
            time.sleep(70)
    try:
        return BeautifulSoup(html_index.text, 'lxml')
    except Exception:
        logger.error(f'Cannot make soup for {page}', exc_info=True)
        return None

def get_htm_links(soup, index_path):
    """[Get page address for the first webpage from an EDGAR file index page]

    Arguments:
        soup {[BeautifulSoup]} -- [the soup for the EDGAR file index page]
        index_path {[str]} -- [the page address for the EDGAR file index page]

    Returns:
        [str] -- [the page address for the first webpage in the EDGAR file index page]
    """
    try:
        links = soup.find_all(href=re.compile(r"Archives.*\.htm"))
        page_links = enumerate([SEC_PREFIX + link['href'] for link in links],1)
        return [(n,re.sub(r'/ix\?doc=/','/',link)) for (n,link) in page_links]
    except:
        logger.exception(f'Unable to get htm pages for {index_path}')
        return None

form_headers = {
    "type_description":re.compile(r'Form\s(.*)(?=:)', re.IGNORECASE),
    "report_period":re.compile(r'\sPeriod of Report\s(\d{4}-\d{2}-\d{2})',re.IGNORECASE),
    "file_date":re.compile(r'\sFiling Date\s(\d{4}-\d{2}-\d{2})',re.IGNORECASE),
    "accepted_time":re.compile(r'\sAccepted\s(\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2})',re.IGNORECASE),
    "accession_num":re.compile(r'SEC Accession No.\s(\d{10}-\d{2}-\d{6})', re.IGNORECASE),
    "items":re.compile(r'Items\s((Item.*))', re.IGNORECASE),
}

def get_form_info(soup, index_path, headers=form_headers):
    """[Get form information from an EDGAR file index page
        the form information is set through "form_headers"
        ]

    Arguments:
        soup {[BeautifulSoup]} -- [the soup for the EDGAR file index page]
        index_path {[str]} -- [the page address for the EDGAR file index page]

    Returns:
        [dict] -- [dictionary representing the form information on the EDGAR file index page]
    """
    form_infos = {}
    form_texts = []
    form_texts.append(soup.find(id="formName").get_text())
    form_texts.append(soup.find(id="secNum").get_text())
    formgroup = soup.find_all("div", {"class":"formGrouping"})
    for group in formgroup:
        form_texts.append(group.get_text())
    tems = headers.items()
    for form_text in form_texts:
        for k, v in tems:
            matches = v.search(form_text)
            if matches:
                form_infos[k] = matches.group(1)
    if len(form_infos) == 0:
        logger.warning(f'No form infos for {index_path}')
    elif len(form_infos) < len(headers):
        logger.debug(f'Incomplete form infos for {index_path}')
    return form_infos

filer_headers = {
    "cname":re.compile(r'\n(.+)\(Filer\)', re.IGNORECASE),
    "fcik":re.compile(r'CIK:\s(\d{10})', re.IGNORECASE),
    "sic":re.compile(r'SIC:\s(\d{4})', re.IGNORECASE),
    "irs_num":re.compile(r'IRS No.:\s(\d+)\s', re.IGNORECASE),
    "year_end":re.compile(r'Fiscal Year End:\s(\d{4})', re.IGNORECASE),
    "state_incorp":re.compile(r'State of Incorp.:\s(\w{2})', re.IGNORECASE),
    "type":re.compile(r'Type:\s(.+?)\s(?=\|)', re.IGNORECASE)
}

fcik_pattern = filer_headers.get("fcik")
fcname_pattern = filer_headers.get("cname")

def get_filer_info(soup, record):
    """[Get filer information from an EDGAR file index page
        the filer information is set through "filer_headers"
        the filer cik needs to be the same as cik in the EDGAR file record]

    Arguments:
        soup {[BeautifulSoup]} -- [the soup for the EDGAR file index page]
        record {[dict]} -- [dictionary representing the EDGAR file record]

    Returns:
        [dict] -- [dictionary representing the filer information on the EDGAR file index page]
    """
    filer_infos = {}
    co_filers_cik = []
    co_filers_cname = []
    all_filer_texts = soup.find_all("div",id="filerDiv")
    tems = filer_headers.items()
    for filer_texts in all_filer_texts:
        filer_text = filer_texts.get_text()
        fcik_info = fcik_pattern.search(filer_text)
        if fcik_info:
            fcik = fcik_info.group(1)
            if str(int(fcik)) != record.get("cik"):
                logger.debug(f'found co-filer in {record.get("html_index")}')
                co_filers_cik.append(fcik)
                cname_info = fcname_pattern.search(filer_text)
                if cname_info:
                    cname = cname_info.group(1)
                    co_filers_cname.append(cname.strip())
                continue
        for k, v in tems:
            matches = v.search(filer_text)
            if matches:
                info_raw = matches.group(1)
                info = info_raw.replace('\n', ' ').replace('\r', ' ')
                filer_infos[k] = info.strip()
        filer_infos = {**filer_infos, **extract_address(filer_texts)}
    if len(filer_infos) == 0:
        logger.warning(f'No filer infos for {record.get("html_index")}')
    if co_filers_cik:
        for i in range(len(co_filers_cik)):
            filer_infos[f'co_filers_cik_{i}'] = co_filers_cik[i]
            try:
                filer_infos[f'co_filers_cname_{i}'] = co_filers_cname[i]
            except IndexError:
                filer_infos[f'co_filers_cname_{i}'] = None
    return filer_infos

def select_items(csv_in,filters):
    csv_out = csv_in.resolve().parent / f'item_{csv_in.name[5:]}'
    if csv_out.exists() == False:
        mc.text_filter(csv_in,csv_out,'items',filters)
        logger.info(f'{csv_out} created')
    else:
        logger.info(f'{csv_out} already exists')

def extract_address(filer_info):
    addresses = filer_info.find_all('div','mailer')
    if addresses:
        for address in addresses:
            add_des = address.contents[0].strip()
            mailer_address = []
            phone = None
            adds = address.find_all('span',class_='mailerAddress')
            if adds:
                for add in adds:
                    add_text = add.get_text().strip()
                    if any(c.isalpha() for c in add_text):
                        mailer_address.append(add_text)
                    else:
                        phone = ''.join([c for c in add_text if c.isdigit()])
            if add_des == 'Mailing Address':
                mail_add = ','.join(mailer_address)
            elif add_des == 'Business Address':
                busi_add = ','.join(mailer_address)
                busi_phone = phone
        return {'mail_add':mail_add, 'busi_add':busi_add,'busi_phone':busi_phone}
    return {'mail_add':None, 'busi_add':None,'busi_phone':None}


import math
from PyQt5.QtWidgets import QGraphicsView, QGraphicsLineItem, QApplication, QMenu, QAction
from PyQt5.QtGui import QColor, QBrush, QPen
from PyQt5.QtCore import pyqtSlot, QLineF, QRectF, QPoint, QPointF, Qt
from pyqtgraph import GraphicsLayoutWidget, PlotItem, ViewBox, GraphicsItem, GraphicsView, PlotDataItem, TextItem, ButtonItem
from orbit import Orbit, BPM
from bpm_line_item import BPMLineItem
from magnet_view import MagnetView
from PyQt5.QtCore import QTimer

class OrbitView(GraphicsLayoutWidget):
    def __init__(self, orbit=None, axis="X", use_sector_ticks=True, parent=None, ymin=-1.0, ymax=1.0, name=None, label=None, units=None, draw_timer=None, magnet_list=None):
        super(OrbitView, self).__init__(parent=parent)
        axis = axis.lower()
        if axis not in ["x", "y", "tmit"]:
            raise Exception("Axis must be 'x', 'y', or 'tmit'")
        self.axis = axis
        self.use_sector_ticks = use_sector_ticks
        self.sector_ticks = [[],[]]
        self.ci.layout.setSpacing(0.0)
        self.plotLabel = self.addLabel(text=name, row=0, col=0, rowspan=3, angle=-90)
        self.up_magnet_view = MagnetView(magnet_list=orbit, direction="up")
        self.up_magnet_view.hideAxis('left')
        self.up_magnet_view.hideAxis('bottom')
        self.ci.layout.setRowStretchFactor(0,3)
        self.plotItem = self.addPlot(name=name, row=0, col=1)
        self.ci.layout.setRowStretchFactor(1,0)
        self.down_magnet_view = MagnetView(magnet_list=orbit, direction="down")
        self.down_magnet_view.hideAxis('left')
        self.up_magnet_view.setXLink(self.plotItem)
        self.down_magnet_view.setXLink(self.plotItem)
        self.ci.layout.setRowStretchFactor(2,0)
        self.show_magnet_buttons = False
        if axis != "tmit" and magnet_list is not None:
            self.show_magnet_views(True)
        self.setViewportUpdateMode(QGraphicsView.BoundingRectViewportUpdate) # Greatly improves drawing performance.
        self.plotItem.setMouseEnabled(y=False)
        #Customize the right-click menu.
        self.plotItem.setMenuEnabled(enableMenu=False, enableViewBoxMenu=None)
        reset_view_range = QAction("Reset View Range", self.plotItem.vb.menu)
        reset_view_range.triggered.connect(self.reset_range)
        self.plotItem.vb.scene().contextMenu = []
        existing_menu_actions = self.plotItem.vb.menu.actions()
        self.plotItem.vb.menu.insertAction(existing_menu_actions[0], reset_view_range)
        for action in existing_menu_actions:
            if str(action.text()) == "View All":
                self.plotItem.vb.menu.removeAction(action)
        self.plotItem.showGrid(y=True)
        #self.plotItem.getAxis('left').setStyle(tickTextWidth=60)
        #self.plotItem.getAxis('left').setStyle(autoExpandTextSpace=False)
        #if label is not None:
        #   self.plotItem.getAxis('left').enableAutoSIPrefix(enable=False)
        #   self.plotItem.getAxis('left').setLabel(text=label, units=units)
        self.bpm_brush = QBrush(QColor(0,255,0))
        self.energy_bpm_brush = QBrush(QColor(100,200,255))
        self.ymin = ymin #Y axis goes from self.ymin to self.ymax by default.
        self.ymax = ymax
        self.yminlimit = 10.0*ymin #This is the limit on the Y axis range.
        self.ymaxlimit = 10.0*ymax #This is the upper limit on the Y axis range.
        self.plotItem.setLimits(minYRange=0.04, maxYRange=abs(self.ymaxlimit - self.yminlimit))
        self.axis_pen = QPen(QBrush(QColor(255,255,255)), 0)
        self.axis_pen.setCapStyle(Qt.FlatCap)
        self.bpm_pen = QPen(self.bpm_brush, 2)
        self.bpm_pen.setCosmetic(True)
        self.bpm_pen.setCapStyle(Qt.FlatCap)
        self.no_beam_brush = QBrush(QColor(0,255,0,45))
        self.no_beam_pen = QPen(self.no_beam_brush, 2)
        self.no_beam_pen.setCosmetic(True)
        self.no_beam_pen.setCapStyle(Qt.FlatCap)
        self.energy_bpm_pen = QPen(self.energy_bpm_brush, 2)
        self.energy_bpm_pen.setCosmetic(True)
        self.energy_bpm_pen.setCapStyle(Qt.FlatCap)
        self.fit_brush = QBrush(QColor(255,255,255,255))
        self.fit_pen = QPen(self.fit_brush, 0)
        self.fit_pen.setCosmetic(True)
        self.fit_pen.setCapStyle(Qt.FlatCap)
        self.axis_line = QGraphicsLineItem(0.0,0.0,1.0,0.0)
        self.axis_line.setPen(self.axis_pen)
        self.plotItem.addItem(self.axis_line, ignoreBounds=True)
        self.lines = {}
        self.orbit = None
        self.needs_initial_range = True
        self.set_draw_timer(draw_timer)
        self._display_fit = False
        self.fit_data_item = None
        self.fit_options = {}
        if orbit is not None:
            self.set_orbit(orbit)

    def make_right_click_menu(self):
        menu = QMenu(self)
        return menu

    def display_fit(self, enabled=True):
        if enabled and self.fit_data_item is None:
            self.fit_data_item = PlotDataItem(pen=self.fit_pen)
            self.plotItem.addItem(self.fit_data_item)
        elif not enabled:
            self.plotItem.removeItem(self.fit_data_item)
            self.fit_data_item = None
        self._display_fit = enabled

    def set_draw_timer(self, new_timer, start=False):
        try:
            self.draw_timer.timeout.disconnect(self.redraw_bpms)
        except:
            pass
        if new_timer is None:
            new_timer = QTimer(self)
            new_timer.setInterval(int(1000/60))
        self.draw_timer = new_timer
        self.draw_timer.timeout.connect(self.redraw_bpms)
        if start:
            self.draw_timer.start()

    def set_orbit(self, orbit, reset_range=True):
        if self.orbit == orbit:
            return
        old_range = None
        old_zmax = None
        old_zmin = None
        if self.orbit is not None:
            old_range = self.plotItem.viewRect()
            old_zmax = self.orbit.zmax()
            old_zmin = self.orbit.zmin()
        self.clear_orbit()
        self.orbit = orbit
        extent = self.orbit.zmax() - self.orbit.zmin()
        self.plotItem.setLimits(xMin=self.orbit.zmin()-(0.02*extent), xMax=self.orbit.zmax()+(0.02*extent))
        self.plotItem.enableAutoRange(enable=False)
        self.axis_line.setLine(self.orbit.zmin(),0.0,self.orbit.zmax(),0.0)
        for bpm in self.orbit:
            line = BPMLineItem(bpm)
            self.lines[bpm.name] = line
            self.set_pen_for_bpm(bpm)
            self.plotItem.addItem(self.lines[bpm.name])
        if self.use_sector_ticks and (old_zmax != orbit.zmax() or old_zmin != orbit.zmin()):
            self.sector_ticks = [[],[]]
            self.sector_ticks[0] = self.orbit.sector_locations()
            unit_nums = [name.split(":")[-1] for name in self.orbit.names()]
            self.sector_ticks[1] = zip(self.orbit.z_vals(), unit_nums)
            self.plotItem.getAxis('bottom').setTicks(self.sector_ticks)
            self.plotItem.getAxis('bottom').setStyle(textFillLimits=[(0,0.72)])
            self.plotItem.showGrid(x=True)
        if reset_range or self.needs_initial_range:
            self.reset_range()
            self.needs_initial_range = False
        else:
            self.plotItem.setRange(old_range, padding=0.0, update=False)
        self.draw_timer.start()

    def show_magnet_views(self, enabled):
        if enabled == self.show_magnet_buttons:
            return
        self.show_magnet_buttons = enabled
        if enabled:
            self.addItem(self.up_magnet_view, row=1, col=1)
            self.addItem(self.down_magnet_view, row=2, col=1)
            self.up_magnet_view.setXLink(self.plotItem)
            self.down_magnet_view.setXLink(self.plotItem)
        else:
            self.removeItem(self.up_magnet_view)
            self.removeItem(self.down_magnet_view)

    def set_magnet_list(self, magnet_list):
        self.up_magnet_view.set_magnets(magnet_list, reset_range=False)
        self.down_magnet_view.set_magnets(magnet_list, reset_range=False)

    @pyqtSlot(bool)
    def reset_range(self, checked=False):
        self.plotItem.enableAutoRange(axis=ViewBox.XAxis)
        self.plotItem.setYRange(self.ymin, self.ymax)

    def wheelEvent(self, event):
        if event.modifiers() == Qt.ShiftModifier:
            numPixels = event.pixelDelta()
            numDegrees = event.angleDelta()
            if not numPixels.isNull():
                s = (1.005) ** (numPixels.y())
            else:
                s = (1.005) ** (numDegrees.y() * (-1.0/8.0))
            self.plotItem.vb.scaleBy(y=s)
        else:
            super(OrbitView, self).wheelEvent(event)

    def clear_orbit(self):
        self.draw_timer.stop()
        auto_range_x_enabled = self.plotItem.vb.state['autoRange'][0]
        auto_range_y_enabled = self.plotItem.vb.state['autoRange'][1]
        self.plotItem.enableAutoRange(enable=False)
        if self.orbit is None:
            return
        for bpm in self.orbit:
            self.plotItem.removeItem(self.lines[bpm.name])
        self.plotItem.enableAutoRange(x=auto_range_x_enabled, y=auto_range_y_enabled)
        self.lines = {}

    @pyqtSlot()
    def redraw_bpms(self):
        for bpm in self.orbit:
            self.set_pen_for_bpm(bpm)
            self.lines[bpm.name].setLine(bpm.z,0.0,bpm.z,bpm[self.axis])
        self.update_fit()

    def set_pen_for_bpm(self, bpm):
        if bpm.severity(self.axis) != 0:
            self.lines[bpm.name].setPen(self.no_beam_pen)
        else:
            if bpm.is_energy_bpm:
                self.lines[bpm.name].setPen(self.energy_bpm_pen)
            else:
                self.lines[bpm.name].setPen(self.bpm_pen)

    def update_fit(self):
        if not self._display_fit:
            return
        if self.orbit.fit_data is None:
            if self.fit_data_item is not None:
                self.fit_data_item.hide()
            return
        fit_data = None
        if self.axis == 'x':
            fit_data = self.orbit.fit_data['xpos']
        elif self.axis == 'y':
            fit_data = self.orbit.fit_data['ypos']
        self.fit_data_item.show()
        self.fit_data_item.setData(x=self.orbit.fit_data['zs'], y=fit_data)

    @pyqtSlot()
    def stop(self):
        self.draw_timer.stop()

    @pyqtSlot()
    def start(self):
        if self.orbit is not None:
            self.draw_timer.start()

    def setXLink(self, view):
        return self.plotItem.setXLink(view.plotItem)

    def setYLink(self, view):
        return self.plotItem.setYLink(view.plotItem)


from django import forms
from .models import Post
from django.core.validators import FileExtensionValidator

class PostForm(forms.ModelForm):
    thumbnail = forms.FileField(required=False, widget=forms.ClearableFileInput(attrs={'class': 'input'}))
    video = forms.FileField(required=False, validators=[FileExtensionValidator(allowed_extensions=['MOV','avi','mp4','webm','mkv'])], widget=forms.ClearableFileInput(attrs={'class': 'input'}))

    class Meta:
        model = Post
        fields = ['title', 'intro', 'body', 'contribution_amount', 'thumbnail', 'video']
        widgets = {
            'title': forms.TextInput(attrs={'class': 'input', 'placeholder': 'The title of your post', 'required': True}),
            'intro': forms.TextInput(attrs={'class': 'input', 'placeholder': 'The intro of your post', 'required': True}),
            'body': forms.Textarea(attrs={'class': 'textarea', 'placeholder': 'Include details about your cause'}),
            'contribution_amount': forms.TextInput(attrs={'class': 'input', 'placeholder': 'Contribution amount'}),
        }
        validators = {
            'video': FileExtensionValidator(allowed_extensions=['MOV','avi','mp4','webm','mkv'])
        }


input_file = open('day-8/input.txt', 'r')
lines = input_file.readlines()
lines = [line.rstrip() for line in lines if line.rstrip()]
lines = [[*line] for line in lines]

column = 1
row = 1

up = True
down = True
left = True
right = True

visible_trees = 0

while row <= len(lines) - 2:
  while column <= len(lines[row]) - 2:
    #check right
    i = column + 1
    while i <= len(lines[row]) - 1 and right:
      if lines[row][column] <= lines[row][i]:
        right = False
      else:
        i = i+1
    #check left
    i = column - 1
    while i >= 0 and left:
      if lines[row][column] <= lines[row][i]:
        left = False
      else:
        i = i-1
    #check down
    i = row + 1
    while i <= len(lines) - 1 and down:
      if lines[row][column] <= lines[i][column]:
        down = False
      else:
        i = i+1
    #check up
    i = row - 1
    while i >= 0 and up:
      if lines[row][column] <= lines[i][column]:
        up = False
      else:
        i = i-1
    if up or down or left or right:
      visible_trees = visible_trees + 1
    column = column + 1
    up = True
    down = True
    left = True
    right = True
  row = row+1
  column = 1

visible_trees = visible_trees + (2*len(lines)) + (2*(len(lines)-2))

print(visible_trees)


import argparse
import collections
import glob
import json
import math
import numpy as np
import random
from ordered_set import OrderedSet
import os
import pickle
import shutil
from sklearn.metrics import average_precision_score
import sys
import termcolor
import time
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.nn.utils.rnn as rnn
import torch.optim as optim
from tqdm import tqdm

from NAACL import vocabulary
from NAACL import settings
from NAACL import util

WORD_VEC_FILE = 'wordvec/PubMed-and-PMC-w2v.txt'
WORD_VEC_NUM_LINES = 4087447
EMB_SZIE = 200 # size of word embeddings
PARA_EMB_SIZE = 100 # size of paragraph index embeddings
PARA_EMB_MAX_SPAN = 1000
MAX_ENTITIES_PER_TYPE = 200
MAX_NUM_PARAGRAPHS = 200
MAX_NUM_CANDIDATES = 10000
ALL_ENTITY_TYPES = ['drug', 'gene', 'variant']
ALL_ENTITY_TYPES_PAIRS = [('drug', 'gene'), ('drug', 'variant'), ('gene', 'variant')]
MAX_PARAGRAPH_LENGTH = 800
CLIP_THRESH = 5 # Gradient clipping (on L2 norm)
JAX_DEV_PMIDS_FILE = 'jax/jax_dev_pmids.txt'
JAX_TEST_PMIDS_FILE = 'jax/jax_test_pmids.txt'

log_file = None

def log(msg):
    print(msg, file=sys.stderr)
    if log_file:
        print(msg, file=log_file)

ParaMention = collections.namedtuple(
    'ParaMention',['start', 'end', 'type', 'name'])

class Candidate(object):
    def __init__(self, drug=None, gene=None, variant=None, label=None):
        self.drug = drug
        self.gene = gene
        self.variant = variant
        self.label = label

    def remove_entity(self, i, new_label=None):
        '''
        :param i:
        :param new_label:
        :return: Return new Candidate with entity |i| replaced with None.
        '''
        triple = (self.drug, self.gene, self.variant)
        new_triple = triple[:i] + (None,) + triple[i+1:]
        return Candidate(*new_triple, label=new_label)

    def get_entities(self):
        return (self.drug, self.gene, self.variant)

    def is_triple(self):
        return self.drug and self.gene and self.variant

    def get_types(self):
        out = []
        if self.drug:
            out.append('drug')
        if self.gene:
            out.append('gene')
        if self.variant:
            out.append('variant')
        return tuple(out)

    def __key(self):
        return (self.drug, self.gene, self.variant, self.label)

    def __eq__(x, y):
        return x.__key() == y.__key()

    def __hash__(self):
        return hash(self.__key())

class Example(object):
    def __init__(self, pmid, paragraphs, mentions, triple_candidates, pair_candidates):
        self.pmid = pmid
        self.paragraphs = paragraphs
        self.mentions = mentions
        self.triple_candidates = triple_candidates
        self.pair_candidates = pair_candidates
        self.entities = collections.defaultdict(OrderedSet)
        for m_list in mentions:
            for m in m_list:
                self.entities[m.type].add(m.name)

    @classmethod
    def read_examples(cls, example_json_file):
        results = []
        with open(os.path.join(settings.DATA_DIR, example_json_file)) as f:
            for line in f:
                ex = cls.read_examples(line)
                results.append(ex)

        return results

    @classmethod
    def read_examples(cls, example_json_str):
        example_json = json.loads(example_json_str)
        mentions = [[ParaMention(**mention) for mention in paragraph_mentions]
                    for paragraph_mentions in example_json['mentions']]
        pair_candidates = {}

        for pair_key in example_json['pair_candidates']:
            pair_key_tuple = tuple(json.loads(pair_key))
            pair_candidates[pair_key_tuple] = OrderedSet(Candidate(**x)
                                                         for x in example_json['pair_candidates'][pair_key])
            triple_candidates = {}
            triple_candidates = [Candidate(**x)
                                 for x in example_json['triple_candidates']]

            return cls(example_json['pmid'],
                       example_json['paragraphs'],
                       mentions,
                       triple_candidates,
                       pair_candidates)

class Preprocessor(object):

    def __init__(self, entity_lists, vacab, device):
        self.entity_lists = entity_lists
        self.vocab = vacab
        self.device = device

    def count_labels(self, ex, pair_only=None):
        if pair_only:
            candidates = ex.pair_candidates[pair_only]
        else:
            candidates = ex.triple_candidates

        num_pos = sum(c.label for c in candidates)
        num_neg = sum(1 - c.label for c in candidates)
        return num_neg, num_pos

    def shuffle_entities(self, ex):
        entity_map = {}
        for e_type in ex.entities:
            cur_ents = ex.entities[e_type]
            replacements = random.sample(self.entity_lists[e_type], len(cur_ents))
            for e_old, e_new in zip(cur_ents, replacements):
                entity_map[(e_type, e_old)] = e_new

        new_paras = []
        new_mentions = []
        for p, m_list in zip(ex.paragraphs, ex.mentions):
            new_para = []
            new_m_list =[]
            mentions_at_loc = collections.defaultdict(list)
            in_mention = [False] * len(p)
            for m in m_list:
                mentions_at_loc[m.start].append((m.type, m.name))
                for i in range(m.start, m.end):
                    in_mention[i] = True
            for i in range(len(p)):
                if mentions_at_loc[i]:
                    for e_type, name in mentions_at_loc[i]:
                        e_new = entity_map[(e_type, name)]
                        m = ParaMention(len(new_para), len(new_para)+1, e_type, name)
                        new_m_list.append(m)
                        new_para.append(e_new)
                if not in_mention[i]:
                    new_paras.append(p[i])
            new_paras.append(new_para)
            new_mentions.append(new_m_list)
        return new_paras, new_mentions

    def preprocess(self, ex, pair_only):
        new_paras, new_mentions = self.shuffle_entities(ex)
        para_prep = []
        for para_idx, (para, m_list) in enumerate(zip(new_paras, new_mentions)):
            word_idxs = torch.tensor(self.vocab.indexify_list(para),
                                     dtype=torch.long, device=self.device)

            para_from_start = [
                para_idx / math.pow(PARA_EMB_MAX_SPAN, 2*i / (PARA_EMB_SIZE // 4))
                for i in range(PARA_EMB_SIZE // 4)
            ]

            para_from_end = [
                (len(new_paras)- para_idx) / math.pow(PARA_EMB_MAX_SPAN, 2*i / (PARA_EMB_SIZE // 4))
                for i in range(PARA_EMB_SIZE // 4)
            ]

            para_args = torch.cat([torch.tensor(x, dtype=torch.float, device=self.device)
                                   for x in (para_from_start, para_from_end)])

            para_vec = torch.cat([torch.sin(para_args), torch.cos(para_args)])
            para_prep.append((word_idxs ,para_vec, m_list))

            # sort for pack_padded_sequence
            para_prep.sort(key=lambda x:len(x[0]), reverse=True)
            T, P = len(para_prep[0][0]), len(para_prep)
            para_mat = torch.zeros((T, P), device=self.device, dtype=torch.long)
            for i, x in enumerate(para_prep):
                cur_words = x[0]
                para_mat[:len(cur_words), i] = cur_words

            lenghts = torch.tensor([len(x[0]) for x in para_prep], device=self.device)
            triple_labels = torch.tensor([c.label for c in ex.triple_candidates],
                                         dtype=torch.float, device=self.device)
            pair_labels = {k: torch.tensor([c.label for c in ex.pair_candidates[k]],
                                          dtype=torch.float, device=self.device)
                                    for k in ex.pair_candidates}
            para_vecs = torch.stack([x[1] for x in para_prep], dim=0)
            unlabeled_triple_cands = [Candidate(ex.drug, ex.gene, ex.variant)
                                      for ex in ex.triple_candidates]
            unlabeled_pair_cands = {k: [Candidate(ex.drug, ex.gene, ex.variant)
                                        for ex in ex.pair_candidates[k]]
                                    for k in ex.pair_candidates}
            return (para_mat, lenghts, para_vecs, [x[2] for x in para_prep],
                    unlabeled_triple_cands, unlabeled_pair_cands, triple_labels, pair_labels)

def logsumexp(inputs, dim=None, keepdim=False):
    '''

    :param inputs: A variable with any shape.
    :param dim: An integer.
    :param keepdim: A boolean.
    :return: Equivalent of log(sum(exp(inputs), dim=dim, keepdim=keepdim)).
    '''
    if dim is None:
        inputs = inputs.view(-1)
        dim = 0
    s, _ = torch.max(inputs, dim=dim, keepdim=True)
    outputs = s + (inputs - s).exp().sum(dim=dim, keepdim=True).log()
    if not keepdim:
        outputs = outputs.squeeze(dim)
    return outputs

class BackoffModel(nn.Module):
    '''
    Combine triple and pairwise information.
    '''

    def __init__(self, emb_mat, lstm_size, lstm_layers, device, use_lstm=True,
                 use_position=True, pool_method='max', dropout_prob=0.5, vocab=None,
                 pair_only=None):

        super(BackoffModel, self).__init__()
        self.device = device
        self.use_lstm = use_lstm
        self.use_position = use_position
        self.pool_method - pool_method
        self.embs = nn.Embedding.from_pretrained(emb_mat, freeze=False)
        self.vocab = vocab
        self.pair_only =pair_only
        self.dropout = nn.Dropout(p=dropout_prob)
        para_emb_size = PARA_EMB_SIZE if use_position else 0
        if use_lstm:
            self.lstm_layers = lstm_layers
            self.lstm = nn.LSTM(EMB_SZIE + para_emb_size, lstm_size,
                                bidirectional=True, num_layers=lstm_layers)
        else:
            self.emb_linear = nn.Linear(EMB_SZIE + para_emb_size, 2 * lstm_size)
        for t1 ,t2 in ALL_ENTITY_TYPES_PAIRS:
            setattr(self, 'hidden_%s_%s' %
                    (t1, t2), nn.Linear(4 * lstm_size, 2 * lstm_size))
            setattr(self, 'out_%s_%s' % (t1, t2), nn.Linear(2 * lstm_size, 1))
            setattr(self, 'backoff_%s_%s' % (t1, t2), nn.Parameter(
                torch.zeros(1, 2 * lstm_size)))
        self.hidden_triple = nn.Linear(3 * 2 * lstm_size, 2 * lstm_size)
        self.backoff_triple = nn.Parameter(torch.zeros(1, 2 * lstm_size))
        self.hidden_all = nn.Linear(4 * 2 * lstm_size, 2 * lstm_size)
        self.out_triple = nn.Linear(2 * lstm_size, 1)

    def pool(self, grouped_vecs):
        '''

        :param grouped_vecs:
        :return:
        '''
        if self.pool_method == 'mean':
            return torch.stack([torch.mean(g, dim=0) for g in grouped_vecs])
        elif self.pool_method == 'sum':
            return torch.stack([torch.sum(g, dim=0) for g in grouped_vecs])
        elif self.pool_method == 'max':
            return torch.stack([torch.max(g, dim=0)[0] for g in grouped_vecs])
        elif self.pool_method == 'softmax':
            return torch.stack([logsumexp(g, dim=0) for g in grouped_vecs])
        raise NotImplementedError

    def forward(self, word_idx_mat, lens, para_vecs, mentions,
                triple_candidates, pair_candidates):
        '''

        :param word_idx_mat: list of word indices, size(T, P)
        :param lens: list of paragraph lengths, size(P)
        :param para_vecs: list of paragraph vectors, size(P, pe)
        :param mentions: list of list of ParaMention
        :param triple_candidates: list of unlabeled Candidate
        :param pair_candidates: list of unlabeled Candidate
        :return:
        '''
        T, P = word_idx_mat.shape # T=num_toks, P=num_paras

        # Organize the candidate pairs and triples
        pair_to_idx = {}
        pair_sets = collections.defaultdict(set)
        for(t1, t2), cands in pair_candidates.items():
            pair_to_idx[(t1, t2)] = {c: i for i, c, in enumerate(cands)}
            for c in cands:
                pair_sets[(t1, t2)].add(c)
        triple_to_idx = {c: i for i, c in enumerate(triple_candidates)}

        # Build local embeddings of each word
        embs = self.embs(word_idx_mat) # T, P, e
        if self.use_position:
            para_embs = para_vecs.unsqueeze(0).expand(T, -1,-1) # T, P, pe
            embs = torch.cat([embs, para_embs], dim=2) # T, P, e + pe
        if self.use_lstm:
            lstm_in = rnn.pack_padded_sequence(embs, lens) # T, P, e + pe
            lstm_out_packed, _ = self.lstm(lstm_in)
            embs, _ = rnn.pad_packed_sequence(lstm_out_packed) # T, P, 2*h
        else:
            embs = self.emb_linear(embs) # T, P, 2*h

        # Gather co-occurring mention pairs and triples
        pair_inputs = {(t1, t2):[[] for i in range(len(cands))]
                       for(t1, t2), cands in pair_candidates.items()}
        triple_inputs = [[] for i in range(len(triple_candidates))]

        for para_idx, m_list in enumerate(mentions):
            typed_mentions = collections.defaultdict(list)
            for m in m_list:
                typed_mentions[m.type].append(m)
            for t1, t2 in ALL_ENTITY_TYPES_PAIRS:
                if self.pair_only and self.pair_only !=(t1 ,t2):
                    continue
                for m1 in typed_mentions[t1]:
                    for m2 in typed_mentions[t2]:
                        query_cand = Candidate(**{t1: m1.name, t2: m2.name})
                        if query_cand in pair_to_idx[(t1, t2)]:
                            idx = pair_to_idx[(t1, t2)][query_cand]
                            cur_vecs = torch.cat([embs[m1.start, para_idx, :],
                                                  embs[m2.start, para_idx, :]]) # 4*h
                            pair_inputs[(t1, t2)][idx].append(cur_vecs)
            if self.pair_only:
                continue
            for m1 in typed_mentions['drug']:
                for m2 in typed_mentions['gene']:
                    for m3 in typed_mentions['variant']:
                        query_cand = Candidate(m1.name, m2.name, m3.name)
                        if query_cand in triple_to_idx:
                            idx = triple_to_idx[query_cand]
                            cur_vecs = torch.cat(
                                                [embs[m1.start, para_idx, :],
                                                embs[m2.start, para_idx, :],
                                                embs[m3.start, para_idx, :]]) # 6*h
                            triple_inputs[idx].append(cur_vecs)

        # Compute local mention pair/triple representations
        pair_vecs = {}
        for t1, t2 in ALL_ENTITY_TYPES_PAIRS:
            if self.pair_only and self.pair_only != (t1, t2):
                continue
            cur_group_sizes = [len(vecs) for vecs in pair_inputs[(t1, t2)]]
            if sum(cur_group_sizes) > 0:
                cur_stack = torch.stack([
                    v for vecs in pair_inputs[(t1, t2)] for v in vecs]) # M, 4*h
                cur_m_reps = getattr(self, 'hidden_%s_%s' %
                                     (t1, t2))(cur_stack) # M, 2*h
                cur_pair_grouped_vecs = list(torch.split(cur_m_reps, cur_group_sizes))
                for i in range(len(cur_pair_grouped_vecs)):
                    if cur_pair_grouped_vecs[i].shape[0] == 0: # Back off
                        cur_pair_grouped_vecs[i] = getattr(self,
                                                           'backoff_%s_%s' % (t1, t2))
            else:
                cur_pair_grouped_vecs = [getattr(self, 'backoff_%s_%s' % (t1, t2))
                                         for vecs in pair_inputs[(t1, t2)]]
            pair_vecs[(t1, t2)] = torch.tanh(
                self.pool(cur_pair_grouped_vecs)) # P, 2*h

        if not self.pair_only:
            triple_group_sizes = [len(vecs) for vecs in triple_inputs]
            if sum(triple_group_sizes) > 0:
                triple_stack = torch.stack([
                    v for vecs in triple_inputs for v in vecs]) # M, 6*h
                triple_m_reps = self.hidden_triple(triple_stack) # M, 2*h
                triple_grouped_vecs = list(
                    torch.split(triple_m_reps, triple_group_sizes))
                for i in range(len(triple_grouped_vecs)):
                    if triple_grouped_vecs[i].shape[0] == 0: # back off
                        triple_grouped_vecs[i] = self.backoff_triple

            else:
                triple_grouped_vecs = [self.backoff_triple for vecs in triple_inputs]
            triple_vecs = torch.tanh(self.pool(triple_grouped_vecs)) # C, 2*h

        # Score candidate pairs
        pair_logits = {}
        for t1, t2 in ALL_ENTITY_TYPES_PAIRS:
            if self.pair_only and self.pair_only != (t1, t2):
                continue
            pair_logits[(t1, t2)] = getattr(self, 'out_%s_%s' % (t1, t2))(
                pair_vecs[(t1, t2)])[:, 0] #M
        if self.pair_only:
            return None, pair_logits

        # Score candidate triples
        pair_feats_per_triple = [[], [], []]
        for c in triple_candidates:
            for i in range(3):
                pair = c.remove_entity(i)
                t1, t2 = pair.get_types()
                pair_idx = pair_to_idx[(t1, t2)](pair)
                pair_feats_per_triple[i].append(
                    pair_vecs[(t1, t2)][pair_idx, :]) # 2*h
        triple_feats = torch.cat(
            [torch.stack(pair_feats_per_triple[0]),
             torch.stack(pair_feats_per_triple[1]),
             torch.stack(pair_feats_per_triple[2]),
             triple_vecs],
            dim=1) # C, 8*h
        final_hidden = F.relu(self.hidden_all(triple_feats)) # C, 2*h
        triple_logits = self.out_triple(final_hidden)[:, 0] # C
        return triple_logits, pair_logits

def get_entity_lists():
    entity_lists = {}
    for et in ALL_ENTITY_TYPES:
        entity_lists[et] = ['__%s__' % et
                        for i in range(MAX_ENTITIES_PER_TYPE)]
    # Can streamline, since we're just using single placeholder per entity type
    return entity_lists

def count_labels(name, data, preprocessor, pair_only=None):
    num_neg, num_pos = 0, 0
    for ex in data:
        cur_neg, cur_pos = preprocessor.count_labels(ex, pair_only=pair_only)
        num_neg += cur_neg
        num_pos += cur_pos
    log('%s data: +%d, -%d' % (name, num_pos, num_neg))
    return num_neg, num_pos

def print_data_stats(data, name):
    print(name)
    print('    Max num paragraphs: %d' % max(len(ex.paragraphs) for ex in data))
    print('    Max num triple candidates: %d' % max(
        len(ex.triple_candidates) for ex in data))

def init_word_vecs(device, vocab, all_zero=False):
    num_pretrained = 0
    embs = torch.zeros((len(vocab), EMB_SZIE), dtype=torch.float, device=device)
    if not all_zero:
        with open(os.path.join(settings.DATA_DIR,WORD_VEC_FILE)) as f:
            for line in tqdm(f, total=WORD_VEC_NUM_LINES):
                toks = line.strip().split(' ')
                if len(toks) != EMB_SZIE + 1:
                    continue
                word = toks[0]
                if word in vocab:
                    idx = vocab.get_index(word)
                    embs[idx, :] = torch.tensor([float(x) for x in toks[1:]],
                                                dtype=torch.float, device=device)
                    num_pretrained += 1
        log('Found pre-trained vectors for %d/%d = %.2f%% words' % (
            num_pretrained, len(vocab), 100*0 * num_pretrained /len(vocab)))

    return embs

def train(model, train_data, dev_data, preprocessor, num_epochs, lr, ckpt_iters,
          downsample_to, out_dir, lr_decay=1.0, pos_weight=None, use_pair_loss=True,
          pair_only=None):
    model.train()
    if ckpt_iters > len(train_data):
        ckpt_iters = len(train_data) # Checkpoint at least once per epoch
    loss_func = nn.BCEWithLogitsLoss(pos_weight=pos_weight)
    params = [p for p in model.paraments() if p.requires_grad]
    optimizer = optim.Adam(params, lr=lr)
    scheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=lr_decay)
    train_data = list(train_data) # Copy before shuffling
    num_iters = 0
    best_ap = 0.0 # Choose checkpoint based on dev average precision
    train_loss = 0.0
    for t in range(num_epochs):
        t0 = time.time()
        random.shuffle(train_data)
        if not downsample_to:
            cur_train = tqdm(train_data)
        else:
            cur_train = train_data # tqdm is annoyingn on downsampled data
        for ex in cur_train:
            model.zero_grad()
            ex_torch = preprocessor.preprocess(ex, pair_only)
            triple_labels, pair_labels = ex_torch[-2:]
            triple_logits, pair_logits = model(*ex_torch[:-2])
            if pair_only:
                loss = loss_func(pair_logits[pair_only], pair_labels[pair_only])
            else:
                loss = loss_func(triple_logits, triple_labels)
                if use_pair_loss:
                    for t1, t2 in ALL_ENTITY_TYPES_PAIRS:
                        loss += loss_func(pair_logits[(t1, t2)], pair_labels[(t1, t2)])
            train_loss += loss.item()
            loss.backward()
            torch.nn.utils.clip_grad_norm(model.paraments(),CLIP_THRESH)
            optimizer.step()
            num_iters += 1
            if num_iters % ckpt_iters == 0:
                model.eval()
                dev_preds, dev_loss = predict(
                    model, dev_data, preprocessor, loss_func=loss_func,
                    use_pair_loss=use_pair_loss, pair_only=pair_only)
                log('Iter %d: train loss = %.6f, dev loss = %.6f' % (
                    num_iters, train_loss / ckpt_iters, dev_loss))

                train_loss = 0.0
                p_doc, r_doc, f1_doc, ap_doc = evaluate(dev_data, dev_preds,
                                                        pair_only=pair_only)
                log('    Document-level : p=%.2f%% r=%.2f%% f1=%.2f%% ap=%.2f%%' % (
                    100 * p_doc, 100 * r_doc, 100 * f1_doc, 100 * ap_doc))
                if out_dir:
                    save_model(model, num_iters, out_dir)
                    model.train()
                    scheduler.step()
            t1 = time.time()
            log('Epoch %s: took %s' % (str(t).rjust(3), util.secs_to_str(t1 - t0)))

def predict(model, data, preprocessor, loss_func=None, use_pair_loss=True, pair_only=None):
    loss = 0.0
    preds = []
    with torch.no_grad():
        for ex in data:
            all_logits = []
            ex_torch = preprocessor.preprocess(ex, pair_only)
            triple_labels, pair_labels = ex_torch[-2:]
            triple_logits, pair_logits = model(*ex_torch[:-2])
            if loss_func:
                if pair_only:
                    loss += loss_func(pair_logits[pair_only], pair_labels[pair_only])
                else:
                    loss += loss_func(triple_logits, triple_labels)
                    if use_pair_loss:
                        for t1 ,t2 in ALL_ENTITY_TYPES_PAIRS:
                            loss += loss_func(pair_logits[(t1 ,t2)], pair_labels[(t1, t2)])
            if pair_only:
                cur_pred = [1 / (1 + np.exp(-z.item())) for z in pair_logits[pair_only]]
            else:
                cur_pred = [1 / (1 + np.exp(-z.item())) for z in pair_logits[pair_only]]
            preds.append(cur_pred)
        out = [preds]
        if loss_func:
            out.append(loss / len(data))
        if len(out) == 1:
            return out[0]
        return out

COLORS = {'drug': 'red', 'variant': 'cyan', 'gene': 'green'}

def pprint_example(ex, f=sys.stdout):
  print('PMID %s' % ex.pmid, file=f)
  for para_idx, (paragraph, m_list) in enumerate(zip(ex.paragraphs, ex.mentions)):
    word_to_type = {}
    for m in m_list:
      for i in range(m.start, m.end):
        word_to_type[i] = m.type
    para_toks = []
    for i in range(len(paragraph)):
      if i in word_to_type:
        para_toks.append(termcolor.colored(
            paragraph[i], COLORS[word_to_type[i]]))
      else:
        para_toks.append(paragraph[i])
    print('    Paragraph %d: %s' % (para_idx, ' '.join(para_toks)), file=f)

def evaluate(data, probs, name=None, threshold=0.5, pair_only=None):
  def get_candidates(ex):
    if pair_only:
      return ex.pair_candidates[pair_only]
    else:
      return ex.triple_candidates
  if name:
    log('== %s, document-level: %d documents, %d candidates (+%d, -%d) ==' % (
        name, len(data), sum(len(get_candidates(ex)) for ex in data),
        sum(1 for ex in data for c in get_candidates(ex) if c.label == 1),
        sum(1 for ex in data for c in get_candidates(ex) if c.label == 0)))
  tp = fp = fn = 0
  y_true = []
  y_pred = []
  for ex, prob_list in zip(data, probs):
    for c, prob in zip(get_candidates(ex), prob_list):
      y_true.append(c.label)
      y_pred.append(prob)
      pred = int(prob > threshold)
      if pred == 1:
        if c.label == 1:
          tp += 1
        else:
          fp += 1
      else:
        if c.label == 1:
          fn += 1
  ap = average_precision_score(y_true, y_pred)
  if name:
    log(util.get_prf(tp, fp, fn, get_str=True))
    log('AvgPrec  : %.2f%%' % (100.0 * ap))
  p, r, f = util.get_prf(tp, fp, fn)
  return p, r, f, ap

def predict_write(model, data, preprocessor, out_dir, ckpt, data_name, pair_only):
  if out_dir:
    if ckpt:
      out_path = os.path.join(out_dir, 'pred_%s_%07d.tsv' % (data_name, ckpt))
    else:
      out_path = os.path.join(out_dir, 'pred_%s.tsv' % data_name)
    # Only one pprint necessary
    pprint_out = os.path.join(out_dir, 'dev_pprint.txt')
  else:
    pprint_out = None
  pred = predict(model, tqdm(data), preprocessor, pair_only=pair_only)
  pprint_predictions(data, pred, preprocessor, fn=pprint_out)
  if out_path:
    write_predictions(data, pred, out_path, pair_only=pair_only)

def pprint_predictions(data, preds, preprocessor, threshold=0.5, fn=None):
  if fn:
    f = open(fn, 'w')
  else:
    f = sys.stdout
  for i, (ex, pred_list) in enumerate(zip(data, preds)):
    pprint_example(ex, f=f)
    new_paras, new_mentions = ex.paragraphs, ex.mentions
    for j, (c, pred) in enumerate(zip(ex.triple_candidates, pred_list)):
      pred_label = pred > threshold
      print('    (%s, %s, %s): pred=%s (p=%.4f), gold=%s, correct=%s' % (
          c.drug, c.gene, c.variant, pred_label, pred,
          c.label == 1, pred_label == (c.label == 1)), file=f)
    print('', file=f)
  if fn:
    f.close()

def write_predictions(data, preds, fn, pair_only=None):
  i = 0
  with open(fn, 'w') as f:
    for ex, pred_list in zip(data, preds):
      if pair_only:
        candidates = ex.pair_candidates[pair_only]
      else:
        candidates = ex.triple_candidates
      for c, pred in zip(candidates, pred_list):
        print('%d\t%s\t%s\t%s\t%s\t%.6f' % (
            i, ex.pmid, c.drug, c.gene, c.variant, pred), file=f)
        i += 1

def make_vocab(train_data, entity_lists, unk_thresh):
  vocab = vocabulary.Vocabulary(unk_threshold=unk_thresh)
  for ents in list(entity_lists.values()):
    for e in ents:
      vocab.add_word_hard(e)
  for ex in tqdm(train_data):
    for p, m_list in zip(ex.paragraphs, ex.mentions):
      in_mention = [False] * len(p)
      for m in m_list:
        for i in range(m.start, m.end):
          in_mention[i] = True
      for i, w in enumerate(p):
        if not in_mention[i]:
          vocab.add_word(w)
  return vocab

def save_model(model, num_iters, out_dir):
  fn = os.path.join(out_dir, 'model.%07d.pth' % num_iters)
  torch.save(model.state_dict(), fn)

def load_model(model, load_dir, device, load_ckpt):
  # if not load_ckpt:
  #   with open(os.path.join(load_dir, 'best_model.txt')) as f:
  #     load_ckpt = int(f.read().strip().split('\t')[0])
  fn = os.path.join(load_dir, 'model.%07d.pth' % load_ckpt)
  log('Loading model from %s' % fn)
  model.load_state_dict(torch.load(fn, map_location=device))

def predict_write(model, data, preprocessor, out_dir, ckpt, data_name, pair_only):
  if out_dir:
    if ckpt:
      out_path = os.path.join(out_dir, 'pred_%s_%07d.tsv' % (data_name, ckpt))
    else:
      out_path = os.path.join(out_dir, 'pred_%s.tsv' % data_name)
    # Only one pprint necessary
    pprint_out = os.path.join(out_dir, 'dev_pprint.txt')
  else:
    pprint_out = None
  pred = predict(model, tqdm(data), preprocessor, pair_only=pair_only)
  pprint_predictions(data, pred, preprocessor, fn=pprint_out)
  if out_path:
    write_predictions(data, pred, out_path, pair_only=pair_only)

def get_ds_train_dev_pmids(pmid_file):
  with open(os.path.join(settings.DATA_DIR, pmid_file)) as f:
    pmids = sorted([pmid.strip() for pmid in f if pmid.strip()])
  random.shuffle(pmids)
  num_train = int(round(len(pmids) * 0.7))
  num_train_dev = int(round(len(pmids) * 0.8))
  train_pmids = set(pmids[:num_train])
  dev_pmids = set(pmids[num_train:num_train_dev])
  return train_pmids, dev_pmids

def parse_args(args):
  parser = argparse.ArgumentParser()
  # Required params
  # parser.add_argument('para_file', help='JSON object storing paragraph text')
  # parser.add_argument('mention_file', help='List of mentions for relevant paragraphs')
  parser.add_argument('--ds-train-dev-file', help='Training examples')
  parser.add_argument('--jax-dev-test-file', help='Dev examples')
  parser.add_argument('--init-pmid-file', default='pmid_lists/init_pmid_list.txt',
                      help='Dev examples')

  # Model architecture
  parser.add_argument('--lstm-size', '-c', default=200,
                      type=int, help='LSTM hidden state size.')
  parser.add_argument('--lstm-layers', '-l', default=1,
                      type=int, help='LSTM number of layers.')
  parser.add_argument('--pool', '-p', choices=['softmax', 'max', 'mean', 'sum'], default='softmax',
                      help='How to pool across mentions')
  parser.add_argument('--no-position', action='store_true',
                      help='Ablate paragraph index encodings')
  parser.add_argument('--no-lstm', action='store_true', help='Ablate LSTM')
  # Training
  parser.add_argument('--num-epochs', '-T', type=int,
                      default=10, help='Training epochs')
  parser.add_argument('--learning-rate', '-r', type=float,
                      default=1e-5, help='Learning rate.')
  parser.add_argument('--dropout-prob', '-d', type=float,
                      default=0.5, help='Dropout probability')
  parser.add_argument('--lr-decay', '-g', type=float, default=1.0,
                      help='Decay learning rate by this much each epoch.')
  parser.add_argument('--balanced', '-b', action='store_true',
                      help='Upweight positive examples to balance dataset')
  parser.add_argument('--pos-weight', type=float, default=None,
                      help='Upweight postiive examples by this much')
  parser.add_argument('--use-pair-loss', action='store_true',
                      help="Multi-task on pair objective")
  # Data
  #parser.add_argument('--data-cache', default=DEFAULT_CACHE)
  parser.add_argument('--data-cache', default=None)
  parser.add_argument('--rng-seed', default=0, type=int, help='RNG seed')
  parser.add_argument('--torch-seed', default=0,
                      type=int, help='torch RNG seed')
  parser.add_argument('--downsample-to', default=None, type=int,
                      help='Downsample to this many examples per split')
  parser.add_argument('--unk-thresh', '-u', default=5, type=int,
                      help='Treat words with fewer than this many counts as <UNK>.')
  parser.add_argument('--print-dev', action='store_true',
                      help='Test on dev data')
  parser.add_argument('--jax', action='store_true', help='Test on JAX data')
  parser.add_argument('--jax-out', default='pred_jax.tsv')
  parser.add_argument('--text-level', choices=['document', 'paragraph', 'sentence'],
                      default='document', help='Split documents paragraph-wise or sentence-wise')
  parser.add_argument('--pair-only', default=None,
                      help='Comma-separated pair of entities to focus on only')
  # CPU vs. GPU
  parser.add_argument('--cpu-only', action='store_true',
                      help='Run on CPU only')
  parser.add_argument('--gpu-id', type=int, default=0,
                      help='GPU ID (default=0)')
  # Saving and loading
  parser.add_argument('--out-dir', '-o', default=None,
                      help='Where to write all output')
  parser.add_argument('--ckpt-iters', '-i', default=10000, type=int,
                      help='Checkpoint after this many training steps.')
  parser.add_argument(
      '--load', '-L', help='Directory to load model parameters and vocabulary')
  parser.add_argument('--load-ckpt', type=int, default=None,
                      help='Which checkpoint to use (default: use best_model.txt)')
  parser.add_argument('--try-all-checkpoints', action='store_true',
                      help='Make predictions for every checkpoint')
  parser.add_argument('--data-dir', help='root data directory')
  # Other
  parser.add_argument('--no-w2v', action='store_true',
                      help='No pre-trained word vectors')
  if len(sys.argv) == 1:
    parser.print_help()
    sys.exit(1)
  return parser.parse_args(args)

def get_all_checkpoints(out_dir):
  fns = glob.glob(os.path.join(out_dir, 'model.*.pth'))
  return sorted([int(os.path.basename(x).split('.')[1]) for x in fns])

def run(OPTS, device):
  # Process pair-only mode
  pair_only = None
  if OPTS.pair_only:
    pair_only = tuple(OPTS.pair_only.split(','))
    if pair_only not in ALL_ENTITY_TYPES_PAIRS:
      raise ValueError('Bad value for pair_only: %s' % OPTS.pair_only)
  entity_lists = get_entity_lists()
  # Read data
  train_pmids_set, dev_ds_pmids_set = get_ds_train_dev_pmids(
      OPTS.init_pmid_file)
  ds_train_dev_data = Example.read_examples(OPTS.ds_train_dev_file)
  # Filter out examples that doesn't contain pair or triple candidates
  if pair_only:
    ds_train_dev_data = [x for x in ds_train_dev_data if pair_only in
                         x.pair_candidates and x.pair_candidates[pair_only]]
  else:
    ds_train_dev_data = [x for x in ds_train_dev_data if x.triple_candidates]
  train_data = [x for x in ds_train_dev_data if x.pmid in train_pmids_set]
  dev_ds_data = [x for x in ds_train_dev_data if x.pmid in dev_ds_pmids_set]
  random.shuffle(train_data)
  random.shuffle(dev_ds_data)

  jax_dev_test_data = Example.read_examples(OPTS.jax_dev_test_file)
  if pair_only:
    jax_dev_test_data = [x for x in jax_dev_test_data if pair_only in
                         x.pair_candidates and x.pair_candidates[pair_only]]
  else:
    jax_dev_test_data = [x for x in jax_dev_test_data if x.triple_candidates]
  random.shuffle(jax_dev_test_data)

  with open(os.path.join(settings.DATA_DIR, JAX_DEV_PMIDS_FILE)) as f:
    dev_jax_pmids_set = set(x.strip() for x in f if x.strip())
  with open(os.path.join(settings.DATA_DIR, JAX_TEST_PMIDS_FILE)) as f:
    test_pmids_set = set(x.strip() for x in f if x.strip())

  dev_jax_data = [x for x in jax_dev_test_data if x.pmid in dev_jax_pmids_set]
  test_data = [x for x in jax_dev_test_data if x.pmid in test_pmids_set]
  log('Read %d train, %d dev dist sup, %d dev jax, %d test examples' %
      (len(train_data), len(dev_ds_data), len(dev_jax_data), len(test_data)))

  vocab = make_vocab(train_data, entity_lists, OPTS.unk_thresh)
  log('Vocab size = %d.' % len(vocab))
  preprocessor = Preprocessor(entity_lists, vocab, device)
  num_neg, num_pos = count_labels('train', train_data, preprocessor,
                                  pair_only=pair_only)
  word_vecs = init_word_vecs(device, vocab, all_zero=OPTS.load or OPTS.no_w2v)
  log('Finished reading data.')

  # Run model
  model = BackoffModel(
      word_vecs, OPTS.lstm_size, OPTS.lstm_layers, device,
      use_lstm=not OPTS.no_lstm, use_position=not OPTS.no_position,
      pool_method=OPTS.pool, dropout_prob=OPTS.dropout_prob,
      vocab=vocab, pair_only=pair_only).to(device=device)
  if OPTS.load:
    load_model(model, OPTS.load, device, OPTS.load_ckpt)
  if OPTS.num_epochs > 0:
    log('Starting training.')
    pos_weight = None
    if OPTS.balanced:
      pos_weight = torch.tensor(float(num_neg) / num_pos, device=device)
    elif OPTS.pos_weight:
      pos_weight = torch.tensor(OPTS.pos_weight, device=device)
    train(model, train_data, dev_ds_data, preprocessor, OPTS.num_epochs,
          OPTS.learning_rate, OPTS.ckpt_iters, OPTS.downsample_to, OPTS.out_dir,
          pos_weight=pos_weight, lr_decay=OPTS.lr_decay,
          use_pair_loss=OPTS.use_pair_loss, pair_only=pair_only)
    log('Finished training.')
  model.eval()
  if OPTS.try_all_checkpoints:
    ckpts = get_all_checkpoints(OPTS.out_dir)
  else:
    ckpts = [None]
  for ckpt in ckpts:
    if ckpt:
      print('== Checkpoint %s == ' % ckpt, file=sys.stderr)
      load_model(model, OPTS.out_dir, device, ckpt)
    predict_write(model, dev_jax_data, preprocessor,
                  OPTS.out_dir, ckpt, 'dev', pair_only)
    predict_write(model, test_data, preprocessor,
                  OPTS.out_dir, ckpt, 'test', pair_only)

def main(OPTS):
  if OPTS.out_dir:
    if os.path.exists(OPTS.out_dir):
      shutil.rmtree(OPTS.out_dir)
    os.makedirs(OPTS.out_dir)
    global log_file
    log_file = open(os.path.join(OPTS.out_dir, 'log.txt'), 'w')
  log(OPTS)
  random.seed(OPTS.rng_seed)
  torch.manual_seed(OPTS.torch_seed)
  if OPTS.cpu_only:
    device = torch.device('cpu')
  else:
    device = torch.device('cuda:%d' % OPTS.gpu_id)
  try:
    run(OPTS, device)
  finally:
    if log_file:
      log_file.close()

if __name__ == '__main__':
  OPTS = parse_args(sys.argv[1:])
  main(OPTS)


import requests
import discord
from webdriver import keep_alive
from bs4 import BeautifulSoup
import pandas as pd
from discord.ext import commands

bot = commands.Bot(command_prefix='!')
bot.remove_command("help")

@bot.event
async def on_ready():
	await bot.change_presence(status=discord.Status.online, activity=discord.Activity(type=discord.ActivityType.watching, name=""))
	print(f'Logged in as {bot.user.name}')

@commands.command(name="walmart")
async def walmart(ctx, SKU, ZIP):
    url = 'https://brickseek.com/walmart-inventory-checker/'
    payload = {'search_method': 'sku', 'sku': SKU, 'zip': ZIP, 'sort': 'distance'}
    df_record = pd.DataFrame(columns=['Store','City','Availability','Quantity'])
    r = requests.post(url, data=payload).text    # Make a POST request with data
    bs = BeautifulSoup(r, 'html.parser')
    j=0
    a = bs.find_all('div', class_='table__body')
    if a == []:
        print('No results found in the searched area.')
    else:
        store=[]
        city=[]
        q=[]
        stock=[]
        for tag in bs.find_all('div', class_='table__body'):
            for i in range(20):
                m_Store = tag.findAll('strong', class_='address-location-name')
                m=str(m_Store)
                if  i < m.count('/strong'):
                    m_s= m_Store[i].get_text().replace("\nWalmart","")
                    m_add = tag.findAll('address',class_="address")
                    m_Address = m_add[i].contents[2]
                    m_Availability = tag.findAll('span',class_="availability-status-indicator__text")
                    m_a = m_Availability[i].get_text()
                    if m_a =='Out of Stock'or m_a == 'Limited Stock':
                        m_q = str(0)
                        j=j-1
                    else:
                        m_Quantity = tag.findAll('span',class_="table__cell-quantity")
                        m_q = m_Quantity[j].get_text()[9:]
                    j=j+1
                    df_record = df_record.append({'Store':m_s, 'City':m_Address, 'Availability':m_a, 'Quantity':m_q }, ignore_index=True)
                    store.append(str(m_s))
                    city.append(str(m_Address))
                    q.append(str(m_a))
                    stock.append(str(m_q))
                else:
                    # df_record=str(df_record)
                    # df_record=str(df_record)
                    # df_record=str(df_record)
                    # df_record=str(df_record)
                    # df_record=str(df_record)
                    # df_record=str(df_record)
                    # df_record=str(df_record)
                    # df_record=str(df_record)
                    print(store)
                    print(city)
                    print(q)
                    print(str(df_record))
                    print()
                    break
        s='\n'
        store = s.join(store)
        city = s.join(city)
        q = s.join(q)
        stock = s.join(stock)
        embed1 = discord.Embed(title='Walmart Stock Checker', color=3447003)
        embed1.add_field(name = 'Store', value=store , inline = True)
        embed1.add_field(name = 'City', value=city , inline = True)
        embed1.add_field(name = 'Availability', value=q , inline = True)
        # embed1.add_field(name = 'Quantity', value=stock , inline = True)
        r = requests.get("https://brickseek.com/walmart-inventory-checker/?sku={}".format(SKU))
        soup = BeautifulSoup(r.content, 'html.parser')
        for tag in soup.find_all("div", "item-overview__image-wrap"):
        	link = tag.img.get("src")
        s=str(link)
        embed1.set_thumbnail(url=s)
        embed1.set_footer(text='odin#9999')
        await ctx.channel.send(embed=embed1)

@commands.command(name="target")
async def target(ctx, SKU, ZIP):
    SKU=str(SKU)
    if '-' not in SKU:
        SKU = ('{}-{}-{}'.format(SKU[0:3], SKU[3:5], SKU[5:9]))
    url = 'https://brickseek.com/target-inventory-checker/'
    payload = {'search_method': 'sku', 'sku': SKU, 'zip': ZIP, 'sort': 'distance'}
    r = requests.post(url, data=payload).text
    bs = BeautifulSoup(r, 'html.parser')
    print(" Store                Availability      Quantity ")
    j=0
    list1=[]
    for tag in bs.find_all('div', class_='table__body'):
        for i in range(10):
            #print(tag)
            m_Store = tag.findAll('strong', class_='address-location-name')
            m=str(m_Store)
            if i < m.count('/strong'):
                m_s= m_Store[i].get_text()
                m_add = tag.findAll('address',class_="address")
                m_Address = m_add[i].contents[0]
                m_Availability = tag.findAll('span',class_="availability-status-indicator__text")
                m_a = m_Availability[i].get_text()
                m_q = 'Unknown'
                j=j+1
                embeder = m_s+"   "  +  m_Address   + "      " + m_a + "        " + m_q
                list1.append(embeder)
            else:break
    s='\n'
    targ = s.join(list1)
    r = requests.get("https://brickseek.com/target-inventory-checker/?sku={}".format(SKU))
    soup = BeautifulSoup(r.content, 'html.parser')
    for tag in soup.find_all("div", "item-overview__image-wrap"):
    	link = tag.img.get("src")
    link=str(link)
    embed1 = discord.Embed(title='Target Stock Checker',description=targ,color=3447003)
    embed1.set_thumbnail(url=link)
    embed1.set_footer(text='odin#9999')
    # embed1.set_image("https://images-ext-2.discordapp.net/external/tKWiJKemxuuUUMoiRarsbbJGCABzsHXGHGFnkzBF5_g/%3Fwidth%3D608%26height%3D612/https/media.discordapp.net/attachments/765387136122880021/783219912453128202/13753__3_.png")
    await ctx.channel.send(embed=embed1)

@commands.command(name="checking")
async def checking(ctx):
	embed = discord.Embed(title='Stock Checker Instructions', description='**Requesting Channel**: <#783828598129295452>\n\n**__Walmart Stock Checker__**\n\nUsage:\n```!walmart sku ZIP```\n\nExample: ```!walmart 781200042 95928 ```\n\n**__Target Stock Checker__**\n\nUsage\n```!target DPCI Zip```\n\nExample: ```!target 057-10-0162 95928```', color=0x32a852)
	embed.set_thumbnail(url="https://media.discordapp.net/attachments/765387136122880021/783219912453128202/13753__3_.png?width=608&height=612")
	await ctx.send(embed=embed)

bot.add_command(walmart)
bot.add_command(target)
bot.add_command(checking)
keep_alive()

bot.run('token')


from collections import defaultdict
import os
from pathlib import Path
from urllib.request import urlretrieve
import xml.etree.ElementTree as ET

# import the countries xml file
tmp = Path(os.getenv("TMP", "/tmp"))
countries = tmp / 'countries.xml'

if not countries.exists():
    urlretrieve(
        'https://bites-data.s3.us-east-2.amazonaws.com/countries.xml',
        countries
    )

def get_income_distribution(xml=countries):
    """
    - Read in the countries xml as stored in countries variable.
    - Parse the XML
    - Return a dict of:
      - keys = incomes (wb:incomeLevel)
      - values = list of country names (wb:name)
    """
    country_dict = defaultdict(list)
    countries_data = ET.parse(countries).getroot()
    for country in countries_data:
        country_dict[country[4].text].append(country[1].text)
    return country_dict

if __name__ == "__main__":
    data = get_income_distribution()
    print(data)


import numpy as np
import torch
import torch.nn as nn

class RNNBaseSTFTMask(nn.Module):
    def __init__(self,
                num_spk=2,
                audio_channels=2,
                n_fft=512,
                hop_length=256,
                sample_rate=16000,
                rnn_hidden=256,
                rnn_layer=2,
                rnn_type="rnn",
                drop_out=0.5,
                activation="relu",
                bidirectional=False,
                *args,
                **kwarg):
        super(RNNBaseSTFTMask, self).__init__()

        self.audio_channels = audio_channels
        self.num_spk = num_spk
        self.n_fft = n_fft
        self.hop_length = hop_length
        self.sample_rate = sample_rate

        n_features = n_fft//2+1
        self.ampltude = Amplitude()
        self.phase = Phase()

        # torch.rnn batch_first
        # https://discuss.pytorch.org/t/could-someone-explain-batch-first-true-in-lstm/15402/9
        # (batch, seq, feature)
        # Without batch_first=True it will use the first dimension as the sequence dimension.
        # With batch_first=True it will use the second dimension as the sequence dimension.
        # out[-1]    # If batch_first=True OR
        # out[:, -1] # If batch_dirst=False
        if rnn_type == "rnn":
            self.rnn = nn.RNN(input_size=n_features,
                        hidden_size=rnn_hidden,
                        num_layers=rnn_layer,
                        bias=False,
                        dropout=drop_out,
                        batch_first=False,
                        bidirectional=bidirectional)
        elif rnn_type == "lstm":
            self.rnn = nn.LSTM(input_size=n_features,
                        hidden_size=rnn_hidden,
                        num_layers=rnn_layer,
                        bias=False,
                        dropout=drop_out,
                        batch_first=False,
                        bidirectional=bidirectional)
        elif rnn_type == "gru":
            self.rnn = nn.GRU(input_size=n_features,
                        hidden_size=rnn_hidden,
                        num_layers=rnn_layer,
                        bias=False,
                        dropout=drop_out,
                        batch_first=False,
                        bidirectional=bidirectional)

        self.batchnorm = nn.BatchNorm1d(num_features=rnn_hidden if not bidirectional else rnn_hidden*2)

        linear = nn.Linear(in_features=rnn_hidden if not bidirectional else rnn_hidden*2,
                                out_features=n_features*num_spk,
                                bias=True)
        if activation=="relu":
            activation = nn.ReLU()

        self.fc_layers = nn.Sequential(
            linear,
            activation,
        )

    def forward(self, inputs):
        # print(inputs.shape)
        mask = self.ampltude(inputs)
        batch, nchannel, nfeature, nframe = mask.shape
        # batch, features, seq
        mask = torch.reshape(mask, shape=(batch*nchannel, nfeature, nframe)) # merge channel
        # print(mask.shape)
        mask = torch.transpose(mask, 1, 2)
        # print(mask.shape)
        mask, _ = self.rnn(mask) # batch, seq, features
        # print(mask.shape)
        mask = torch.transpose(mask, 1, 2)
        # print(mask.shape)
        mask = self.batchnorm(mask) # batch, features, seq
        # print(mask.shape)
        mask = torch.transpose(mask, 1, 2)
        # print(mask.shape)
        mask = self.fc_layers(mask) # batch, seq, features
        # print(mask.shape)
        mask = torch.transpose(mask, -1, -2) # batch, seq, features
        # print(mask.shape)
        batch, nfeature, nframe = mask.shape
        mask = torch.reshape(mask, shape=(batch, self.num_spk, int(nfeature//self.num_spk), nframe))
        # mask = mask.view(batch, self.num_spk, int(nfeature//self.num_spk), nframe)
        # print(mask.shape)
        mask = torch.reshape(mask, shape=(batch//nchannel, nchannel, self.num_spk, int(nfeature//self.num_spk), nframe))
        # mask = mask.view(batch//nchannel, nchannel, self.num_spk, int(nfeature//self.num_spk), nframe)
        # print(mask.shape)
        mask = torch.transpose(mask, 1, 2)
        # print(mask.shape)

        mask = torch.unsqueeze(mask, dim=-1) # dtype expand
        out = mask*torch.unsqueeze(inputs, dim=1)
        return out

class Amplitude(nn.Module):
    def __init__(self,
                *args,
                **kwarg):
        super(Amplitude, self).__init__()
    def forward(self, inputs):
        assert inputs.size()[-1] == 2, f"Tensor needs real and imag in the last rank..."
        return torch.abs(torch.pow(inputs[..., 0], exponent=2) - torch.pow(inputs[..., 1], exponent=2))

class Phase(nn.Module):
    def __init__(
        self,
        *args,
        **kwargs,
    ):
        super().__init__(**kwargs)
        self.eps = torch.tensor(np.ones(1, dtype=np.float32)*1e-5, dtype=torch.float32)

    def call(self, inputs, training=True):
        inputs[..., 1] += self.eps
        outputs = torch.angle(torch.complex(real=inputs[..., 0], imag=inputs[..., 1]))
        return outputs

if __name__ == "__main__":
    # First checking if GPU is available
    train_on_gpu=torch.cuda.is_available()

    def get_model():
        return RNNBaseSTFTMask

    if(train_on_gpu):
        print('Training on GPU.')
    else:
        print('No GPU available, training on CPU.')

    device = torch.device('cuda' if train_on_gpu else 'cpu')

    import argparse
    parser = argparse.ArgumentParser(
        "denoiser.demucs",
        description="Benchmark the streaming Demucs implementation, "
                    "as well as checking the delta with the offline implementation.")
    parser.add_argument("--sample_rate", default=16000, type=int)
    parser.add_argument("--segment", default=1.024, type=float)
    parser.add_argument("--audio_channels", default=2, type=int)
    parser.add_argument("--num_spk", default=2, type=int)
    parser.add_argument("--n_fft", default=512, type=int)
    parser.add_argument("--hop_length", default=256, type=int)

    parser.add_argument("--rnn_hidden", default=896, type=int)
    parser.add_argument("--rnn_layer", default=3, type=int)
    parser.add_argument("--rnn_type", default="lstm", type=str)
    parser.add_argument("--activation", default="relu", type=str)
    parser.add_argument("--bidirectional", default=True, type=bool)
    parser.add_argument("--drop_out", default=0.5, type=float)
    parser.add_argument("--device", default="cpu", type=str)

    args = parser.parse_args()

    model = get_model()(num_spk=args.num_spk,
                    audio_channels=args.audio_channels,
                    n_fft=args.n_fft,
                    hop_length=args.hop_length,
                    sample_rate=args.sample_rate,
                    rnn_hidden=args.rnn_hidden,
                    rnn_layer=args.rnn_layer,
                    rnn_type=args.rnn_type,
                    drop_out=args.drop_out,
                    activation=args.activation,
                    bidirectional=args.bidirectional,
                    ).to(args.device)

    nframe = int(int(args.sample_rate*args.segment) // args.hop_length) + 1
    nfeature = int(args.n_fft//2)+1

    x = torch.randn(args.audio_channels, nfeature, nframe, 2).to(args.device) # channel, T, F, real/imag
    out = model(x[None])[0]
    model_size = sum(p.numel() for p in model.parameters()) * 4 / 2**20
    print(f"model size: {model_size:.1f}MB")


import threading
import socket
import sys
import time

host = ''
port = 9000
locaddr = (host, port)

# Create a UDP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind(locaddr)

tello_address = ('192.168.10.1', 8889)
COMMANDS = []
ALL_COMMANDS = 0
NEXT = True

def _recv():
    while True:
        try:
            data, server = sock.recvfrom(1518)
            ans = data.decode(encoding="utf-8")
            print(ans)

            if ans == "ok":
                time.sleep(1)
                global NEXT
                NEXT = True

        except Exception:
            print ('\Landolás . . .\n')
            break

#recvThread create
recvThread = threading.Thread(target=_recv)
recvThread.start()

def elore(n):
    COMMANDS.append(f"forward {n}")

def hatra(n):
    COMMANDS.append(f"back {n}")

def fel(n):
    COMMANDS.append(f"up {n}")

def le(n):
    COMMANDS.append(f"down {n}")

def jobbra(alfa):
    COMMANDS.append(f"cw {alfa}")

def balra(alfa):
    COMMANDS.append(f"ccw {alfa}")

def szalto():
    COMMANDS.append("flip f")

def hatra_szalto():
    COMMANDS.append("flip b")

def allj():
    COMMANDS.append("stop")

def inditas():
    COMMANDS.append("command")
    COMMANDS.append("takeoff")

def leallitas():
    COMMANDS.append("land")
    COMMANDS.reverse()

    global ALL_COMMANDS
    ALL_COMMANDS = len(COMMANDS)

    start()

def start():
    while len(COMMANDS) >= 1:
        global NEXT
        if NEXT:
            NEXT = False
            current = COMMANDS.pop()
            msg = current.encode(encoding="utf-8")
            sent = sock.sendto(msg, tello_address)
            print(f"{ALL_COMMANDS - len(COMMANDS)}. parancs végrehajtása ({current}):", end=" ")

    print('A program véget ért.')
    sock.close()


import os
import numpy as np

from mtuq import read, open_db, download_greens_tensors
from mtuq.event import Origin
from mtuq.graphics import plot_data_greens2, plot_beachball, plot_misfit_dc
from mtuq.grid import DoubleCoupleGridRegular
from mtuq.grid_search import grid_search
from mtuq.misfit import Misfit
from mtuq.process_data import ProcessData
from mtuq.util import fullpath, merge_dicts, save_json
from mtuq.util.cap import parse_station_codes, Trapezoid

if True:
    #
    # Creates example data structures
    #
    # Rather than being executed as a script, this code is designed to be
    # imported.  After importing this module, users can access the example data
    # and functions listed in __all__
    #
    # Note that some I/O and data processing are involved in creating the
    # example data, so importing this module may take significantly longer than
    # other modules
    #

    __all__ = [
        'process_bw'
        'process_bw'
        'misfit_bw',
        'misfit_bw',
        'data_bw',
        'data_sw',
        'greens_bw',
        'greens_sw',
        'stations',
        'origin',
        ]

    path_data=    fullpath('data/examples/20090407201255351/*.[zrt]')
    path_weights= fullpath('data/examples/20090407201255351/weights.dat')
    event_id=     '20090407201255351'
    model=        'ak135'

    process_bw = ProcessData(
        filter_type='Bandpass',
        freq_min= 0.1,
        freq_max= 0.333,
        pick_type='taup',
        taup_model=model,
        window_type='body_wave',
        window_length=15.,
        capuaf_file=path_weights,
        )

    process_sw = ProcessData(
        filter_type='Bandpass',
        freq_min=0.025,
        freq_max=0.0625,
        pick_type='taup',
        taup_model=model,
        window_type='surface_wave',
        window_length=150.,
        capuaf_file=path_weights,
        )

    misfit_bw = Misfit(
        norm='L2',
        time_shift_min=-2.,
        time_shift_max=+2.,
        time_shift_groups=['ZR'],
        )

    misfit_sw = Misfit(
        norm='L2',
        time_shift_min=-10.,
        time_shift_max=+10.,
        time_shift_groups=['ZR','T'],
        )

    #
    # Next, we specify the moment tensor grid and source-time function
    #

    grid = DoubleCoupleGridRegular(
        npts_per_axis=10,
        magnitudes=[4.5])

    wavelet = Trapezoid(
        magnitude=4.5)

    #
    # The main I/O work starts now
    #

    data = read(path_data, format='sac',
        event_id=event_id,
        station_id_list=station_id_list,
        tags=['units:m', 'type:velocity'])

    data.sort_by_distance()
    stations = data.get_stations()

    data_bw = data.map(process_bw)
    data_sw = data.map(process_sw)

    greens = download_greens_tensors(stations, origin, model)

    greens.convolve(wavelet)
    greens_bw = greens.map(process_bw)
    greens_sw = greens.map(process_sw)


import tensorflow as tf
from tensorflow.keras.models import Sequential,Model
from tensorflow.keras.layers import Conv2D,BatchNormalization,Dropout,Activation,LeakyReLU,UpSampling2D,Input,Dense,Reshape,Flatten,Conv2DTranspose,ReLU,concatenate,ZeroPadding2D
import numpy as np
def BaseUnet_modeling(kernel_size=4,dropout=0.5):
    initializer = tf.random_normal_initializer(0.,0.02)
    inputs = Input(shape=(256,256,3))
    layer1 = Conv2D(filters=64,kernel_size=kernel_size,strides=2,padding='same',use_bias=False,kernel_initializer=initializer)(inputs)
    layer1 = LeakyReLU()(layer1)
    layer1_ = layer1

    layer2 = Conv2D(filters=128,kernel_size=kernel_size,strides=2,padding='same',use_bias=False,kernel_initializer=initializer)(layer1)
    layer2_ = BatchNormalization()(layer2)
    layer2 = LeakyReLU()(layer2_)

    layer3 = Conv2D(filters=256,kernel_size=kernel_size,strides=2,padding='same',use_bias=False,kernel_initializer=initializer)(layer2)
    layer3_ = BatchNormalization()(layer3)
    layer3 = LeakyReLU()(layer3_)

    layer4 = Conv2D(filters=512,kernel_size=kernel_size,strides=2,padding='same',use_bias=False,kernel_initializer=initializer)(layer3)
    layer4_ = BatchNormalization()(layer4)
    layer4 = LeakyReLU()(layer4_)

    layer5 = Conv2D(filters=512,kernel_size=kernel_size,strides=2,padding='same',use_bias=False,kernel_initializer=initializer)(layer4)
    layer5_ = BatchNormalization()(layer5)
    layer5 = LeakyReLU()(layer5_)

    layer6 = Conv2D(filters=512,kernel_size=kernel_size,strides=2,padding='same',use_bias=False,kernel_initializer=initializer)(layer5)
    layer6_ = BatchNormalization()(layer6)
    layer6 = LeakyReLU()(layer6_)

    layer7 = Conv2D(filters=512,kernel_size=kernel_size,strides=2,padding='same',use_bias=False,kernel_initializer=initializer)(layer6)
    layer7_ = BatchNormalization()(layer7)
    layer7 = LeakyReLU()(layer7_)

    layer8 = Conv2D(filters=512,kernel_size=kernel_size,strides=2,padding='same',use_bias=False,kernel_initializer=initializer)(layer7)
    layer8_ = BatchNormalization()(layer8)
    layer8 = LeakyReLU()(layer8_)

    layer9 = Conv2DTranspose(filters=512,kernel_size=kernel_size,strides=2,padding='same',kernel_initializer=initializer,use_bias=False)(layer8)
    layer9 = BatchNormalization()(layer9)
    layer9 = layer9+layer7_
    layer9 = Dropout(dropout)(layer9)
    layer9 = ReLU()(layer9)

    layer10 = Conv2DTranspose(filters=512,kernel_size=kernel_size,strides=2,padding='same',kernel_initializer=initializer,use_bias=False)(layer9)
    layer10 = BatchNormalization()(layer10)
    layer10 = concatenate([layer10,layer6_])
    layer10 = Dropout(dropout)(layer10)
    layer10 = ReLU()(layer10)

    layer11 = Conv2DTranspose(filters=512,kernel_size=kernel_size,strides=2,padding='same',kernel_initializer=initializer,use_bias=False)(layer10)
    layer11 = BatchNormalization()(layer11)
    layer11 = layer11+layer5_
    layer11 = Dropout(dropout)(layer11)
    layer11 = ReLU()(layer11)

    layer12 = Conv2DTranspose(filters=512,kernel_size=kernel_size,strides=2,padding='same',kernel_initializer=initializer,use_bias=False)(layer11)
    layer12 = BatchNormalization()(layer12)
    layer12 = layer12+layer4_
    layer12 = ReLU()(layer12)

    layer13 = Conv2DTranspose(filters=256,kernel_size=kernel_size,strides=2,padding='same',kernel_initializer=initializer,use_bias=False)(layer12)
    layer13 = BatchNormalization()(layer13)
    layer13 = layer13+layer3_
    layer13 = ReLU()(layer13)

    layer14 = Conv2DTranspose(filters=128,kernel_size=kernel_size,strides=2,padding='same',kernel_initializer=initializer,use_bias=False)(layer13)
    layer14 = BatchNormalization()(layer14)
    layer14 = layer14+layer2_
    layer14 = ReLU()(layer14)

    layer15 = Conv2DTranspose(filters=64,kernel_size=kernel_size,strides=2,padding='same',kernel_initializer=initializer,use_bias=False)(layer14)
    layer15 = BatchNormalization()(layer15)
    layer15 = layer15+layer1_
    layer15 = ReLU()(layer15)

    outputs = Conv2DTranspose(3,4,strides=2,padding='same',kernel_initializer=initializer,activation='tanh')(layer15)

    Generator = Model(inputs=inputs,outputs=outputs)
    return Generator
model = BaseUnet_modeling()
model.summary()


class Node:
    def __init__(self, data):
        self.data = data
        self.left = None
        self.right = None

    def __str__(self):
        return str(self.data)

class BST:
    def __init__(self):
        self.root = None

    def insert(self, data):
        if self.root is None:
            self.root = Node(data)
        else:
            self._insert(data,self.root)
        return self.root

    def _insert(self,data,currentNode):
        if data<currentNode.data:
            if currentNode.left==None:
                currentNode.left = Node(data)
            else:
                self._insert(data,currentNode.left)
        elif data>currentNode.data:
            if currentNode.right==None:
                currentNode.right = Node(data)
            else:
                self._insert(data,currentNode.right)

    def printTree(self, node, level = 0):
        if node != None:
            self.printTree(node.right, level + 1)
            print('     ' * level, node)
            self.printTree(node.left, level + 1)

    def findBelow(self,node,num):
        global list
        if node:
            if int(node.data) < int(num):
                list.append(node.data)
                self.findBelow(node.right,num)
                self.findBelow(node.left,num)
            else:
                self.findBelow(node.right,num)
                self.findBelow(node.left,num)

        return list

def listSorting(l):
        toStr = ''
        for i in range(len(l)):
            for j in range(i + 1, len(l)):

                if l[i] > l[j]:
                    l[i], l[j] = l[j], l[i]

        if len(l)==0:
            print('Not have')
        else:
            for i in l:
                toStr += str(i)+' '
        print(toStr)

list = []
T = BST()
inp,num = [i for i in input('Enter Input : ').split('|')]
inpdata = [int(i) for i in inp.split()]
for i in inpdata:
    root = T.insert(i)
T.printTree(root)
print('--------------------------------------------------')
print('Below '+num+' : ',end='')
res = T.findBelow(root,num)
listSorting(res)


from __future__ import print_function

import asyncio
import random
import threading
import time
import numpy as np
import websockets
from websocket import create_connection
from robot_dqnagent import DQNAgent
from robot_arena import Arena

class MSGWorker (threading.Thread):

  def __init__(self):
    self.coords = [100, 100]
    state_size = 2
    action_size = 5
    self.currentAction = ""
    self.agent = DQNAgent(state_size, action_size)
    self.agent.load('trekker_5000_refactored.h5')
    threading.Thread.__init__(self)
    self.connected = set()

  def run(self):
    while True:
      time.sleep(0.001)

  def act(self,x,y):

      state = np.array([x,y])
      #arena.drawRobot(state)

      #arena.setPos(np.array_str(state))

      mops= np.reshape(state, [1, 2])
      arena.drawRobot(state)

      action = self.agent.act_execute(mops)
      if action == 0 and self.currentAction != 0:  # up
          print("up")
          self.currentAction = 0
          ws = create_connection("ws://192.168.178.61:8080")
          ws.send("camup")
          ws.close()

      if action == 1 and self.currentAction != 1:  # down
          print("down")
          self.currentAction = 1
          ws = create_connection("ws://192.168.178.61:8080")
          ws.send("camdown")
          ws.close()

      if action == 2 and self.currentAction != 2:  # left
           print("left")
           self.currentAction = 2
           ws = create_connection("ws://192.168.178.61:8080")
           ws.send("left,1")
           ws.close()
      if action == 3 and self.currentAction != 3:  # right
           print("right")
           self.currentAction = 3
           ws = create_connection("ws://192.168.178.61:8080")
           ws.send("right,1")
           ws.close()

  @asyncio.coroutine
  def handler(self, websocket, path):
    self.connected.add(websocket)
    try:
      name = yield from websocket.recv()
      commaindex = name.find(",")
      commandlength = len(name)
      direction = name[0:commaindex]
      self.speed = name[commaindex+1:commandlength]

      self.act(direction,self.speed)

      #print(direction+','+self.speed)

      # here are the coordinates coming -> handled to the message worker !

    except websockets.exceptions.ConnectionClosed:
      pass
    finally:
      self.connected.remove(websocket)

  def sendData(self, data):
    for websocket in self.connected.copy():
      #print("Sending data: %s" % data)
      coro =yield from websocket.send(data)
      future = asyncio.run_coroutine_threadsafe(coro, loop)

if __name__ == "__main__":
  print('AI Server')
  msgWorker = MSGWorker()
  arena = Arena()

  try:

    msgWorker.start()
    ws_server = websockets.serve(msgWorker.handler, '192.168.178.67', 8080)
    loop = asyncio.get_event_loop()
    loop.run_until_complete(ws_server)
    loop.run_forever()
  except KeyboardInterrupt:
    stopFlag = True
    #TODO: close ws server and loop correctely
    print("Exiting program...")