# Custom zeroconf implementation based on python-zeroconf
# (https://github.com/jstasiak/python-zeroconf) that supports Python 2

import errno
import logging
import select
import socket
import struct
import sys
import threading
import time

import ifaddr

log = logging.getLogger(__name__)

# Some timing constants

_LISTENER_TIME = 200

# Some DNS constants

_MDNS_ADDR = '224.0.0.251'
_MDNS_PORT = 5353

_MAX_MSG_ABSOLUTE = 8966

_FLAGS_QR_MASK = 0x8000  # query response mask
_FLAGS_QR_QUERY = 0x0000  # query
_FLAGS_QR_RESPONSE = 0x8000  # response

_FLAGS_AA = 0x0400  # Authoritative answer
_FLAGS_TC = 0x0200  # Truncated
_FLAGS_RD = 0x0100  # Recursion desired
_FLAGS_RA = 0x8000  # Recursion available

_FLAGS_Z = 0x0040  # Zero
_FLAGS_AD = 0x0020  # Authentic data
_FLAGS_CD = 0x0010  # Checking disabled

_CLASS_IN = 1
_CLASS_CS = 2
_CLASS_CH = 3
_CLASS_HS = 4
_CLASS_NONE = 254
_CLASS_ANY = 255
_CLASS_MASK = 0x7FFF
_CLASS_UNIQUE = 0x8000

_TYPE_A = 1
_TYPE_NS = 2
_TYPE_MD = 3
_TYPE_MF = 4
_TYPE_CNAME = 5
_TYPE_SOA = 6
_TYPE_MB = 7
_TYPE_MG = 8
_TYPE_MR = 9
_TYPE_NULL = 10
_TYPE_WKS = 11
_TYPE_PTR = 12
_TYPE_HINFO = 13
_TYPE_MINFO = 14
_TYPE_MX = 15
_TYPE_TXT = 16
_TYPE_AAAA = 28
_TYPE_SRV = 33
_TYPE_ANY = 255

# Mapping constants to names
int2byte = struct.Struct(">B").pack

# Exceptions
class Error(Exception):
    pass

class IncomingDecodeError(Error):
    pass

# pylint: disable=no-init
class QuietLogger:
    _seen_logs = {}

    @classmethod
    def log_exception_warning(cls, logger_data=None):
        exc_info = sys.exc_info()
        exc_str = str(exc_info[1])
        if exc_str not in cls._seen_logs:
            # log at warning level the first time this is seen
            cls._seen_logs[exc_str] = exc_info
            logger = log.warning
        else:
            logger = log.debug
        if logger_data is not None:
            logger(*logger_data)
        logger('Exception occurred:', exc_info=True)

    @classmethod
    def log_warning_once(cls, *args):
        msg_str = args[0]
        if msg_str not in cls._seen_logs:
            cls._seen_logs[msg_str] = 0
            logger = log.warning
        else:
            logger = log.debug
        cls._seen_logs[msg_str] += 1
        logger(*args)

class DNSEntry:
    """A DNS entry"""

    def __init__(self, name, type_, class_):
        self.key = name.lower()
        self.name = name
        self.type = type_
        self.class_ = class_ & _CLASS_MASK
        self.unique = (class_ & _CLASS_UNIQUE) != 0

class DNSQuestion(DNSEntry):
    """A DNS question entry"""

    def __init__(self, name, type_, class_):
        DNSEntry.__init__(self, name, type_, class_)

    def answered_by(self, rec):
        """Returns true if the question is answered by the record"""
        return (self.class_ == rec.class_ and
                (self.type == rec.type or self.type == _TYPE_ANY) and
                self.name == rec.name)

class DNSRecord(DNSEntry):
    """A DNS record - like a DNS entry, but has a TTL"""

    def __init__(self, name, type_, class_, ttl):
        DNSEntry.__init__(self, name, type_, class_)
        self.ttl = 15
        self.created = time.time()

    def write(self, out):
        """Abstract method"""
        raise NotImplementedError

    def is_expired(self, now):
        return self.created + self.ttl <= now

    def is_removable(self, now):
        return self.created + self.ttl * 2 <= now

class DNSAddress(DNSRecord):
    """A DNS address record"""

    def __init__(self, name, type_, class_, ttl, address):
        DNSRecord.__init__(self, name, type_, class_, ttl)
        self.address = address

    def write(self, out):
        """Used in constructing an outgoing packet"""
        out.write_string(self.address)

class DNSText(DNSRecord):
    """A DNS text record"""

    def __init__(self, name, type_, class_, ttl, text):
        assert isinstance(text, (bytes, type(None)))
        DNSRecord.__init__(self, name, type_, class_, ttl)
        self.text = text

    def write(self, out):
        """Used in constructing an outgoing packet"""
        out.write_string(self.text)

class DNSIncoming(QuietLogger):
    """Object representation of an incoming DNS packet"""

    def __init__(self, data):
        """Constructor from string holding bytes of packet"""
        self.offset = 0
        self.data = data
        self.questions = []
        self.answers = []
        self.id = 0
        self.flags = 0  # type: int
        self.num_questions = 0
        self.num_answers = 0
        self.num_authorities = 0
        self.num_additionals = 0
        self.valid = False

        try:
            self.read_header()
            self.read_questions()
            self.read_others()
            self.valid = True

        except (IndexError, struct.error, IncomingDecodeError):
            self.log_exception_warning((
                'Choked at offset %d while unpacking %r', self.offset, data))

    def unpack(self, format_):
        length = struct.calcsize(format_)
        info = struct.unpack(
            format_, self.data[self.offset:self.offset + length])
        self.offset += length
        return info

    def read_header(self):
        """Reads header portion of packet"""
        (self.id, self.flags, self.num_questions, self.num_answers,
         self.num_authorities, self.num_additionals) = self.unpack(b'!6H')

    def read_questions(self):
        """Reads questions section of packet"""
        for _ in range(self.num_questions):
            name = self.read_name()
            type_, class_ = self.unpack(b'!HH')

            question = DNSQuestion(name, type_, class_)
            self.questions.append(question)

    def read_character_string(self):
        """Reads a character string from the packet"""
        length = self.data[self.offset]
        self.offset += 1
        return self.read_string(length)

    def read_string(self, length):
        """Reads a string of a given length from the packet"""
        info = self.data[self.offset:self.offset + length]
        self.offset += length
        return info

    def read_unsigned_short(self):
        """Reads an unsigned short from the packet"""
        return self.unpack(b'!H')[0]

    def read_others(self):
        """Reads the answers, authorities and additionals section of the
        packet"""
        n = self.num_answers + self.num_authorities + self.num_additionals
        for _ in range(n):
            domain = self.read_name()
            type_, class_, ttl, length = self.unpack(b'!HHiH')

            rec = None
            if type_ == _TYPE_A:
                rec = DNSAddress(
                    domain, type_, class_, ttl, self.read_string(4))
            elif type_ == _TYPE_TXT:
                rec = DNSText(
                    domain, type_, class_, ttl, self.read_string(length))
            elif type_ == _TYPE_AAAA:
                rec = DNSAddress(
                    domain, type_, class_, ttl, self.read_string(16))
            else:
                # Try to ignore types we don't know about
                # Skip the payload for the resource record so the next
                # records can be parsed correctly
                self.offset += length

            if rec is not None:
                self.answers.append(rec)

    def is_query(self):
        """Returns true if this is a query"""
        return (self.flags & _FLAGS_QR_MASK) == _FLAGS_QR_QUERY

    def is_response(self):
        """Returns true if this is a response"""
        return (self.flags & _FLAGS_QR_MASK) == _FLAGS_QR_RESPONSE

    def read_utf(self, offset, length):
        """Reads a UTF-8 string of a given length from the packet"""
        return str(self.data[offset:offset + length], 'utf-8', 'replace')

    def read_name(self):
        """Reads a domain name from the packet"""
        result = ''
        off = self.offset
        next_ = -1
        first = off

        while True:
            length = self.data[off]
            off += 1
            if length == 0:
                break
            t = length & 0xC0
            if t == 0x00:
                result = ''.join((result, self.read_utf(off, length) + '.'))
                off += length
            elif t == 0xC0:
                if next_ < 0:
                    next_ = off + 1
                off = ((length & 0x3F) << 8) | self.data[off]
                if off >= first:
                    raise IncomingDecodeError(
                        f"Bad domain name (circular) at {off}")
                first = off
            else:
                raise IncomingDecodeError(f"Bad domain name at {off}")

        if next_ >= 0:
            self.offset = next_
        else:
            self.offset = off

        return result

class DNSOutgoing:
    """Object representation of an outgoing packet"""

    def __init__(self, flags):
        self.finished = False
        self.id = 0
        self.flags = flags
        self.names = {}
        self.data = []
        self.size = 12
        self.state = False

        self.questions = []
        self.answers = []

    def add_question(self, record):
        """Adds a question"""
        self.questions.append(record)

    def pack(self, format_, value):
        self.data.append(struct.pack(format_, value))
        self.size += struct.calcsize(format_)

    def write_byte(self, value):
        """Writes a single byte to the packet"""
        self.pack(b'!c', int2byte(value))

    def insert_short(self, index, value):
        """Inserts an unsigned short in a certain position in the packet"""
        self.data.insert(index, struct.pack(b'!H', value))
        self.size += 2

    def write_short(self, value):
        """Writes an unsigned short to the packet"""
        self.pack(b'!H', value)

    def write_int(self, value):
        """Writes an unsigned integer to the packet"""
        self.pack(b'!I', int(value))

    def write_string(self, value):
        """Writes a string to the packet"""
        assert isinstance(value, bytes)
        self.data.append(value)
        self.size += len(value)

    def write_utf(self, s):
        """Writes a UTF-8 string of a given length to the packet"""
        utfstr = s.encode('utf-8')
        length = len(utfstr)
        self.write_byte(length)
        self.write_string(utfstr)

    def write_character_string(self, value):
        assert isinstance(value, bytes)
        length = len(value)
        self.write_byte(length)
        self.write_string(value)

    def write_name(self, name):
        # split name into each label
        parts = name.split('.')
        if not parts[-1]:
            parts.pop()

        # construct each suffix
        name_suffices = ['.'.join(parts[i:]) for i in range(len(parts))]

        # look for an existing name or suffix
        for count, sub_name in enumerate(name_suffices):
            if sub_name in self.names:
                break
        else:
            count = len(name_suffices)

        # note the new names we are saving into the packet
        name_length = len(name.encode('utf-8'))
        for suffix in name_suffices[:count]:
            self.names[suffix] = self.size + name_length - len(suffix.encode('utf-8')) - 1

        # write the new names out.
        for part in parts[:count]:
            self.write_utf(part)

        # if we wrote part of the name, create a pointer to the rest
        if count != len(name_suffices):
            # Found substring in packet, create pointer
            index = self.names[name_suffices[count]]
            self.write_byte((index >> 8) | 0xC0)
            self.write_byte(index & 0xFF)
        else:
            # this is the end of a name
            self.write_byte(0)

    def write_question(self, question):
        self.write_name(question.name)
        self.write_short(question.type)
        self.write_short(question.class_)

    def packet(self):
        if not self.state:
            for question in self.questions:
                self.write_question(question)
            self.state = True

            self.insert_short(0, 0)  # num additionals
            self.insert_short(0, 0)  # num authorities
            self.insert_short(0, 0)  # num answers
            self.insert_short(0, len(self.questions))
            self.insert_short(0, self.flags)  # _FLAGS_QR_QUERY
            self.insert_short(0, 0)
        return b''.join(self.data)

class Engine(threading.Thread):
    def __init__(self, zc):
        threading.Thread.__init__(self, name='zeroconf-Engine')
        self.daemon = True
        self.zc = zc
        self.readers = {}
        self.timeout = 5
        self.condition = threading.Condition()
        self.start()

    def run(self):
        while not self.zc.done:
            # pylint: disable=len-as-condition
            with self.condition:
                rs = self.readers.keys()
                if len(rs) == 0:
                    # No sockets to manage, but we wait for the timeout
                    # or addition of a socket
                    self.condition.wait(self.timeout)

            if len(rs) != 0:
                try:
                    rr, _, _ = select.select(rs, [], [], self.timeout)
                    if not self.zc.done:
                        for socket_ in rr:
                            reader = self.readers.get(socket_)
                            if reader:
                                reader.handle_read(socket_)

                except OSError as e:
                    # If the socket was closed by another thread, during
                    # shutdown, ignore it and exit
                    if e.args[0] != socket.EBADF or not self.zc.done:
                        raise

    def add_reader(self, reader, socket_):
        with self.condition:
            self.readers[socket_] = reader
            self.condition.notify()

    def del_reader(self, socket_):
        with self.condition:
            del self.readers[socket_]
            self.condition.notify()

class Listener(QuietLogger):
    def __init__(self, zc):
        self.zc = zc
        self.data = None

    def handle_read(self, socket_):
        try:
            data, (addr, port) = socket_.recvfrom(_MAX_MSG_ABSOLUTE)
        except Exception:  # pylint: disable=broad-except
            self.log_exception_warning()
            return

        log.debug('Received from %r:%r: %r ', addr, port, data)

        self.data = data
        msg = DNSIncoming(data)
        if not msg.valid or msg.is_query():
            pass
        else:
            self.zc.handle_response(msg)

class RecordUpdateListener:
    def update_record(self, zc, now, record):
        raise NotImplementedError()

class HostResolver(RecordUpdateListener):
    def __init__(self, name):
        self.name = name
        self.address = None

    def update_record(self, zc, now, record):
        if record is None:
            return
        if record.type == _TYPE_A:
            assert isinstance(record, DNSAddress)
            if record.name == self.name:
                self.address = record.address

    def request(self, zc, timeout):
        now = time.time()
        delay = 0.2
        next_ = now + delay
        last = now + timeout

        try:
            zc.add_listener(self)
            while self.address is None:
                if last <= now:
                    # Timeout
                    return False
                if next_ <= now:
                    out = DNSOutgoing(_FLAGS_QR_QUERY)
                    out.add_question(
                        DNSQuestion(self.name, _TYPE_A, _CLASS_IN))
                    zc.send(out)
                    next_ = now + delay
                    delay *= 2

                zc.wait(min(next_, last) - now)
                now = time.time()
        finally:
            zc.remove_listener(self)

        return True

class DashboardStatus(RecordUpdateListener, threading.Thread):
    def __init__(self, zc, on_update):
        threading.Thread.__init__(self)
        self.zc = zc
        self.query_hosts = set()
        self.key_to_host = {}
        self.cache = {}
        self.stop_event = threading.Event()
        self.query_event = threading.Event()
        self.on_update = on_update

    def update_record(self, zc, now, record):
        if record is None:
            return
        if record.type in (_TYPE_A, _TYPE_AAAA, _TYPE_TXT):
            assert isinstance(record, DNSEntry)
            if record.name in self.query_hosts:
                self.cache.setdefault(record.name, []).insert(0, record)
            self.purge_cache()

    def purge_cache(self):
        new_cache = {}
        for host, records in self.cache.items():
            if host not in self.query_hosts:
                continue
            new_records = [rec for rec in records if not rec.is_removable(time.time())]
            if new_records:
                new_cache[host] = new_records
        self.cache = new_cache
        self.on_update({key: self.host_status(key) for key in self.key_to_host})

    def request_query(self, hosts):
        self.query_hosts = set(hosts.values())
        self.key_to_host = hosts
        self.query_event.set()

    def stop(self):
        self.stop_event.set()
        self.query_event.set()

    def host_status(self, key):
        return self.key_to_host.get(key) in self.cache

    def run(self):
        self.zc.add_listener(self)
        while not self.stop_event.is_set():
            self.purge_cache()
            for host in self.query_hosts:
                if all(record.is_expired(time.time()) for record in self.cache.get(host, [])):
                    out = DNSOutgoing(_FLAGS_QR_QUERY)
                    out.add_question(
                        DNSQuestion(host, _TYPE_A, _CLASS_IN))
                    self.zc.send(out)
            self.query_event.wait()
            self.query_event.clear()
        self.zc.remove_listener(self)

def get_all_addresses():
    return list({
        addr.ip
        for iface in ifaddr.get_adapters()
        for addr in iface.ips
        if addr.is_IPv4 and addr.network_prefix != 32  # Host only netmask 255.255.255.255
    })

def new_socket():
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

    # SO_REUSEADDR should be equivalent to SO_REUSEPORT for
    # multicast UDP sockets (p 731, "TCP/IP Illustrated,
    # Volume 2"), but some BSD-derived systems require
    # SO_REUSEPORT to be specified explicitly.  Also, not all
    # versions of Python have SO_REUSEPORT available.
    # Catch OSError and socket.error for kernel versions <3.9 because lacking
    # SO_REUSEPORT support.
    try:
        reuseport = socket.SO_REUSEPORT
    except AttributeError:
        pass
    else:
        try:
            s.setsockopt(socket.SOL_SOCKET, reuseport, 1)
        except OSError as err:
            # OSError on python 3, socket.error on python 2
            if err.errno != errno.ENOPROTOOPT:
                raise

    # OpenBSD needs the ttl and loop values for the IP_MULTICAST_TTL and
    # IP_MULTICAST_LOOP socket options as an unsigned char.
    ttl = struct.pack(b'B', 255)
    s.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl)
    loop = struct.pack(b'B', 1)
    s.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, loop)

    s.bind(('', _MDNS_PORT))
    return s

class Zeroconf(QuietLogger):
    def __init__(self):
        # hook for threads
        self._GLOBAL_DONE = False

        self._listen_socket = new_socket()
        interfaces = get_all_addresses()

        self._respond_sockets = []

        for i in interfaces:
            try:
                _value = socket.inet_aton(_MDNS_ADDR) + socket.inet_aton(i)
                self._listen_socket.setsockopt(
                    socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, _value)
            except OSError as e:
                _errno = e.args[0]
                if _errno == errno.EADDRINUSE:
                    log.info(
                        'Address in use when adding %s to multicast group, '
                        'it is expected to happen on some systems', i,
                    )
                elif _errno == errno.EADDRNOTAVAIL:
                    log.info(
                        'Address not available when adding %s to multicast '
                        'group, it is expected to happen on some systems', i,
                    )
                    continue
                elif _errno == errno.EINVAL:
                    log.info(
                        'Interface of %s does not support multicast, '
                        'it is expected in WSL', i
                    )
                    continue

                else:
                    raise

            respond_socket = new_socket()
            respond_socket.setsockopt(
                socket.IPPROTO_IP, socket.IP_MULTICAST_IF, socket.inet_aton(i))

            self._respond_sockets.append(respond_socket)

        self.listeners = []

        self.condition = threading.Condition()

        self.engine = Engine(self)
        self.listener = Listener(self)
        self.engine.add_reader(self.listener, self._listen_socket)

    @property
    def done(self):
        return self._GLOBAL_DONE

    def wait(self, timeout):
        """Calling thread waits for a given number of milliseconds or
        until notified."""
        with self.condition:
            self.condition.wait(timeout)

    def notify_all(self):
        """Notifies all waiting threads"""
        with self.condition:
            self.condition.notify_all()

    def resolve_host(self, host, timeout=3.0):
        info = HostResolver(host)
        if info.request(self, timeout):
            return socket.inet_ntoa(info.address)
        return None

    def add_listener(self, listener):
        self.listeners.append(listener)
        self.notify_all()

    def remove_listener(self, listener):
        """Removes a listener."""
        try:
            self.listeners.remove(listener)
            self.notify_all()
        except Exception as e:  # pylint: disable=broad-except
            log.exception('Unknown error, possibly benign: %r', e)

    def update_record(self, now, rec):
        """Used to notify listeners of new information that has updated
        a record."""
        for listener in self.listeners:
            listener.update_record(self, now, rec)
        self.notify_all()

    def handle_response(self, msg):
        """Deal with incoming response packets.  All answers
        are held in the cache, and listeners are notified."""
        now = time.time()
        for record in msg.answers:
            self.update_record(now, record)

    def send(self, out):
        """Sends an outgoing packet."""
        packet = out.packet()
        log.debug('Sending %r (%d bytes) as %r...', out, len(packet), packet)
        for s in self._respond_sockets:
            if self._GLOBAL_DONE:
                return
            try:
                bytes_sent = s.sendto(packet, 0, (_MDNS_ADDR, _MDNS_PORT))
            except Exception:  # pylint: disable=broad-except
                # on send errors, log the exception and keep going
                self.log_exception_warning()
            else:
                if bytes_sent != len(packet):
                    self.log_warning_once(
                        '!!! sent %d out of %d bytes to %r' % (
                            bytes_sent, len(packet), s))

    def close(self):
        """Ends the background threads, and prevent this instance from
        servicing further queries."""
        if not self._GLOBAL_DONE:
            self._GLOBAL_DONE = True
            # shutdown recv socket and thread
            self.engine.del_reader(self._listen_socket)
            self._listen_socket.close()
            self.engine.join()

            # shutdown the rest
            self.notify_all()
            for s in self._respond_sockets:
                s.close()


"""empty message

Revision ID: 6b2cc6652042
Revises: a3e3a35deb56
Create Date: 2023-04-12 06:05:42.474902

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '6b2cc6652042'
down_revision = 'a3e3a35deb56'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('people',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(length=120), nullable=False),
    sa.Column('gender', sa.String(length=20), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('name')
    )
    op.create_table('planets',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(length=120), nullable=False),
    sa.Column('rotation', sa.Integer(), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('name')
    )
    op.create_table('fav_people',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('id_people', sa.Integer(), nullable=True),
    sa.Column('id_user', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['id_people'], ['people.id'], ),
    sa.ForeignKeyConstraint(['id_user'], ['user.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('fav_planets',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('id_planets', sa.Integer(), nullable=True),
    sa.Column('id_user', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['id_planets'], ['planets.id'], ),
    sa.ForeignKeyConstraint(['id_user'], ['user.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('fav_planets')
    op.drop_table('fav_people')
    op.drop_table('planets')
    op.drop_table('people')
    # ### end Alembic commands ###


"""Combine documents by doing a first pass and then refining on more documents."""

from __future__ import annotations

from typing import Any, Dict, List, Tuple

from langchain_core.documents import Document
from langchain_core.prompts import BasePromptTemplate, format_document
from langchain_core.prompts.prompt import PromptTemplate
from langchain_core.pydantic_v1 import Extra, Field, root_validator

from langchain.callbacks.manager import Callbacks
from langchain.chains.combine_documents.base import (
    BaseCombineDocumentsChain,
)
from langchain.chains.llm import LLMChain

def _get_default_document_prompt() -> PromptTemplate:
    return PromptTemplate(input_variables=["page_content"], template="{page_content}")

class RefineDocumentsChain(BaseCombineDocumentsChain):
    """Combine documents by doing a first pass and then refining on more documents.

    This algorithm first calls `initial_llm_chain` on the first document, passing
    that first document in with the variable name `document_variable_name`, and
    produces a new variable with the variable name `initial_response_name`.

    Then, it loops over every remaining document. This is called the "refine" step.
    It calls `refine_llm_chain`,
    passing in that document with the variable name `document_variable_name`
    as well as the previous response with the variable name `initial_response_name`.

    Example:
        .. code-block:: python

            from langchain.chains import RefineDocumentsChain, LLMChain
            from langchain_core.prompts import PromptTemplate
            from langchain.llms import OpenAI

            # This controls how each document will be formatted. Specifically,
            # it will be passed to `format_document` - see that function for more
            # details.
            document_prompt = PromptTemplate(
                input_variables=["page_content"],
                 template="{page_content}"
            )
            document_variable_name = "context"
            llm = OpenAI()
            # The prompt here should take as an input variable the
            # `document_variable_name`
            prompt = PromptTemplate.from_template(
                "Summarize this content: {context}"
            )
            initial_llm_chain = LLMChain(llm=llm, prompt=prompt)
            initial_response_name = "prev_response"
            # The prompt here should take as an input variable the
            # `document_variable_name` as well as `initial_response_name`
            prompt_refine = PromptTemplate.from_template(
                "Here's your first summary: {prev_response}. "
                "Now add to it based on the following context: {context}"
            )
            refine_llm_chain = LLMChain(llm=llm, prompt=prompt_refine)
            chain = RefineDocumentsChain(
                initial_llm_chain=initial_llm_chain,
                refine_llm_chain=refine_llm_chain,
                document_prompt=document_prompt,
                document_variable_name=document_variable_name,
                initial_response_name=initial_response_name,
            )
    """

    initial_llm_chain: LLMChain
    """LLM chain to use on initial document."""
    refine_llm_chain: LLMChain
    """LLM chain to use when refining."""
    document_variable_name: str
    """The variable name in the initial_llm_chain to put the documents in.
    If only one variable in the initial_llm_chain, this need not be provided."""
    initial_response_name: str
    """The variable name to format the initial response in when refining."""
    document_prompt: BasePromptTemplate = Field(
        default_factory=_get_default_document_prompt
    )
    """Prompt to use to format each document, gets passed to `format_document`."""
    return_intermediate_steps: bool = False
    """Return the results of the refine steps in the output."""

    @property
    def output_keys(self) -> List[str]:
        """Expect input key.

        :meta private:
        """
        _output_keys = super().output_keys
        if self.return_intermediate_steps:
            _output_keys = _output_keys + ["intermediate_steps"]
        return _output_keys

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.forbid
        arbitrary_types_allowed = True

    @root_validator(pre=True)
    def get_return_intermediate_steps(cls, values: Dict) -> Dict:
        """For backwards compatibility."""
        if "return_refine_steps" in values:
            values["return_intermediate_steps"] = values["return_refine_steps"]
            del values["return_refine_steps"]
        return values

    @root_validator(pre=True)
    def get_default_document_variable_name(cls, values: Dict) -> Dict:
        """Get default document variable name, if not provided."""
        if "document_variable_name" not in values:
            llm_chain_variables = values["initial_llm_chain"].prompt.input_variables
            if len(llm_chain_variables) == 1:
                values["document_variable_name"] = llm_chain_variables[0]
            else:
                raise ValueError(
                    "document_variable_name must be provided if there are "
                    "multiple llm_chain input_variables"
                )
        else:
            llm_chain_variables = values["initial_llm_chain"].prompt.input_variables
            if values["document_variable_name"] not in llm_chain_variables:
                raise ValueError(
                    f"document_variable_name {values['document_variable_name']} was "
                    f"not found in llm_chain input_variables: {llm_chain_variables}"
                )
        return values

    def combine_docs(
        self, docs: List[Document], callbacks: Callbacks = None, **kwargs: Any
    ) -> Tuple[str, dict]:
        """Combine by mapping first chain over all, then stuffing into final chain.

        Args:
            docs: List of documents to combine
            callbacks: Callbacks to be passed through
            **kwargs: additional parameters to be passed to LLM calls (like other
                input variables besides the documents)

        Returns:
            The first element returned is the single string output. The second
            element returned is a dictionary of other keys to return.
        """
        inputs = self._construct_initial_inputs(docs, **kwargs)
        res = self.initial_llm_chain.predict(callbacks=callbacks, **inputs)
        refine_steps = [res]
        for doc in docs[1:]:
            base_inputs = self._construct_refine_inputs(doc, res)
            inputs = {**base_inputs, **kwargs}
            res = self.refine_llm_chain.predict(callbacks=callbacks, **inputs)
            refine_steps.append(res)
        return self._construct_result(refine_steps, res)

    async def acombine_docs(
        self, docs: List[Document], callbacks: Callbacks = None, **kwargs: Any
    ) -> Tuple[str, dict]:
        """Async combine by mapping a first chain over all, then stuffing
         into a final chain.

        Args:
            docs: List of documents to combine
            callbacks: Callbacks to be passed through
            **kwargs: additional parameters to be passed to LLM calls (like other
                input variables besides the documents)

        Returns:
            The first element returned is the single string output. The second
            element returned is a dictionary of other keys to return.
        """
        inputs = self._construct_initial_inputs(docs, **kwargs)
        res = await self.initial_llm_chain.apredict(callbacks=callbacks, **inputs)
        refine_steps = [res]
        for doc in docs[1:]:
            base_inputs = self._construct_refine_inputs(doc, res)
            inputs = {**base_inputs, **kwargs}
            res = await self.refine_llm_chain.apredict(callbacks=callbacks, **inputs)
            refine_steps.append(res)
        return self._construct_result(refine_steps, res)

    def _construct_result(self, refine_steps: List[str], res: str) -> Tuple[str, dict]:
        if self.return_intermediate_steps:
            extra_return_dict = {"intermediate_steps": refine_steps}
        else:
            extra_return_dict = {}
        return res, extra_return_dict

    def _construct_refine_inputs(self, doc: Document, res: str) -> Dict[str, Any]:
        return {
            self.document_variable_name: format_document(doc, self.document_prompt),
            self.initial_response_name: res,
        }

    def _construct_initial_inputs(
        self, docs: List[Document], **kwargs: Any
    ) -> Dict[str, Any]:
        base_info = {"page_content": docs[0].page_content}
        base_info.update(docs[0].metadata)
        document_info = {k: base_info[k] for k in self.document_prompt.input_variables}
        base_inputs: dict = {
            self.document_variable_name: self.document_prompt.format(**document_info)
        }
        inputs = {**base_inputs, **kwargs}
        return inputs

    @property
    def _chain_type(self) -> str:
        return "refine_documents_chain"


import setuptools

with open('README.md', 'r') as fh:
    long_description = fh.read()

setuptools.setup(
    name='django-admin-left-menu',
    version='0.1.1',
    include_package_data=True,
    license='BSD License',
    description='A simple left menu for DjangoAdmin',
    long_description=long_description,
    long_description_content_type='text/markdown',
    url='https://github.com/ChanMo/django-admin-left-menu',
    author='ChanMo',
    author_email='chan.mo@outlook.com',
    packages=setuptools.find_packages(),
    classifiers=[
        'Environment :: Web Environment',
        'Framework :: Django',
        'Framework :: Django :: 2.2',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: BSD License',  # example license
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        'Topic :: Internet :: WWW/HTTP',
        'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
    ]
)


import random

colors = ['red', 'green', 'green', 'red' , 'red']

#measurements = ['green']                                            # test 1
#motions = [[1]]                                                     # test 1
measurements = ['green', 'green', 'green' ,'green', 'green','red']  # test 2
motions = [[1],[0],[-1],[1],[1],[0]]                                # test 2

sensor_right = {}
sensor_right['green'] = 0.6
sensor_right['red'] = 0.8

p_move = 0.8

def sense(p, Z):
    """Update belief array p according to new measurement Z"""

    # weights
    w = [sensor_right[colors[i]] if Z == sensor_right[colors[i]] else 1 - sensor_right[colors[i]] for i in p]

    # resampling
    particles = random.choices(p, w, k=len(p))
    return sorted(particles)

def move(p, U):
    """Update p after movement U"""

    if U[0] == 0:
        return p

    # weights
    w = [(1-p_move)+(p_move*(p.count(i-U[0])/len(p))) for i in p]

    # resampling
    particles = random.choices(p, w, k=len(p))
    return sorted(particles)

#main
num_particles = 100
p = [int(i//(num_particles/len(colors))) for i in range(num_particles)]

width  = len(colors)
n = width

for s in range(len(measurements)):
    print("sense ",measurements[s])
    p = sense(p,measurements[s])
    print(p)
    print("move ", motions[s])
    p = move(p,motions[s])
    print(p)


# -*- coding: utf-8 -*-
"""
Created on Tue Mar  2 18:38:57 2021

@author: Licht
"""

# Examen Python
# En una llantera se ha establecido una promoción de las llantas marca
# Ponchadas, dicha promoción consiste en lo siguiente: Si se compran menos
# de cinco llantas el precio es de $300 cada una, de $250 si se compran de
# cinco a 10 y de $200 si se compran más de 10. Obtener la cantidad de dinero
# que una persona tiene que pagar por cada una de las llantas que compra y la
# que tiene que pagar por el total de la compra.

clla = int(input('Digite la cantidad de llantas a comprar: '))
p1 = 300
p2 = 250
p3 = 200
if clla < 5:
    f = p1 * clla
    p = p1
elif clla >= 5 and clla <= 10:
    f = p2 * clla
    p = p2
else:
    f = p3 * clla
    p = p3
print(f'Por cada llanta debe pagar el valor de: ${p}')
print(f'La cantidad total de dinero a pagar es de: ${f}')

# Hacer algoritmo que de el valor final por la compra de televisores. El
# descuento lo hace basado en los dos numeros finales a la cédula, si
# termina en 01 el descuento es del 10% y si termina en 25 el descuento es
# del 20%, si termina en 44 el descuento es 35% y si es 57 el 50%

vc = float(input('Digite valor de la compra: '))
nc1 = int(input('Digite el penúltimo número de su cédula: '))
nc2 = int(input('Digite el último número de su cédula: '))
if nc1 == 0 and nc2 == 1:
    f = vc * 0.90
else:
    if nc1 == 2 and nc2 == 5:
        f = vc * 0.80
    else:
        if nc1 == 4 and nc2 == 4:
            f = vc * 0.65
        else:
            if nc1 == 5 and nc2 == 7:
                f = vc * 0.50
            else:
                f = vc
print(f'El valor de la compra es de: {vc}')
print(f'El valor de la compra con descuento es de: ${f}')

# Una empresa de colchones ofrece descuento según la siguiente tabla
# Numero de colchones comprados % Descuento
# De 0 a menos de 3              0%
# De 3 hasta menos de 6          20%
# De 6 hasta menos de 8          25%
# De 8 en adelante               30%
# Determine cuanto pagará una persona que compre colchones

cc = int(input('Digite el número de colchones comprados: '))
vc = 50000
d1 = 0
d2 = 0.80
d3 = 0.75
d4 = 0.70
if cc >= 1 and cc < 3:
    p = vc
else:
    if cc >= 3 and cc < 6:
        p = (vc * cc) * d2
    else:
        if 6 >= cc < 8:
            p = (vc * cc) * d3
        else:
            p = (vc * cc) * d4
print(f'El valor a pagar por la compra de {cc} colchones')
print(f'es de {p}')

# Una universidad desea seleccionar una muestra de estudiantes para
# realizar una encuesta. Si el número de estudiantes es menor que 20 se
# tomará el 50%, si el salón tiene entre 20 y 30 estudiantes se tomará 60%,
# si la cantidad es mayor a 30 tomará 70%. ¿Que cantidad de estudiantes
# serán seleccionados para la encuesta?.

ce = int(input('Digite la cantidad de estudiantes para la muestra: '))
c1 = 0.50
c2 = 0.60
c3 = 0.70

if ce < 20:
    f = ce * c1
    ff = round(f)
elif ce >= 20 and cc <= 30:
    f = ce * c2
    ff = round(f)
else:
    f = ce * c3
    ff = round(f)
print(f'La cantidad de estudiantes seleccionados para la encuestas es: {ff}')


import numpy as np

x=np.array([
    [1,-1,2,1],
    [3,2,1,4],
    [5,-8,6,3],
    [4,2,5,3] ],dtype=np.float64)

b=np.array([1,1,1,-1],dtype=np.float64)

def s_l(x):
    s=[] #scale vector
    l=[] #index vector
    for i in range(len(x)):
        # adding max values in each rows to scale vector
        # signs regarded
        s.append(np.max(abs(x[i])))
        #adding index
        l.append(i)
    return s,l

#get scale vector,and index vector
s,l=s_l(x)
#initialize ratios
ratios=[0,0,0,0]

for step in range(len(x)-1):

    #update ratios
    for r in range(step,len(l)):
        ratios[r]= (abs(x[l[r]][step])/s[l[r]])

    #find max ratio
    max_in=( np.argmax(ratios[step:len(x)]))
    if(max_in>=step):
    #exchange order for determine pivot
        l[step],l[max_in]=l[max_in],l[step]
    #forward elimination
    for i in l[step+1:len(x)]:
        rate=(x[i][step])/(x[l[step]][step])
        rate*=-1
        for j in range(step,len(x)):
            x[i][j] +=  x[l[step]][j]*rate

        b[i] += b[l[step]]*rate

# i can't write as a for loop due to complex index notation
x4= b[l[3]]/x[l[3]][3]
x3=( b[l[2]]-x[l[2]][3]*x4)/(x[l[2]][2])
x2=( b[l[1]]- x[l[1]][3]*x4-x[l[1]][2]*x3) / (x[l[1]][1])

x1=( b[l[0]]-x[l[0]][3]*x4-x[l[0]][2]*x3-x[l[0]][1]*x2)/ x[l[0]][0]

print("x4 : ",x4)
print("x3 : ",x3)
print("x2 : ",x2)
print("x1 : ",x1)


our_list = []
clean_list = []
marked_tasks = []

def print_task_list(our_list, marked_tasks):
    for task_no in range(len(our_list)):
        print(str(task_no + 1) + ". [", end = "")
        if task_no in marked_tasks:
            print("X", end="")
        else:
            print(" ", end="");
        print("]", our_list[task_no])

def add_item():
    new_task = input("Add an item: ")
    our_list.append(new_task)
    print("Item added")

def list_items():
    if not our_list:
       print("You dont have things to do")
    else:
        print("You saved the following to-do items: ")
        print_task_list(our_list, marked_tasks)

def mark_item():
    print("you saved the following to-do items: ")
    print_task_list(our_list, marked_tasks)
    d = int(input("Which one do you want to mark as completed? Press the number: ")) - 1
    marked_tasks.append(d)

def archive_item():
    for task_no in range(len(our_list)):
        if task_no not in marked_tasks:
            clean_list.append(our_list[task_no])

while True:

    c = input("Please specify a command [list, add, mark, archive, exit]: ")

    if c == "add":
        add_item()

    elif c == "list":
        list_items()

    elif c == "mark":
        mark_item()

    elif c == "archive":
        archive_item()
        our_list = clean_list
        clean_list = []
        marked_tasks = []
        print_task_list(our_list, marked_tasks)

    elif c == "exit":
        exit();


import torch
import torch.nn as nn
from vit_pytorch.vit import Transformer, repeat

class ViT(nn.Module):
    def __init__(self, *, num_views, num_classes, dim, depth, heads, mlp_dim, pool='cls',
                 dim_head=64, dropout=0., emb_dropout=0.):
        # super().__init__()
        super(ViT, self).__init__()

        assert pool in {'cls', 'mean'}, 'pool type must be either cls (cls token) or mean (mean pooling)'
        self.num_views = num_views
        self.cls_token = nn.Parameter(torch.randn(1, 1, dim))
        self.dropout = nn.Dropout(emb_dropout)

        self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim, dropout)

        self.pool = pool
        self.to_latent = nn.Identity()

        self.mlp_head = nn.Sequential(
            nn.LayerNorm(dim),
            nn.Linear(dim, num_classes)
        )

    def forward(self, img):
        _, C, N = img.shape
        x = img.contiguous().view(-1, self.num_views, C, N)
        x = x.permute(0, 3, 1, 2).contiguous().view(-1, self.num_views, C)
        b, n, _ = x.shape
        cls_tokens = repeat(self.cls_token, '() n d -> b n d', b=b)
        x = torch.cat((cls_tokens, x), dim=1)
        # x = self.dropout(x)

        x = self.transformer(x)
        x = x.mean(dim=1) if self.pool == 'mean' else x[:, 0]

        x = self.to_latent(x)
        x = self.mlp_head(x)

        x = x.view(-1, N, x.shape[1]).permute(0, 2, 1).contiguous()
        return x

if __name__ == '__main__':
    v = ViT(
        num_views=1,
        num_classes=2,
        dim=1024,
        depth=6,
        heads=16,
        mlp_dim=2048,
        dropout=0.1,
        emb_dropout=0.1
    )

    img = torch.randn(1, 4, 1024)

    preds = v(img)  # (1, 1000)
    print(preds.shape)


from flask import render_template, flash, redirect, url_for, request
from flask_login import current_user, login_user, logout_user, login_required
from app import app
from app.forms import LoginForm, RegistrationForm
from app.models import User
from werkzeug.urls import url_parse
from app import db

@app.route('/hello')
def hello():
    return 'Hello World'

@app.route('/')
@app.route('/index')
@login_required
def index():
    user = {'username' :  'Tom'}
    posts = [{
                'author': {'username': 'John'},
                'body': 'Beautiful day in Portland!'
            },
            {
                'author': {'username': 'Susan'},
                'body': 'The Avengers movie was so cool!'
            }]
    return render_template('index.html', title='Home', user=user, posts=posts)

@app.route('/login', methods = ['POST', 'GET'])
def login():
    print('Login called')
    if current_user.is_authenticated:
        redirect(url_for('index'))
    login_form = LoginForm()
    if login_form.validate_on_submit():
        # flash('Login requested for user {}, remember_me={}'.format(
        #     login_form.username.data,
        #     login_form.password.data))
        user = User.query.filter_by(username=login_form.username.data).first()
        if user is None or not user.check_password(login_form.password.data):
            flash('Invalid username or password')
            return redirect(url_for('login'))
        login_user(user, remember=login_form.remember_me.data)
        next_page = request.args.get('next')
        if not next_page or url_parse(next_page).netloc != '':
            next_page = url_for('index')
        return redirect(next_page)
        #return redirect('/index')
        return redirect(url_for('index'))
    return render_template('login.html', form=login_form)

@app.route('/logout')
def logout():
    logout_user()
    return redirect(url_for('login'))

@app.route('/register', methods=['GET', 'POST'])
def register():
    if current_user.is_authenticated:
        return redirect(url_for('index'))
    form = RegistrationForm()
    if form.validate_on_submit():
        user = User(username=form.username.data, email=form.email.data)
        user.set_password(form.password.data)
        db.session.add(user)
        db.session.commit()
        flash('Congratulations, you are now a registered user!')
        return redirect(url_for('login'))
    return render_template('register.html', title='Register', form=form)

@app.route('/user/<username>')
@login_required
def user(username):
    user = User.query.filter_by(username=username).first_or_404()
    posts = [
        {'author': user, 'body': 'Test post #1'},
        {'author': user, 'body': 'Test post #2'}
    ]
    return render_template('user.html', user=user, posts=posts)


import os

import torch
import torchvision
import torchvision.transforms as transforms
import torch.optim as optim
from torch import nn

# from torch.utils.tensorboard import SummaryWriter

from torch.utils.data import Dataset
from config import config
import logging

# ============================= Logger Settings ========================================================================

save_path = "generated_files/training_logs/"
if not os.path.exists(save_path):
    os.makedirs(save_path)

logger = logging.getLogger('pytorch')
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

file_handler = logging.FileHandler(save_path + 'torch.log', mode='w')
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(formatter)

logger.addHandler(file_handler)

# ========================================================================================================

def train_model(model):
    transform = transforms.Compose(
        [transforms.ToTensor(),
         transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

    trainset = torchvision.datasets.CIFAR10(root='./data-sets/cifar10', train=True, download=True, transform=transform)
    trainloader = torch.utils.data.DataLoader(trainset, batch_size=config['batch_size'], shuffle=True, num_workers=2)

    testset = torchvision.datasets.CIFAR10(root='./data-sets/cifar10', train=False, download=True, transform=transform)
    testloader = torch.utils.data.DataLoader(testset, batch_size=config['batch_size'], shuffle=False, num_workers=2)

    criterion = nn.CrossEntropyLoss()
    optimizer = optim.Adam(model.parameters())

    for epoch in range(2):  # loop over the dataset multiple times
        running_loss = 0.0
        for i, data in enumerate(trainloader, 0):
            # get the inputs; data is a list of [inputs, labels]
            inputs, labels = data

            # zero the parameter gradients
            optimizer.zero_grad()

            # forward + backward + optimize
            outputs = model(inputs)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()

            # print statistics
            running_loss += loss.item()
            if i % 200 == 199:  # print every 200 mini-batches
                print('[Epoch : %d Iteration : %5d loss: %.3f]' %
                      (epoch + 1, i + 1, running_loss / 200))
                running_loss = 0.0

            # printing the network weights every 50 iterations
            if i % 50 == 0:
                for layer_name, layer_params in model.named_parameters():
                    print(layer_name, layer_params.size(), layer_params)

                logger.info('[Epoch : %d Iteration : %5d loss: %.3f]' %
                            (epoch + 1, i + 1, running_loss / 200))

    print('Finished Training')


# Import required packages
import pandas as pd
import numpy as np
from scipy.stats import t

class OLS:
    """Retrieves OLS estimator, standard errors and confidence intervals

    - Initialization parameters:
        - target = Pandas dataframe with target variable
        - regressors = Pandas dataframe with predictors
        - intercept = Whether or not to include a constant

    - Methods:
        - data_matrices = Target, regressors and inverted cross-matrix data
        - ols_estimator = Calculates OLS estimator vector
        - ols_se = Computes standard errors of OLS estimator
        - ols_t = Retrieves t-statistic for individual significance
        - ols_F = Obtains F-statistic for joint significance
        - ols_ci = 5% significance level confidence interval of OLS estimator
        - ols_summary = Data matrix with all previous methods' results
    """
    def __init__(self, target, regressors, intercept=True):
        self.target = target
        self.regressors = regressors

        # Include intercept in data matrix
        if intercept and 'intercept' not in self.regressors.columns:
            constant = [1]*len(self.target)
            self.regressors.insert(loc=0, column='intercept', value=constant)

    def data_matrices(self):
        """"Sets up data matrices for modeling"""
        regressors = self.regressors
        target = self.target
        regressors, target = regressors.to_numpy(), target.to_numpy()
        cross_matrix = np.matmul(regressors.T, regressors)

        # Invert cross matrix if it is nonsingular
        try:
            cross_matrix_inv = np.linalg.inv(cross_matrix)
        except:
            cross_matrix_inv = np.linalg.pinv(cross_matrix)

        return regressors, target, cross_matrix_inv

    def ols_estimator(self):
        """Retrieves OLS estimator vector"""
        regressors, target, cross_matrix_inv = self.data_matrices()
        dot_prod = np.dot(regressors.T, target)
        b = np.matmul(cross_matrix_inv, dot_prod)
        b = [i.round(4) for i in b]
        return b

    def ols_se(self):
        """Retrieves OLS estimator standard errors (assumes homoskedasticity)"""
        b = self.ols_estimator()
        regressors, target, cross_matrix_inv = self.data_matrices()
        n, k = len(target), len(b)
        resid = target - np.matmul(regressors, b)
        s2 = sum(resid**2)/(n-k)
        se = np.sqrt(s2*np.diag(cross_matrix_inv))
        return se

    def ols_t(self):
        """Retrieves t-statistic for individual significance"""
        b, target = self.ols_estimator(), self.target
        se = self.ols_se()
        t_stat = np.divide(b, se)
        n, k = len(target), len(b)
        return t_stat, 2*t.sf(abs(t_stat), n - k).round(3)

    def ols_F(self):
        """Calculates F-statistic for joint significance"""
        b = self.ols_estimator()
        regressors, target, cross_matrix_inv = self.data_matrices()
        n, k = len(target), len(b)
        resid_u = target - np.matmul(regressors, b)
        SSR_U = sum(resid_u**2)
        resid_r = target - [np.mean(target)]*n
        SSR_R = sum(resid_r**2)
        F = ((SSR_R - SSR_U)/(k-1))/(SSR_U/(n - k))
        return F

    def ols_ci(self):
        """Computes onfidence intervals with 5% significance level"""
        b = self.ols_estimator()
        se = self.ols_se()
        lb, ub = b - se*1.96, b + se*1.96
        lb = [i.round(3) for i in lb]
        ub = [j.round(3) for j in ub]
        return lb, ub

    def ols_summary(self):
        """Provides summary of OLS estimation"""
        cols = self.regressors.columns
        b = self.ols_estimator()
        se = self.ols_se()
        t, p_value = self.ols_t()
        lb, ub = self.ols_ci()
        summary = pd.DataFrame(data=np.column_stack((
            cols, b, se, t, p_value, lb, ub)),
            columns=['features', 'coef', 'std err', 't', 'P>|t|', '[0.025', '0.975]'])
        return summary

class OLS_tests(OLS):
    """Performs statistical significance inference on OLS estimates

        - Initialization parameters (inherited):
            - target = Pandas dataframe with target variable
            - regressors = Pandas dataframe with predictors
            - intercept = Whether or not to include a constant

        - Methods:
            - individual_test = Individual statistical significance test
            - joint_test = Joint significance hypothesis test
    """
    def __init__(self, target, regressors, intercept=True):
        super().__init__(
            target, regressors, intercept
        )

    def individual_test(self, coef: str, value: float, sign: str):
        """Returns outcome of individual significance test"""
        assert sign in ['two.sided', 'less', 'great'], f'Sign argument must be either ' \
            f"'two.sided', 'less' or 'great' but '{sign}' was introduced"
        self.coef = coef
        self.value = value
        self.sign = sign

        # Set up data matrices
        regressors, target = self.regressors, self.target

        # Retrieve OLS estimator and variance estimator
        b = self.ols_estimator()
        se = self.ols_se()

        # Compute degrees of freedom
        n, k = len(target), len(b)
        df = n - k

        # Obtain OLS estimate
        cols = list(regressors.columns)
        idx = cols.index(self.coef)
        b_k = b[idx]

        # Calculate test-statistic
        t_stat = (b_k - self.value)/se[idx]

        # Print results depending on sign
        if self.sign == 'two.sided':
            pvalue = 2*t.sf(abs(t_stat), n - k)
        elif self.sign == 'less':
            pvalue = 1 - t.cdf(t_stat, df)
        else:
            pvalue = t.cdf(t_stat, df)

        # Check if H0 is rejected
        if pvalue <= 0.05:
            print(f'H0 is rejected with p-value {pvalue.round(3)}')
        else:
            print(f'Failed to reject H0 with p-value {pvalue.round(3)}')

    def joint_test(self, coefs: list, values: list):
        """Computes outcome of joint statistical significance test"""
        assert len(coefs) == len(values), f'Number of estimates to be tested ' \
            f'should coincide with their hypothesized values!'
        self.coefs = coefs
        self.values = values

        # Set up data matrices
        regressors, target = self.regressors, self.target

        # Retrieve OLS estimator and variance estimator
        b = self.ols_estimator()
        se = np.diag(self.ols_se())

        # Compute degrees of freedom
        n, k = len(target), len(b)
        df1 = n - k

        # Define restriction matrices and value vector
        r = values
        R = [[0] * k for i in range(len(r))]
        cols = list(regressors.columns)
        idx = [cols.index(i) for i in self.coefs]
        for i, j in zip(range(len(r)), idx):
            R[i][j] = 1
        R = np.asarray(R)
        df2 = len(r)
        assert np.linalg.matrix_rank(R) == df2, f'Failure of rank condition! ' \
            f'Check for linearly dependent restrictions!'

        # Set up test-statistic
        null = np.matmul(R, b) - r
        var = np.matmul(np.matmul(R, se), R.T)

        # Invert variance if it is nonsingular
        try:
            var_inv = np.linalg.inv(var)
        except:
            var_inv = np.linalg.pinv(var)

        # Compute F-statistic and associated p-value
        F = np.matmul(np.matmul(null.T, var_inv), null)/df2
        pvalue = f.sf(F, df2, df1)

        # Check if H0 is rejected
        if pvalue <= 0.05:
            print(f'H0 is rejected with p-value {pvalue.round(3)}')
        else:
            print(f'Failed to reject H0 with p-value {pvalue.round(3)}')


import pandas as pd
import dash
from dash import dcc , html
from dash.dependencies import Input, Output

# Load the CSV file into a Pandas DataFrame
df = pd.read_csv("airline_routes.csv")

# Initialize the Dash app
app = dash.Dash(__name__)

# Layout of the web application
app.layout = html.Div([
    html.H1("Airline Route Optimization"),
    dcc.Dropdown(
        id="origin-dropdown",
        options=[{'label': airport, 'value': airport} for airport in df['origin_airport'].unique()],
        value=df['origin_airport'].iloc[0]
    ),
    dcc.Graph(id="route-graph")
])

# Callback to update the graph based on the selected origin airport
@app.callback(
    Output("route-graph", "figure"),
    [Input("origin-dropdown", "value")]
)
def update_graph(selected_origin):
    filtered_df = df[df['origin_airport'] == selected_origin]
    figure = {
        'data': [
            {'x': filtered_df['destination_airport'], 'y': filtered_df['ticket_price'], 'type': 'bar', 'name': 'Ticket Price'},
            {'x': filtered_df['destination_airport'], 'y': filtered_df['demand'], 'type': 'line', 'name': 'Demand'},
            {'x': filtered_df['destination_airport'], 'y': filtered_df['fuel_cost'], 'type': 'line', 'name': 'Fuel Cost'}
        ],
        'layout': {
            'title': f"Flight Routes from {selected_origin}",
            'xaxis': {'title': 'Destination Airport'},
            'yaxis': {'title': 'Value'},
        }
    }
    return figure

if __name__ == '__main__':
    app.run_server(debug=True)


# cook your dish here
t = int(input())

for i in range(t):
    A = list(map(int,input().split()))
    if A[2] in A[0:2] and A[3] in A[0:2]:
        print(1, end = '')
    elif A[4] in A[0:2] and A[5] in A[0:2]:
        print(2, end = '')
    else:
        print(0)

    print()


import string

import nltk
from nltk.collocations import *
from nltk.corpus import stopwords

def compare_types_reviews(pos_reviews, neg_reviews):
    """
    Compare two lists of texts and find bigrams unique to each one
    :param pos_reviews: List of positive reviews
    :param neg_reviews: List of negative reviews
    :return: Pair of sets (s1, s2) where s1 is bigrams exclusive to positive reviews and s2 is exclusive to negative reviews
    """
    # Extract bigrams for both review types
    positive_bigrams = extract(pos_reviews)
    negative_bigrams = extract(neg_reviews)

    # Find bigrams which appear in both positive and negative reviews
    intersection = positive_bigrams.intersection(negative_bigrams)

    # Keep only pairs exclusive to each review type
    positive_bigrams_set = positive_bigrams.difference(intersection)
    negative_bigrams_set = negative_bigrams.difference(intersection)

    return positive_bigrams_set, negative_bigrams_set

def extract(texts):
    """
    Find most popular bigrams in given texts
    :param texts: List of texts for analysis
    :return: Set of pairs (w1, w2) of most common consecutive words
    """
    all_words = []
    table = str.maketrans('', '', string.punctuation)
    stop_words = set(stopwords.words('english'))

    # Remove punctuation from texts, convert to lowercase, remove stopwords and merge texts into one list
    for text in texts:
        text = text.translate(table)
        tokens = nltk.word_tokenize(text.lower())
        stripped = [w.translate(table) for w in tokens]
        words = [word for word in stripped if word.isalpha()]
        words = [w for w in words if not w in stop_words]
        all_words += words

    # Generate set of top bigrams based on their frequency
    bigram_measures = nltk.collocations.BigramAssocMeasures()
    finder2 = BigramCollocationFinder.from_words(all_words, window_size=6)
    finder2.apply_freq_filter(6)
    tuples = finder2.nbest(bigram_measures.likelihood_ratio, 40)
    tuples_set = set()
    for a, b in tuples:
        if a != b:
            tuples_set.add((a, b))

    return tuples_set


# -*- coding: utf-8 -*-
"""
Created on Sat Apr 25 22:29:14 2020

@author: sombanerjee
"""

import pandas as pd
import os

my_cols = [str(j) for j in range(35)]
#frequency_code = pd.read_excel(r"C:\Users\sombanerjee\Documents\power purchase MIS\2303_2903\New folder\JSEB\frequency.csv")
#code = frequency_code.to_dict()
temp = pd.DataFrame()
frequency = pd.DataFrame()
path_original = r"C:\Users\sombanerjee\Documents\power purchase MIS"
#os.chdir(r"C:\Users\sombanerjee\Documents\power purchase MIS\WEEK 140920-200920\WEEK 140920-200920\JSEB")
start_date = 15
end_date = 21
month = 4
if month <= 9:
    file_string = str(0)+str(month)
else:
    file_string = str(month)
if start_date<=9:
    start_date1 = str("0")+ str(start_date)
else:
    start_date1 = str(start_date)
if end_date <=9:
    end_date1 = str("0") + str(end_date)
else:
    end_date1 = str(end_date)
folder_name = 'WEEK '+str(start_date1)+file_string+str('21')+"-"+str(end_date1)+file_string+str('21')
destination_folder1 = os.path.join(path_original,folder_name,folder_name,"JSEB")
os.chdir(destination_folder1)
for i in range (start_date,end_date+1,1):
    a = i
    if i<10:
        file = str('0')+str(a) + file_string+"21.JSB"
    else:
        file = str(a) + file_string+"21.JSB"
    df1 =pd.read_csv(file,header = None, sep = "\s+|\t+|\s+\t+|\t+\s+", names = my_cols, engine = "python")
    #column_name = str('0') + str(a)+ str('0320')
    column_name = str(a)+ str(file_string)
    temp[column_name] = df1.iloc[6:102,-3].astype(float)*-4
    column_name =  str(a)+ str(file_string)+ "_f"
    frequency[column_name] = df1.iloc[:,1]
temp_reverse = temp[temp.columns[::-1]]
consumption_filename = str(start_date)+"_"+ str(end_date)+"consumption.csv"
temp_reverse.to_csv(consumption_filename)
#frequency1 = pd.merge(frequency,frequency_code, on = '')
frequency_reverse = frequency[frequency.columns[::-1]]
frequency_filename = str(start_date)+"_"+ str(end_date)+"frequency.csv"
frequency_reverse.iloc[6:102,:].to_csv(frequency_filename)

destination_folder2 = os.path.join(path_original,folder_name,folder_name,"JSEB","schedule")
#dir1 = r"C:\Users\sombanerjee\Documents\power purchase MIS\WEEK 070920-130920\WEEK 070920-130920\JSEB\schedule"
dir1 = destination_folder2
temp1 = pd.DataFrame()

for j in range (start_date,end_date+1,1) :
     #a = j+1
     a =j
     if j<9:
        substring = str('0')+str(a)+"-"+ file_string+"-2021"
     else:
        substring = str(a) + "-"+file_string+"-2021"

     for root, subdirs,files  in os.walk(dir1):
        for filename in files:
            if substring in filename:
                name_path2 = os.path.join(root,filename)
     column_name1 = substring
     df_schedule = pd.read_csv(name_path2, encoding= 'UTF-16 LE', header = 0)
     temp1[column_name1] = df_schedule['Net Total']

temp1_reverse = temp1[temp1.columns[::-1]].iloc[0:96,:]
schedule_filename = str(start_date)+"_"+ str(end_date)+"schedule.csv"
temp1_reverse.to_csv(schedule_filename)


from glob import glob

import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.colors
import os
import skimage.draw
import numpy as np
from tqdm.auto import tqdm
import cv2
import random
from sklearn.model_selection import train_test_split
import warnings
from awesometools.single2coco import convert2coco

warnings.filterwarnings('ignore')

def create_directory(directory):
    '''
    Creates a new folder in the specified directory if the folder doesn't exist.
    INPUT
        directory: Folder to be created, called as "folder/".
    OUTPUT
        New folder in the current directory.
    '''
    if not os.path.exists(directory):
        os.makedirs(directory)
        # os.mkdir(directory)

def pad(img, pad_size=96):
    """
    Load image from a given path and pad it on the sides, so that eash side is divisible by 80 (network requirement)
    if pad = True:
        returns image as numpy.array, tuple with padding in pixels as(x_min_pad, y_min_pad, x_max_pad, y_max_pad)
    else:
        returns image as numpy.array
    """

    if pad_size == 0:
        return img

    height, width = img.shape[:2]

    if height % pad_size == 0:
        y_min_pad = 0
        y_max_pad = 0
    else:
        y_pad = pad_size - height % pad_size
        y_min_pad = int(y_pad / 2)
        y_max_pad = y_pad - y_min_pad

    if width % pad_size == 0:
        x_min_pad = 0
        x_max_pad = 0
    else:
        x_pad = pad_size - width % pad_size
        x_min_pad = int(x_pad / 2)
        x_max_pad = x_pad - x_min_pad

    # print(img.shape, y_min_pad, y_max_pad, x_min_pad, x_max_pad)

    # 对于第3个channel大于512的情况，copyMakeBorder会报错（主要是细胞数量多时）
    if len(img.shape) == 3 and img.shape[2] > 512:
        image_list = []

        # 迭代的将channel放入list
        while img.shape[2] > 512:
            image_list.append(img[:, :, :512])
            img = img[:, :, 512:]

        image_list.append(img)
        image_pad_list = []
        # 分别 makeborder
        for image in image_list:
            pad_image = cv2.copyMakeBorder(image, y_min_pad, y_max_pad, x_min_pad, x_max_pad, cv2.BORDER_REFLECT_101)
            if len(pad_image.shape) != len(image.shape):
                pad_image = pad_image[:, :, np.newaxis]
            image_pad_list.append(pad_image)
        # img = image_pad_list
        # 拼接回去
        img = np.concatenate(image_pad_list, 2)
    else:
        img = cv2.copyMakeBorder(img, y_min_pad, y_max_pad, x_min_pad, x_max_pad, cv2.BORDER_REFLECT_101)

    return img, (x_min_pad, y_min_pad, x_max_pad, y_max_pad)

def unpad(img, pads):
    """
    img: numpy array of the shape (height, width)
    pads: (x_min_pad, y_min_pad, x_max_pad, y_max_pad)
    @return padded image
    """
    (x_min_pad, y_min_pad, x_max_pad, y_max_pad) = pads
    height, width = img.shape[:2]

    return img[y_min_pad:height - y_max_pad, x_min_pad:width - x_max_pad]

def read_nuclei(path):
    "read raw data"

    # Load 4-channel image
    img = skimage.io.imread(path)

    # input image
    if len(img.shape) > 2:
        img = img[:, :, :3]
    # mask
    else:
        # do nothing
        pass

    return img

def save_nuclei(path, img):
    "save image"
    skimage.io.imsave(path, img)

def sliding_window(image, step):
    x_loc = []
    y_loc = []
    # cells = []

    for y in range(0, image.shape[0], step):
        for x in range(0, image.shape[1], step):
            # cells.append(image[y:y + window[1], x:x + window[0]])
            x_loc.append(x)
            y_loc.append(y)
    return x_loc, y_loc

def extract_patches(image, step, patch_size):
    # print("*******************************")
    # print(image.shape)
    patches = []
    # Get locations
    x_pos, y_pos = sliding_window(image, step)
    for (x, y) in zip(x_pos, y_pos):
        # Get patch
        # patch = image[y:y + patch_size[0], x:x + patch_size[0]]
        patch = image[y:y + patch_size[0], x:x + patch_size[1]]

        # Get size
        # raw_dim = (patch.shape[1], patch.shape[0])  # W, H
        raw_dim = (patch.shape[0], patch.shape[1])  # W, H

        # print(raw_dim)
        # print(patch.shape)

        if raw_dim != (patch_size[0], patch_size[1]):

            # Resize to 64x64
            # patch = cv2.resize(patch, (64, 64), interpolation = cv2.INTER_AREA)
            old_shape = patch.shape
            patch, pad_locs = pad(patch, pad_size=patch_size[0])
            # print(old_shape, patch.shape)
            # 对于 h x w x 1, padding 后最后一个维度会消失
            if len(old_shape) != len(patch.shape):
                # print(old_shape, patch.shape)
                patch = patch[:, :, np.newaxis]

            # Do stuffffff
            patches.append(patch)

        else:
            # Do stuffffff
            patches.append(patch)

    patches = np.array(patches)

    return patches

# Sanity check

def show_image_mask_5class(image, mask):
    # Stolen from https://stackoverflow.com/questions/16834861/create-own-colormap-using-matplotlib-and-plot-color-scale
    norm = plt.Normalize(0, 4)  # 5 classes including BG
    map_name = matplotlib.colors.LinearSegmentedColormap.from_list("", ["black", "red", "yellow", "blue", "green"])
    h, w = mask.shape

    f, axarr = plt.subplots(1, 2, figsize=(16 * w / h * 2, 16))
    axarr[0].imshow(image, cmap=map_name, norm=norm)
    axarr[1].imshow(mask, cmap=map_name, norm=norm)
    plt.show()

def generate_patches_and_select():
    # label_map = {'Epithelial': 1,
    #              'Lymphocyte': 2,
    #              'Macrophage': 4,
    #              'Neutrophil': 3,
    #              }

    label_map = {'Epithelial': 1, 'Macrophage': 2, 'Neutrophil': 3, 'Lymphocyte': 4}
    # Root directory of the project
    ROOT_DIR = os.path.abspath("/home/pzsuen/MoNuSAC/")

    # Directory of images to run detection on
    IMAGES_SOURCE = os.path.join(ROOT_DIR, "image")
    MASKS_SOURCE = os.path.join(ROOT_DIR, "mask")
    CELLS_SOURCE = os.path.join(ROOT_DIR, "cell_masks")

    # Make new folders
    """
    Source File Tree:
        -- images
            -- TCGA-UZ-A9PO-01Z-00-DX1_1.png
            -- TCGA-UZ-A9PO-01Z-00-DX1_2.png
            -- ......
        -- masks
            -- TCGA-UZ-A9PO-01Z-00-DX1_1.png
            -- TCGA-UZ-A9PO-01Z-00-DX1_2.png
            -- ......
        -- cells
            -- TCGA-UZ-A9PO-01Z-00-DX1_1
                -- Epithelial
                    -- 1.png
                    -- 2.png
                    -- ......
                -- Macrophage
                -- Neutrophil
                -- Lymphocyte
            -- TCGA-UZ-A9PO-01Z-00-DX1_2
                -- Epithelial
                -- Macrophage
                -- Neutrophil
                -- Lymphocyte
    Destination File Tree:
        -- patches
            -- images
                -- TCGA-UZ-A9PO-01Z-00-DX1_1.png
                -- TCGA-UZ-A9PO-01Z-00-DX1_2.png
                -- ......
            -- masks
                -- TCGA-UZ-A9PO-01Z-00-DX1_1.png
                -- TCGA-UZ-A9PO-01Z-00-DX1_2.png
                -- ......
            -- cells
                -- TCGA-UZ-A9PO-01Z-00-DX1_1
                    -- Epithelial
                        -- 1.png
                        -- 2.png
                        -- ......
                    -- Macrophage
                    -- Neutrophil
                    -- Lymphocyte
                -- TCGA-UZ-A9PO-01Z-00-DX1_2
                    -- Epithelial
                    -- Macrophage
                    -- Neutrophil
                    -- Lymphocyte

    """
    IMAGES_DEST = os.path.join(ROOT_DIR, "patches", "images/")
    MASKS_DEST = os.path.join(ROOT_DIR, "patches", "masks/")
    CELLS_DEST = os.path.join(ROOT_DIR, "patches", "cells/")
    # Create destination folders
    create_directory(IMAGES_DEST)
    create_directory(MASKS_DEST)
    create_directory(MASKS_DEST)

    # Get all image file names
    image_fns = sorted(next(os.walk(IMAGES_SOURCE))[2])
    gt_fns = sorted(next(os.walk(MASKS_SOURCE))[2])

    assert len(image_fns) == len(gt_fns), "len(image_fns) != len(gt_fns)"

    # # Load a random image from the images folder
    # idx = random.randrange(len(image_fns))  # 94
    # print("Index: ", idx)
    #
    # image = skimage.io.imread(os.path.join(IMAGES_SOURCE, image_fns[idx]))
    # gt = skimage.io.imread(os.path.join(MASKS_SOURCE, gt_fns[idx]))
    #
    # assert image.shape[:2] == gt.shape, "Wrong image or ground truth!"
    # assert image.dtype == gt.dtype, "Wrong data types!"
    #
    # print(image.shape, gt.shape)
    #
    # val1 = gt.flatten()
    # print("Ground truth classes: ", np.unique(val1))
    # show_image_mask_5class(image, mask=gt)

    img_patches = None
    gt_patches = None

    # patch_size = (96, 96)
    # step = 16
    # img_patches = extract_patches(image, step, patch_size)
    # gt_patches = extract_patches(gt, step, patch_size)
    #
    # print('Patches shape: {}, {}'.format(img_patches.shape, gt_patches.shape))
    #
    # idxs = random.sample(range(0, len(img_patches)), 5)
    # # idxs = [random.randrange(len(img_patches)) for x in range(len(img_patches))]
    # print(idxs)
    # for img, msk in zip(img_patches[idxs], gt_patches[idxs]):
    #     print("Patch mask mean:", np.mean(msk))
    #
    #     # Set threshold for mask
    #     thresh = 0.1  # 0.099
    #     if np.mean(msk) < thresh:
    #         pass
    #
    #     else:
    #         show_image_mask_5class(img, msk)

    # Patch size and stride step
    patch_size = (256, 256)
    step = 128
    threshold = 0.1

    # Iterate over all image and masks
    ct = 0

    for img_path, gt_path in tqdm(zip(image_fns[:], gt_fns[:])):
        # print(img_path, gt_path)
        # img_path = 'TCGA-A2-A0ES-01Z-00-DX1_2.png'
        # gt_path = 'TCGA-A2-A0ES-01Z-00-DX1_2.png'
        patient_case_id = img_path.split('.')[0]
        print(patient_case_id)
        if len(patient_case_id.split('_')) == 2:  # TCGA-55-1594-01Z-00-DX1_001.png
            patient_id = patient_case_id.split('_')[0]
            case_id = patient_case_id.split('_')[1]
        else:  # TCGA-55-1594-01Z-00-DX1-001.png
            patient_id = '-'.join(patient_case_id.split('-')[:-1])
            case_i = patient_case_id.split('-')[-1]

        cell_root = os.path.join(CELLS_SOURCE, patient_id, patient_case_id)
        # if len(os.listdir(cell_root)) != 4:
        #     print(patient_case_id)

        cell_classes = ('Epithelial', 'Macrophage', 'Neutrophil', 'Lymphocyte')
        all_cell_dict = dict()
        # 遍历每个类别
        for cls in cell_classes:
            cell_path = os.path.join(cell_root, cls)
            if os.path.exists(cell_path):
                cell_fnames = glob(os.path.join(cell_path, '*.png'))
                if len(cell_fnames) > 0:
                    all_cells = []
                    # 遍历该类别下所有细胞
                    for cf in cell_fnames:
                        cell = read_nuclei(cf)
                        all_cells.append(cell)
                    all_cells = np.array(all_cells)

                    all_cell_dict[cls] = all_cells
            # else:
            #     all_cell_dict[cls] = None
        # print('\n$##########################################')
        # print(patient_case_id)
        # for k, v in all_cell_dict.items():
        #     print(k, v.shape)

        # Read image and ground truth
        image = skimage.io.imread(os.path.join(IMAGES_SOURCE, img_path))
        gt = skimage.io.imread(os.path.join(MASKS_SOURCE, gt_path))

        # # Extract patches
        img_patches = extract_patches(image, step, patch_size)
        gt_patches = extract_patches(gt, step, patch_size)
        # print(img_patches.shape)
        # print(gt_patches.shape)
        # print("############################################")
        # print(img_patches.shape, gt_patches.shape)
        cell_patches_dict = dict()
        for k, v in all_cell_dict.items():
            # print(k, v.shape)
            # cell_patches = None
            v = v.transpose((1, 2, 0))
            # print(v.shape)
            cell_patches = extract_patches(v, step, patch_size)
            # print(cell_patches.shape)
            cell_patches_dict[k] = cell_patches
            # print(cell_patches.shape)
        # print("############################################")

        # if len(gt_patches) == 0:
        #     print("Not included")

        for i in range(len(img_patches)):
            # for im, msk,cpd in zip(img_patches, gt_patches,cell_patches_dict):
            im = img_patches[i]
            msk = gt_patches[i]

            # Threshold
            if np.mean(msk) < threshold:
                pass

            else:
                # show image and mask with a certain probability
                # if np.random.rand(1) < 1:
                #     show_image_mask_5class(im, msk)

                # Save image patch
                save_nuclei(IMAGES_DEST + "{}.png".format(ct), im)
                # Save mask patch
                save_nuclei(MASKS_DEST + "{}.png".format(ct), msk)
                # Save cell patch

                for k, v in cell_patches_dict.items():
                    this_cell_path = os.path.join(CELLS_DEST, str(ct), k)
                    # print(v.shape)
                    cell_masks = v[i]
                    cell_masks = cell_masks.transpose(2, 0, 1)
                    ci = 1
                    for cm in cell_masks:
                        if np.sum(cm) > 0:
                            create_directory(this_cell_path)
                            save_nuclei(os.path.join(this_cell_path, str(ci) + '.png'), cm)
                            ci += 1
                    # create_directory(os.path.join(CELLS_DEST, str(ct), '1'.png))

                ct += 1

def generate_patches_from_1image(img_fname, IMAGE_SOURCE, MASK_SOURCE, CELLS_SOURCE, IMAGE_DEST, MASK_DEST, CELL_DEST,
                                 PATCH_SIZE=(256, 256), STEP=128, THRESHOLD=0.1):
    # Create destination folders
    create_directory(IMAGE_DEST)
    create_directory(MASK_DEST)
    create_directory(CELL_DEST)

    # print(img_path, gt_path)
    # img_path = 'TCGA-A2-A0ES-01Z-00-DX1_2.png'
    # gt_path = 'TCGA-A2-A0ES-01Z-00-DX1_2.png'
    patient_case_id = img_fname.split('.')[0]
    print(patient_case_id)

    if len(patient_case_id.split('_')) == 2:  # TCGA-55-1594-01Z-00-DX1_001.png
        patient_id = patient_case_id.split('_')[0]
        case_id = patient_case_id.split('_')[1]
    else:  # TCGA-55-1594-01Z-00-DX1-001.png
        patient_id = '-'.join(patient_case_id.split('-')[:-1])
        case_id = patient_case_id.split('-')[-1]
    # print("######################################################")
    # print(patient_id, case_id)
    new_patient_case_id = '_'.join([patient_id, case_id])

    # cell source path
    cell_root = os.path.join(CELLS_SOURCE, patient_id, patient_case_id)

    cell_classes = ('Epithelial', 'Macrophage', 'Neutrophil', 'Lymphocyte')
    all_cell_dict = dict()
    # 遍历每个类别
    for cls in cell_classes:
        cell_path = os.path.join(cell_root, cls)
        if os.path.exists(cell_path):
            cell_fnames = glob(os.path.join(cell_path, '*.png'))
            if len(cell_fnames) > 0:
                all_cells = []
                # 遍历该类别下所有细胞
                for cf in cell_fnames:
                    cell = read_nuclei(cf)
                    all_cells.append(cell)
                all_cells = np.array(all_cells)

                all_cell_dict[cls] = all_cells

    # Read image and ground truth
    image = skimage.io.imread(os.path.join(IMAGE_SOURCE, img_fname))
    gt = skimage.io.imread(os.path.join(MASK_SOURCE, img_fname))

    # # Extract patches
    img_patches = extract_patches(image, STEP, PATCH_SIZE)
    gt_patches = extract_patches(gt, STEP, PATCH_SIZE)
    cell_patches_dict = dict()
    for k, v in all_cell_dict.items():
        # print(k, v.shape)
        # cell_patches = None
        v = v.transpose((1, 2, 0))
        # print(v.shape)
        cell_patches = extract_patches(v, STEP, PATCH_SIZE)
        # print(cell_patches.shape)
        cell_patches_dict[k] = cell_patches

    ct = 0
    for i in range(len(img_patches)):
        # for im, msk,cpd in zip(img_patches, gt_patches,cell_patches_dict):

        patch_id = new_patient_case_id + '_' + str(ct)
        im = img_patches[i]
        msk = gt_patches[i]

        # Threshold
        if np.mean(msk) < THRESHOLD:
            pass

        else:

            # Save image patch
            save_nuclei(IMAGE_DEST + patch_id + '.png', im)
            # Save mask patch
            save_nuclei(MASK_DEST + patch_id + ".png", msk)
            # Save cell patch
            for k, v in cell_patches_dict.items():
                this_cell_path = os.path.join(CELL_DEST, patch_id, k)
                cell_masks = v[i]
                cell_masks = cell_masks.transpose(2, 0, 1)
                ci = 1
                for cm in cell_masks:
                    # if new_patient_case_id == 'TCGA-V1-A9O9-01Z-00-DX1_3':
                    #     print(np.sum(cm))
                    if np.sum(cm) > 0:
                        create_directory(this_cell_path)
                        save_nuclei(os.path.join(this_cell_path, str(ci) + '.png'), cm)
                        ci += 1

            ct += 1

def select_images_and_generate_patches():
    # label_map = {'Epithelial': 1,
    #              'Lymphocyte': 2,
    #              'Macrophage': 4,
    #              'Neutrophil': 3,
    #              }

    label_map = {'Epithelial': 1, 'Macrophage': 2, 'Neutrophil': 3, 'Lymphocyte': 4}
    # Root directory of the project
    ROOT_DIR = os.path.abspath("/home/pzsuen/MoNuSAC/")

    # Directory of images to run detection on
    IMAGE_SOURCE = os.path.join(ROOT_DIR, "image")
    MASK_SOURCE = os.path.join(ROOT_DIR, "mask")
    CELL_SOURCE = os.path.join(ROOT_DIR, "cell_masks")

    # Make new folders
    """
    Source File Tree:
        -- images
            -- TCGA-UZ-A9PO-01Z-00-DX1_1.png
            -- TCGA-UZ-A9PO-01Z-00-DX1_2.png
            -- ......
        -- masks
            -- TCGA-UZ-A9PO-01Z-00-DX1_1.png
            -- TCGA-UZ-A9PO-01Z-00-DX1_2.png
            -- ......
        -- cells
            -- TCGA-UZ-A9PO-01Z-00-DX1_1
                -- Epithelial
                    -- 1.png
                    -- 2.png
                    -- ......
                -- Macrophage
                -- Neutrophil
                -- Lymphocyte
            -- TCGA-UZ-A9PO-01Z-00-DX1_2
                -- Epithelial
                -- Macrophage
                -- Neutrophil
                -- Lymphocyte
    Destination File Tree:
        -- patches
            -- images
                -- TCGA-UZ-A9PO-01Z-00-DX1_1.png
                -- TCGA-UZ-A9PO-01Z-00-DX1_2.png
                -- ......
            -- masks
                -- TCGA-UZ-A9PO-01Z-00-DX1_1.png
                -- TCGA-UZ-A9PO-01Z-00-DX1_2.png
                -- ......
            -- cells
                -- TCGA-UZ-A9PO-01Z-00-DX1_1
                    -- Epithelial
                        -- 1.png
                        -- 2.png
                        -- ......
                    -- Macrophage
                    -- Neutrophil
                    -- Lymphocyte
                -- TCGA-UZ-A9PO-01Z-00-DX1_2
                    -- Epithelial
                    -- Macrophage
                    -- Neutrophil
                    -- Lymphocyte

    """
    """
    Source File Tree:
        -- images
            -- TCGA-UZ-A9PO-01Z-00-DX1_1.png
            -- TCGA-UZ-A9PO-01Z-00-DX1_2.png
            -- ......
        -- masks
            -- TCGA-UZ-A9PO-01Z-00-DX1_1.png
            -- TCGA-UZ-A9PO-01Z-00-DX1_2.png
            -- ......
        -- cells
            -- TCGA-UZ-A9PO-01Z-00-DX1_1
                -- Epithelial
                    -- 1.png
                    -- 2.png
                    -- ......
                -- Macrophage
                -- Neutrophil
                -- Lymphocyte
            -- TCGA-UZ-A9PO-01Z-00-DX1_2
                -- Epithelial
                -- Macrophage
                -- Neutrophil
                -- Lymphocyte
    Destination File Tree:
        -- patches
            -- images
                -- TCGA-UZ-A9PO-01Z-00-DX1_1.png
                -- TCGA-UZ-A9PO-01Z-00-DX1_2.png
                -- ......
            -- masks
                -- TCGA-UZ-A9PO-01Z-00-DX1_1.png
                -- TCGA-UZ-A9PO-01Z-00-DX1_2.png
                -- ......
            -- cells
                -- TCGA-UZ-A9PO-01Z-00-DX1_1
                    -- Epithelial
                        -- 1.png
                        -- 2.png
                        -- ......
                    -- Macrophage
                    -- Neutrophil
                    -- Lymphocyte
                -- TCGA-UZ-A9PO-01Z-00-DX1_2
                    -- Epithelial
                    -- Macrophage
                    -- Neutrophil
                    -- Lymphocyte

    """

    # Get all image file names
    image_fns = sorted(next(os.walk(IMAGE_SOURCE))[2])
    gt_fns = sorted(next(os.walk(MASK_SOURCE))[2])

    assert len(image_fns) == len(gt_fns), "len(image_fns) != len(gt_fns)"

    # Patch size and stride step
    PATCH_SIZE = (256, 256)
    STEP = 128
    THRESHOLD = 0.1

    # Iterate over all image and masks
    train_fnames, val_fnames, _, _ = train_test_split(image_fns, range(len(image_fns)), test_size=0.05,
                                                      random_state=2020, shuffle=True)
    for img_fname in tqdm(train_fnames):
        ROOT = os.path.join(ROOT_DIR, "patches_new", "train")
        IMAGE_DEST = os.path.join(ROOT, "images/")
        MASK_DEST = os.path.join(ROOT, "masks/")
        CELL_DEST = os.path.join(ROOT, "cells/")
        generate_patches_from_1image(img_fname, IMAGE_SOURCE, MASK_SOURCE, CELL_SOURCE, IMAGE_DEST, MASK_DEST,
                                     CELL_DEST, PATCH_SIZE=(256, 256), STEP=128, THRESHOLD=0.15)

    for img_fname in tqdm(val_fnames):
        ROOT = os.path.join(ROOT_DIR, "patches_new", "validation")
        IMAGE_DEST = os.path.join(ROOT, "images/")
        MASK_DEST = os.path.join(ROOT, "masks/")
        CELL_DEST = os.path.join(ROOT, "cells/")
        generate_patches_from_1image(img_fname, IMAGE_SOURCE, MASK_SOURCE, CELL_SOURCE, IMAGE_DEST, MASK_DEST,
                                     CELL_DEST, PATCH_SIZE=(256, 256), STEP=128, THRESHOLD=0.1)

    # convert train to coco
    TRAIN_ROOT = os.path.join(ROOT_DIR, "patches_new", "train")
    TRAIN_IMAGE_DEST = os.path.join(TRAIN_ROOT, "images/")
    image_fnames = os.listdir(TRAIN_IMAGE_DEST)
    convert2coco(image_fnames, TRAIN_ROOT, is_train=True)

    # convert test to coco
    VAL_ROOT = os.path.join(ROOT_DIR, "patches_new", "validation")
    VAL_IMAGE_DEST = os.path.join(VAL_ROOT, "images/")
    image_fnames = os.listdir(VAL_IMAGE_DEST)
    convert2coco(image_fnames, VAL_ROOT, is_train=False)

if __name__ == "__main__":
    select_images_and_generate_patches()


import scipy.io as sio
import sklearn.preprocessing as prep
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data as Data
import matplotlib.pyplot as plt
import torch.backends.cudnn as cudnn
import torch.nn.init as init

# torch.set_default_tensor_type(torch.DoubleTensor)

def init_weights(m):
    if type(m) == nn.Linear or type(m) == nn.Conv2d:
        init.xavier_normal_(m.weight)
        if m.bias is not None:
            init.zeros_(m.bias)

def load_data(fold):
    data = sio.loadmat(r'C:/Users/xhx20/Desktop/BNC-DGHL/BNC-DGHL/Datasets/ALLASD{}_NETFC_SG_Pear.mat'.format(fold+1))
    X = data['net']
    X_train = data['net_train']
    X_valid = data['net_valid']
    X_test = data['net_test']

    Y_train2 = data['phenotype_train'][:, 2]
    Y_valid2 = data['phenotype_valid'][:, 2]
    Y_test2 = data['phenotype_test'][:, 2]

    ln = nn.LayerNorm(normalized_shape=[90, 90], elementwise_affine=False)
    X_train = ln(torch.tensor(X_train)).view(-1, 1, 90, 90).type(torch.FloatTensor)
    X_valid = ln(torch.tensor(X_valid)).view(-1, 1, 90, 90).type(torch.FloatTensor)
    X_test = ln(torch.tensor(X_test)).view(-1, 1, 90, 90).type(torch.FloatTensor)
    Y_train2 = torch.tensor(Y_train2)
    Y_valid2 = torch.tensor(Y_valid2)
    Y_test2 = torch.tensor(Y_test2)

    return X, X_train, X_valid, X_test, Y_train2, Y_valid2, Y_test2

# class E2EBlock(torch.nn.Module):
#     '''E2Eblock.'''

#     def __init__(self, in_planes, planes, bias=False):
#         super(E2EBlock, self).__init__()
#         self.cnn1 = torch.nn.Conv2d(in_planes, planes, (1, 90), bias=bias)
#         self.cnn2 = torch.nn.Conv2d(in_planes, planes, (90, 1), bias=bias)

#     def forward(self, x):
#         a = self.cnn1(x)
#         b = self.cnn2(x)
#         return torch.cat([a] * 90, 3) + torch.cat([b] * 90, 2)

class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()
        self.E2E = nn.Conv2d(1, 32, kernel_size=(90, 1))
        # self.E2E = E2EBlock(1, 32, bias=True)
        self.E2N = nn.Conv2d(32, 64, kernel_size=(90, 1))
        self.N2G = nn.Conv2d(64, 128, kernel_size=(90, 1))
        self.fc1 = nn.Linear(128, 96)
        self.fc2 = nn.Linear(96, 2)
        self.softmax = nn.Softmax(dim=1)
        self.dropout = nn.Dropout(p=0.5)

    def forward(self, x):
        x = F.relu(self.E2E(x) + self.E2E(x).transpose(3, 2))
        x = self.dropout(x)
        x = F.relu(self.E2N(x).transpose(3, 2)*2)
        x = self.dropout(x)
        x = F.relu(self.N2G(x))
        x = self.dropout(x)
        x = x.view(x.size(0), -1)
        x = F.relu(self.fc1(x))
        x = self.dropout(x)
        x = self.softmax(self.fc2(x))
        return x

def train(train_loader, net, optimizer, loss_func):
    acc_sum = 0.
    loss_sum = 0.
    net.train()
    for step, (batch_x, batch_y) in enumerate(train_loader):
        if use_cuda:
            batch_x, batch_y = batch_x.cuda(), batch_y.cuda()
        optimizer.zero_grad()
        out = net(batch_x)
        loss_step = loss_func(out, batch_y)
        loss_step.backward()
        optimizer.step()
        acc_step = (out.argmax(dim=1) == batch_y).float().mean().item()
        loss_sum += loss_step.data.item()
        acc_sum += acc_step
        # print("step {}: loss: {}; acc: {}".format(epoch + 1, step + 1, loss_step.data.item(), acc_step))
    loss = loss_sum / len(train_loader)
    acc = acc_sum / len(train_loader)
    return loss, acc

def infer(loader, net, loss_func):
    acc_sum = 0.
    loss_sum = 0.
    net.eval()
    with torch.no_grad():
        for step, (batch_x, batch_y) in enumerate(loader):
            if use_cuda:
                batch_x, batch_y = batch_x.cuda(), batch_y.cuda()
            out = net(batch_x)
            loss_step = loss_func(out, batch_y)
            acc_step = (out.argmax(dim=1) == batch_y).float().mean().item()
            loss_sum += loss_step.data.item()
            acc_sum += acc_step
            # print("step {}: loss: {}; acc: {}".format(epoch + 1, step + 1, loss_step.data.item(), acc_step))
        loss = loss_sum / len(loader)
        acc = acc_sum / len(loader)
    return loss, acc

if __name__ == '__main__':
    # 定义参数
    fold = 5
    BATCH_SIZE = 96
    EPOCH = 500
    n_evaluation_epochs = 2
    n_patience = 50
    ax = [[] for _ in range(fold)]
    ay = [[] for _ in range(fold)]
    color_list = ['royalblue', 'darkorange', 'mediumpurple', 'forestgreen', 'firebrick']
    acc_list = []

    # 实例化网络，并且定义loss和优化器
    net = Net()

    use_cuda = torch.cuda.is_available()
    if use_cuda:
        net = net.cuda()
        #net = torch.nn.DataParallel(net, device_ids=[0])
        cudnn.benchmark = True

    loss_func = nn.CrossEntropyLoss()
    optimizer = torch.optim.SGD(net.parameters(), lr=1e-3, momentum=0.9, weight_decay=1e-2)  # 1e-4

    for f in range(fold):
        net.apply(init_weights)
        X, X_train, X_valid, X_test, Y_train2, Y_valid2, Y_test2 = load_data(f)
        train_dataset = Data.TensorDataset(X_train, Y_train2.long())
        train_loader = Data.DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True)

        valid_dataset = Data.TensorDataset(X_valid, Y_valid2.long())
        valid_loader = Data.DataLoader(dataset=valid_dataset, batch_size=BATCH_SIZE, shuffle=True)

        test_dataset = Data.TensorDataset(X_test, Y_test2.long())
        test_loader = Data.DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=True)

        best_valid_acc = 0.0
        min_valid_loss = 10000
        p = 0
        print("-----------------------------------------------------")
        print("fold:", f+1)
        print("-----------------------------------------------------")

        for epoch in range(EPOCH):
            train_loss, train_acc = train(train_loader, net, optimizer, loss_func)
            print("epoch {} : train_loss= {}; train_acc= {}".format(epoch + 1, train_loss, train_acc))
            valid_loss, valid_acc = infer(valid_loader, net, loss_func)
            print('valid_loss = ', valid_loss, 'valid_acc = ', valid_acc, "p =", p)

            if epoch % n_evaluation_epochs == 0:
                plt.figure(1)
                ax[f].append(epoch + 1)
                ay[f].append(np.mean(train_loss))
                plt.clf()
                for i in range(fold):
                    if len(ax[i]) > 0:
                        plt.plot(ax[i], ay[i], color=color_list[i], label=f'Fold {i + 1}')
                plt.legend()
                plt.pause(0.01)
                # plt.ioff()
            if valid_loss <= min_valid_loss:
                p = 0

                min_valid_loss = valid_loss
                best_valid_acc = valid_acc

            else:
                p += 1

                if p > n_patience:
                    test_loss, test_acc = infer(test_loader, net, loss_func)
                    print('test_loss = ', test_loss, 'test_acc = ', test_acc)
                    print("fold {} : train_acc= {}; valid_acc= {}; test_acc= {}".format(f + 1, train_acc, valid_acc, test_acc))
                    acc_list.append(test_acc)
                    break

    print("ave_test_acc =", round(sum(acc_list) / len(acc_list), 5))
    # 保存图像
    plt.savefig('plot.png')
    plt.ioff()
    plt.show()

"""
EPOCH = 500:
    lr = 1e-3, wd = 1e-2, drop_out = 0.4: ave_test_acc = 63.803%
    lr = 1e-3, wd = 1e-2, drop_out = 0.5: ave_test_acc = 64.679%
    lr = 1e-3, wd = 1e-2, drop_out = 0.6: ave_test_acc = 64.00%
"""


import ntpath
import json
import os
from inventory import Inventory
from random import randrange

inventory = Inventory()

print("Ingesting images...")

current_path = os.path.abspath(os.path.dirname(__file__))
input_path = os.path.join(current_path, "../front_images/")

output_path = os.path.join(current_path, "../web_monitor/output_front_images/")

for file in sorted(os.listdir(input_path)):
    #print((path + str(files)))
    if file.startswith('.') is False:
        image_to_scan_path = input_path + str(file)
        inventory.ingest_image(image_to_scan_path, output_path)

# Get currently owned products
print("Listing detected products...")
products = inventory.get_products()
for product in products:
    print("Name: " + product.name())
    print("Available? " + str(product.status()))

test_temp = randrange(20)
inventory.close_inventory(test_temp)
print("Close inventory at "+str(test_temp)+" degree and print yml")

print( inventory.inventory_as_string() )


import requests
from bs4 import BeautifulSoup as bs

response = requests.get('https://www.pycon.kr/2017/')
html = response.text

#print(response.text)

#soup = bs(html, 'html.parser')
soup = bs(html, 'lxml')

title = soup.select('body > div.frontpage > div.onsky > div > div > p')

print(title[0].text)


"""
Problem Statement : Codestdio
https://www.codingninjas.com/codestudio/problems/intersection-of-2-arrays_1082149?leftPanelTab=1?source=youtube&campaign=LoveBabbar_Codestudiovideo1&utm_source=youtube&utm_medium=affiliate&utm_campaign=LoveBabbar_Codestudiovideo1
TIME COMPLEXITY : O(max(M, N))
"""
import  sys
def findArrayIntersection(arr: list, n: int, brr: list, m: int):
    # Write your code here
    # Return a list containing all the common elements in arr and brr.
    intersec = []

    p = 0 ; q = 0
    flag = False

    while p < n and q < m:
        if arr[p]  == brr[q]:
            intersec.append(arr[p])
            p += 1 ; q += 1
            flag = True
        elif arr[p] < brr[q]:
            p += 1
        elif arr[p] > brr[q]:
            q += 1

    #if there at least one element is common
    if flag:
        return intersec

    return -1

def main():

    #will take fast input in console/terminal than input()
    t = int(sys.stdin.readline().rstrip())
    while t > 0:
        n, m = map(int, sys.stdin.readline().rstrip().split())
        arr = list(map(int, sys.stdin.readline().rstrip().split()))
        brr = list(map(int, sys.stdin.readline().rstrip().split()))
        print(findArrayIntersection(arr, n, brr, m))
        t -= 1

if __name__ == "__main__":
    main()

"""
CPP SOLUTION:

vector<int> findArrayIntersection(vector<int> &arr1, int n, vector<int> &arr2, int m)
{
	// Write your code here.
    int p = 0, q = 0;
    vector<int> intersec;

    while (p < n && q < m){

        if(arr1[p] == arr2[q]){

            intersec.push_back(arr1[p]);
            p++;
            q++;
        }
          else if(arr1[p] > arr2[q]){

              q++;
          }
          else{

              p++;
          }
    }

    return intersec;

"""