import os
import sys
import six
import copy
import socket

from eventlet import greenthread
from oslo_config import cfg
from oslo_log import log as logging

from ryu.lib import dpid as dpid_lib
from ryu.lib import hub
from ryu.ofproto import ether
from ryu.ofproto import inet

from daolicontroller import exception
from daolicontroller.lib import BMAX
from daolicontroller.lib import FILTERS
from daolicontroller.lib import PORT_MIN, PORT_MAX
from daolicontroller.lib import PR_FIREWALL, PR_INTERNET
from daolicontroller.objects import HashPort
from daolicontroller.openstack.common import threadgroup
from daolicontroller.openstack.common import timeutils
from daolicontroller.services.base import PacketBase

port_opts = [
    cfg.ListOpt('static_ports', default=[],
                help='The static port range'),
    cfg.IntOpt('port_timeout', default=10,
               help='The timeout value of a port'),
    cfg.IntOpt('threads', default=10000,
               help='The threads number'),
    cfg.FloatOpt('threshold', default=0.8,
                 help='The threshold of port set'),
]

CONF = cfg.CONF
LOG = logging.getLogger(__name__)
CONF.register_opts(port_opts)

class PacketIPv4(PacketBase):
    priority = 1

    def __init__(self, manager, ryuapp):
        super(PacketIPv4, self).__init__(manager, ryuapp)
        self.hashport = HashPort()
        self.static_ports = CONF.static_ports

        self.tg = threadgroup.ThreadGroup(CONF.threads)
        self.tg.add_thread(self._port_clean)

    def _port_clean(self):
        while True:
            greenthread.sleep(CONF.port_timeout)
            for key in self.hashport.keys():
                val = self.hashport.get(key)
                if val and timeutils.is_older_than(val, CONF.port_timeout):
                    self.hashport.remove(key)

    def filter_port(self, port):
        # When the threshold is reached, reset port set
        amount = int((PORT_MAX - PORT_MIN) * CONF.threshold)
        if len(self.hashport) > amount:
            self.hashport.clear()

        while True:
            if port <= PORT_MIN:
                port = PORT_MAX - port
                continue

            if port >= PORT_MAX:
                port = PORT_MAX - port / 2
                continue

            if port in FILTERS or self.hashport.has_key(port):
                port += 1
                continue

            self.hashport.set(port, timeutils.utcnow())

            return port

    def _in_out_kwargs(self, pkt_ipv4, pkt_tp, ofp_set, src_port, dst_port=None):
        if pkt_ipv4.proto == inet.IPPROTO_TCP:
            inkwargs = {'tcp_src': pkt_tp.dst_port,
                        'tcp_dst': pkt_tp.src_port}
            outkwargs = {'tcp_src': pkt_tp.src_port,
                         'tcp_dst': pkt_tp.dst_port}

            mod_inkwargs = {'tcp_dst': src_port}
            mod_outkwargs = {'tcp_src': src_port}

            if dst_port is not None:
                mod_inkwargs['tcp_src'] = dst_port
                mod_outkwargs['tcp_dst'] = dst_port

        elif pkt_ipv4.proto == inet.IPPROTO_UDP:
            inkwargs = {'udp_src': pkt_tp.dst_port,
                        'udp_dst': pkt_tp.src_port}
            outkwargs = {'udp_src': pkt_tp.src_port,
                         'udp_dst': pkt_tp.dst_port}

            mod_inkwargs = {'udp_dst': src_port}
            mod_outkwargs = {'udp_src': src_port}

            if dst_port is not None:
                mod_inkwargs['udp_src'] = dst_port
                mod_outkwargs['udp_dst'] = dst_port

        else:
            #outkwargs = inkwargs = {'icmpv4_id': pkt_tp.id}
            outkwargs = {'icmpv4_id': pkt_tp.id}
            inkwargs = {'icmpv4_id': socket.ntohs(pkt_tp.id)}
            mod_inkwargs = mod_outkwargs = {'icmpv4_id': src_port}

        return (inkwargs, outkwargs, mod_inkwargs, mod_outkwargs)

    def _in_out_keys(self, pkt_ipv4, pkt_tp, ofp_set, src_port, dst_port=None):
        if pkt_ipv4.proto == inet.IPPROTO_TCP:
            inkeys = [ofp_set(tcp_src=pkt_tp.dst_port),
                      ofp_set(tcp_dst=pkt_tp.src_port)]
            outkeys = [ofp_set(tcp_src=pkt_tp.src_port),
                       ofp_set(tcp_dst=pkt_tp.dst_port)]

            mod_inkeys = [ofp_set(tcp_dst=src_port)]
            mod_outkeys = [ofp_set(tcp_src=src_port)]

            if dst_port is not None:
                mod_inkeys.append(ofp_set(tcp_src=dst_port))
                mod_outkeys.append(ofp_set(tcp_dst=dst_port))

        elif pkt_ipv4.proto == inet.IPPROTO_UDP:
            inkeys = [ofp_set(udp_src=pkt_tp.dst_port),
                      ofp_set(udp_dst=pkt_tp.src_port)]
            outkeys = [ofp_set(udp_src=pkt_tp.src_port),
                       ofp_set(udp_dst=pkt_tp.dst_port)]

            mod_inkeys = [ofp_set(udp_dst=src_port)]
            mod_outkeys = [ofp_set(udp_src=src_port)]

            if dst_port is not None:
                mod_inkeys.append(ofp_set(udp_src=dst_port))
                mod_outkeys.append(ofp_set(udp_dst=dst_port))
        else:
            #inkeys = [ofp_set(icmpv4_id=pkt_tp.id)]
            #outkeys = inkeys
            inkeys = [ofp_set(icmpv4_id=socket.ntohs(pkt_tp.id))]
            outkeys = [ofp_set(icmpv4_id=pkt_tp.id)]

            #mod_inkeys = [ofp_set(icmpv4_id=src_port)]
            mod_inkeys = [ofp_set(icmpv4_id=socket.ntohs(src_port))]

            if dst_port is not None:
                mod_inkeys.append(ofp_set(icmpv4_id=dst_port))

            mod_outkeys = mod_inkeys

        return (inkeys, outkeys, mod_inkeys, mod_outkeys)

    def setup_ip_flow(self, msg, dp, in_port, src_key, pkt_ether, pkt_ipv4, pkt_tp,
                      src, src_gateway, dst, dst_gateway,
                      priority=PR_INTERNET):
        ofp, ofp_parser, ofp_set, ofp_out = self.ofp_get(dp)
        liport = self.port_get(dp, devname=src_gateway['int_dev']).port_no

        rdp = self.ryuapp.dps[dpid_lib.str_to_dpid(dst_gateway['datapath_id'])]
        rofp, rofp_parser, rofp_set, rofp_out = self.ofp_get(rdp)
        riport = self.port_get(rdp, devname=dst_gateway['int_dev']).port_no

        server_port = self.port_get(rdp, dst.devname)
        if not server_port:
            return

        port_len = len(self.static_ports)

        if port_len == 0:
            port = None
        else:
            port = self.static_ports[pkt_ipv4.identification % port_len]

        inkeys, outkeys, mod_inkeys, mod_outkeys = self._in_out_keys(
                pkt_ipv4, pkt_tp, ofp_set, src_key, port)

        inkwargs, outkwargs, mod_inkwargs, mod_outkwargs = self._in_out_kwargs(
                pkt_ipv4, pkt_tp, ofp_set, src_key, port)

        if pkt_ether.dst == dst.mac_address:
            dst_srcmac = src.mac_address
        else:
            dst_srcmac = dst_gateway['vint_mac']

        if src_gateway['idc_id'] == dst_gateway['idc_id']:
            output_local_actions = [
                    ofp_set(eth_src=src_gateway['int_mac']),
                    ofp_set(eth_dst=dst_gateway['int_mac']),
                    ofp_set(ipv4_src=src_gateway['int_ip']),
                    ofp_set(ipv4_dst=dst_gateway['int_ip'])]

            output_local_actions.extend(mod_outkeys)
            output_local_actions.append(ofp_out(liport))

            input_local_match = ofp_parser.OFPMatch(
                    in_port=liport,
                    eth_type=ether.ETH_TYPE_IP,
                    eth_dst=src_gateway['int_mac'],
                    ip_proto=pkt_ipv4.proto,
                    ipv4_src=dst_gateway['int_ip'],
                    ipv4_dst=src_gateway['int_ip'],
                    **mod_inkwargs)

            input_remote_match = rofp_parser.OFPMatch(
                    in_port=riport,
                    eth_type=ether.ETH_TYPE_IP,
                    eth_dst=dst_gateway['int_mac'],
                    ip_proto=pkt_ipv4.proto,
                    ipv4_src=src_gateway['int_ip'],
                    ipv4_dst=dst_gateway['int_ip'],
                    **mod_outkwargs)

            output_remote_actions = [
                    rofp_set(eth_src=dst_gateway['int_mac']),
                    rofp_set(eth_dst=src_gateway['int_mac']),
                    rofp_set(ipv4_src=dst_gateway['int_ip']),
                    rofp_set(ipv4_dst=src_gateway['int_ip'])]

            output_remote_actions.extend(mod_inkeys)
            output_remote_actions.append(rofp_out(riport))
        else:
            src_gateways = self.db.gateway_get_by_idc(src_gateway['idc_id'])
            dst_gateways = self.db.gateway_get_by_idc(dst_gateway['idc_id'])

            if not src_gateways or not dst_gateways:
                return

            src_dps = dict((g.datapath_id, g) for g in src_gateways)
            dst_dps = dict((g.datapath_id, g) for g in dst_gateways)

            lgdp_id = (src_gateways[pkt_ipv4.identification % len(src_gateways)]
                       if src_gateway.datapath_id not in src_dps else None)
            rgdp_id = (dst_gateways[pkt_ipv4.identification % len(dst_gateways)]
                       if dst_gateway.datapath_id not in dst_dps else None)

            input_remote_match, output_remote_actions = self._remote_gateway(
                    rdp, in_port, pkt_ether, pkt_ipv4, src_gateway, dst_gateway,
                    mod_inkeys, mod_inkwargs, mod_outkwargs,
                    riport=riport, lgdp_id=lgdp_id, rgdp_id=rgdp_id)

            input_local_match, output_local_actions = self._local_gateway(
                    dp, pkt_ether, pkt_ipv4, src_gateway, dst_gateway,
                    mod_outkeys, mod_inkwargs, mod_outkwargs,
                    liport=liport, lgdp_id=lgdp_id, rgdp_id=rgdp_id)

        output_remote_match, input_remote_actions = self._remote_host(rdp,
                server_port.port_no, pkt_ipv4, src, dst, dst_srcmac, outkeys,
                **inkwargs)

        output_local_match, input_local_actions = self._local_host(dp,
                in_port, pkt_ether, pkt_ipv4, inkeys, **outkwargs)

        self.add_flow(rdp, input_remote_match, input_remote_actions, priority=priority)
        self.add_flow(rdp, output_remote_match, output_remote_actions, priority=priority)

        self.add_flow(dp, input_local_match, input_local_actions, priority=priority)
        self.add_flow(dp, output_local_match, output_local_actions, priority=priority)
        self.packet_out(msg, dp, output_local_actions)

    def _redirect(self, msg, dp, in_port, pkt_ether, pkt_ipv4, output, **kwargs):
        ofp, ofp_parser, ofp_set, ofp_out = self.ofp_get(dp)
        match = ofp_parser.OFPMatch(
                in_port=in_port, eth_type=ether.ETH_TYPE_IP,
                ip_proto=pkt_ipv4.proto,
                eth_src=pkt_ether.src, eth_dst=pkt_ether.dst,
                ipv4_src=pkt_ipv4.src, ipv4_dst=pkt_ipv4.dst,
                **kwargs)

        actions = [ofp_parser.OFPActionOutput(output)]

        self.add_flow(dp, match, actions)
        self.packet_out(msg, dp, actions)

        return True

    def firewall(self, msg, dp, in_port, key, pkt_ether, pkt_ipv4, pkt_tp, gateway):
        ofp, ofp_parser, ofp_set, ofp_out = self.ofp_get(dp)

        inkwargs, outkwargs, _, _ = self._in_out_kwargs(
                    pkt_ipv4, pkt_tp, ofp_set, key)

        port = self.port_get(dp, devname=gateway.ext_dev)
        if not port:
            raise exception.DevicePortNotFound(device=gateway.ext_dev)

        if in_port == port.port_no:
            if pkt_ipv4.dst != gateway.ext_ip and \
                    pkt_ipv4.dst.rpartition('.')[2] != str(BMAX):
                raise exception.IPAddressNotMatch(address=pkt_ipv4.dst)

            # ICMP Packet redirect to physical host
            if pkt_ipv4.proto != inet.IPPROTO_ICMP:
                server = self.db.firewall_get_by_packet(gateway.hostname, pkt_tp.dst_port)
                if server:
                    self._firewall(msg, dp, in_port, pkt_ether, pkt_ipv4, pkt_tp, server, gateway)
                    return True

            return self._redirect(msg, dp, in_port, pkt_ether, pkt_ipv4,
                                  dp.ofproto.OFPP_LOCAL, **outkwargs)

        if gateway.int_dev != gateway.ext_dev:
            int_port = self.port_get(dp, devname=gateway.int_dev)
            tap_port = self.port_get(dp, devname=gateway.vint_dev)

            if not int_port or not tap_port:
                device = gateway.int_dev if not int_port else gateway.vint_dev
                raise exception.DevicePortNotFound(device=device)

            if in_port == int_port.port_no:
                if pkt_ipv4.dst != gateway.int_ip and \
                        pkt_ipv4.dst.rpartition('.')[2] != str(BMAX):
                    raise exception.IPAddressNotMatch(address=pkt_ipv4.dst)

                return self._redirect(msg, dp, in_port, pkt_ether, pkt_ipv4,
                                      tap_port.port_no, **outkwargs)

            if in_port == tap_port.port_no:
                if pkt_ipv4.src != gateway.int_ip:
                    raise exception.IPAddressNotMatch(address=pkt_ipv4.src)

                return self._redirect(msg, dp, in_port, pkt_ether, pkt_ipv4,
                                      int_port.port_no, **outkwargs)

        if in_port == dp.ofproto.OFPP_LOCAL:
            if pkt_ipv4.src != gateway['ext_ip']:
                raise exception.IPAddressNotMatch(address=pkt_ipv4.src)

            return self._redirect(msg, dp, in_port, pkt_ether, pkt_ipv4,
                                  port.port_no, **outkwargs)

    def _firewall(self, msg, dp, in_port, pkt_ether, pkt_ipv4, pkt_tp, server, gateway,
                  priority=PR_FIREWALL):
        ofp, ofp_parser, ofp_set, ofp_out = self.ofp_get(dp)

        if pkt_ipv4.proto == inet.IPPROTO_TCP:
            output_key = [ofp_set(tcp_src=pkt_tp.dst_port)]
            input_key = [ofp_set(tcp_dst=server.service_port)]
            output_kwargs = {
                'tcp_src': server.service_port,
                'tcp_dst': pkt_tp.src_port}
            input_kwargs = {
                'tcp_src': pkt_tp.src_port,
                'tcp_dst': server.service_port}
        else:
            output_key = [ofp_set(udp_src=pkt_tp.dst_port)]
            input_key = [ofp_set(udp_dst=server.service_port)]
            output_kwargs = {
                'udp_src': server.service_port,
                'udp_dst': pkt_tp.src_port}
            input_kwargs = {
                'udp_src': pkt_tp.src_port,
                'udp_dst': server.service_port}

        inkwargs, outkwargs = self._in_out_kwargs(
                pkt_ipv4, pkt_tp, ofp_set, None)[:2] ##

        input_match, output_actions = self._local_host(dp, in_port,
                pkt_ether, pkt_ipv4, output_key, **outkwargs)

        def server_port_wrap(dpath):
            server_port = self.port_get(dpath, server.devname)

            # Virtual machine may be poweroff
            if not server_port:
                raise exception.DevicePortNotFound(device=server.devname)

            return server_port

        if server.host == gateway.hostname:
            server_port = server_port_wrap(dp)

            input_actions = [
                    ofp_set(eth_src=gateway['vint_mac']),
                    ofp_set(eth_dst=server.mac_address),
                    ofp_set(ipv4_dst=server.address)]

            input_actions.extend(input_key)
            input_actions.append(ofp_out(server_port.port_no))

            output_match = ofp_parser.OFPMatch(
                    in_port=server_port.port_no,
                    eth_type=ether.ETH_TYPE_IP,
                    ip_proto=pkt_ipv4.proto,
                    ipv4_src=server.address,
                    ipv4_dst=pkt_ipv4.src,
                    **output_kwargs)
        else:
            server_gateway = self.gateway_get(server.host)
            ldp = self.ryuapp.dps[dpid_lib.str_to_dpid(server_gateway.datapath_id)]
            lofp, lofp_parser, lofp_set, lofp_out = self.ofp_get(ldp)
            liport = self.port_get(ldp, devname=server_gateway.int_dev).port_no
            server_port = server_port_wrap(ldp)

            giport = self.port_get(dp, devname=gateway['int_dev']).port_no

            input_actions = [
                    ofp_set(eth_src=gateway['int_mac']),
                    ofp_set(eth_dst=server_gateway.int_mac),
                    ofp_set(ipv4_src=gateway['int_ip']),
                    ofp_set(ipv4_dst=server_gateway.int_ip)]

            input_actions.extend(input_key)
            input_actions.append(ofp_out(giport))

            output_match = ofp_parser.OFPMatch(
                    in_port=giport,
                    eth_type=ether.ETH_TYPE_IP,
                    eth_dst=gateway['int_mac'],
                    ip_proto=pkt_ipv4.proto,
                    ipv4_src=server_gateway.int_ip,
                    ipv4_dst=gateway['int_ip'],
                    **output_kwargs)

            input_local_match = lofp_parser.OFPMatch(
                    in_port=liport,
                    eth_type=ether.ETH_TYPE_IP,
                    eth_dst=server_gateway.int_mac,
                    ip_proto=pkt_ipv4.proto,
                    ipv4_src=gateway['int_ip'],
                    ipv4_dst=server_gateway.int_ip,
                    **input_kwargs)

            input_local_actions = [
                    lofp_set(eth_src=server_gateway.vint_mac),
                    lofp_set(eth_dst=server.mac_address),
                    lofp_set(ipv4_src=pkt_ipv4.src),
                    lofp_set(ipv4_dst=server.address),
                    lofp_out(server_port.port_no)]

            output_local_match = lofp_parser.OFPMatch(
                    in_port=server_port.port_no,
                    eth_type=ether.ETH_TYPE_IP,
                    eth_src=server.mac_address,
                    ip_proto=pkt_ipv4.proto,
                    ipv4_src=server.address,
                    ipv4_dst=pkt_ipv4.src,
                    **output_kwargs)

            output_local_actions = [
                    lofp_set(eth_src=server_gateway.int_mac),
                    lofp_set(eth_dst=gateway['int_mac']),
                    lofp_set(ipv4_src=server_gateway.int_ip),
                    lofp_set(ipv4_dst=gateway['int_ip']),
                    ofp_out(liport)]

            self.add_flow(ldp, input_local_match, input_local_actions,
                          priority=priority)
            self.add_flow(ldp, output_local_match, output_local_actions,
                          priority=priority)

        self.add_flow(dp, output_match, output_actions, priority=priority)
        self.add_flow(dp, input_match, input_actions, priority=priority)
        #self.packet_out(msg, dp, input_actions)

    def _local_host(self, dp, in_port, pkt_ether, pkt_ipv4,
                    inkeys, **kwargs):
        ofp, ofp_parser, ofp_set, ofp_out = self.ofp_get(dp)

        output_match = ofp_parser.OFPMatch(
                in_port=in_port,
                eth_type=ether.ETH_TYPE_IP,
                eth_src=pkt_ether.src,
                ip_proto=pkt_ipv4.proto,
                ipv4_src=pkt_ipv4.src,
                ipv4_dst=pkt_ipv4.dst,
                **kwargs)

        input_actions = [
                ofp_set(eth_src=pkt_ether.dst),
                ofp_set(eth_dst=pkt_ether.src),
                ofp_set(ipv4_src=pkt_ipv4.dst),
                ofp_set(ipv4_dst=pkt_ipv4.src)]

        input_actions.extend(inkeys)
        input_actions.append(ofp_out(in_port))

        return (output_match, input_actions)

    def _local_gateway(self, dp, pkt_ether, pkt_ipv4, src_gateway, dst_gateway,
                       outkeys, mod_inkwargs, mod_outkwargs,
                       liport=None, lgdp_id=None, rgdp_id=None, public=False):
        ofp, ofp_parser, ofp_set, ofp_out = self.ofp_get(dp)

        if not public:
            if rgdp_id is None:
                ipv4_dst = dst_gateway.vext_ip
            else:
                if isinstance(lgdp_id, six.string_types):
                    remote_gateway = self.gateway_get(rgdp_id)
                else:
                    remote_gateway = rgdp_id
                ipv4_dst = remote_gateway.vext_ip
        else:
            ipv4_dst = pkt_ipv4.dst

        if lgdp_id is not None:
            if liport is None:
                liport = self.port_get(dp, devname=src_gateway.int_dev).port_no

            if isinstance(lgdp_id, six.string_types):
                local_gateway = self.gateway_get(lgdp_id)
                lgdp = self.ryuapp.dps[dpid_lib.str_to_dpid(lgdp_id)]
            else:
                local_gateway = lgdp_id
                lgdp = self.ryuapp.dps[dpid_lib.str_to_dpid(lgdp_id.datapath_id)]

            lgofp, lgofp_parser, lgofp_set, lgofp_out = self.ofp_get(lgdp)
            lgiport = self.port_get(lgdp, devname=local_gateway.int_dev).port_no
            lgeport = self.port_get(lgdp, devname=local_gateway.ext_dev).port_no

            input_match = ofp_parser.OFPMatch(
                    in_port=liport,
                    eth_type=ether.ETH_TYPE_IP,
                    eth_dst=src_gateway.int_mac,
                    ip_proto=pkt_ipv4.proto,
                    ipv4_src=local_gateway.int_ip,
                    ipv4_dst=src_gateway.int_ip,
                    **mod_inkwargs)

            output_actions = [
                    ofp_set(eth_src=src_gateway.int_mac),
                    ofp_set(eth_dst=local_gateway.int_mac),
                    ofp_set(ipv4_src=src_gateway.int_ip),
                    ofp_set(ipv4_dst=local_gateway.int_ip)]

            output_actions.extend(outkeys)
            output_actions.append(ofp_out(liport))

            input_gateway_match = lgofp_parser.OFPMatch(
                    in_port=lgeport,
                    eth_type=ether.ETH_TYPE_IP,
                    eth_dst=local_gateway.ext_mac,
                    ip_proto=pkt_ipv4.proto,
                    ipv4_src=ipv4_dst,
                    ipv4_dst=local_gateway.ext_ip,
                    **mod_inkwargs)

            input_gateway_actions = [
                    lgofp_set(eth_src=local_gateway.int_mac),
                    lgofp_set(eth_dst=src_gateway.int_mac),
                    lgofp_set(ipv4_src=local_gateway.int_ip),
                    lgofp_set(ipv4_dst=src_gateway.int_ip),
                    lgofp_out(lgiport)]

            output_gateway_match = lgofp_parser.OFPMatch(
                    in_port=lgiport,
                    eth_type=ether.ETH_TYPE_IP,
                    eth_dst=local_gateway.int_mac,
                    ip_proto=pkt_ipv4.proto,
                    ipv4_src=src_gateway.int_ip,
                    ipv4_dst=local_gateway.int_ip,
                    **mod_outkwargs)

            output_gateway_actions = [
                    lgofp_set(eth_src=local_gateway.ext_mac),
                    lgofp_set(eth_dst=local_gateway.idc_mac),
                    lgofp_set(ipv4_src=local_gateway.ext_ip),
                    lgofp_set(ipv4_dst=ipv4_dst),
                    lgofp_out(lgeport)]

            self.add_flow(lgdp, input_gateway_match, input_gateway_actions,
                          priority=PR_INTERNET)
            self.add_flow(lgdp, output_gateway_match, output_gateway_actions,
                          priority=PR_INTERNET)
        else:
            leport = self.port_get(dp, devname=src_gateway.ext_dev).port_no

            input_match = ofp_parser.OFPMatch(
                    in_port=leport,
                    eth_type=ether.ETH_TYPE_IP,
                    eth_dst=src_gateway.ext_mac,
                    ip_proto=pkt_ipv4.proto,
                    ipv4_src=ipv4_dst,
                    ipv4_dst=src_gateway.ext_ip,
                    **mod_inkwargs)

            output_actions = [
                    ofp_set(eth_src=src_gateway.ext_mac),
                    ofp_set(eth_dst=src_gateway.idc_mac),
                    ofp_set(ipv4_src=src_gateway.ext_ip),
                    ofp_set(ipv4_dst=ipv4_dst)]

            output_actions.extend(outkeys)
            output_actions.append(ofp_out(leport))

        return (input_match, output_actions)

    def _remote_host(self, dp, in_port, pkt_ipv4, src, dst, dst_srcmac,
                     outkeys, **kwargs):
        ofp, ofp_parser, ofp_set, ofp_out = self.ofp_get(dp)

        output_match = ofp_parser.OFPMatch(
                in_port=in_port,
                eth_type=ether.ETH_TYPE_IP,
                eth_src=dst.mac_address,
                ip_proto=pkt_ipv4.proto,
                ipv4_src=dst.address,
                ipv4_dst=src.address,
                **kwargs)

        input_actions = [ofp_set(eth_src=dst_srcmac),
                         ofp_set(eth_dst=dst.mac_address),
                         ofp_set(ipv4_src=src.address),
                         ofp_set(ipv4_dst=dst.address)]

        input_actions.extend(outkeys)
        input_actions.append(ofp_out(in_port))

        return (output_match, input_actions)

    def _remote_gateway(self, dp, in_port, pkt_ether, pkt_ipv4, src_gateway, dst_gateway,
                        inkeys, mod_inkwargs, mod_outkwargs,
                        riport=None, lgdp_id=None, rgdp_id=None):
        ofp, ofp_parser, ofp_set, ofp_out = self.ofp_get(dp)

        if lgdp_id is None:
            ipv4_src = src_gateway.vext_ip
        else:
            if isinstance(lgdp_id, six.string_types):
                local_gateway = self.gateway_get(lgdp_id)
            else:
                local_gateway = lgdp_id
            ipv4_src = local_gateway.vext_ip

        if rgdp_id is not None:
            if riport is None:
                riport = self.port_get(dp, devname=dst_gateway.int_dev).port_no

            if isinstance(rgdp_id, six.string_types):
                remote_gateway = self.gateway_get(rgdp_id)
                rgdp = self.ryuapp.dps[dpid_lib.str_to_dpid(rgdp_id)]
            else:
                remote_gateway = rgdp_id
                rgdp = self.ryuapp.dps[dpid_lib.str_to_dpid(rgdp_id.datapath_id)]

            rgofp, rgofp_parser, rgofp_set, rgofp_out = self.ofp_get(rgdp)
            rgiport = self.port_get(rgdp, devname=remote_gateway.int_dev).port_no
            rgeport = self.port_get(rgdp, devname=remote_gateway.ext_dev).port_no

            input_gateway_match = rgofp_parser.OFPMatch(
                    in_port=rgeport,
                    eth_type=ether.ETH_TYPE_IP,
                    eth_dst=remote_gateway.ext_mac,
                    ip_proto=pkt_ipv4.proto,
                    ipv4_src=ipv4_src,
                    ipv4_dst=remote_gateway.ext_ip,
                    **mod_outkwargs)##

            input_gateway_actions = [
                    rgofp_set(eth_src=remote_gateway.int_mac),
                    rgofp_set(eth_dst=dst_gateway.int_mac),
                    rgofp_set(ipv4_src=remote_gateway.int_ip),
                    rgofp_set(ipv4_dst=dst_gateway.int_ip),
                    rgofp_out(rgiport)]

            output_gateway_match = rgofp_parser.OFPMatch(
                    in_port=rgiport,
                    eth_type=ether.ETH_TYPE_IP,
                    eth_dst=remote_gateway.int_mac,
                    ip_proto=pkt_ipv4.proto,
                    ipv4_src=dst_gateway.int_ip,
                    ipv4_dst=remote_gateway.int_ip,
                    **mod_inkwargs)##

            output_gateway_actions = [
                    rgofp_set(eth_src=remote_gateway.ext_mac),
                    rgofp_set(eth_dst=remote_gateway.idc_mac),
                    rgofp_set(ipv4_src=remote_gateway.ext_ip),
                    rgofp_set(ipv4_dst=ipv4_src),
                    rgofp_out(rgeport)]

            input_match = ofp_parser.OFPMatch(
                    in_port=riport,
                    eth_type=ether.ETH_TYPE_IP,
                    eth_dst=dst_gateway.int_mac,
                    ip_proto=pkt_ipv4.proto,
                    ipv4_src=remote_gateway.int_ip,
                    ipv4_dst=dst_gateway.int_ip,
                    **mod_outkwargs) ##

            output_actions =[
                    ofp_set(eth_src=dst_gateway.int_mac),
                    ofp_set(eth_dst=remote_gateway.int_mac),
                    ofp_set(ipv4_src=dst_gateway.int_ip),
                    ofp_set(ipv4_dst=remote_gateway.int_ip)]

            if inkeys is not None:
                output_actions.extend(inkeys)

            output_actions.append(ofp_out(riport))

            self.add_flow(rgdp, input_gateway_match, input_gateway_actions,
                          priority=PR_INTERNET)
            self.add_flow(rgdp, output_gateway_match, output_gateway_actions,
                          priority=PR_INTERNET)
        else:
            report = self.port_get(dp, devname=dst_gateway.ext_dev).port_no

            input_match = ofp_parser.OFPMatch(
                    in_port=report,
                    eth_type=ether.ETH_TYPE_IP,
                    eth_dst=dst_gateway.ext_mac,
                    ip_proto=pkt_ipv4.proto,
                    ipv4_src=ipv4_src,
                    ipv4_dst=dst_gateway.ext_ip,
                    **mod_outkwargs) ##

            output_actions = [
                    ofp_set(eth_src=dst_gateway.ext_mac),
                    ofp_set(eth_dst=dst_gateway.idc_mac),
                    ofp_set(ipv4_src=dst_gateway.ext_ip),
                    ofp_set(ipv4_dst=ipv4_src)]

            if inkeys is not None:
                output_actions.extend(inkeys)

            output_actions.append(ofp_out(report))

        return (input_match, output_actions)

    def run(self, msg, pkt_ether, pkt_ipv4, pkt_tp, src_gateway, **kwargs):
        dp = msg.datapath
        in_port = msg.match['in_port']

        if pkt_ipv4.proto == inet.IPPROTO_ICMP:
            src_key = socket.ntohs(self.filter_port(pkt_tp.id))
            #src_key = pkt_tp.id
        else:
            src_key = self.filter_port(pkt_tp.src_port)
            #src_key = pkt_tp.src_port

        try:
            ret = self.firewall(msg, dp, in_port, src_key, pkt_ether,
                                pkt_ipv4, pkt_tp, src_gateway)
            if ret:
                return True
        except (exception.DevicePortNotFound, exception.IPAddressNotMatch) as e:
            LOG.debug(e.message)
            return False

        servers = self.db.server_get_by_mac(pkt_ether.src, pkt_ipv4.dst)
        src, dst, has_more = (servers['src'], servers['dst'], servers['has_more'])

        if src is None or (dst is not None and not has_more):
            return False

        # Create nat flow if 'dst' instance is not null, else public flow
        if dst is not None:
            if src.host == dst.host:
                ofp, ofp_parser, ofp_set, ofp_out = self.ofp_get(dp)
                submac = (pkt_ether.dst if pkt_ether.dst != dst.mac_address
                                            else None)

                def same_host_flow(smac, dmac, sip, dip, iport, oport):
                    match = ofp_parser.OFPMatch(
                            in_port=iport,
                            eth_type=ether.ETH_TYPE_IP,
                            eth_src=smac,
                            ipv4_src=sip,
                            ipv4_dst=dip)
                    actions = ([ofp_set(eth_src=submac)] if submac is not None
                                                             else [])
                    actions += [ofp_set(eth_dst=dmac), ofp_out(oport)]
                    self.add_flow(dp, match, actions, priority=2)

                    return actions

                dst_port = self.port_get(dp, dst.devname).port_no
                same_host_flow(dst.mac_address, src.mac_address, pkt_ipv4.dst,
                               pkt_ipv4.src, dst_port, in_port)
                self.packet_out(msg, dp, same_host_flow(
                                src.mac_address, dst.mac_address, pkt_ipv4.src,
                                pkt_ipv4.dst, in_port, dst_port))
            else:
                dst_gateway = self.db.gateway_get_by_name(dst.host)

                self.setup_ip_flow(msg, dp, in_port, src_key, pkt_ether, pkt_ipv4,
                                   pkt_tp, src, src_gateway, dst, dst_gateway)
        else:
            self._public_ip(msg, dp, in_port, src_key, pkt_ether, pkt_ipv4, pkt_tp,
                            src, src_gateway)

    def _public_ip(self, msg, dp, in_port, src_key, pkt_ether, pkt_ipv4, pkt_tp,
                   src, src_gateway, priority=PR_INTERNET):
        ofp, ofp_parser, ofp_set, ofp_out = self.ofp_get(dp)

        inkwargs, outkwargs, mod_inkwargs, mod_outkwargs = self._in_out_kwargs(
                    pkt_ipv4, pkt_tp, ofp_set, src_key)

        if pkt_ipv4.proto == inet.IPPROTO_UDP and pkt_tp.dst_port == 67:
            gateway = src.address.split('.')
            gateway[2] = str(BMAX)
            gateway[3] = str(BMAX - 1)
            match = ofp_parser.OFPMatch(in_port=in_port,
                                        eth_type=ether.ETH_TYPE_IP,
                                        ip_proto=pkt_ipv4.proto,
                                        ipv4_src='.'.join(gateway),
                                        **outkwargs)
            actions = [ofp_set(eth_src=src_gateway['vint_mac']),
                       ofp_set(eth_dst=pkt_ether.src),
                       ofp_set(ipv4_src='.'.join(gateway)),
                       ofp_set(ipv4_dst=src.address),
                       ofp_set(udp_src=pkt_tp.dst_port),
                       ofp_set(udp_dst=pkt_tp.src_key),
                       ofp_out(ofp.OFPP_IN_PORT)]

            self.add_flow(dp, match, actions, priority=priority)
            return self.packet_out(msg, dp, actions)

        greenthread.sleep(0)
        gateways = self.db.gateway_get_by_idc(src_gateway.idc_id)

        if gateways:
            gateway_dps = dict((g.datapath_id, g) for g in gateways)
            lgdp_id = (gateways[pkt_ipv4.identification % len(gateways)]
                       if src_gateway.datapath_id not in gateway_dps else None)

            inkeys, _, _, outkeys = self._in_out_keys(
                    pkt_ipv4, pkt_tp, ofp_set, src_key)

            input_match, output_actions = self._local_gateway(
                    dp, pkt_ether, pkt_ipv4, src_gateway, None,
                    outkeys, mod_inkwargs, mod_outkwargs, lgdp_id=lgdp_id,
                    public=True)

            output_match, input_actions = self._local_host(
                    dp, in_port, pkt_ether, pkt_ipv4, inkeys, **outkwargs)

            self.add_flow(dp, input_match, input_actions, priority=priority)
            self.add_flow(dp, output_match, output_actions, priority=priority)
            self.packet_out(msg, dp, output_actions)
        else:
            LOG.warn("Gateway not found - %s", src_gateway.hostname)


from flask import Flask, render_template, request
import pickle
import jsonify
import requests
import numpy as np
import sklearn
from datetime import date

app = Flask(__name__)

model = pickle.load(open('random_forest_regression_model.pkl', 'rb'))

@app.route('/', methods=['GET'])
def Home():
    return render_template('index.html')

@app.route('/predict', methods=['POST'])
def predict():
    today = date.today()
    if request.method == "POST":
        Duration = int(today.year) - int(request.form['year'])
        Present_Price = float(request.form['present_price'])
        Kms_Driven = int(request.form['kms_driven'])
        Owner = int(request.form['ownership'])
        Fuel_Type_Petrol = 0
        Fuel_Type_Diesel = 0
        if(request.form['fuel_type'] == "Petrol"):
            Fuel_Type_Petrol = 1
        elif(request.form['fuel_type'] == "Diesel"):
            Fuel_Type_Diesel = 1
        # else:
        #     fuel_petrol = 0
        #     fuel_diesel = 0
        Seller_Type_Individual = 0
        if(request.form['seller_type'] == "Individual"):
            Seller_Type_Individual = 1

        Transmission_Manual = 0
        if(request.form['transmission_type'] == "Automatic"):
            Transmission_Manual = 0
        else:
            Transmission_Manual = 1

        important = {
            "Purchased Year": int(request.form['year']),
            "Present Price": f"₹ {float(request.form['present_price'])} Lacs",
            "KMs Driven": f"{Kms_Driven} KM",
            "Ownership": Owner,
            "Fuel Type": request.form['fuel_type'],
            "Seller Information": request.form['seller_type'],
            "Transmission Type": request.form['transmission_type']
        }

        prediction = model.predict([[Duration, Present_Price, Kms_Driven, Owner, Fuel_Type_Diesel, Fuel_Type_Petrol, Seller_Type_Individual, Transmission_Manual]])
        output = round(prediction[0], 2)

        if(output < 0):
            return render_template('prediction.html', predicted="Sorry you cannot sell this car !!", table_content=important)
        else:
            return render_template('prediction.html', predicted=f"You can sell this car @ ₹{output} Lacs", table_content=important)

    else:
        return render_template('index.html')

if __name__ == "__main__":
    app.run(debug=True)


import chainer
import chainer.functions as F
import chainer.links as L

class MLP(chainer.Chain):

    def __init__(self, n_in=34, n_unit=1024, mode='generator',
                 use_bn=False, activate_func=F.leaky_relu):
        if n_in % 2 != 0:
            raise ValueError("'n_in' must be divisible by 2.")
        if not mode in ['generator', 'discriminator']:
            raise ValueError("only 'generator' and 'discriminator' are valid "
                             "for 'mode', but '{}' is given.".format(mode))
        super(MLP, self).__init__()
        n_out = n_in // 2 if mode == 'generator' else 1
        print('MODEL: {}, N_OUT: {}, N_UNIT: {}'.format(mode, n_out, n_unit))
        self.mode = mode
        self.use_bn = use_bn
        self.activate_func = activate_func
        w = chainer.initializers.Normal(0.02)
        with self.init_scope():
            self.l1 = L.Linear(n_in, n_unit, initialW=w)
            self.l2 = L.Linear(n_unit, n_unit, initialW=w)
            self.l3 = L.Linear(n_unit, n_unit, initialW=w)
            self.l4 = L.Linear(n_unit, n_out, initialW=w)

            if self.use_bn:
                self.bn1 = L.BatchNormalization(n_unit)
                self.bn2 = L.BatchNormalization(n_unit)
                self.bn3 = L.BatchNormalization(n_unit)

    def __call__(self, x):
        if self.use_bn:
            h1 = self.activate_func(self.bn1(self.l1(x)))
            h2 = self.activate_func(self.bn2(self.l2(h1)))
            h3 = self.activate_func(self.bn3(self.l3(h2)) + h1)
        else:
            h1 = self.activate_func(self.l1(x))
            h2 = self.activate_func(self.l2(h1))
            h3 = self.activate_func(self.l3(h2) + h1)
        return self.l4(h3)


import cv2
image = cv2.imread("ironman.jpg")
grey_img = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
invert = cv2.bitwise_not(grey_img)
blur = cv2.GaussianBlur(invert, (21,21),0)
invertedblur = cv2.bitwise_not(blur)
sketch = cv2.divide(grey_img, invertedblur, scale=256.0)

cv2.imwrite("sketch.png",sketch)
cv2.imwrite("sketchgrey.png",grey_img)
cv2.imwrite("sketchblur.png",blur)
cv2.imwrite("sketchinvert.png",invert)
cv2.imwrite("sketchinblur.png",invertedblur)


# -*- coding: utf-8 -*-
import numpy as np
from vasc import vasc
from helpers import clustering,measure,print_2D
from config import config

if __name__ == '__main__':
    DATASET = 'biase' #sys.argv[1]
    PREFIX = 'biase' #sys.argv[2]

    filename = DATASET+'.txt'
    data = open( filename )
    head = data.readline().rstrip().split()

    label_file = open( DATASET+'_label.txt' )
    label_dict = {}
    for line in label_file:
        temp = line.rstrip().split()
        label_dict[temp[0]] = temp[1]
    label_file.close()

    label = []
    for c in head:
        if c in label_dict.keys():
            label.append(label_dict[c])
        else:
            print(c)

    label_set = []
    for c in label:
        if c not in label_set:
            label_set.append(c)
    name_map = {value:idx for idx,value in enumerate(label_set)}
    id_map = {idx:value for idx,value in enumerate(label_set)}
    label = np.asarray( [ name_map[name] for name in label ] )

    expr = []
    for line in data:
        temp = line.rstrip().split()[1:]
        temp = [ float(x) for x in temp]
        expr.append( temp )

    expr = np.asarray(expr).T
    n_cell,_ = expr.shape
    if n_cell > 150:
        batch_size=config['batch_size']
    else:
        batch_size=32
    #expr = np.exp(expr) - 1
    #expr = expr / np.max(expr)

#
#    percentage = [0.5]
#
#    for j in range(1):
#        print(j)
#        p = percentage[j]
#        samples = np.random.choice( n_cell,size=int(n_cell*p),replace=True )
#        expr_train = expr[ samples,: ]
#        label_train = label[samples]

    #latent = 2
    for i in range(1):
        print("Iteration:"+str(i))
        res = vasc( expr,var=False,
                    latent=config['latent'],
                    annealing=False,
                    batch_size=batch_size,
                    prefix=PREFIX,
                    label=label,
                    scale=config['scale'],
                    patience=config['patience']
                )
#            res_file = PREFIX+'_res.h5'
#            res_data = h5py.File( name=res_file,mode='r' )
#            dim2 = res_data['RES5']
#            print(np.max(dim2))

        print(res.shape)
        k = len( np.unique(label) )
        cl,_ = clustering( res,k=k)
        dm = measure( cl,label )

#            res_data.close()
        ### analysis results
        # plot loss

        # plot 2-D visulation
        fig = print_2D( points=res,label=label,id_map=id_map )
#        fig.savefig('embryo.eps')
#        fig = print_2D( points=res_data['RES5'],label=label,id_map=id_map )
#        fig.show()
#        res_data.close()
#        time.sleep(30)
        #res_data.close()
    # plot NMI,ARI curve
#
#    pollen = h5py.File( name=DATASET+'_'+str(latent)+'_.h5',mode='w' )
#    pollen.create_dataset( name='NMI',data=nmi)
#    pollen.create_dataset( name='ARI',data=ari )
#    pollen.create_dataset( name='HOM',data=hom )
#    pollen.create_dataset( name='COM',data=com )
#    pollen.close()
#

    #print("============SUMMARY==============")
    #k = len(np.unique(label))
    #for r in res:
    #    print("======"+str(r.shape[1])+"========")
    #    pred,si = clustering( r,k=k )
    #    if label is not None:
    #        metrics = measure( pred,label )
    #


import os

import pyexcel.internal.garbagecollector as gc
from pyexcel import iget_array

from nose.tools import eq_

def test_gc():
    gc.free_resources()
    data = iget_array(
        file_name=os.path.join("tests", "fixtures", "bug_01.csv")
    )
    data = list(data)
    eq_(len(gc.GARBAGE), 1)
    gc.free_resources()
    assert len(gc.GARBAGE) == 0

def test_gc_custom():
    gc.free_resources()
    f = open(os.path.join("tests", "fixtures", "bug_01.csv"), "r")
    gc.append(f)
    eq_(len(gc.GARBAGE), 1)
    gc.free_resources()
    assert len(gc.GARBAGE) == 0


from __future__ import annotations

import os
from datetime import datetime
from typing import cast

from google.cloud import storage

from airflow import models
from airflow.decorators import task
from airflow.models.xcom_arg import XComArg
from airflow.providers.google.cloud.hooks.automl import CloudAutoMLHook
from airflow.providers.google.cloud.operators.automl import (
    AutoMLCreateDatasetOperator,
    AutoMLDeleteDatasetOperator,
    AutoMLDeleteModelOperator,
    AutoMLImportDataOperator,
    AutoMLTrainModelOperator,
)
from airflow.providers.google.cloud.operators.gcs import (
    GCSCreateBucketOperator,
    GCSDeleteBucketOperator,
)
from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator
from airflow.utils.trigger_rule import TriggerRule

DAG_ID = "example_automl_vision_clss"
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
GCP_PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default")
GCP_AUTOML_LOCATION = "us-central1"
DATA_SAMPLE_GCS_BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}".replace("_", "-")
RESOURCE_DATA_BUCKET = "airflow-system-tests-resources"

DATASET_NAME = f"ds_vision_clss_{ENV_ID}".replace("-", "_")
DATASET = {
    "display_name": DATASET_NAME,
    "image_classification_dataset_metadata": {"classification_type": "MULTILABEL"},
}
AUTOML_DATASET_BUCKET = f"gs://{DATA_SAMPLE_GCS_BUCKET_NAME}/automl/vision_classification.csv"
IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [AUTOML_DATASET_BUCKET]}}

MODEL_NAME = "vision_clss_test_model"
MODEL = {
    "display_name": MODEL_NAME,
    "image_classification_model_metadata": {"train_budget": 1},
}

CSV_FILE_NAME = "vision_classification.csv"
GCS_FILE_PATH = f"automl/datasets/vision/{CSV_FILE_NAME}"
DESTINATION_FILE_PATH = f"/tmp/{CSV_FILE_NAME}"

extract_object_id = CloudAutoMLHook.extract_object_id

# Example DAG for AutoML Vision Classification
with models.DAG(
    DAG_ID,
    schedule="@once",  # Override to match your needs
    start_date=datetime(2021, 1, 1),
    catchup=False,
    user_defined_macros={"extract_object_id": extract_object_id},
    tags=["example", "automl", "vision-clss"],
) as dag:
    create_bucket = GCSCreateBucketOperator(
        task_id="create_bucket",
        bucket_name=DATA_SAMPLE_GCS_BUCKET_NAME,
        storage_class="REGIONAL",
        location=GCP_AUTOML_LOCATION,
    )

    @task
    def upload_csv_file_to_gcs():
        # download file to local storage
        storage_client = storage.Client()
        bucket = storage_client.bucket(RESOURCE_DATA_BUCKET)
        blob = bucket.blob(GCS_FILE_PATH)
        blob.download_to_filename(DESTINATION_FILE_PATH)

        # update file content
        with open(DESTINATION_FILE_PATH) as file:
            lines = file.readlines()

        updated_lines = [line.replace("template-bucket", DATA_SAMPLE_GCS_BUCKET_NAME) for line in lines]

        with open(DESTINATION_FILE_PATH, "w") as file:
            file.writelines(updated_lines)

        # upload updated file to bucket storage
        destination_bucket = storage_client.bucket(DATA_SAMPLE_GCS_BUCKET_NAME)
        destination_blob = destination_bucket.blob(f"automl/{CSV_FILE_NAME}")
        generation_match_precondition = 0
        destination_blob.upload_from_filename(
            DESTINATION_FILE_PATH, if_generation_match=generation_match_precondition
        )

    upload_csv_file_to_gcs_task = upload_csv_file_to_gcs()

    copy_folder_tasks = [
        GCSToGCSOperator(
            task_id=f"copy_dataset_folder_{folder}",
            source_bucket=RESOURCE_DATA_BUCKET,
            source_object=f"automl/datasets/vision/{folder}",
            destination_bucket=DATA_SAMPLE_GCS_BUCKET_NAME,
            destination_object=f"automl/{folder}",
        )
        for folder in ("cirrus", "cumulonimbus", "cumulus")
    ]

    create_dataset_task = AutoMLCreateDatasetOperator(
        task_id="create_dataset_task",
        dataset=DATASET,
        location=GCP_AUTOML_LOCATION,
    )

    dataset_id = cast(str, XComArg(create_dataset_task, key="dataset_id"))
    import_dataset_task = AutoMLImportDataOperator(
        task_id="import_dataset_task",
        dataset_id=dataset_id,
        location=GCP_AUTOML_LOCATION,
        input_config=IMPORT_INPUT_CONFIG,
    )

    MODEL["dataset_id"] = dataset_id

    create_model = AutoMLTrainModelOperator(task_id="create_model", model=MODEL, location=GCP_AUTOML_LOCATION)
    model_id = cast(str, XComArg(create_model, key="model_id"))

    delete_model = AutoMLDeleteModelOperator(
        task_id="delete_model",
        model_id=model_id,
        location=GCP_AUTOML_LOCATION,
        project_id=GCP_PROJECT_ID,
        trigger_rule=TriggerRule.ALL_DONE,
    )

    delete_dataset = AutoMLDeleteDatasetOperator(
        task_id="delete_dataset",
        dataset_id=dataset_id,
        location=GCP_AUTOML_LOCATION,
        project_id=GCP_PROJECT_ID,
        trigger_rule=TriggerRule.ALL_DONE,
    )

    delete_bucket = GCSDeleteBucketOperator(
        task_id="delete_bucket",
        bucket_name=DATA_SAMPLE_GCS_BUCKET_NAME,
        trigger_rule=TriggerRule.ALL_DONE,
    )

    (
        # TEST SETUP
        create_bucket
        >> upload_csv_file_to_gcs_task
        >> copy_folder_tasks
        # TEST BODY
        >> create_dataset_task
        >> import_dataset_task
        >> create_model
        # TEST TEARDOWN
        >> delete_model
        >> delete_dataset
        >> delete_bucket
    )

    from tests.system.utils.watcher import watcher

    # This test needs watcher in order to properly mark success/failure
    # when "tearDown" task with trigger rule is part of the DAG
    list(dag.tasks) >> watcher()

from tests.system.utils import get_test_run  # noqa: E402

# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
test_run = get_test_run(dag)


# -*- coding: utf-8 -*-
"""
Created on Mon Jul 10 00:14:26 2017

@author: Madhu
"""
# Import libraries necessary for this project
import sklearn
#import networkx as nx
import numpy  as np
import pandas as pd
import seaborn as sns; sns.set()
import datetime as dt
from matplotlib import pyplot as plt
import matplotlib.ticker as ticker
from pandas import compat

compat.PY3 = True
print ("-----------------------------------------------------------------------")
print('The scikit-learn version is {}.'.format(sklearn.__version__))

#load functions from
from projectFunctions import loadData, missingValues

path = r'C:\Users\pmspr\Documents\HS\MS\Sem 3\EECS 731\Week 8\HW\Git\EECS-731-Project-6\Data'
filename = "cpu.csv"
data = loadData(path,filename)

#Check the missing values
misVal, mis_val_table_ren_columns = missingValues(data)
print(mis_val_table_ren_columns.head(20))

##Remove rows with missing target values
#ind = data_raw[data_raw['Date'].isnull()].index.tolist()
#data = data_raw.drop(index=ind, axis=0)

#Remove special characters in the column
#data['target'] = pd.to_numeric(data['target'].str.replace('\(|\)',""),errors='coerce').isnull()
#data['value'] = data['value'].str.replace('\(|\)',"")

#Change the datatypes accordingly
data['value'] = data['value'].astype('float')
data['timestamp'] = pd.to_datetime(data['timestamp'])

#sort the dataframe by timestamp
data = data.sort_values(by='timestamp')
data.set_index(np.arange(len(data.index)))

#compute the elapsed time feature
md = data['timestamp'].min()
ed = md - dt.timedelta(hours=1)
data['elapsed'] = data['timestamp'] - ed
data['elapsed'] = data['elapsed'] / pd.Timedelta('1 hour')
data['elapsed'] = data['elapsed'].apply(lambda x: round(x,2))

#Compute year,month,day features
data['year'] = data['timestamp'].dt.year
data['year'] = data['year'].astype('int')

data['month'] = data['timestamp'].dt.month
data['month'] = data['month'].astype('int')

data['day'] = data['timestamp'].dt.day
data['day'] = data['day'].astype('int')

#plots
#plt.figure(figsize=(5,5))
#sns.countplot(x='year', data=data)
#plt.title('Year wise observings')
#plt.ylabel('Count', fontsize=12)
#plt.xlabel('Year', fontsize=12)

#Drop rows for year 2013
ind = data[data['timestamp'].dt.year == 2013].index.tolist()
data = data.drop(index=ind, axis=0)

#fig,axes = plt.subplots(1,2,sharex=True, figsize=(16,8))
#lineplot = sns.relplot(x="timestamp", y="value", kind="line", data=data)
#lineplot.fig.autofmt_xdate()

ax = sns.relplot(x="day", y="value", hue="month", data=data);
ax.set(xlim=(0,31))
#Drop the timestamp column as model cannot process
data = data.drop(columns = ['timestamp'], axis=1)

from projectFunctions import exploreData, transformData, splitData
exploreData(data)

data_raw = data
features = transformData(data_raw)
features_s = features
features_s = features_s.drop(columns=['year','elapsed'],axis=1)

X_train, X_test = splitData(features_s,0.3)

from projectFunctions import bayesGauMix, gaussMix, isoForest, locOutlier

#Implement Bayessian gaussian mixture
clusters = bayesGauMix(features_s,20)
#print(clusters)

#Implement Gaussian mixture clustering
gm = gaussMix(X_train,4,10)
weights = np.round(gm.weights_,2)
predict = gm.predict(features_s)
prob    = gm.predict_proba(features_s)
density = gm.score_samples(features_s)
print ("-----------------------------------------------------------------------")
print ("Gaussian mixture converged %s in %d iterations" %(gm.converged_, gm.n_iter_))

data['gclus'] = pd.Series(predict.tolist())
print(data['gclus'].unique())
data['gprob'] = pd.Series(prob.tolist())
data['gdensity'] = pd.Series(density.tolist())

ax = sns.relplot(x="day", y="gdensity", hue="gclus", data=data);
ax.set(xlim=(0,31))

#Implement Isolation Forest
ifo = isoForest(X_train)
predict = ifo.predict(features_s)
density = ifo.score_samples(features_s)

data['iclus'] = pd.Series(predict.tolist())
data['idensity'] = pd.Series(density.tolist())

ax = sns.relplot(x="day", y="value", hue="iclus", data=data);
ax.set(xlim=(0,31))

#Implement local outlier
lof = locOutlier(X_train,1)
#density = lof.decision_function(features_s)
clus = lof.fit_predict(features_s)

data['lclus'] = pd.Series(clus.tolist())

ax = sns.relplot(x="day", y="value", hue="lclus", data=data);
ax.set(xlim=(0,31))

data.to_csv('test.csv',index=False)


import numpy as np
from model import HouseParams, AttackParams, PADM_Params
from model import Control
import matplotlib.pyplot as plt

PV_availabilities = np.loadtxt("time_series/TS_PVAvail.csv")
demands = np.loadtxt("time_series/TS_Demand.csv")
PV_availabilities = PV_availabilities[0:24]
demands = demands[0:24]

house_params = HouseParams(
    life_time=12*10*30*4,
    price_PV = 1000,
    price_battery = 140,
    cost_buy = 0.25,
    sell_price = 0.05,
    total_demand = 3500,
    demands= demands,
    PV_availabilities= PV_availabilities
)

ub = 0.8
lb = -0.8
attack_params = AttackParams(ub=ub,lb=lb,capacity_battery=1)
PADM_params = PADM_Params()

control = Control(house_params, attack_params)
print(50*"-")
control.primal_model()
print(50*"-")
# control.dual_model()
print(50*"-")
# output = control.bigM_attack(100000)
# print(50*"-")
output = control.sos_attack()
# print(50*"-")
# control.sos_valid_ineq_attack()
# control.PADM_attack(PADM_params)

# prepare data for visualization
demands = np.array(output["demands"])
changed_demands = np.array(output["changed_demands"])
demands_ub = demands * (1+ub)
demands_lb = demands * (1+lb)

# plot
f = plt.figure(1)
plt.subplot(4,1,1)
x = range(len(demands))
plt.plot( x, demands, color="blue", alpha=0.2, label="demands")
# plt.plot(x, demands, color="blue", alpha=0.2)
plt.plot( x, changed_demands, color="red", alpha=0.2, label="changed demands")
plt.plot(x, demands_ub, color="green", linestyle="dashed", alpha=0.1)
plt.plot(x, demands_lb, color="green", linestyle="dashed", alpha=0.1)
plt.ylabel("Demand(kWh)")
plt.title("Demand Curve")
plt.subplot(4,1,2)
plt.plot(x, output["PV"], color="blue", alpha=0.2)
plt.ylabel("PV(kWh)")
plt.subplot(4,1,3)
plt.plot(x, output["battery"], color="blue", alpha=0.2)
plt.ylabel("Battery(kWh)")
plt.subplot(4,1,4)
plt.plot(x, output["buy"], color="blue", alpha=0.2)
plt.ylabel("Buy(kWh)")

g = plt.figure(2)
x = range(len(demands))
plt.plot( x, demands, color="blue", alpha=0.2, label="demands")
# plt.plot(x, demands, color="blue", alpha=0.2)
plt.plot( x, changed_demands, color="red", alpha=0.2, label="changed demands")
plt.plot(x, demands_ub, color="green", linestyle="dashed", alpha=0.1)
plt.plot(x, demands_lb, color="green", linestyle="dashed", alpha=0.1)
plt.ylabel("Demand(kWh)")
plt.title("Demand Curve")
plt.legend()
plt.show()


# -*- coding: utf-8 -*-
"""API endpoint for combatant import."""

# standard library imports
import csv
from io import TextIOWrapper

# third-party imports
from flask import request, abort, current_app
from flask_restful import Resource

# application imports
from emol.models import Combatant, Discipline
from emol.utility.value_tools import yes_or_no
from emol.utility.date import add_years, string_to_date

@current_app.api.route('/api/import')
class ImportApi(Resource):
    """Endpoint for combatant import.

    Permitted methods: POST

    Allows for upload of a CSV file full of combatant data

    """

    @staticmethod
    def post():
        """Import some combatants."""
        slug = request.form['discipline']
        discipline = Discipline.find(slug)

        combatant_data = request.files['file']
        f = TextIOWrapper(combatant_data, encoding='utf-8')
        reader = csv.DictReader(f, skipinitialspace=True)
        for row in reader:
            try:
                combatant = Combatant.create(row)
                current_app.logger.info('Import {}'.format(combatant))
                card = combatant.get_card(discipline, create=True)

                card_date = row.get('card_date')
                if card_date is None:
                    card.card_date = None
                else:
                    card.card_date = add_years(card_date, -2)

                for auth in discipline.authorizations:
                    if yes_or_no(row.get(auth.slug)):
                        card.add_authorization(auth)
                    else:
                        card.remove_authorization(auth)

                if row.get('member_number') and row.get('member_expiry'):
                    for marshal in discipline.marshals:
                        if yes_or_no(row.get(marshal.slug)):
                            card.add_warrant(marshal)
                        else:
                            card.remove_warrant(marshal)
            except Exception as exc:
                current_app.logger.exception(exc)


import sys
import pygame
import random

# # Set Color
Blue = (0,250,0)
Green = (0,0,250)
Black = (0,0,0)

# # Initial Setup
WIDTH, HEIGHT = (500, 500)
FPS = 60
pygame.init()           # initialise game
pygame.mixer.init()     # fro sound
screen = pygame.display.set_mode((HEIGHT, WIDTH))
pygame.display.set_caption("Snake Game")
clock = pygame.time.Clock()
all_sprite = pygame.sprite.Group()

# # Load Images:
head = pygame.image.load("img/body.png").convert()
head = pygame.transform.scale(head, (25,25))
head.set_colorkey(head.get_at((0,0)))
rect = head.get_rect()

game_over = False
while not game_over:
    screen.fill(Black)
    clock.tick(FPS)
    for e in pygame.event.get():
        if e.type == pygame.QUIT:       # type ( event ) isequals playgame.QUIT
            print("Quit")
            game_over = True

    screen.blit(head,rect)
    # all_sprite.update()
    # all_sprite.draw(screen)
    pygame.display.flip()

pygame.quit()


"""
This script is an implementation of Murison's method for solving Kepler's equation.
The key part of this method is the initial guess.
The iterations stop after a precision epsilon on the functions value (which we wish to be f(x)=0) is reached.

This measure of error is discussed on a pdf file available on:
https://github.com/martimmcdc/solving-keplers-equation

In order to minimize calculations but still properly evaluate the precision of each iteration,
the error is considered to be that of the previous iteration, so that,
when the desired precision is reached, the algorithm runs one additional time.
"""

### imports
import numpy as np

def KeplerStart3(e,M):
    t34 = e*e
    t35 = e*t34
    t33 = np.cos(M)
    return M+(-0.5*t35 + e + (t34+1.5*t33*t35)*t33)*np.sin(M)

def eps3(e,M,x):
    t1 = np.cos(x)
    t2 = -1+e*t1
    t3 = np.sin(x)
    t4 = e*t3
    t5 = -x+t4+M
    t6 = t5/(0.5*t5*t4/t2+t2)
    return t5/((0.5*t3 - 1/6*t1*t6)*e*t6+t2)

def murison_solver(e,M,epsilon=1e-9,iter_counter=False):
    Mnorm = M%(2*np.pi)
    E = KeplerStart3(e,Mnorm)
    filt = np.ones(len(M),bool)
    counter = 0
    while filt.any():
        E[filt] -= eps3(e,Mnorm[filt],E[filt])
        filt[filt] = np.abs(E[filt] - e*np.sin(E[filt]) - M[filt]) > epsilon
        counter += 1
    if iter_counter:
        return E,counter
    else:
        return E

# Test function
if __name__=="__main__":

    import matplotlib.pyplot as plt,time

    # Parameters
    N = 1000000
    e = 0.5

    print("\n##### PARAMETERS #####")
    print("# len(M) = %d"%N)
    print("# e = %.2f"%e)
    #print("# Iterations: %d"%N_it)
    print("######################")

    # Create ell array from E
    E_true = (np.pi*(np.arange(N)+0.5))/N
    M_input = E_true - e*np.sin(E_true)

    # Time the function
    init = time.time()
    E_out,iters = murison_solver(e,M_input,iter_counter=True)
    runtime = time.time()-init
    print("\nEstimation complete after %.1f millseconds, achieving mean error %.2e."%(runtime*1000.,np.mean(np.abs(E_out-E_true))))
    print(iters,'iterations\n')

    # Plot values calculated and compare with pre-calculated grid
    sympy_grid = np.loadtxt('../sympy_200x200grid.txt')[::30,:]
    M = np.linspace(0,np.pi,200)
    e = np.arange(0,1,1/200)[::30]
    for power in range(6,16,3):
        for i in range(len(e)):
            vals = murison_solver(e[i],M,epsilon=10**(-power))
            sympy_vals = sympy_grid[i,:]
            plt.plot(M,vals-sympy_vals,label='e = {}'.format(e[i]))
        plt.xlabel('$M$')
        plt.ylabel('$\Delta E$')
        plt.title('$\\epsilon = 10^{}$'.format('{'+str(-power)+'}'))
        plt.legend()
        plt.grid()
        plt.show()


import os
import json

# Specify the directory where your dataset is stored
dataset_directory = 'D:\leaves'

# Initialize an empty dictionary to store class labels
class_labels = {}
cnt=0
# Iterate through the subdirectories (class directories) in the dataset directory
for class_name in os.listdir(dataset_directory):
    # Check if it's a directory
    if os.path.isdir(os.path.join(dataset_directory, class_name)):
        # Use the class directory name (class_name) as the label
        # You can modify this to extract the label from the file name if needed
        class_labels[cnt] = class_name
        cnt+=1

# Save the dictionary to a JSON file
with open('class_labels.json', 'w') as json_file:
    json.dump(class_labels, json_file, indent=4)

print("Class labels saved to 'class_labels.json'")


from flask import Flask,request,render_template
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity
#initailze app
app = Flask(__name__)

#read data file
df = pd.read_csv('model/movies.csv')

#generalize data
df['cast'] = df['cast'].str.replace(' ','')
df['cast'] = df['cast'].str.replace(',',' ')
df['genre'] = df['genre'].str.replace('&',' ')
df['genre'] = df['genre'].str.replace(',',' ')
df['country'] = df['country'].str.replace(' ','')
df['country'] = df['country'].str.replace(',',' ')
df.sort_values(by=('year'),axis=0,inplace=True,ascending=False)
df.reset_index(drop=True,inplace=True)
df['year'] = df['year'].astype(str)

#train features
df['features'] = df['cast'] +' '+df['description']+' '+df['genre']+' '+df['year']+' '+df['country']
cv = CountVectorizer()
count_matrix = cv.fit_transform(df['features'])
sim = cosine_similarity(count_matrix)

clean_df  = pd.read_csv('model/movies_updated.csv')
#define function
def recommender(name):
    name = name.lower()
    df['title']=df['title'].str.lower()
    if name not in df['title'].unique():
        return ('Try Aboslute Spelling or it may not be in our database')
    else:
        i = df.loc[df['title']==name].index[0]
        mov_id = list(enumerate(sim[i]))
        mov_id = sorted(mov_id,key = lambda x:x[1],reverse=True)
        mov_id = mov_id[1:11]
        rec_mov = []
        for i in range(len(mov_id)):
            a = mov_id[i][0]
            rec_mov.append(clean_df['title'][a])
        return rec_mov

@app.route('/', methods=['GET', 'POST'])
def home():
    return render_template('main.html')

@app.route('/predict', methods=["GET","POST"])
def predict():
    movie_name = request.form.get('movie')
    predictions = recommender(movie_name)
    if type(predictions)==type('str'):
        return render_template('trueReturn.html',predictions=[''],message=predictions)
    else :
        return render_template('trueReturn.html', predictions=predictions,message='')
#@app.route('/moviecard',methods=['GET','POST'])
#def moviecard():
 #   selc_movie = 'after'
  #  i = clean_df[clean_df['title']==selc_movie]
  #  values = []
  #  for index in ['title','genre','rating','year','cast','director', 'country','duration','description']:
  #     values.append(clean_df[index][4])
  #   return render_template('movie_card.html',movielist = values)
if __name__=='__main__':
    app.run(debug=True)


from copy import deepcopy

class PuzzleGame:
    def __init__(self, initial_state):
        self.initial_state = initial_state
        self.matrix = []

        self.MOVE = {
            'UP'    :   (-1, 0),
            'DOWN'  :   (1, 0),
            'LEFT'  :   (0, -1),
            'RIGHT' :   (0, 1)
            }

    def get_matrix(self):
        matrix = ''
        for row in self.matrix:
            matrix += str(row) + '\n'

        return matrix

    def __str__(self):
        return self.get_matrix()

    def check_moves(self, prev_move):
        if prev_move == None:
            return tuple(self.MOVE.keys())

        if prev_move == 'UP':
            return ('UP', 'LEFT', 'RIGHT')

        if prev_move == 'DOWN':
            return ('DOWN', 'LEFT', 'RIGHT')

        if prev_move == 'LEFT':
            return ('UP', 'DOWN', 'LEFT')

        if prev_move == 'RIGHT':
            return ('UP', 'DOWN', 'RIGHT')

    def move_allowed(self, place, move):
        row, col = self.new_place(place, move)
        if row > 0 and row < 5 and col > 0 and col < 5:
            return True

        else:
            return False

    def new_place(self, place, move):
        return (place[0]+move[0], place[1]+move[1])

    def apply_move(self, place, new_place):
        o_r, o_c = place
        n_r, n_c = new_place

        new_matrix = deepcopy(self.matrix)

        new_matrix[n_r-1][n_c-1] = new_matrix[o_r-1][o_c-1]
        new_matrix[o_r-1][o_c-1] = self.matrix[n_r-1][n_c-1]

        return new_matrix

    def value_replaced(self, new_place):
        n_r, n_c = new_place
        return self.matrix[n_r-1][n_c-1]

    def distance(self, prev_place, new_place):
        x1, y1 = prev_place
        x2, y2 = new_place
        d = abs(x2 - x1) + abs(y2 - y1)

        return d

    def count_dissimilar(self, matrix):
        dissmilar = 0
        count = 1
        for row in matrix:
            for col in row:
                if col and col is not count:
                    dissmilar += 1

                count += 1

        return dissmilar

    def generate_matrix(self, initial_state):
        return [initial_state[i:i+4] for i in range(0, len(initial_state), 4)]

    def generate_numbers(self, grid_size):
        return [i for i in range(1, grid_size)] + [None]

    def is_solvable(self):
        self.place = self.get_block_place(block=None, matrix=self.matrix)
        result = self.sum_less_i() + self.x(block_place=self.place)

        return (result % 2 == 0)

    def get_block_place(self, block, matrix):
        for row in range(0, 4):
            for col in range(0, 4):
                if (matrix[row][col] is block):
                    return (row+1, col+1)

    def sum_less_i(self):
        sumission = 0
        for num in range(0, 16):
            if self.initial_state[num] is None:
                sumission += (16 - (num + 1))
                continue

            for current in range(num + 1, 16):
                if (self.initial_state[current] is None):
                    continue

                if (self.initial_state[num] > self.initial_state[current]):
                    sumission += 1

        return sumission

    def x(self, block_place):
        row, col = block_place
        # block is at even row and even col
        # or odd row and odd col
        if (row + col) % 2 == 0:
            return 0

        else:
            return 1


"""
This implementation of DEPSYM is copied directly from the code notebook in https://github.com/RakshaAg/DEPSYM.

Some code using StanzaLanguage have been modified as it is no longer supported in spacy-stanza v1.x.x.

SpaCy model is changed to use en_core_web_lg instead of en_core_web_sm

@inproceedings{depsym,
  author = {Chatterjee, Niladri and Agarwal, Raksha},
  title        = {DEPSYM: A Lightweight Syntactic Text Simplification Approach using Dependency Trees},
  booktitle    = {Proceedings of the First Workshop on Current Trends in Text Simplification (CTTS 2021), co-located with SEPLN 2021},
  year         = {2021},
  editor       = {Saggion, Horacio and Štajner, Sanja and Ferrés, Daniel and Kim Cheng Sheang},
  pages        = {42-56},
  month        = {09},
}
"""

import spacy
from spacy import displacy
import stanza
import spacy_stanza
from pattern.en import comparative, superlative, conjugate, INFINITIVE, PRESENT, PAST, FUTURE, pluralize
import truecase
import re
import textstat
import pattern

##### MONKEY PATCH OF PATTERN 3.6.0 #####
# Retrieved from https://github.com/clips/pattern/issues/308#issuecomment-1308344763
import os

from pattern.helpers import decode_string
from codecs import BOM_UTF8

from simplertimes.simplify.simplifier import AbstractSimplifier

BOM_UTF8 = BOM_UTF8.decode("utf-8")
decode_utf8 = decode_string

def _read(path, encoding="utf-8", comment=";;;"):
    """ Returns an iterator over the lines in the file at the given path,
        strippping comments and decoding each line to Unicode.
    """
    if path:
        if isinstance(path, str) and os.path.exists(path):
            # From file path.
            f = open(path, "r", encoding="utf-8")
        elif isinstance(path, str):
            # From string.
            f = path.splitlines()
        else:
            # From file or buffer.
            f = path
        for i, line in enumerate(f):
            line = line.strip(BOM_UTF8) if i == 0 and isinstance(line, str) else line
            line = line.strip()
            line = decode_utf8(line, encoding)
            if not line or (comment and line.startswith(comment)):
                continue
            yield line
    return
# The _read method uses old style of raising StopIteration
# Raising stop iteration breaks the code in newer Python versions
pattern.text._read = _read
##### MONKEY PATCH OF PATTERN 3.6.0 #####

def inorder_traversal(token, removed=None):
  if(len(list(token.children)) == 0):
     return token.text
  s = ""
  if(len(list(token.lefts)) != 0):
    for child in token.lefts:
      if(child != removed):
        s = s + " " + inorder_traversal(child, removed)

  s = s + " " + token.text

  if(len(list(token.rights)) != 0):
    for child in token.rights:
      if(child != removed):
        s = s + " " + inorder_traversal(child, removed)
  return s

def inorder_traversal_2(token, removed):
  if(len(list(token.children)) == 0):
     return token.text
  s = ""
  if(len(list(token.lefts)) != 0):
    for child in token.lefts:
      if(child not in removed):
        s = s + " " + inorder_traversal_2(child, removed)

  s = s + " " + token.text

  if(len(list(token.rights)) != 0):
    for child in token.rights:
      if(child not in removed):
        s = s + " " + inorder_traversal_2(child, removed)
  return s

def inorder_traversal_3(token, removed):
  if(len(list(token.children)) == 0):
     return token.text
  s = ""
  if(len(list(token.lefts)) != 0):
    for child in token.lefts:
      if(child not in removed):
        s = s + " " + inorder_traversal_3(child, removed)
  if(token not in removed):
    s = s + " " + token.text

  if(len(list(token.rights)) != 0):
    for child in token.rights:
      if(child not in removed):
        s = s + " " + inorder_traversal_3(child, removed)
  return s

def left_subtree(token):
  if(len(list(token.lefts)) == 0):
     return token.text
  s = ""
  if(len(list(token.lefts)) != 0):
    for child in token.lefts:
      s = s + " " + inorder_traversal(child)
  s = s + " " + token.text
  return s

def left_subtree_2(token, removed=None):
  if(len(list(token.children)) == 0):
     return token.text
  s = ""
  if(len(list(token.lefts)) != 0):
    for child in token.lefts:
      if(child != removed):
        s = s + " " + inorder_traversal(child, removed)

  s = s + " " + token.text
  return s

def left_subtree_3(token, removed):
  if(len(list(token.children)) == 0):
     return token.text
  s = ""
  if(len(list(token.lefts)) != 0):
    for child in token.lefts:
      if(child not in removed):
        s = s + " " + inorder_traversal_3(child, removed)

  if(token not in removed):
    s = s + " " + token.text
  return s

def right_subtree_3(token, removed):
  if(len(list(token.children)) == 0):
     return token.text
  s = ""
  if(len(list(token.rights)) != 0):
    for child in token.rights:
      if(child not in removed):
        s = s + " " + inorder_traversal_3(child, removed)

  if(token not in removed):
    s = s + " " + token.text
  return s

def get_root(doc):
  root  = None
  for token in doc:
    if(token.dep_ == "ROOT"):
      root = token
      return root

def get_root_stanza(doc):
  root  = None
  for token in doc:
    if(token.dep_ == "root"):
      root = token
      return root

def get_tag(token, tag):
  for child in token.children:
    if(child.dep_ == tag):
      return child

def find_in_subtree(token, tag):
  if(token == None):
     return
  if(token.tag_ == tag):
     return token
  for child in token.subtree:
    if(child.dep_ == tag):
      return child

def find_in_children(token, tag):
  if(token == None):
     return
  if(token.tag_ == tag):
     return token
  for child in token.children:
    if(child.dep_ == tag):
      return child

def pos_in_subtree(token, tag):
  if(token == None):
     return False
  if(token.pos_ == tag):
     return True
  for child in token.subtree:
    if(child.pos_ == tag):
      return True
  return False

def auxiliary_verb(verb, subj):
  if subj.tag_ in ("NN", "NNP"):
    if verb.tag_ in ("VBP", "VBZ", "VB"):
      aux = "is "
    elif verb.tag_ in ("VBD", "VBG", "VBN"):
      aux = "was "
    else:
      aux = "are "
  elif subj.tag_ in ("NNS", "NNPS"):
    if verb.tag_ in ("VBP", "VBZ", "VB"):
      aux = "are "
    elif verb.tag_ in ("VBD", "VBG", "VBN"):
      aux = "were "
    else:
      aux = "are "
  elif subj.tag_ in ("PRP") and subj.text.lower() == "they":
    if verb.tag_ in ("VBP", "VBZ", "VB") :
      aux = "are "
    elif verb.tag_ in ("VBD", "VBG", "VBN"):
      aux = "were "
  elif subj.tag_ in ("PRP"):
    if verb.tag_ in ("VBP", "VBZ", "VB"):
      aux = "is "
    elif verb.tag_ in ("VBD", "VBG", "VBN"):
      aux = "was "
  else:
      aux = "are "
  return aux

def remove_punct(s):
    s_list = list(s.split(" "))
    if(s_list[0] in ['?', '!', ',', ';', '.']):
        s_list[0] = ""
    for i in range(len(s_list)):
        if(len(s_list[i]) > 0 and (s_list[i][0] == "." or s_list[i][0] == ",")):
            s_list[i] = "."
    s = " ".join(c for c in s_list)
    fix_spaces = re.compile(r'\s*([?!.,]+(?:\s+[?!.,]+)*)\s*')
    s = fix_spaces.sub(lambda x: "{} ".format(x.group(1).replace(" ", "")), s)
    s = truecase.get_true_case(s)
    return s

# Share a global level nlp pipeline to avoid repeated instantiation
spacy_nlp = spacy.load("en_core_web_lg")

def appositive_simplification_2(comp_sent):
  # nlp = spacy.load("en_core_web_sm")
  doc = spacy_nlp(comp_sent)

  root = get_root(doc)
  subj = get_tag(root, "nsubj")

  apposcl = None

  apposcl = find_in_subtree(root, "appos")

  if(apposcl == None):
    apposcl = find_in_subtree(root, "amod")
    if(apposcl != None):
      if(apposcl.head.dep_ != "nsubj"):
        print("Appos: No simplification(no subject)")
        return False, comp_sent
    else:
      print("Appos: No simplification")
      return False, comp_sent

  appos_subj = apposcl.head

  appos_phrase = inorder_traversal(apposcl)
  aux_verb = str(auxiliary_verb(root, appos_subj))
  noun_phrase = left_subtree_2(appos_subj, apposcl)
  sentence1 = inorder_traversal(root, apposcl)
  sentence1 = ' '.join(sentence1.split(","))
  sentence1 = ' '.join(sentence1.split()) + "."

  sentence2 = noun_phrase + " " + aux_verb + appos_phrase
  sentence2 = ' '.join(sentence2.split(","))
  sentence2 = ' '.join(sentence2.split()) + "."

  sentence1 = remove_punct(sentence1)
  sentence2 = remove_punct(sentence2)

  print(sentence1)
  print(sentence2)
  return True, sentence1 + " " + sentence2

def rel_order(token):
  if(token.dep_ == "nsubj" or token.dep_ == "nsubjpass"):
    return True
  else:
    return False

def relative_clause_simplify(cmp_snt):
  # nlp = spacy.load("en_core_web_sm")
  doc = spacy_nlp(cmp_snt)
  root = get_root(doc)
  relcl = None
  relcl = find_in_subtree(root, "relcl")
  if(relcl == None):
    relcl = find_in_subtree(root, "rcmod")

  if(relcl == None):
    print("Relcl: No simplification")
    return False, cmp_snt

  subj = relcl.head

  rel_phrase = doc[relcl.left_edge.i : relcl.right_edge.i+1].text
  noun_phrase = left_subtree_2(subj, relcl)
  sentence1 = inorder_traversal(root, relcl)
  sentence1 = ' '.join(sentence1.split(","))
  sentence1 = ' '.join(sentence1.split()) + "."
  sentence2 = noun_phrase + " " + rel_phrase
  sentence2 = ' '.join(sentence2.split(","))
  sentence2 = ' '.join(sentence2.split())
  sentence2 = ' '.join(sentence2.split()) + "."

  sent1_doc = spacy_nlp(sentence1)
  sent1_root = get_root(sent1_doc)
  if(sent1_root.pos_ != "VERB" and sent1_root.pos_ != "AUX"):
    print("Relcl: No simplification")
    return False, cmp_snt

  order = rel_order(subj)

  sentence1 = remove_punct(sentence1)
  sentence2 = remove_punct(sentence2)

  if(not order):
    print(sentence1)
    print(sentence2)
    return True, sentence1 + " " + sentence2
  else:
    print(sentence2)
    print(sentence1)
    return True, sentence2 + " " + sentence1

def conjoint_clause_simplification(cmp_snt,Qnlp):
  cmp_snt = cmp_snt.rstrip(" ,.")
  print('the input is ',cmp_snt)
  doc = Qnlp(cmp_snt)
  root = get_root_stanza(doc)

  clause_type = None

  clause_type = find_in_children(root, "conj")
  if(clause_type == None):
    clause_type = find_in_children(root, "advcl")
  if(clause_type == None):
    clause_type = find_in_children(root, "parataxis")
  if(clause_type == None):
      clause_type = find_in_subtree(root, "conj")
  if(clause_type == None):
    clause_type = find_in_subtree(root, "advcl")
  if(clause_type == None):
    clause_type = find_in_subtree(root, "parataxis")
  if(clause_type == None):
    print("Conjoint: No simplification(no second clause found)")
    return False, cmp_snt

  clause_root = None

  if(clause_type.dep_ == "conj"):
    clause_root = find_in_subtree(root, "cc")
    if(clause_root != None):
      if((not pos_in_subtree(root, "VERB")) or (not (pos_in_subtree(clause_type, "VERB") or pos_in_subtree(clause_type, "AUX")))):
        print("Conjoint: No simplification 2(no verb in second clause)")
        return False, cmp_snt
    else:
      clause_root = find_in_children(clause_type, "advmod")
  elif(clause_type.dep_ == "advcl"):
    print("advcl")
    clause_root = find_in_children(clause_type, "mark")
    if(clause_root == None):
      clause_root = find_in_subtree(root, "mark")
    if(clause_root == None):
      clause_root = find_in_children(clause_type, "advmod")
    if(clause_root != None):
        if(clause_root.text == "to"):
            print("Conjoint: No simplification 3(no conjunction found)")
            return False, cmp_snt
  elif(clause_type.dep_ == "parataxis"):
    print("parataxis")
    clause_root = find_in_children(clause_type, "advmod")
  else:
    print("Conjoint: No simplification 3(no conjunction found)")
    return False, cmp_snt

  if(clause_root == None):
    print("Conjoint: No simplification 4")
    return False, cmp_snt

  clause_subj = None

  clause_subj = find_in_children(clause_type, "nsubj")
  if(clause_subj == None):
    clause_subj = find_in_children(clause_type, "nsubj:pass")
  clause_subj_flag = False
  if(clause_subj == None):
    clause_subj_flag = True
    clause_subj = find_in_children(root, "nsubj")
    if(clause_subj == None):
        clause_subj = find_in_children(root, "nsubj:pass")
    if(clause_subj == None):
        print("Conjoint: No simplification(no clause subject found)")
        return False, cmp_snt
    aux_str = ""
    for child in root.children:
        if(child.dep_ == "aux"):
            aux_str += child.text + " "
  modal = None
  if(clause_root.text.lower() in ['when', 'after', 'since', 'before', 'once']):
    modal = find_in_subtree(root, "aux")

  marker1 = ""
  marker2 = ""
  if(clause_root.text.lower() == "if"):
    marker1 = "Then"
    for child in root.subtree:
      if(child.text.lower() == "then"):
        marker1 = ""

  sentence1 = inorder_traversal_2(root, [clause_type, clause_root])

  sentence2 = ""
  marker2 = add_marker(clause_type, clause_root, modal) + " "
  if(clause_subj_flag):
    sentence2 = clause_subj.text + " " + aux_str + inorder_traversal(clause_type, clause_root)
  else:
    sentence2 = inorder_traversal(clause_type, clause_root)

  reverse = None

  if((root.i > clause_type.i) or (clause_root.text.lower() in ["because", "as"] and clause_root.i > 0)):
    if(clause_root.text.lower() in ['when', 'after', 'since', 'before', 'once'] and clause_root.i == 0):
      reverse = False
    else:
      reverse = True
  else:
    reverse = False

  if(clause_root.text.lower() in ["because", "as"] and clause_root.i > 0):
    sentence1 = marker2 + sentence1
    sentence2 = marker1 + sentence2
  else:
    sentence1 = marker1 + sentence1
    sentence2 = marker2 + sentence2

  s1 = remove_punct(sentence1)
  s2 = remove_punct(sentence2)
  exception_list = ["This happened", "This happens", "This was", "This is"]
  for c in ['when', 'after', 'since', 'before', 'once']:
        exception_list.append("This is " + c)
        exception_list.append("This was " + c)
        exception_list.append("This happens " + c)
        exception_list.append("This happened " + c)
        if(modal != None):
             exception_list.append('This ' + modal.text + ' happen ' + c)
  if(s1 in exception_list or s2 in exception_list):
        return False, cmp_snt

  if(reverse):
    print(sentence2)
    print(sentence1)
    return True, sentence2 + ". " + sentence1 + "."
  else:
    print(sentence1)
    print(sentence2)
    return True, sentence1 + ". " + sentence2 + "."

def add_marker(root, conj,  modal=None):
  conj_text = conj.text.lower()
  if(conj_text in ["and", "moreover"]):
    return "And"
  elif(conj_text == "if"):
    return "Suppose"
  elif(conj_text in ['though', 'although', 'but', 'whereas', 'however']):
    return "But"
  elif(conj_text in ['when', 'after', 'since', 'before', 'once']):
    if(conj.i > 0):
      if root.tag_ == 'VBP' or root.tag_ == 'VBZ' or root.tag_ == 'VB':
        return 'This is ' + conj_text + " "
      else:
        return 'This was ' + conj_text + " "
    else:
      if root.tag_ == 'VBP' or root.tag_ == 'VBZ':
        return 'This happens ' + conj_text + " "
      elif root.tag_ == 'VB' and modal != None:
        return 'This ' + modal.text + ' happen ' + conj_text + " "
      else:
        return 'This happened ' + conj_text + " "
  elif conj_text in ['because', 'so', 'while', "therefore"]:
    return 'So '
  elif conj_text in ["nevertheless", "nonetheless", "yet"]:
    return 'But'
  elif conj_text == "otherwise":
    return 'or'
  elif conj_text == "unless":
    return 'unless'
  else:
     return ""

def change_verb_form(verb, v_aux, auxpass, subj_tag, subj_word):
    word_list = v_aux.split()
    new_verb = ""
    modal_list = ["can", "could", "must", "should", "will", "would", "may", "might", "shall"]
    if ('will' in word_list or 'would' in word_list) and auxpass != "been":
      print("future simple")
      new_verb = v_aux + conjugate(verb, 'pl')
    elif auxpass == 'been':
      if 'will' in word_list or 'would' in word_list:
        print("future perfect")
        new_verb = v_aux + verb
      elif 'has' in word_list or 'have' in word_list:
        print("present perfect")
        if bool(set(word_list) & set(modal_list)):
            new_verb = v_aux + verb
        elif subj_tag in ("NNS", "NNPS") :
            new_verb = "have " + verb
        elif subj_tag in ("NN", "NNP"):
            new_verb = "has " + verb
        elif subj_tag in ("PRP") and subj_word.lower() in ("them", "me", "you"):
            new_verb = "have " + verb
        elif subj_tag in ("PRP") and subj_word.lower() in ("it", "her", "him"):
            new_verb = "has " + verb
        else:
            new_verb = v_aux + verb
      elif 'had' in word_list:
        print("past perfect")
        new_verb = v_aux + verb
    elif auxpass == "being":
      if('was' in word_list or 'were' in word_list):
        print("past continous")
        if subj_tag in ("NNS", "NNPS"):
            new_verb = "were " + conjugate(verb, tense='present', aspect='progressive')
        elif subj_tag in ("NN", "NNP"):
            new_verb = "was " + conjugate(verb, tense='present', aspect='progressive')
        elif subj_tag in ("PRP") and subj_word.lower() in ("them", "you"):
            new_verb = "were " + conjugate(verb, tense='present', aspect='progressive')
        elif subj_tag in ("PRP") and subj_word.lower() in ("me", "it", "her", "him"):
            new_verb = "was " + conjugate(verb, tense='present', aspect='progressive')
        else:
            new_verb = "was " + conjugate(verb, tense='present', aspect='progressive')
      else:
        print("present continous")
        if subj_tag in ("NNS", "NNPS"):
            new_verb = "are " + conjugate(verb, tense='present', aspect='progressive')
        elif subj_tag in ("NN", "NNP"):
            new_verb = "is " + conjugate(verb, tense='present', aspect='progressive')
        elif subj_tag in ("PRP") and subj_word.lower() in ("them", "you"):
            new_verb = "are " + conjugate(verb, tense='present', aspect='progressive')
        elif subj_tag in ("PRP") and subj_word.lower() in ("it", "her", "him"):
            new_verb = "is " + conjugate(verb, tense='present', aspect='progressive')
        elif subj_tag in ("PRP") and subj_word.lower() in ("me"):
            new_verb = "am " + conjugate(verb, tense='present', aspect='progressive')
        else:
            new_verb = "is " + conjugate(verb, tense='present', aspect='progressive')
    elif auxpass in ['was', 'were']:
      print("simple past")
      if subj_tag in ("NNS", "NNPS"):
        new_verb = conjugate(verb, 'ppl') + " "
      elif subj_tag in ("NN", "NNP"):
        new_verb = conjugate(verb, '3sgp') + " "
      elif subj_tag in ("PRP") and subj_word.lower() in ("them", "me", "you"):
        new_verb = conjugate(verb, 'ppl') + " "
      elif subj_tag in ("PRP") and subj_word.lower() in ("it", "her", "him"):
        new_verb = conjugate(verb, '3sgp') + " "
      else:
        new_verb = conjugate(verb, '3sgp') + " "
    else:
      print("simple present")
      if subj_tag in ("NNS", "NNPS"):
        new_verb = conjugate(verb, 'pl') + " "
      elif subj_tag in ("NN", "NNP"):
        new_verb = conjugate(verb, '3sg') + " "
      elif subj_tag in ("PRP") and subj_word.lower() in ("them", "me", "you"):
        new_verb = conjugate(verb, 'pl') + " "
      elif subj_tag in ("PRP") and subj_word.lower() in ("it", "her", "him"):
        new_verb = conjugate(verb, '3sg') + " "
      else:
        new_verb = conjugate(verb, '3sg') + " "

    return new_verb

def change_pronoun_object(word):
  pron_list = {'me':'I', 'you':'you', 'him':'he', 'her':'she', 'it':'it', 'them':'they', 'us':'we'}
  word_list = word.split()
  for i in range(len(word_list)):
    if(word_list[i] in pron_list.keys()):
        word_list[i] = pron_list[word_list[i]]
  return " ".join(str(e) for e in word_list)

def change_pronoun_subject(word):
  pron_list = {'i':'me', 'you':'you', 'he': 'him', 'she':'her', 'it':'it', 'they':'them', 'we':'us'}
  word_list = word.split()
  word_list = word.split()
  for i in range(len(word_list)):
    if(word_list[i].lower() in pron_list.keys()):
        word_list[i] = pron_list[word_list[i].lower()]
  return " ".join(str(e) for e in word_list)

def passive_simplify(cmp_snt):
    # nlp=spacy.load('en_core_web_sm')
    doc = spacy_nlp(cmp_snt)
    simplify_flag = False
    root = get_root(doc)
    auxpass = find_in_children(root, "auxpass")
    if(auxpass == None):
        print("No passive clause found")
        return simplify_flag, cmp_snt
    agent = find_in_children(root, "agent")
    if(agent == None):
        print("No agent found")
        return simplify_flag, cmp_snt
    subj = find_in_children(root, "nsubjpass")
    if(subj == None):
        print("No subject found")
        return simplify_flag, cmp_snt

    simplify_flag = True

    aux = find_in_children(root, "aux")

    aux_tense = auxpass.tag_
    if aux_tense == 'VB' and aux != None:
        aux_tense = aux.tag_
    elif aux_tense == 'VBN' and aux != None:
        if aux.text.lower() not in ("has", "have", "had"):
            aux_tense = 'MD'

    new_obj = inorder_traversal(subj)
    obj = find_in_children(agent, "pobj")
    new_subj = inorder_traversal(obj)
    removed_aux = []
    for child in root.children:
        if(child.dep_ == "aux"):
            removed_aux.append(child)

    left_of_subj = ""
    right_of_subj = ""
    left_of_obj = ""
    right_of_obj = ""

    flag = 0
    for child in root.lefts:
        if(child.dep_ == "nsubjpass"):
            flag = 1
            continue
        if(flag == 0 and child not in [root, subj, auxpass, agent] + removed_aux):
            left_of_subj += inorder_traversal(child) + " "
        elif(flag == 1 and child not in [root, subj, auxpass, agent] + removed_aux):
            right_of_subj += inorder_traversal(child) + " "

    flag = 0
    for child in root.rights:
        if(child.dep_ == "agent"):
            continue
        if(((child.dep_ == "prt" or child.dep_ == "prep") and (len(list(child.children)) == 0)) and child not in [root, subj, auxpass, agent] + removed_aux):
            left_of_obj += inorder_traversal(child) + " "
        elif(child not in [root, subj, auxpass, agent] + removed_aux):
            right_of_obj += inorder_traversal(child) + " "

    v_aux = ""
    if(aux != None):
        v_aux = aux.text
    aux_str = ""
    for child in root.children:
        if(child.dep_ == "aux"):
            aux_str += child.text + " "
    new_verb = change_verb_form(root.text, aux_str, auxpass.text, obj.tag_, obj.text)
    new_subj = change_pronoun_object(new_subj)
    new_obj = change_pronoun_subject(new_obj)
    final_sent = left_of_subj + " " + new_subj + " " + right_of_subj + " " + new_verb + " " + left_of_obj + " " + new_obj + " " + right_of_obj
    final_sent = ' '.join(final_sent.split())
    final_sent = truecase.get_true_case(final_sent)
    final_sent = remove_punct(final_sent)
    print(final_sent)
    return simplify_flag, final_sent

def simplify(snt,Qnlp):
    doc=Qnlp(snt)
    flag = False
    final_snt = ""
    for s in doc.sents:
        flag, simp_snt = appositive_simplification_2(s.text)
        if(flag == True):
            print(simp_snt)
            final_snt += (simplify(simp_snt,Qnlp) + " ")
            continue
        else:
            flag, simp_snt = conjoint_clause_simplification(s.text,Qnlp)
            if(flag == True):
                print(simp_snt)
                final_snt += (simplify(simp_snt,Qnlp) + " ")
                continue
            else:
                flag, simp_snt = relative_clause_simplify(s.text)
                if(flag == True):
                    final_snt += (simplify(simp_snt,Qnlp) + " ")
                    continue
                else:
                    flag, simp_snt = passive_simplify(s.text)
                    if(flag == True):
                        final_snt += (simplify(simp_snt,Qnlp) + " ")
                        continue
                    else:
                        final_snt += (simp_snt + " ")
    return final_snt

# Custom simplifier wrapper around the DEPSYM code
from nltk.tokenize import sent_tokenize
from typing import *
import os, contextlib # For silencing output. From https://stackoverflow.com/a/28321717
from ..utils import detokenize_for_output

class DEPSYMSimplifier(AbstractSimplifier):

    def __init__(self):
        # Will hold onto the Stanza NLP object
        # Reuse across simplifications because this is heavy to create
        self.Qnlp = None

    def simplify_documents(self, documents:Union[List[str], str]) -> Union[List[List[str]], List[List[Tuple[str,str]]]]:
        """Simplifies the documents given

        Args:
            documents : str | [str]
                The document(s) to simplify

        Returns:
            A list containing the simplified documents. `out[0]` gives the simplified documents.
            `out[1]` gives the complex-simple sentence pairs. Accessing complex-simple pairs will
            follow the format: `pairs[doc_idx][pair_idx][orig=0|simp=1]`
        """

        # Ensure list
        if isinstance(documents, str):
            documents = [documents]

        # Split a document to its individual sentences
        orig_sents = []
        for document in documents:
            # Split document into sentences using NLTK
            doc_sents = sent_tokenize(document, language='english')
            orig_sents.append(doc_sents)

        # Initialize stanza if not yet initialized
        if self.Qnlp is None:
            self.Qnlp = spacy_stanza.load_pipeline("en")

        # Create simplified predictions
        simp_docs = []
        simp_sents = []
        # For each document
        for doc_idx, doc_sents in enumerate(orig_sents):
            print(f"Simplifying document {doc_idx+1}")
            simp_doc_sents = []
            # Output silencing from https://stackoverflow.com/a/28321717
            with open(os.devnull, 'w') as devnull:
                with contextlib.redirect_stdout(devnull):
                    # For each sentence in the document
                    for comp_sent in doc_sents:
                        # Simplify the sentences
                        sentence = simplify(comp_sent, self.Qnlp)
                        # Fix formatting
                        sentence = detokenize_for_output(sentence)
                        simp_doc_sents.append(sentence)
            # Collect the simplified sentences per doument
            simp_docs.append(' '.join(simp_doc_sents))
            simp_sents.append(simp_doc_sents)

        # Create complex-simple sentence pairs
        orig_simp_pairs = []
        # For each doc that was processed
        for doc_idx in range(len(documents)):
            # Get all the orig and simplified sentences
            doc_orig_sents = orig_sents[doc_idx]
            doc_simp_sents = simp_sents[doc_idx]
            # Store as pair
            orig_simp_pairs.append([ (doc_orig_sents[sent_idx], doc_simp_sents[sent_idx] )
                                    for sent_idx in range(len(doc_orig_sents)) ])

        return simp_docs, orig_simp_pairs


'''
This script load the PokeAPI database with the Pokemons in docs/pokemon.json
To run just load the enviroment and execute

python load_data.py

'''

import json
import os
import django
from django.db import transaction
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pokeapi.settings")
django.setup()

from apiV1.models.pokemon import *

with open('../docs/pokemon.json') as json_file:
    data = json.load(json_file)
    for pokemon in data['pokemon']:
        with transaction.atomic():
            _id = pokemon.pop('id')
            types_list = pokemon.pop('types')
            obj = Pokemon.objects.create(**pokemon)

            for tp in types_list:
                t, created = PokemonType.objects.get_or_create(name=tp)
                obj.types.add(t)


import os

# Flask
TESTING = True
DEBUG = True
ENV = 'development'
SECRET_KEY = os.environ['FLASK_SECRET_KEY']

# SQLAlchemy
SQLALCHEMY_DATABASE_URI = os.environ['FLASK_DATABASE_URL']
SQLALCHEMY_TRACK_MODIFICATIONS = False

# Auth
AUTH_DOMAIN = os.environ['AUTH_DOMAIN']
AUTH_ALGORITHM = os.environ['AUTH_ALGORITHM']
AUTH_API_AUDIENCE = os.environ['AUTH_API_AUDIENCE']
AUTH_CLIENT_ID = os.environ['AUTH_CLIENT_ID']


import os.path

from Orange.data.table import Table
from Orange.widgets import gui, widget
from Orange.widgets.settings import Setting
from Orange.data.io import FileFormat
from Orange.widgets.utils import filedialogs
from Orange.widgets.widget import Input

class OWSave(widget.OWWidget):
    name = "Save Data"
    description = "Save data to an output file."
    icon = "icons/Save.svg"
    category = "Data"
    keywords = ["data", "save"]

    class Inputs:
        data = Input("Data", Table)

    want_main_area = False
    resizing_enabled = False

    last_dir = Setting("")
    last_filter = Setting("")
    auto_save = Setting(False)

    @classmethod
    def get_writers(cls, sparse):
        return [
            f
            for f in FileFormat.formats
            if getattr(f, "write_file", None)
            and getattr(f, "EXTENSIONS", None)
            and (not sparse or getattr(f, "SUPPORT_SPARSE_DATA", False))
        ]

    def __init__(self):
        super().__init__()
        self.data = None
        self.filename = ""
        self.writer = None

        self.save = gui.auto_commit(
            self.controlArea,
            self,
            "auto_save",
            "Save",
            box=False,
            commit=self.save_file,
            callback=self.adjust_label,
            disabled=True,
            addSpace=True,
        )
        self.saveAs = gui.button(
            self.controlArea,
            self,
            "Save As...",
            callback=self.save_file_as,
            disabled=True,
        )
        self.saveAs.setMinimumWidth(220)
        self.adjustSize()

    def adjust_label(self):
        if self.filename:
            filename = os.path.split(self.filename)[1]
            text = ["Save as '{}'", "Auto save as '{}'"][self.auto_save]
            self.save.button.setText(text.format(filename))

    @Inputs.data
    def dataset(self, data):
        self.data = data
        self.save.setDisabled(data is None)
        self.saveAs.setDisabled(data is None)
        if data is not None:
            self.save_file()

    def save_file_as(self):
        file_name = self.filename or os.path.join(
            self.last_dir or os.path.expanduser("~"), getattr(self.data, "name", "")
        )
        filename, writer, filter = filedialogs.open_filename_dialog_save(
            file_name, self.last_filter, self.get_writers(self.data.is_sparse())
        )
        if not filename:
            return
        self.filename = filename
        self.writer = writer
        self.unconditional_save_file()
        self.last_dir = os.path.split(self.filename)[0]
        self.last_filter = filter
        self.adjust_label()

    def save_file(self):
        if self.data is None:
            return
        if not self.filename:
            self.save_file_as()
        else:
            try:
                self.writer.write(self.filename, self.data)
            except Exception as errValue:
                self.error(str(errValue))
            else:
                self.error()

if __name__ == "__main__":
    import sys
    from AnyQt.QtWidgets import QApplication

    a = QApplication(sys.argv)
    table = Table("iris")
    ow = OWSave()
    ow.show()
    ow.dataset(table)
    a.exec()
    ow.saveSettings()