from uuid import uuid4

import pytest

from pydent.marshaller.base import add_schema
from pydent.marshaller.base import ModelRegistry
from pydent.marshaller.fields import Alias
from pydent.marshaller.fields import Callback
from pydent.marshaller.fields import Field
from pydent.marshaller.fields import Nested
from pydent.marshaller.fields import Relationship

class TestDump:
    def test_dump_empty_data(self, base):
        """Dump should produce an empty dictionary."""

        @add_schema
        class MyModel(base):
            pass

        model = MyModel()
        assert model.dump() == {}

    def test_dump_empty_data_with_non_tracked_attrs(self, base):
        """Expect that non-tracked attributes are excluded from the dump."""

        @add_schema
        class MyModel(base):
            pass

        model = MyModel()
        model.id = 4
        assert model.dump() == {}

    def test_dump_loaded_data(self, base):
        """Manually set data should appear in the dump."""

        @add_schema
        class MyModel(base):
            pass

        model = MyModel._set_data({"id": 5, "name": "MyName"})
        assert model.dump() == {"id": 5, "name": "MyName"}

    def test_dump_loaded_data_and_overwrite(self, base):
        """Manually set data can be overridden by setting attributes."""

        @add_schema
        class MyModel(base):
            pass

        model = MyModel._set_data({"id": 5, "name": "MyName"})
        model.id = 6
        assert model.dump() == {"id": 6, "name": "MyName"}

    def test_dump_empty_field(self, base):
        """Empty fields should return an empty dictionary."""

        @add_schema
        class MyModel(base):
            fields = dict(field=Field())

        model = MyModel()
        assert model.dump() == {}

    def test_dump_field(self, base):
        @add_schema
        class MyModel(base):
            fields = dict(field=Field())

        model = MyModel._set_data({"name": "NAME"})
        assert model.dump() == {"name": "NAME"}

    def test_dump_with_new_data_key(self, base):
        @add_schema
        class MyModel(base):
            fields = {
                "field": Field(),
                "source": Callback(
                    lambda s: getattr(s, "field"),
                    callback_args=(Callback.SELF,),
                    always_dump=True,
                    data_key="field",
                ),
            }

        model = MyModel({"field": 5})
        assert model.field == model.source
        print(model._get_data())
        assert model.dump() == {"field": 5}
        model.source = 6
        assert model.field == model.source
        assert model.dump() == {"field": 6}
        model.field = 7
        assert model.field == model.source
        assert model.dump() == {"field": 7}

        model = MyModel({"source": 5})
        assert model.field == model.source
        print(model._get_data())
        assert model.dump() == {"field": 5}
        model.source = 6
        assert model.field == model.source
        assert model.dump() == {"field": 6}
        model.field = 7
        assert model.field == model.source
        assert model.dump() == {"field": 7}
        print(model._get_data())

    def test_alias(self, base):
        """Expect that alias fields refer to exactly the attribute set in the
        alias.

        That means, the 'source' field should refer to the 'field'
        attribute.
        """

        @add_schema
        class MyModel(base):
            fields = {"field": Field(), "source": Alias("field")}

        model = MyModel({"field": 5})
        assert model.field == model.source
        assert model.dump() == {"field": 5}
        model.source = 6
        assert model.field == model.source
        assert model.dump() == {"field": 6}
        model.field = 7
        assert model.field == model.source
        assert model.dump() == {"field": 7}

        model = MyModel({"source": 5})
        assert model.field == model.source
        print(model._get_data())
        assert model.dump() == {"field": 5}
        model.source = 6
        assert model.field == model.source
        assert model.dump() == {"field": 6}
        model.field = 7
        assert model.field == model.source
        assert model.dump() == {"field": 7}

    def test_dump_marshalling_field(self, base):
        """Expect the custom HTMLTag field to be properly
        serialized/deserialized."""

        class HTMLTag(Field):
            def serialize(self, caller, val):
                return "<{tag}>{val}</{tag}>".format(tag=self.data_key, val=val)

        @add_schema
        class MyModel(base):
            fields = dict(h1=HTMLTag())

        model = MyModel._set_data({"h1": "raw"})
        assert model.h1 == "raw"

        model.h1 = "This is a Heading 1 Title"
        assert model.h1 == "This is a Heading 1 Title"

        assert model.dump() == {"h1": "
This is a Heading 1 Title
"}

    def test_always_dump(self, base):
        """Expect that fields with 'always_dump' are, by default, dumped as
        empty constructors event when they are empty."""

        @add_schema
        class MyModel(base):
            fields = dict(
                field1=Callback("find"), field2=Callback("find", always_dump=True)
            )

            def find(self):
                return 100

        m = MyModel()

        assert m.dump() == {"field2": 100}
        assert m.dump(include="field1") == {"field1": 100, "field2": 100}
        assert m.dump(ignore="field2") == {}

    def test_empty_list_field(self, base):
        """Expect."""

        @add_schema
        class ModelWithList(base):
            fields = dict(mylist=Field())

        model = ModelWithList()
        model.mylist = []
        assert model.mylist == []
        model.mylist.append(5)
        assert model.mylist == [5]

class TestNested:
    """Tests for nested serialization/deserialization."""

    @pytest.fixture(scope="function")
    def Company(self, base):
        @add_schema
        class Company(base):
            pass

        return Company

    @pytest.fixture(scope="function")
    def Publisher(self, base):
        @add_schema
        class Publisher(base):

            fields = dict(author=Nested("Author"), company=Nested("Company"))

        return Publisher

    @pytest.fixture(scope="function")
    def Author(self, base):
        @add_schema
        class Author(base):

            fields = dict(
                publisher=Nested("Publisher"), id=Field("id", allow_none=True)
            )

        return Author

    def test_simple_nested(self, Author, Publisher):
        author = Author._set_data(
            {"name": "Richard Dawkings", "publisher": {"name": "Scotts Books"}}
        )

        print(author._get_data())
        assert isinstance(author, Author)
        assert isinstance(author.publisher, Publisher)

        assert author.name == "Richard Dawkings"
        assert author.publisher.name == "Scotts Books"

    def test_double_nested(self, Author, Publisher, Company):
        author = Author._set_data(
            {
                "name": "Samuel",
                "publisher": {"name": "Archive 81", "company": {"name": "Damage Inc."}},
            }
        )

        print(author._get_data())
        assert isinstance(author, Author)
        assert isinstance(author.publisher, Publisher)
        assert isinstance(author.publisher.company, Company)

    @pytest.fixture(scope="function")
    def author_example_data(self):
        data = {
            "name": "Samuel",
            "publisher": {"name": "Archive 81", "company": {"name": "Damage Inc."}},
        }
        return data

    @pytest.fixture(scope="function")
    def author_example(self, author_example_data, Author, Publisher, Company):
        author = Author._set_data(author_example_data)
        return author

    def test_shared_data(self, author_example, author_example_data):

        author = author_example
        company = author.publisher.company

        print(id(author_example_data["publisher"]))
        assert author_example_data["publisher"] is author._get_data()["publisher"]
        print(id(author._get_data()["publisher"]))
        publisher = author.publisher
        print(id(publisher._get_data()))
        assert author._get_data()["publisher"] is publisher._get_data()

    def test_double_nested_dump(self, author_example, author_example_data):
        assert author_example._get_data() == author_example_data
        assert author_example.publisher._get_data() == author_example_data["publisher"]
        assert (
            author_example.publisher.company._get_data()
            == author_example_data["publisher"]["company"]
        )

    def test_del_nested(self, author_example, author_example_data):

        author_example.name = "TIM"
        assert author_example.name == "TIM"

        author_example.publisher.name = "Holland"
        assert author_example.publisher.name == "Holland"

        author_example.publisher.company.name = "ABC"
        assert author_example.publisher.company.name == "ABC"
        del author_example.publisher.company

        with pytest.raises(AttributeError):
            author_example.publisher.company

        assert "company" not in author_example._get_data()["publisher"]
        assert "company" not in author_example.publisher._get_data()

    def test_set_none_on_nested(self, author_example):
        author_example.publisher = None
        assert author_example.publisher is None
        assert author_example._get_data()["publisher"] is None

    def test_set_nested_attribute(self, author_example, Publisher):
        author_example.publisher = None
        assert author_example.publisher is None
        assert author_example._get_data()["publisher"] is None
        publisher = Publisher._set_data({"name": "P"})
        author_example.publisher = publisher

        assert author_example.publisher.name == "P"
        assert author_example._get_data()["publisher"] is publisher._get_data()

    def test_nested_dump(self, author_example, author_example_data):

        new_company_name = str(uuid4())

        expected_data = dict(author_example_data)
        expected_data["publisher"]["company"]["name"] = new_company_name

        author_example.publisher.company.name = new_company_name

        expected_data_copy = dict(expected_data)
        expected_data_copy.pop("publisher")
        assert expected_data_copy == author_example.dump()

        expected_data_copy = dict(expected_data["publisher"])
        expected_data_copy.pop("company")
        assert expected_data_copy == author_example.publisher.dump()

        assert (
            expected_data["publisher"]["company"]
            == author_example.publisher.company.dump()
        )

    def test_load_a_model(self, base, author_example):
        @add_schema
        class AuthorList(base):
            fields = dict(author=Nested("Author"))

        author_list = AuthorList()
        author_example.publisher.company.name = "Umbrella Corp"
        author_list.author = author_example
        assert author_list.author.publisher.company.name == "Umbrella Corp"

        author_example.publisher.company.name = "LexCorp"
        assert author_list.author.publisher.company.name == "LexCorp"

class TestRelationship:
    @pytest.fixture(scope="function")
    def Company(self, base):
        @add_schema
        class Company(base):
            pass

        return Company

    @pytest.fixture(scope="function")
    def Publisher(self, base):
        @add_schema
        class Publisher(base):
            fields = dict(
                company=Relationship(
                    "Company", "instantiate_model", 6, {"name": "MyCompany"}
                )
            )

            def instantiate_model(self, model_name, model_id, name="Default"):
                return ModelRegistry.get_model(model_name)._set_data(
                    {"id": model_id, "name": name}
                )

        return Publisher

    @pytest.fixture(scope="function")
    def Author(self, base):
        @add_schema
        class Author(base):
            fields = dict(
                publisher=Relationship(
                    "Publisher", "instantiate_model", 4, {"name": "MyPublisher"}
                )
            )

            def instantiate_model(self, model_name, model_id, name="Default"):
                return ModelRegistry.get_model(model_name)._set_data(
                    {"id": model_id, "name": name}
                )

        return Author

    @pytest.mark.parametrize(
        "model,include,expected",
        [
            ("Company", None, {}),
            ("Publisher", None, {}),
            ("Author", None, {}),
            pytest.param(
                "Publisher",
                "company",
                {"company": {"id": 6, "name": "MyCompany"}},
                id="include 1 layer nested",
            ),
            pytest.param(
                "Author",
                "publisher",
                {"publisher": {"id": 4, "name": "MyPublisher"}},
                id="include 1 layer nested",
            ),
            pytest.param(
                "Author",
                {"publisher": "company"},
                {
                    "publisher": {
                        "id": 4,
                        "name": "MyPublisher",
                        "company": {"id": 6, "name": "MyCompany"},
                    }
                },
                id="include 2 layer nested",
            ),
        ],
    )
    def test_nested_dump_with_include(
        self, base, Author, Publisher, Company, model, include, expected
    ):
        instance = ModelRegistry.get_model(model)()
        assert instance.dump(include=include) == expected

    @pytest.mark.parametrize(
        "model,only,expected",
        [
            pytest.param(
                "Author", "publisher", {"publisher": {"name": "MyPublisher", "id": 4}}
            ),
            pytest.param(
                "Author", {"publisher": "name"}, {"publisher": {"name": "MyPublisher"}}
            ),
            pytest.param("Author", {"publisher": "id"}, {"publisher": {"id": 4}}),
            pytest.param(
                "Author",
                {"publisher": "company"},
                {"publisher": {"company": {"name": "MyCompany", "id": 6}}},
            ),
            pytest.param(
                "Author",
                {"publisher": {"company": "id"}},
                {"publisher": {"company": {"id": 6}}},
            ),
        ],
    )
    def test_relationship_dump_with_only(
        self, base, Author, Publisher, Company, model, only, expected
    ):
        instance = ModelRegistry.get_model(model)()
        assert instance.dump(only=only) == expected

    def test_relationship_dump_ignore(self, base, Author):
        instance = Author._set_data({"name": "MyName", "id": 5})
        assert instance.dump() == {"name": "MyName", "id": 5}
        assert instance.dump(ignore="name") == {"id": 5}
        assert instance.dump(ignore=["name", "id"]) == {}

    def test_basic_relationship(self, base):
        @add_schema
        class Publisher(base):
            pass

        @add_schema
        class Author(base):

            fields = dict(
                publisher=Relationship(
                    "Publisher", "instantiate_model", 4, {"name": "MyPublisher"}
                ),
                cache=True,
            )

            def instantiate_model(self, model_name, model_id, name="Default"):
                return ModelRegistry.get_model(model_name)._set_data(
                    {"id": model_id, "name": name}
                )

        author = Author()
        assert author.dump() == {}
        assert isinstance(
            author.publisher, Publisher
        ), "publisher attribute should be a Publisher type"
        assert (
            author.publisher._get_data() is author._get_data()["publisher"]
        ), "data should be shared between the Author and Publisher"

        assert author._get_data() == {"publisher": {"id": 4, "name": "MyPublisher"}}

        assert author.dump() == {}
        assert author.dump(include=["publisher"]) == author._get_data()

    def test_many_relationship(self, base):
        @add_schema
        class Publisher(base):
            pass

        @add_schema
        class Author(base):

            fields = dict(
                publisher=Relationship(
                    "Publisher",
                    "instantiate_model",
                    4,
                    {"name": "MyPublisher"},
                    many=True,
                )
            )

            def instantiate_model(self, model_name, model_id, name="Default"):
                models = []
                for i in range(3):
                    models.append(
                        ModelRegistry.get_model(model_name)._set_data(
                            {"id": model_id, "name": name}
                        )
                    )
                return models

        author = Author()
        print(author.dump())
        print(author.dump(include="publisher"))
        assert len(author.publisher) == 3
        assert len(author.dump(include="publisher")["publisher"]) == 3
        author.publisher = [
            Publisher._set_data({"id": 3}),
            Publisher._set_data({"id": 5}),
        ]

        assert len(author.publisher) == 2
        assert len(author.dump(include="publisher")["publisher"]) == 2
        assert author.publisher[0].id == 3
        assert author.publisher[1].id == 5

        author.publisher.append(Publisher())
        assert len(author.publisher) == 3
        assert len(author.dump(include="publisher")["publisher"]) == 3

    def test_load_relationship(self, base):
        @add_schema
        class Publisher(base):
            pass

        @add_schema
        class Author(base):

            fields = dict(
                publishers=Relationship(
                    "Publisher",
                    "instantiate_model",
                    4,
                    {"name": "MyPublisher"},
                    many=True,
                )
            )

            def instantiate_model(self, model_name, model_id, name="Default"):
                models = []
                for i in range(3):
                    models.append(
                        ModelRegistry.get_model(model_name)._set_data(
                            {"id": model_id, "name": name}
                        )
                    )
                return models

        publisher_data = {"name": "Torr Books"}

        author = Author._set_data(
            {"name": "Steven King", "publishers": [publisher_data]}
        )

        assert len(author.publishers) == 1
        assert isinstance(author.publishers[0], Publisher)
        assert author.publishers[0]._get_data() is publisher_data
        del author.publishers

        assert len(author.publishers) == 3
        assert isinstance(author.publishers[0], Publisher)


import glob
import io
import logging
import re
import textwrap
import traceback
from contextlib import redirect_stdout

import discord
from discord.ext import commands
from discord.ext.commands import Cog, Bot, Context

from cogs.commands import settings
from utils import embeds
from utils.record import record_usage

# Enabling logs
log = logging.getLogger(__name__)

class AdministrationCog(Cog):
    """ Administration Cog Cog """

    def __init__(self, bot):
        self.bot = bot
        self._last_result = None

    def _cleanup_code(self, content):
        """Automatically removes code blocks from the code."""
        # remove ```py\n```
        if content.startswith('```') and content.endswith('```'):
            return '\n'.join(content.split('\n')[1:-1])

        # remove `foo`
        return content.strip('` \n')

    @commands.before_invoke(record_usage)
    @commands.group(aliases=["u", "ul"])
    async def utilities(self, ctx):
        if ctx.invoked_subcommand is None:
            # Send the help command for this group
            await ctx.send_help(ctx.command)

    @commands.is_owner()
    @utilities.command(name="ping")
    async def ping(self, ctx):
        """Returns the Discord WebSocket latency."""
        await ctx.send(f"{round(self.bot.latency * 1000)}ms.")

    @commands.is_owner()
    @utilities.command(name="say")
    async def say(self, ctx, *, args):
        """Echos the input argument."""
        await ctx.send(args)

    @commands.is_owner()
    @utilities.command(name="eval")
    async def eval(self, ctx, *, body: str):
        """Evaluates input as Python code."""
        # Required environment variables.
        env = {
            'bot': self.bot,
            'ctx': ctx,
            'channel': ctx.channel,
            'author': ctx.author,
            'guild': ctx.guild,
            'message': ctx.message,
            'embeds': embeds,
            '_': self._last_result
        }
        # Creating embed.
        embed = discord.Embed(title="Evaluating.", color=0xb134eb)
        env.update(globals())

        # Calling cleanup command to remove the markdown traces.
        body = self._cleanup_code(body)
        embed.add_field(
            name="Input:", value=f"```py\n{body}\n```", inline=False)
        # Output stream.
        stdout = io.StringIO()

        # Exact code to be compiled.
        to_compile = f'async def func():\n{textwrap.indent(body, "  ")}'

        try:
            # Attempting execution
            exec(to_compile, env)
        except Exception as e:
            # In case there's an error, add it to the embed, send and stop.
            errors = f'```py\n{e.__class__.__name__}: {e}\n```'
            embed.add_field(name="Errors:", value=errors, inline=False)
            await ctx.send(embed=embed)
            return errors

        func = env['func']
        try:
            with redirect_stdout(stdout):
                ret = await func()
        except Exception:
            # In case there's an error, add it to the embed, send and stop.
            value = stdout.getvalue()
            errors = f'```py\n{value}{traceback.format_exc()}\n```'
            embed.add_field(name="Errors:", value=errors, inline=False)
            await ctx.send(embed=embed)

        else:
            value = stdout.getvalue()
            try:
                await ctx.message.add_reaction('\u2705')
            except:
                pass

            if ret is None:
                if value:
                    # Output.
                    output = f'```py\n{value}\n```'
                    embed.add_field(
                        name="Output:", value=output, inline=False)
                    await ctx.send(embed=embed)
            else:
                # Maybe the case where there's no output?
                self._last_result = ret
                output = f'```py\n{value}{ret}\n```'
                embed.add_field(name="Output:", value=output, inline=False)
                await ctx.send(embed=embed)

    @commands.is_owner()
    @utilities.command(name="reload")
    async def reload_cog(self, ctx: commands.Context, name_of_cog: str = None):
        """ Reloads specified cog or all cogs. """

        regex = r"(?<=<).*(?=\..* object at 0x.*>)"
        if name_of_cog is not None and name_of_cog in ctx.bot.cogs:
            # Reload cog if it exists.
            cog = re.search(regex, str(ctx.bot.cogs[name_of_cog]))
            try:
                self.bot.reload_extension(cog.group())

            except commands.ExtensionError as e:
                await ctx.message.add_reaction("❌")
                await ctx.send(f'{e.__class__.__name__}: {e}')

            else:
                await ctx.message.add_reaction("✔")
                await ctx.send(f"Reloaded `{cog.group()}` module!")

        elif name_of_cog is None:
            # Reload all the cogs in the folder named cogs.
            # Skips over any cogs that start with '__' or do not end with .py.
            cogs = []
            try:
                for cog in glob.iglob("cogs/**/[!^_]*.py", recursive=True):
                    if "\\" in cog:  # Pathing on Windows.
                        self.bot.reload_extension(cog.replace("\\", ".")[:-3])
                    else:  # Pathing on Linux.
                        self.bot.reload_extension(cog.replace("/", ".")[:-3])
            except commands.ExtensionError as e:
                await ctx.message.add_reaction("❌")
                await ctx.send(f'{e.__class__.__name__}: {e}')

            else:
                await ctx.message.add_reaction("✔")
                await ctx.send("Reloaded all modules!")
        else:
            await ctx.message.add_reaction("❌")
            await ctx.send("Module not found, check spelling, it's case sensitive.")

    @commands.is_owner()
    @commands.bot_has_permissions(embed_links=True, send_messages=True)
    @commands.before_invoke(record_usage)
    @commands.command(name="rules")
    async def rules(self, ctx: Context):
        """ Generates the #rules channel embeds. """

        # Captain Karen header image embed
        embed = embeds.make_embed(color="quotes_grey")
        embed.set_image(url="https://i.imgur.com/Yk4kwZy.gif")
        await ctx.send(embed=embed)

        # The actual rules embed
        embed = embeds.make_embed(title="📃  Discord Server Rules", color="quotes_grey", description="This list is not all-encompassing and you may be actioned for a reason outside of these rules. Use common sense when interacting in our community.")
        embed.add_field(name="Rule 1: Do not send copyright-infringing material.", inline=False, value="> Linking to torrents, pirated stream links, direct download links, or uploading files over Discord puts our community at risk of being shut down. We are a discussion community, not a file-sharing hub.")
        embed.add_field(name="Rule 2: Be courteous and mindful of others.", inline=False, value="> Do not engage in toxic behavior such as spamming, derailing conversations, attacking other users, or attempting to instigate drama. Bigotry will not be tolerated. Avoid problematic avatars, usernames, or nicknames.")
        embed.add_field(name="Rule 3: Do not post self-promotional content.", inline=False, value="> We are not a billboard nor the place to advertise your Discord server, app, website, service, etc.")
        embed.add_field(name="Rule 4: Do not post unmarked spoilers.", inline=False, value="> Use spoiler tags and include what series or episode your spoiler is in reference to outside the spoiler tag so people don't blindly click a spoiler.")
        embed.add_field(name="Rule 5: Do not backseat moderate.", inline=False, value="> If you see someone breaking the rules or have something to report, please submit a <#829861810999132160> ticket.")
        embed.add_field(name="Rule 6: Do not abuse pings.", inline=False, value="> Do not ping staff outside of conversation unless necessary. Do not ping VIP users for questions or help with their service. Do not spam or ghost ping other users.")
        embed.add_field(name="Rule 7: Do not beg, buy, sell, or trade.", inline=False, value="> This includes, but is not limited to, server ranks, roles, permissions, giveaways, private community invites, or any digital or physical goods.")
        embed.add_field(name="Rule 8: Follow the Discord Community Guidelines and Terms of Service.", inline=False, value="> The Discord Community Guidelines and Terms of Service govern all servers on the platform. Please familarize yourself with them and the restrictions that come with them. \n> \n> https://discord.com/guidelines \n> https://discord.com/terms")
        await ctx.send(embed=embed)

        # /r/animepiracy links embed
        embed = embeds.make_embed(title="🔗  Our Links", color="quotes_grey")
        embed.add_field(name="Reddit:", inline=True, value="> [/r/animepiracy](https://reddit.com/r/animepiracy)")
        embed.add_field(name="Discord:", inline=True, value="> [discord.gg/piracy](https://discord.gg/piracy)")
        embed.add_field(name="Index:", inline=True, value="> [piracy.moe](https://piracy.moe)")
        embed.add_field(name="Wiki:", inline=True, value="> [wiki.piracy.moe](https://wiki.piracy.moe)")
        embed.add_field(name="Seadex:", inline=True, value="> [releases.moe](https://releases.moe)")
        embed.add_field(name="GitHub:", inline=True, value="> [github.com/ranimepiracy](https://github.com/ranimepiracy)")
        embed.add_field(name="Twitter:", inline=True, value="> [@ranimepiracy](https://twitter.com/ranimepiracy)")
        embed.add_field(name="Uptime Status:", inline=True, value="> [status.piracy.moe](https://status.piracy.moe/)")
        await ctx.send(embed=embed)

        # Clean up the command invoker
        await ctx.message.delete()

    @commands.is_owner()
    @commands.bot_has_permissions(embed_links=True, send_messages=True)
    @commands.before_invoke(record_usage)
    @commands.command(name="createticketembed")
    async def create_ticket_embed(self, ctx: Context):
        embed = embeds.make_embed(title="🎫 Create a new modmail ticket",
                                  description="Click the react below to create a new modmail ticket.",
                                  color="default")
        embed.add_field(name="Warning:", value="Serious inquiries only. Abuse may result in warning or ban.")
        spawned = await ctx.send(embed=embed)
        await spawned.add_reaction("🎫")
        await ctx.message.delete()

    @commands.is_owner()
    @commands.bot_has_permissions(embed_links=True, send_messages=True)
    @commands.before_invoke(record_usage)
    @commands.command(name="createcolorrolesembed", aliases=['ccre'])
    async def create_color_roles_embed(self, ctx: Context):
        embed = discord.Embed(description=f"You can react to one of the squares below to be assigned a colored user role. If you are interested in a different color, you can become a <@&{settings.get_value('role_server_booster')}> to receive a custom colored role.")
        msg = await ctx.send(embed=embed)

        # API call to fetch all the emojis to cache, so that they work in future calls
        emotes_guild = await ctx.bot.fetch_guild(settings.get_value("emoji_guild_id"))
        emojis = await emotes_guild.fetch_emojis()

        await msg.add_reaction(":redsquare:805032092907601952")
        await msg.add_reaction(":orangesquare:805032107952308235")
        await msg.add_reaction(":yellowsquare:805032120971165709")
        await msg.add_reaction(":greensquare:805032132325801994")
        await msg.add_reaction(":bluesquare:805032145030348840")
        await msg.add_reaction(":pinksquare:805032162197635114")
        await msg.add_reaction(":purplesquare:805032172074696744")
        await ctx.message.delete()

    @commands.is_owner()
    @commands.bot_has_permissions(embed_links=True, send_messages=True)
    @commands.before_invoke(record_usage)
    @commands.command(name="createassignablerolesembed", aliases=['care'])
    async def create_assignable_roles_embed(self, ctx: Context):
        role_assignment_text = """
        You can react to one of the emotes below to assign yourself an event role.

        🎁  <@&832528733763928094> - Receive giveaway pings.
        📢  <@&827611682917711952> - Receive server announcement pings.
        📽  <@&831999443220955136> - Receive group watch event pings.
        <:kakeraW:830594599001129000>  <@&832512304334766110> - Receive Mudae event and season pings.
        🧩  <@&832512320306675722> - Receive Rin event pings.
        <:pickaxe:831765423455993888>  <@&832512327731118102> - Receive Minecraft server related pings.
        """
        embed = discord.Embed(description=role_assignment_text)
        msg = await ctx.send(embed=embed)

        # API call to fetch all the emojis to cache, so that they work in future calls
        emotes_guild = await ctx.bot.fetch_guild(settings.get_value("emoji_guild_id"))
        await emotes_guild.fetch_emojis()

        await msg.add_reaction("🎁")
        await msg.add_reaction("📢")
        await msg.add_reaction("📽")
        await msg.add_reaction(":kakeraW:830594599001129000")
        await msg.add_reaction("🧩")
        await msg.add_reaction(":pickaxe:831765423455993888")
        await ctx.message.delete()

def setup(bot: Bot) -> None:
    """ Load the AdministrationCog cog. """
    bot.add_cog(AdministrationCog(bot))
    log.info("Commands loaded: administration")


import os
import gzip
import shutil
from inspect import isawaitable
from typing import Optional, Union, Dict, Tuple, Iterable, Iterator, List, Coroutine, NamedTuple, Callable, Generator
from logging import Logger
from pandas import read_csv, DataFrame, isna, Series, concat
import numpy as np
from pathlib import Path
from aiofiles import open as aiofiles_open
from re import compile as re_compile
from pyexcel import Sheet
import asyncio
from unsync import unsync, Unfuture
from itertools import chain
import orjson as json
import logging
from io import StringIO
from operator import itemgetter
from textdistance import overlap, sorensen
from collections import Counter, OrderedDict
from warnings import warn

"""def to_interval(lyst: Union[Iterable, Iterator]) -> List:
    def pass_check(lyst):
        try:
            if (not isinstance(lyst, Generator) and  (len(lyst) == 0)) or isna(lyst):
                return False
            else:
                return True
        except ValueError:
            if isinstance(lyst, float):
                return False
            else:
                return True
    if not pass_check(lyst):
        return ()
    else:
        if not isinstance(lyst, set):
            lyst = frozenset(int(i) for i in lyst if i is not None)
        if not pass_check(lyst):
            return ()
        start = []
        interval_lyst = []
        max_edge = max(lyst)
        min_edge = min(lyst)
        if len(lyst) == (max_edge + 1 - min_edge):
            return ((min_edge, max_edge),)
        else:
            lyst_list = sorted(lyst)
            for j in lyst_list:
                if not start:
                    i = j
                    start.append(j)
                    i += 1
                else:
                    if (i != j) or (j == max(lyst_list)):
                        if j == max(lyst_list):
                            if (i != j):
                                interval_lyst.append(start)
                                interval_lyst.append([j])
                                break
                            else:
                                start.append(j)
                        interval_lyst.append(start)
                        start = [j]
                        i = j + 1
                    else:
                        start.append(j)
                        i += 1
            return tuple((min(li), max(li)) for li in interval_lyst)

def lyst22interval(x, y):
    # x, y = sorted(x), sorted(y)
    data = frozenset({i for i in zip(x,y)})
    x, y = zip(*sorted(data, key=itemgetter(0)))
    start_x, start_y = x[0], y[0]
    index_x, index_y = x[0]-1, y[0]-1
    interval_x, interval_y = [], []
    for i, j in zip(x, y):
        pre_x = index_x + 1
        pre_y = index_y + 1
        if pre_x == i and pre_y == j:
            index_x, index_y = i, j
        else:
            interval_x.append((start_x, index_x))
            interval_y.append((start_y, index_y))
            start_x, start_y = i, j
            index_x, index_y = i, j
    interval_x.append((start_x, index_x))
    interval_y.append((start_y, index_y))
    return interval_x, interval_y"""

@unsync
async def init_semaphore(concurreq) -> Unfuture:
    """
    `semaphore` initiated in the `unsync` event loop
    """
    await asyncio.sleep(.01)
    return asyncio.Semaphore(concurreq)

'''
def decompression(path: str, extension: str =".gz", remove: bool =True, outputPath: Optional[str] = None, logger: Optional[Logger] = None):
    """
    Decompress gz file

    :param path: The file path of the compressed file
    :param extension: Compressed file tail, default value: `.gz`
    :param remove: Whether remove the compressed file, default value: `True`
    :param outputPath: File safe path, default value: `None`
    :param logger: logger Object
    """

    """
    with gzip.GzipFile(mode="rb", fileobj=open(path, 'rb')) as raw:
        with open(path[:-len(extension)], "wb") as file:
            file.write(raw.read())
    """
    if outputPath is None:
        outputPath = path[:-len(extension)]

    with gzip.open(path, 'rb') as raw:
        with open(outputPath, 'wb') as file:
            shutil.copyfileobj(raw, file)
    try:
        if remove:
            os.remove(path)
    except Exception as e:
        if isinstance(logger, Logger):
            logger.error(e)

    return outputPath
'''

def related_dataframe(filters: Optional[Union[Dict, Iterable[Tuple]]] = None, dfrm: Optional[DataFrame] = None, path: Union[str, Path, None] = None, sep: str = '\t', **kwargs):
    '''
    valid symbol: `eq, ne, le, lt, ge, gt, isin, isnull`
    '''

    if dfrm is None:
        if path is not None:
            dfrm = read_csv(path, sep=sep)
        else:
            raise ValueError('path should not be None')
    elif not isinstance(dfrm, DataFrame):
        raise ValueError('dfrm should be a pandas.DataFrame')

    if filters is None:
        return dfrm
    elif isinstance(filters, Dict):
        filters = filters.items()

    for col, (symbol, value) in filters:
        dfrm = dfrm[getattr(getattr(dfrm, col), symbol)(value)]
    return dfrm

def sort_sub_cols(dfrm, cols):
    if set(dfrm.columns) >= set(cols):
        dfrm[cols] = np.sort(dfrm[cols].to_numpy())
        return dfrm.drop_duplicates()
    else:
        return dfrm

@unsync
async def a_read_csv(path, read_mode='r',**kwargs):
    '''
    only suitable for small dataframe
    '''
    try:
        if isinstance(path, (Coroutine, Unfuture)):
            path = await path
        if isinstance(path, (Path, str)):
            async with aiofiles_open(path, read_mode) as file_io:
                with StringIO(await file_io.read()) as text_io:
                    return read_csv(text_io, **kwargs)
        elif isinstance(path, DataFrame):
            return path
        else:
            return DataFrame(path, columns=kwargs.get('columns', None))
    except Exception:
        raise ValueError(f'{path}')

async def a_load_json(path):
    try:
        if isawaitable(path):
            path = await path
        if path is None:
            return None
        async with aiofiles_open(path) as inFile:
            return json.loads(await inFile.read())
    except Exception as e:
        raise ValueError(f"Exception {e} ({path})")

async def pipe_out(df, path, **kwargs):
    if not isinstance(df, Sheet):
        raise TypeError(f"Invalid Object for pipe_out(): {type(df)}")
    if len(df) == 0:
        raise ValueError("Zero record!")
    path = Path(path)
    mode = kwargs.get('mode', 'a')
    clear_headers:bool = path.exists() and mode.startswith('a')
    var_format = kwargs.get('format', 'tsv').lower()
    async with aiofiles_open(path, mode) as file_io:
        """
        if isinstance(df, DataFrame):
            sorted_col = sorted(df.columns)
            if clear_headers:
                headers = None
            else:
                headers = sorted_col
            dataset = Dataset(headers=headers)
            dataset.extend(df[sorted_col].to_records(index=False))
            to_write = dataset.export(var_format, lineterminator='\n')
        elif isinstance(df, Dataset):
            if clear_headers:
                df.headers = None
            to_write = df.export(var_format, lineterminator='\n')
        """
        df = df.project(sorted(df.colnames))
        if clear_headers:
            df.colnames = []
        to_write = getattr(df, f"get_{var_format}")(lineterminator='\n')
        await file_io.write(to_write)

def flatten_dict(data: Dict, root: str, with_root: bool = True):
    if with_root:
        iterator = yield_flatten_dict(data[root], root)
    else:
        iterator = yield_flatten_dict(data[root])
    for key, value in iterator:
        data[key] = value
    del data[root]

def yield_flatten_dict(data: Dict, root: Optional[str] = None):
    if root is None:
        yield from data.items()
    else:
        for key, value in data.items():
            yield f'{root}.{key}', value

def slice_series(se: Iterable) -> Dict:
    '''
    For Sorted Series
    '''
    data = {}
    cur = next(iter(se))
    start = 0
    try:
        for index, i in enumerate(se):
            if i != cur:
                assert cur not in data, "Invalid Series"
                data[cur] = (start, index)
                cur = i
                start = index
        assert cur not in data, "Invalid Series"
        data[cur] = (start, index+1)
    except AssertionError as e:
        logging.error(e)
        raise e
    return data

def split_df_by_chain(df, all_cols, cols_to_split, mode='sep', sep=','):
    '''
    Reference: <https://stackoverflow.com/a/50731258/12876491>
    '''
    def chainer_sep(s):
        return list(chain.from_iterable(s.str.split(sep)))

    def chainer_json(s):
        return list(chain.from_iterable(s.apply(json.loads)))

    def chainer_list(s):
        return list(chain.from_iterable(s))

    if mode == 'sep':
        chainer = chainer_sep
        lens = df[cols_to_split[0]].str.split(sep).map(len)
    elif mode == 'json-list':
        chainer = chainer_json
        lens = df[cols_to_split[0]].apply(json.loads).map(len)
    elif mode == 'list':
        chainer = chainer_list
        lens = df[cols_to_split[0]].map(len)
    else:
        raise ValueError("Invalid mode!")

    repeat_dict = {col: np.repeat(df[col], lens)
                   for col in set(all_cols)-set(cols_to_split)}
    chain_dict = {col: chainer(df[col]) for col in cols_to_split}
    return DataFrame({**repeat_dict, **chain_dict})

def init_folder_from_suffix(folder: Union[Path, str], suffix: str):
    folder = Path(folder)
    new_path = folder/suffix
    new_path.mkdir(parents=True, exist_ok=True)
    return new_path

def init_folder_from_suffixes(folder: Union[Path, str], suffixes: Iterable) -> Iterable[Path]:
    folder = Path(folder)
    for suffix in suffixes:
        new_path = folder/suffix
        new_path.mkdir(parents=True, exist_ok=True)
        yield new_path

def iter_first(df: DataFrame, criteria: Callable[[NamedTuple], bool], **kwargs) -> Optional[NamedTuple]:
    '''
    Implement pandas.DataFrame.itertuples

    Returns the value as soon as you find the first row/record
    that meets the requirements and NOT iterating other rows

    Originated from: https://stackoverflow.com/a/63826677/12876491

    >>> iter_first(df, lambda row: row.A > 4 and row.B > 3)
    '''
    for row in df.itertuples(**kwargs):
        if criteria(row):
            return row

class MMCIF2DictPlus(dict):
    """
    Parse a mmCIF file and return a dictionary

    NOTE: Override methods based on Biopython's `Bio.PDB.MMCIF2Dict.MMCIF2Dict`
    """

    def _check_token_with_focus_keys(self, token: Iterable[str]) -> bool:
        is_key, key_value = token
        return is_key == 0 and ((key_value in self.focus_keys) or any(key in key_value for key in self.focus_keys))

    def __init__(self, handle, focus_keys: Iterable[str]=['']):
        self.focus_keys = set(focus_keys)
        self.quote_chars = ["'", '"']
        self.whitespace_chars = [" ", "\t"]
        # TODO: init first loop
        loop_flag = False
        key = None
        tokens = self._tokenize(handle)
        try:
            token = next(tokens)
        except StopIteration:
            return  # NOTE: annotation from biopython: for Python 3.7 and PEP 479
        self[token[1][0:5]] = token[1][5:]
        i = 0
        n = 0
        use = []
        # TODO: loops
        for token in tokens:
            if token[1].lower() == "loop_":
                loop_flag = True
                keys = []
                i = 0
                n = 0
                use = []
                continue
            elif loop_flag:
                '''
                NOTE: annotation from biopython:
                # The second condition checks we are in the first column
                # Some mmCIF files (e.g. 4q9r) have values in later columns
                # starting with an underscore and we don't want to read
                # these as keys
                '''
                if token[1].startswith("_") and (n == 0 or i % n == 0):
                    if i > 0:
                        loop_flag = False
                    else:
                        if self._check_token_with_focus_keys(token):
                            use.append(n)
                            self[token[1]] = []
                        keys.append(token[1])
                        n += 1
                        continue
                else:
                    key_index = i % n
                    try:
                        if key_index in use:
                            self[keys[key_index]].append(token[1])
                    except Exception:
                        raise ValueError(f"{keys}, {key_index}, {use}")
                    i += 1
                    continue
            if key is None:
                if self._check_token_with_focus_keys(token):
                    key = token[1]
            else:
                # Always returns a list
                self[key] = [token[1]]
                key = None
            if self.keys() >= self.focus_keys:
                break

    def _splitline(self, line: str):
        # NOTE: annotation from biopython: See https://www.iucr.org/resources/cif/spec/version1.1/cifsyntax for the syntax
        in_token = False
        # NOTE: annotation from biopython: quote character of the currently open quote, or None if no quote open
        quote_open_char = None
        start_i = 0
        for (i, c) in enumerate(line):
            if c in self.whitespace_chars:
                if in_token and not quote_open_char:
                    in_token = False
                    yield start_i, line[start_i:i]
            elif c in self.quote_chars:
                if not quote_open_char and not in_token:
                    # raise ValueError(f"{self['data_']}: Opening quote in middle of word: " + line)
                    quote_open_char = c
                    in_token = True
                    start_i = i + 1
                elif c == quote_open_char and (i + 1 == len(line) or line[i + 1] in self.whitespace_chars):
                    quote_open_char = None
                    in_token = False
                    yield start_i, line[start_i:i]
            elif c == "#" and not in_token:
                ''' NOTE: annotation from biopython:
                # Skip comments. "#" is a valid non-comment char inside of a
                # quote and inside of an unquoted token (!?!?), so we need to
                # check that the current char is not in a token.
                '''
                return
            elif not in_token:
                in_token = True
                start_i = i
        if in_token:
            yield start_i, line[start_i:]
        if quote_open_char:
            raise ValueError("Line ended with quote open: " + line)

    def _tokenize(self, handle):
        empty = True
        for line in handle:
            empty = False
            if line.startswith("#"):
                continue
            elif line.startswith(";"):
                '''
                NOTE: annotation from biopython:
                # The spec says that leading whitespace on each line must be
                # preserved while trailing whitespace may be stripped.  The
                # trailing newline must be stripped.
                '''
                token_buffer = [line[1:].rstrip()]
                for line in handle:
                    line = line.rstrip()
                    if line.startswith(";"):
                        yield 1, "\n".join(token_buffer)
                        line = line[1:]
                        if line and not line[0] in self.whitespace_chars:
                            raise ValueError("Missing whitespace")
                        break
                    token_buffer.append(line)
                else:
                    raise ValueError("Missing closing semicolon")
            yield from self._splitline(line.strip())
        if empty:
            raise ValueError("Empty file.")

class DisplayPDB(object):

    a_name = 'Asymmetric unit'

    b_name = 'Biological assembly {assembly_id}'

    a_code = 'model-1'

    b_code = 'assembly-{assembly_id}'

    header_unit = '''

                    {name} of {pdb_id}

    '''

    content_unit = '''

    '''

    css = '''

    '''

    template = '''

            {headers}

            {content}

        '''

    @classmethod
    def setting(cls, pdb_id, assemblies):
        headers = [cls.header_unit.format(name=cls.a_name, pdb_id=pdb_id)]
        content = [cls.content_unit.format(pdb_id=pdb_id, in_code=pdb_id[1:3], code=cls.a_code)]
        for assembly_id in assemblies:
            headers.append(cls.header_unit.format(
                name=cls.b_name.format(assembly_id=assembly_id),
                pdb_id=pdb_id))
            content.append(cls.content_unit.format(
                pdb_id=pdb_id,
                in_code=pdb_id[1:3],
                code=cls.b_code.format(assembly_id=assembly_id)
            ))
        return ''.join(headers), ''.join(content)

    def show(self, pdb_id, assemblies: Iterable[int]= [1]):
        from IPython.display import display, HTML
        assemblies = sorted(int(i) for i in assemblies if int(i) > 0)
        headers, content = self.setting(pdb_id, assemblies)
        self.table = self.template.format(headers=headers, content=content)
        if self.dark:
            self.table = self.css + self.table
        display(HTML(self.table))

    def __init__(self, dark:bool=False):
        self.dark = dark

fasta_pat = re_compile(r'(>.+)\n([A-Z\*\n]+)')
unp_header_pat = re_compile(r'>sp\|(.+)\|')

async def a_seq_reader(path: Union[Unfuture, Union[Path, str]]):
    if isinstance(path, Unfuture):
        path = await path
    async with aiofiles_open(path, 'rt') as handle:
        header, content = fasta_pat.match(await handle.read()).groups()
        content = content.replace('\n', '')
        assert content != '', str(path)
        return header, content

@unsync
async def get_seq_from_parser(res, identifier, seq_only:bool = True):
    async for header, content in await res:
        if (identifier in header) and (f'{identifier}-' not in header):
            return content if seq_only else (header, content)

@unsync
async def get_seqs_from_parser(res, identifiers:Optional[Iterable[str]]=None):
    ret = []
    async for header, content in await res:
        header = unp_header_pat.match(header).group(1)
        if identifiers is None or header in identifiers:
            ret.append((header, content))
    return ret

async def a_seq_parser(path: Union[Unfuture, Coroutine, Path, str]):
    if isawaitable(path):
        path = await path
    async with aiofiles_open(path, 'rt') as handle:
        header, content = None, ''
        async for line in handle:
            if line.startswith('>'):
                if header is not None:
                    yield header.strip(), content.replace('\n', '')
                header, content = line, ''
            else:
                content += line
        assert header is not None, f"\npath: {path}\ncur_content: {content}"
        yield header.strip(), content.replace('\n', '')

nu2aa_dict = {
    'ATA': 'I', 'ATC': 'I', 'ATT': 'I', 'ATG': 'M',
    'ACA': 'T', 'ACC': 'T', 'ACG': 'T', 'ACT': 'T',
    'AAC': 'N', 'AAT': 'N', 'AAA': 'K', 'AAG': 'K',
    'AGC': 'S', 'AGT': 'S', 'AGA': 'R', 'AGG': 'R',
    'CTA': 'L', 'CTC': 'L', 'CTG': 'L', 'CTT': 'L',
    'CCA': 'P', 'CCC': 'P', 'CCG': 'P', 'CCT': 'P',
    'CAC': 'H', 'CAT': 'H', 'CAA': 'Q', 'CAG': 'Q',
    'CGA': 'R', 'CGC': 'R', 'CGG': 'R', 'CGT': 'R',
    'GTA': 'V', 'GTC': 'V', 'GTG': 'V', 'GTT': 'V',
    'GCA': 'A', 'GCC': 'A', 'GCG': 'A', 'GCT': 'A',
    'GAC': 'D', 'GAT': 'D', 'GAA': 'E', 'GAG': 'E',
    'GGA': 'G', 'GGC': 'G', 'GGG': 'G', 'GGT': 'G',
    'TCA': 'S', 'TCC': 'S', 'TCG': 'S', 'TCT': 'S',
    'TTC': 'F', 'TTT': 'F', 'TTA': 'L', 'TTG': 'L',
    'TAC': 'Y', 'TAT': 'Y', 'TAA': '_', 'TAG': '_',
    'TGC': 'C', 'TGT': 'C', 'TGA': '_', 'TGG': 'W',
}

def translate2aa(seq:str, check:bool=False):
    assert len(seq) % 3 == 0, "Invalid length of dna OR rna sequence!"
    seq = seq.replace('U', 'T')
    p_seq = ""
    for i in range(0, len(seq), 3):
        codon = seq[i:i + 3]
        p_seq += nu2aa_dict[codon]
    if check:
        assert "_" not in p_seq, "Invalid Sequence!"
    return p_seq

def unsync_run(arg):
    @unsync
    async def s_unsync_wrap(arg):
        return await arg
    return s_unsync_wrap(arg).result()

@unsync
def unsync_wrap(var):
    return var

class SeqRangeReader(object):
    def __init__(self, name_group):
        self.name = name_group  # ('pdb_id', 'chain_id', 'UniProt')
        self.pdb_range = []
        self.unp_range = []

    def output(self):
        if self.pdb_range:
            pdb_range = json.dumps(self.pdb_range).decode('utf-8')
            unp_range = json.dumps(self.unp_range).decode('utf-8')
            return pdb_range, unp_range
        else:
            return self.default_pdb_range, self.default_unp_range

    def check(self, name_group_to_check, data_group):
        self.default_pdb_range = '[[%s, %s]]' % data_group[:2]
        self.default_unp_range = '[[%s, %s]]' % data_group[2:4]

        if self.name == name_group_to_check:
            self.pdb_range.append([int(data_group[0]), int(data_group[1])])
            self.unp_range.append([int(data_group[2]), int(data_group[3])])
        else:
            self.name = name_group_to_check
            self.pdb_range = [[int(data_group[0]), int(data_group[1])]]
            self.unp_range = [[int(data_group[2]), int(data_group[3])]]

        return self.output()

def sort_2_range(unp_range: List, pdb_range: List):
    unp_range, pdb_range = zip(
        *sorted(zip(unp_range, pdb_range), key=lambda x: x[0][0]))
    return unp_range, pdb_range

def flat_dict_in_df(dfrm:DataFrame, targetCol:Union[str, Series], cols:List):
    try:
        new_cols = list(f'{targetCol.name}.{col}' for col in cols)
        dfrm[new_cols] = DataFrame(
            targetCol.apply(lambda x: list(x[col] for col in cols)).to_list(),
            columns=new_cols)
        return dfrm.drop(columns=[targetCol.name])
    except AttributeError:
        assert isinstance(targetCol, str)
        new_cols = list(f'{targetCol}.{col}' for col in cols)
        dfrm[cols] = DataFrame(
            dfrm[targetCol].apply(lambda x: list(x[col] for col in cols)).to_list(),
            columns=new_cols)
        return dfrm.drop(columns=[targetCol])

class AAThree2One(dict):
    def __missing__(self, key):
        return 'X'

aa_three2one = AAThree2One({
    "GLY": "G", "ALA": "A", "SER": "S", "THR": "T", "CYS": "C", "VAL": "V", "LEU": "L",
    "ILE": "I", "MET": "M", "PRO": "P", "PHE": "F", "TYR": "Y", "TRP": "W", "ASP": "D",
    "GLU": "E", "ASN": "N", "GLN": "Q", "HIS": "H", "LYS": "K", "ARG": "R"})

standardAA = list(aa_three2one.keys())

standardNu = ['DA', 'DT', 'DC', 'DG', 'DI', 'A', 'U', 'C', 'G', 'I']

"""def range_len(lyst: Union[List, str, float]) -> int:
    if isinstance(lyst, float) or lyst is None:
        return 0
    elif isinstance(lyst, str):
        lyst = json.loads(lyst)
    length = 0
    for left, right in lyst:
        assert right >= left, f"\n{lyst}"
        length += right - left + 1
    return length

def interval2set(lyst: Union[Iterable, Iterator, str]):
    if isinstance(lyst, str):
        lyst = json.loads(lyst)
    range_set = frozenset()
    for left, right in lyst:
        range_set |= frozenset(range(left, right+1))
    return range_set"""

def expand_interval(lyst: Union[Iterable, Iterator, str]):
    lyst = json.loads(lyst) if isinstance(lyst, str) else lyst
    yield from (i for start, end in lyst for i in range(start, end+1))

def lyst2range(lyst, add_end=1):
    if isinstance(lyst, str):
        lyst = json.loads(lyst)
    for start, end in lyst:
        yield from range(int(start), int(end)+add_end)

"""def subtract_range(pdb_range: Union[str, Iterable], mis_range: Union[str, Iterable]) -> List:
    if isinstance(mis_range, float) or mis_range is None:
        return pdb_range
    elif len(pdb_range) == 0:
        return ()
    elif len(mis_range) == 0:
        return pdb_range
    pdb_range_set = interval2set(pdb_range)
    mis_range_set = interval2set(mis_range)
    return to_interval(pdb_range_set - mis_range_set)

def check_range(i) -> bool:
    if isinstance(i, float) or (i is None) or (len(i) == 0) or (i == 'nan'):
        return False
    return True

def add_range(left: Union[str, Iterable], right: Union[str, Iterable]) -> List:
    check_left = check_range(left)
    check_right = check_range(right)
    if check_left and not check_right:
        return left
    elif not check_left and check_right:
        return right
    elif not check_left and not check_right:
        return np.nan
    try:
        left_range_set = interval2set(left)
        right_range_set = interval2set(right)
        return to_interval(left_range_set | right_range_set)
    except Exception as e:
        print(left, right)
        print(type(left), type(right))
        raise e

def overlap_range(obs_range:Union[str, Iterable], unk_range: Union[str, Iterable]) -> List:
    if isinstance(unk_range, float) or unk_range is None:
        return ()
    '''
    obs_range_set = interval2set(obs_range)
    unk_range_set = interval2set(unk_range)
    return to_interval(obs_range_set.intersection(unk_range_set))
    '''
    obs_range = json.loads(obs_range) if isinstance(obs_range, str) else obs_range
    unk_range = json.loads(unk_range) if isinstance(unk_range, str) else unk_range

    def unit(i1,i2):
        for start1, end1 in i1:
            for start2, end2 in i2:
                sl = start2 >= start1
                sr = start2 <= end1
                el = end2 >= start1
                er = end2 <= end1
                s_in = sl and sr
                e_in = el and er
                ini = s_in or e_in
                # out = (sl and el) or (sr and er)
                cov = (not sl) and (not er)
                start = start2 if s_in else start1
                end = end2 if e_in else end1
                if ini or cov:
                    yield start, end

    return tuple(unit(obs_range, unk_range))"""

def get_seq_seg(seq, ranges, **kwargs):
    for start,end in ranges:
        if end >= start:
            yield start, seq[start-1:end]
        else:
            warn(f"{kwargs} -> Invalid Order: {ranges}, skip")

def get_diff_index(lseq, lrange, rseq, rrange):
    if isinstance(lrange, str):
        lrange = json.loads(lrange)
    if isinstance(rrange, str):
        rrange = json.loads(rrange)
    for (lstart, lseg), (rstart, rseg) in zip(get_seq_seg(lseq, lrange), get_seq_seg(rseq, rrange)):
        yield from ((lstart+index, rstart+index) for index, (r1, r2) in enumerate(zip(lseg, rseg)) if r1 != r2)

def red_seq_seg(seq, ranges):
    edge = 0
    for start, end in ranges:
        yield f"{seq[edge:start-1]}\x1b[31m{seq[start-1:end]}\x1b[0m"
        edge = end
    yield seq[end:]

"""def outside_range(pdb_range: Union[str, Iterable], seqres_len: int):
    pdb_range = json.loads(pdb_range) if isinstance(pdb_range, str) else pdb_range
    out_head = pdb_range[0][0] - 1
    out_tail = pdb_range[-1][-1] + 1
    ret = [[1, out_head], [out_tail, seqres_len]]
    return [i for i in ret if i[0]<=i[1]]"""

def outside_range_len(pdb_range: Union[str, Iterable], seqres_len: int, omit: int = 5) -> int:
    if isinstance(pdb_range, str):
        lyst = json.loads(pdb_range)
    else:
        lyst = pdb_range
    out_head = lyst[0][0]-1
    out_tail = seqres_len - lyst[-1][-1]
    if out_head <= omit:
        out_head = 0
    else:
        out_head -= omit
    if out_tail <= omit:
        out_tail = 0
    else:
        out_tail -= omit
    return out_head + out_tail

def get_gap_list(li: Union[str,List,Tuple]):
    if isinstance(li, str):
        li = json.loads(li)
    return [li[i+1][0] - li[i][1] - 1 for i in range(len(li)-1)]

def get_range_diff(lyst_a: Union[str, List, Tuple], lyst_b: Union[str, List, Tuple]):
    lyst_a = json.loads(lyst_a) if isinstance(lyst_a, str) else lyst_a
    lyst_b = json.loads(lyst_b) if isinstance(lyst_b, str) else lyst_b
    array_a = np.array([right - left + 1 for left, right in lyst_a])
    array_b = np.array([right - left + 1 for left, right in lyst_b])
    return array_a - array_b

def select_range(ranges, indexes, cutoff=0.2, skip_index=[], selected_ranges=None, similarity_func=overlap.similarity):
    select_index = []
    selected_ranges = [] if selected_ranges is None else selected_ranges
    def unit(cur_index):
        if cur_index in skip_index:
            return
        cur_range = ranges[cur_index]
        cur_range = json.loads(cur_range) if isinstance(cur_range, str) else cur_range
        for selected_range in selected_ranges:
            selected_range = json.loads(selected_range) if isinstance(selected_range, str) else selected_range
            if len(cur_range) == 0:
                return
            score = similarity_func(lyst2range(cur_range),
                               lyst2range(selected_range))
            if score > cutoff:
                return
        select_index.append(cur_index)
        selected_ranges.append(cur_range)

    for index in indexes:
        unit(index)
    return select_index

"""def select_ho_range(ranges1, ranges2, indexes, cutoff=0.2, skip_index=[]):
    from scipy.stats import wasserstein_distance
    select_index = []

    def unit(cur_index):
        if cur_index in skip_index:
            return
        cur_range1, cur_range2 = ranges1[cur_index], ranges2[cur_index]
        c1_1 = Counter(expand_interval(cur_range1))
        c1_2 = Counter(expand_interval(cur_range2))
        if len(c1_1) == 0 or len(c1_2) == 0:
            return
        c1 = c1_1 + c1_2
        for selected in select_index:
            selected_range1,selected_range2 = ranges1[selected], ranges2[selected]
            c_c1 = c1.copy()
            c2 = Counter(expand_interval(selected_range1))+Counter(expand_interval(selected_range2))
            for key in c_c1.keys() | c2.keys():
                if key not in c_c1:
                    c_c1[key] = 0
                if key not in c2:
                    c2[key] = 0
            oc2 = OrderedDict(sorted((item for item in c2.items()), key=lambda x: x[0]))
            oc1 = OrderedDict(sorted((item for item in c_c1.items()), key=lambda x: x[0]))
            score = wasserstein_distance(tuple(oc1.values()), tuple(oc2.values()))
            if score < cutoff:
                return
        select_index.append(cur_index)

    for index in indexes:
        unit(index)
    return select_index"""

def select_ho_max_range(ranges1, ranges2, indexes, cutoff=0.2, skip_index=[]):
    select_range_set = OrderedDict()

    def unit(cur_index):
        if cur_index in skip_index:
            return
        cur_range1, cur_range2 = ranges1[cur_index], ranges2[cur_index]

        c1_1 = frozenset((1, i) for i in expand_interval(cur_range1))
        c1_2 = frozenset((2, i) for i in expand_interval(cur_range2))
        if len(c1_1) == 0 or len(c1_2) == 0:
            return
        c1 = c1_1 | c1_2
        for c2s in select_range_set.values():
            for c2 in c2s:
                score = sorensen.similarity(c1, c2)
                if score > cutoff:
                    return
        select_range_set[cur_index] = (
                frozenset((1, i) for i in expand_interval(cur_range1)) | frozenset((2, i) for i in expand_interval(cur_range2)),
                frozenset((2, i) for i in expand_interval(cur_range1)) | frozenset((1, i) for i in expand_interval(cur_range2)))

    for index in indexes:
        unit(index)
    return list(select_range_set.keys())

def select_he_range(Entry_1, Entry_2, ranges1, ranges2, indexes, cutoff=0.2, skip_index=[]):
    select_index = []

    def unit(cur_index):
        if cur_index in skip_index:
            return
        cur_range1, cur_range2 = ranges1[cur_index], ranges2[cur_index]
        cur_e1, cur_e2 = Entry_1[cur_index], Entry_2[cur_index]
        (cur_e1, cur_range1), (cur_e2, cur_range2) = sorted(((cur_e1, cur_range1), (cur_e2, cur_range2)), key=lambda x: x[0])

        c1_1 = frozenset((1, i) for i in expand_interval(cur_range1))
        c1_2 = frozenset((2, i) for i in expand_interval(cur_range2))
        if len(c1_1) == 0 or len(c1_2) == 0:
            return
        c1 = c1_1 | c1_2
        for selected in select_index:
            selected_range1, selected_range2 = ranges1[selected], ranges2[selected]
            selected_e1, selected_e2 = Entry_1[selected], Entry_2[selected]
            (selected_e1, selected_range1), (selected_e2, selected_range2) = sorted(((selected_e1, selected_range1), (selected_e2, selected_range2)), key=lambda x: x[0])
            c2 = frozenset((1, i) for i in expand_interval(selected_range1)) | frozenset((2, i) for i in expand_interval(selected_range2))
            score = sorensen.similarity(c1, c2)
            if score > cutoff:
                return
        select_index.append(cur_index)

    for index in indexes:
        unit(index)
    return select_index

def dumpsParams(params: Dict) -> str:
    return '&'.join(f'{key}={value}' for key, value in params.items())

@unsync
async def a_concat(pathes, sep='\t', sort=False, ignore_index=True, columns=None):
    if isinstance(pathes, (Unfuture, Coroutine)):
        pathes = await pathes
    res = [await a_read_csv((await path) if isinstance(Unfuture, Coroutine) else path, sep=sep, columns=columns) for path in pathes]
    return concat((i for i in res if i is not None), sort=sort, ignore_index=ignore_index)

def get_str_dict_len(x):
    if isinstance(x, str):
        return x.count(':')
    else:
        return len(x)

def id2score(identifier):
    len_id = len(identifier)
    return -sum(ord(i)*2*(len_id-level) for level, i in enumerate(identifier))


# coding: utf-8

# In[2]:

import argparse
import pandas as pd
import numpy as np
def main():
    # Parse arguments from command line
    parser = argparse.ArgumentParser()

    # Set up required arguments this script
    parser.add_argument('function', type=str, help='function to call')
    parser.add_argument('start_date', type=str, help='first argument')
    parser.add_argument('end_date', type=str, help='second argument')
    parser.add_argument('commodity_type', type=str, help='third argument')

    # Parse the given arguments
    args = parser.parse_args()

    # Get the function based on the command line argument and
    # call it with the other two command line arguments as
    # function arguments
    eval(args.function)(args. start_date, args. end_date, args.commodity_type)

def getCommodityPrice(start_date,end_date,commodity_type):

    if (commodity_type == 'gold'):
      data = pd.read_csv('gold.csv', sep=';')
    else:
      data = pd.read_csv('silver.csv', sep=';')
    mean = data['Price'].mean()
    variance = np.var(data['Price'])
    new = commodity_type + " "+str(mean) +" " +str(variance)
    print (new)

if __name__ == '__main__':
    main()

# In[ ]:


#!/usr/bin/env python3

import sys
import curses
import argparse

import debugoutput
import keyinput
import mapgenfuncs
from gameworld import GameWorld, GameMap
from screenpanels import MessagePanel, ListMenu

def draw_screen(stdscr, gameworld, gamewindow, panellist, show_debug_text=False):
    """Display the current game state on the screen"""

    #Update non-game panels
    for panel in panellist:
        panel.display()

    #Draw the gameworld to its window
    window_height, window_width = gamewindow.getmaxyx()
    view = gameworld.get_view(view_width=window_width, view_height=window_height, center_on_player=True)
    for y, row in enumerate(view):
        for x, tile in enumerate(row):
            gamewindow.addstr(y, x, tile.char, tile.color)
    gamewindow.refresh()

    #Flush debug text
    if show_debug_text:
        debugoutput.flush_debug_text()

def layout_panels(stdscr):
    """Build panel layout and create sub-windows of stdscr

    Return: A tuple with the game window and a list of other panels
    """
    screen_width = curses.COLS-1
    screen_height = curses.LINES-1
    messagepanel_height = 5
    gamewindow_width = 3 * (screen_width // 4)
    #Arguments for creating sub-windows are height, width, y coord of top, x coord of left
    #0,0 is top left corner of the screen
    messagepanel = MessagePanel(stdscr.subwin(messagepanel_height, gamewindow_width, 0, 0))
    gamewindow = stdscr.subwin(screen_height-messagepanel_height, gamewindow_width, messagepanel_height+1, 0)
    menupanel = ListMenu(stdscr.subwin(screen_height, (screen_width // 4), 0, gamewindow_width+1))
    return (gamewindow, [messagepanel, menupanel])

def main(stdscr):
    #SETUP
    args = get_args()
    curses.curs_set(False) #Turn off the cursor
    stdscr.clear() #Clear the screen

    show_debug_text = args.debugging_output
    debugoutput.init(stdscr)

    gamewindow, panellist = layout_panels(stdscr)
    if args.mapfile:
        gameworld = GameWorld(genfunc=mapgenfuncs.load_from_file,
                              mapfile=args.mapfile)
    else:
        gameworld = GameWorld(genfunc=mapgenfuncs.empty_box,
                              width=20, height=20)

    #GAME LOOP
    while True:
        try:
            draw_screen(stdscr, gameworld, gamewindow, panellist, show_debug_text=show_debug_text)
            keyinput.handle_key(stdscr.getkey())
            gameworld.update_world()
        except KeyboardInterrupt:
            #The user pressed Ctrl-C
            stdscr.refresh()
            sys.exit()
        except SystemExit:
            stdscr.refresh()
            sys.exit()

def get_args():
    """Parse the command line arguments and return a dictionary"""
    parser = argparse.ArgumentParser(description="""
        A curses-based roguelike something-or-other

        Movement:
        7 8 9      y k u
         \|/        \|/
        4-@-6  or  h-@-l
         /|\        /|\\
        1 2 3      b j n

        Commands:
        i - list inventory
        d - drop inventory

        Maps to try are in the maps/ subfolder, or make your own.
            """, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("mapfile",
            nargs='?',
            help="Path to a text file describing a game map",
            type=argparse.FileType('r'))
    parser.add_argument("-D", "--debugging-output", help="Print debugging messages", action="store_true")
    return parser.parse_args()

if __name__ == "__main__":
    #Parse the command line arguments before curses so that the help message can show
    get_args()

    #This will run the main function in a curses scope, and clean up
    #the terminal mode when the program ends.
    curses.wrapper(main)


# -*- coding: utf-8 -*-

#
### PATHS DATASETS
#
PATH_DATA = 'data/'
PATH_DATA_RAW = PATH_DATA+'raw/'

# cognitive
PATH_DATA_COGNITIVE = PATH_DATA+'cognitive/'
PATH_DATA_FEATURES01_COGNITIVE = PATH_DATA+'features01_cognitive/'
PATH_FEATURES01_COGNITIVE_CSV = PATH_DATA_FEATURES01_COGNITIVE+'features.csv'

# dlib
PATH_DLIB_MODEL = PATH_DATA+'shape_predictor_68_face_landmarks.dat'
PATH_DATA_DLIB = PATH_DATA+'dlib/'
PATH_DATA_DLIB_CSV = PATH_DATA_DLIB+'features.csv'
PATH_DATA_FEATURES01_DLIB = PATH_DATA+'features01_dlib/'
PATH_FEATURES01_DLIB_CSV = PATH_DATA_FEATURES01_DLIB+'features.csv'

#
### DATASET CONSTANTS
#
FEATURES01_EYES_SIZE = (30,20)

#
### GAME / CALIBRATION
#

# SCREEN
SCREEN_WIDTH = 1920
SCREEN_HEIGHT = 1080
SCREEN_DIAGONAL =  14  # Not used

# WEBCAM
CAM_DEVICE = "/dev/video0"
CAM_WIDTH = 1280
CAM_HEIGHT = 720
"""
-NOT USED-
Position of the camera respect to the screen.
Format is XX where X can be:
T: top
B: bottom
C: center
L: left
R: right
"""
CAM_POSITION = "TC"

# IMAGES
IMG_BACKGROUND = 'img/bg-sky.jpg'
IMG_TARGET = 'img/target01.png'

# OTHER
TIME_GAME = 60  # Seconds
FAILS = 5  # Hits a user can fail
RADIUS = 75.0  # Radius of the target, centered in the image

try:
    from config_local import *
except:
    pass


# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
import os

import fitz
import img2pdf

class WqxtPipeline(object):
    def process_item(self, item, spider):
        return item

    def open_spider(self, spider):
        print('爬虫开始了...')

    def close_spider(self, spider):
        self.img_to_pdf(spider)

        print('爬虫结束了...')

    def img_to_pdf(self, spider):
        path = spider.path
        if path:
            book_name = spider.book_name
            pdf_path_ = "{}/{}_.pdf".format(path, book_name)
            pdf_path = "{}/{}.pdf".format(path, book_name)
            print(pdf_path_)
            with open(pdf_path_, "wb") as f:
                img_list = []
                for img_name in range(0, spider.pages):
                    img_name = "%s/%s.jpeg" % (path, img_name)
                    img_list.append(img_name)
                    print(img_name)
                pfn_bytes = img2pdf.convert(img_list)
                f.write(pfn_bytes)
            print("转换完成")

            doc = fitz.open(pdf_path_)
            toc = spider.toc
            doc.setToC(toc)
            doc.save(pdf_path)
            doc.close()
            print('添加目录完成')
            print(pdf_path_, pdf_path)

            if (os.path.exists(pdf_path_)):
                os.remove(pdf_path_)
            for img_name in range(0, spider.pages):
                img_name = "%s/%s.jpeg" % (path, img_name)
                if (os.path.exists(img_name)):
                    os.remove(img_name)
            print('已删除')


import os
import socket
import json
import requests
import datetime
import threading
from urllib import parse
from flask import Flask,request,redirect,url_for,render_template

from Core import Request
from Classifier import Classifier
from Configuration import Configuration
from Log import LogController
class TransparentProxy():

    def __init__(self,Sock):
        self.Config = Configuration()
        self.ServerIP,self.ServerPort,self.FirewallIP,self.FirewallPort = self.Config.SetTransparentConfig()
        self.Sock = Sock

    def ParseThreat(self,Req):
        if not isinstance(Req, Request):
            raise TypeError("Object should be a Request")

        Model = Classifier()
        return Model.Run(Req)

    def __ParseUrl(self,Url):
        Detail = parse.urlparse(Url)
        Param = Detail.query
        Param = Param.split("&")
        Params = {}
        for p in Param:
            if p == "":
                continue
            label, value = p.split('=')
            label = label.strip()
            value = value.strip()
            Params[label] = value
        return Params

    def __ParseBody(self,Body):
        Body = Body.split('\r\n')
        Params = {}
        for Each in Body:
            if Each == "":
                continue
            Param = Each.split("&")
            # print("Param=",Param)
            for p in Param:
                label, value = p.split("=")
                label = label.strip()
                value = value.strip()
                Params[label] = value
        return Params

    def __GenerateUrl(self,Req):
        Host = Req.Header['Host']
        if Req.Protocol == 'HTTP/1.1':
            Url = 'http://' + Host + Req.Url
        return Url

    def __SendMessage(self,Req):
        Cookies = ''
        if "cookies" in Req.Header:
            Cookies = Req.Header['Cookies']
            Req.Header.pop('cookies')
        BodyParam = self.__ParseBody(Req.Body)

        UrlParam = self.__ParseUrl(Req.Url)

        Method = Req.Method.lower()

        Url = self.__GenerateUrl(Req)

        Respone = requests.request(Method,Url,params=UrlParam,data=BodyParam,cookies=Cookies)

        return Respone

    def __GenerateRespone(self,Respone,Req):
        Protocol = Req.Protocol + ' '
        Statue = Respone.status_code
        Explain = ' OK'
        ResponeLine = Protocol + str(Statue) + Explain + '\r\n'
        ResponeBody = Respone.text + '\r\nTransparent'

        ResponeHeader = ""
        Headers = Respone.headers
        for index, content in Headers.items():
            if index == "Content-Length":
                content = str(len(ResponeBody))
            ResponeHeader = ResponeHeader + index + ": " + content + "\r\n"

        ResponeHeader = ResponeHeader + "\r\n"

        return ResponeLine + ResponeHeader + ResponeBody

    def __GenerateSecurityRespone(self,Req):

        ResponeLine = "HTTP/1.1 200 OK\r\n"
        ResponeBody = "Attack Detected!!!<br/>"
        AttackResult = ""
        for key,value in Req.ThreatType.items():
            Location,value = value.split(':')
            Attack = Location + '->' + value + ':' + key

            AttackResult = AttackResult + '<xmp>' + Attack +'</xmp>'
        ResponeBody = ResponeBody + AttackResult

        GMT_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
        date = datetime.datetime.utcnow().strftime(GMT_FORMAT)
        ResponHeaders = "Content-Type: text/html; charset=utf-8\r\nContent-Length: " + str(len(ResponeBody)) + "\r\nServer: WebFirewall/1.0 Python/3.7.9\r\nDate: " + str(date) + "\r\n\r\n"

        Respone = ResponeLine + ResponHeaders +ResponeBody

        return Respone

    def Proxy(self):
        connection,addr = self.Sock.accept()

        SourceIP = addr[0]
        Data = connection.recv(2048)

        if len(Data) == 0:
            return Request(Url='' , Header='', Body='')
        #print('Recv!')
        Req = Request()
        Req.ParseRequest(Data)
        Req.ParaseSourceIP(SourceIP)
        Req.ThreatType = self.ParseThreat(Req)

        if not Req.ThreatType:
            Respone = self.__SendMessage(Req)

            Respone = self.__GenerateRespone(Respone,Req)

            connection.send(Respone.encode())
        else:

            Rspone = self.__GenerateSecurityRespone(Req)

            connection.send(Rspone.encode())

        return Req

class TransparentProxyServer():
    def __init__(self):
        self.Config = Configuration()
        self.ServerIP, self.ServerPort, self.FirewallIP, self.FirewallPort = self.Config.SetTransparentConfig()
        self.LogStatue,self.LogPath = self.Config.SetLogSystem()
    def ParseThreat(self,Req):
        if not isinstance(Req, Request):
            raise TypeError("Object should be a Request")

        Model = Classifier()
        return Model.Run(Req)

    def __ParseUrl(self, Url):
        Detail = parse.urlparse(Url)
        Param = Detail.query
        Param = Param.split("&")
        Params = {}
        for p in Param:
            if p == "":
                continue
            label, value = p.split('=',1)
            label = label.strip()
            value = value.strip()
            Params[label] = value
        return Params

    def __ParseBody(self, Body):
        Body = Body.split('\r\n')
        Params = {}
        for Each in Body:
            if Each == "":
                continue
            Param = Each.split("&")
            # print("Param=",Param)
            for p in Param:
                label, value = p.split("=")
                label = label.strip()
                value = value.strip()
                Params[label] = value
        return Params

    def __GenerateUrl(self, Req):
        Host = Req.Header['Host']
        if not os.listdir("./ca"):
            Url = 'http://' + Host + Req.Url
        else:
            Url = 'https://' + Host + Req.Url
        return Url

    def __SendMessage(self,Req):
        Cookies = ''
        if "cookies" in Req.Header:
            Cookies = Req.Header['Cookies']
            Req.Header.pop('cookies')
        BodyParam = self.__ParseBody(Req.Body)

        UrlParam = self.__ParseUrl(Req.Url)

        Method = Req.Method.lower()

        Url = self.__GenerateUrl(Req)

        Respone = requests.request(Method,Url,verify=False,params=UrlParam,data=BodyParam,cookies=Cookies)

        return Respone

    def proxy(self):
        app = Flask(__name__)

        @app.before_request
        def GetData():
            Req = Request(ID='',Timestamp='',SourceIP ='',Method = '',Url = '',Protocol='',Header={},Body='',ThreatType={})
            Req.SetTime()
            from builtins import str
            Req.SourceIP = str(request.remote_addr)
            Req.Method = str(request.method)
            try:
                parm ='?'+str(request.url).split('?')[1]
            except:
                parm=''
            Req.Url = str(request.path) + parm
            Req.Header = Req.Str2Dic(str(request.headers))
            json_str = json.dumps(request.form)
            data = json.loads(json_str)
            str = ""
            for key,value in data.items():
                str = str + key + "=" + value + "&"
            Req.Body = str[0:-1]
            Req.ThreatType = self.ParseThreat(Req)
            if Req.ThreatType == {}:
                Req.Operation = 'Pass'
            else:
                Req.Operation = 'Intercept'
            if self.LogStatue == True:
                Log = LogController()
                Log.Save(Req)
                Log.close()
            if Req.ThreatType == {}:
                respone = self.__SendMessage(Req)
                return respone.text
            else:
                return render_template('Attack.html',log=Req)

        if not os.listdir("./ca"):
            app.run(host=str(self.FirewallIP), port=int(self.FirewallPort))
        else:
            app.run(host=str(self.FirewallIP), port=int(self.FirewallPort),ssl_context=('./ca/server.crt', './ca/server.key'))

if __name__=='__main__':
    server = TransparentProxyServer()
    server.proxy()


# coding: utf-8

"""

This module contains all data objects getted from forum and wiki API. These API provides JSON data, and this module enhances data model by mirroring data attributes to data model. Here is an exemple :

.. code-block:: python

    from campbot import CampBot

    bot = CampBot(use_demo=True)
    waypoint = bot.wiki.get_waypoint(107702)

    # this two lines are perfectly equivalents
    elevation = waypoint["elevation"]  # standard way to access data
    elevation = waypoint.elevation  # and sexier way.

    # set values is possible
    waypoint.elevation = 1000
    assert waypoint["elevation"] == 1000  # it's true!

We try to use the second way every times it's possible in this documentation.
"""

from __future__ import print_function, unicode_literals, division

import re
import logging
from .differ import get_diff_report

def _input(message):  # pragma: no cover
    try:  # py 2
        return raw_input(message)
    except NameError:  # py 3
        return input(message)

def get_constructor(document_type):
    return {
        "u": WikiUser,
        "a": Area,
        "w": Waypoint,
        "o": Outing,
        "i": Image,
        "m": Map,
        "x": Xreport,
        "c": Article,
        "b": Book,
        "r": Route,
    }[document_type]

class BotObject(dict):
    """
    Base class for all data object
    """

    def __init__(self, campbot, data):
        super(BotObject, self).__init__(data)
        self._campbot = campbot

    # make instance.key equivalent to instance["key"]
    def __getattr__(self, item):
        if item.startswith("_"):
            raise AttributeError(
                "Object {} has not attribute {}".format(self.__class__.__name__, item)
            )

        if item not in self:  # pragma: no cover
            print("666777", self)
            raise AttributeError(
                "Object {} has not attribute {}".format(self.__class__.__name__, item)
            )

        return self[item]

    def __setattr__(self, key, value):
        if key in self:
            self[key] = value
        else:
            super(BotObject, self).__setattr__(key, value)

    def _convert_list(self, name, constructor):
        if name in self:
            self[name] = [constructor(self._campbot, data) for data in self[name]]

    # def _convert_dict(self, name, constructor):
    #     if name in self:
    #         self[name] = {
    #             key: constructor(self._campbot, self[name][key]) for key in self[name]
    #         }

class Version(BotObject):
    """
    A historical version of one wiki document.
    """

    def __init__(self, campbot, data):
        super(Version, self).__init__(campbot, data)
        if self["document"]:
            self["document"] = get_constructor(self["document"]["type"])(
                campbot, self["document"]
            )
        else:
            self["document"] = None

    def get_diff_url(self, lang):
        constructor = get_constructor(document_type=self.document.type)

        if not self.previous_version_id:
            return self.document.get_url(lang)

        return "{}/{}/diff/{}/{}/{}/{}".format(
            self._campbot.wiki.ui_url,
            constructor.url_path,
            self.document.document_id,
            lang,
            self.previous_version_id,
            self.version["version_id"],
        )

    def get_locale_length(self, lang):
        locale = self.document.get_locale(lang)

        return locale.get_length() if locale else 0

class Contribution(BotObject):
    def __init__(self, campbot, data):
        super(Contribution, self).__init__(campbot, data)
        self["document"] = get_constructor(self["document"]["type"])(
            campbot, self["document"]
        )
        self["user"] = ShortWikiUser(campbot, self["user"])

    def get_full_document(self):
        return self._campbot.wiki.get_wiki_object(
            self.document["document_id"], document_type=self.document["type"]
        )

class Locale(BotObject):
    """
    Locale is a set of field, given a lang.
    """

    def get_title(self):
        """
        Get the title, with prefix if it exists.

        :return: String, pretty title
        """

        if "title_prefix" in self:
            return "{} : {}".format(self.title_prefix, self.title)
        else:
            return self.title

    def get_locale_fields(self):
        return (
            "description",
            "gear",
            "remarks",
            "route_history",
            "summary",
            "access",
            "access_period",
            "title",
            "external_resources",
            "other_comments",
            "slope",
            "slackline_anchor1",
            "slackline_anchor2",
        )

    def get_length(self):
        """
        Get text length

        :return: Integer, number of characters
        """

        result = 0
        for field in self.get_locale_fields():
            if field in self and self[field]:
                result += len(self[field])

        return result

class WikiObject(BotObject):
    """
    Base object for all wiki documents
    """

    url_path = None

    def __init__(self, campbot, data):
        super(WikiObject, self).__init__(campbot, data)

        if "associations" in self and self["associations"] is not None:
            self["associations"] = BotObject(campbot=campbot, data=self["associations"])
            self.associations._convert_list("images", Image)

        self._convert_list("locales", Locale)
        self._data = data

    def get_url(self, lang=None):
        """
        :return: camptocamp.org URL.
        """

        return "{}/{}/{}{}".format(
            self._campbot.wiki.ui_url,
            self.url_path,
            self.document_id,
            "" if lang is None else "/" + lang,
        )

    def get_history_url(self, lang):
        """

        :return: camptocamp.org version list URL
        """

        return "{}/{}/history/{}/{}".format(
            self._campbot.wiki.ui_url, self.url_path, self.document_id, lang
        )

    def get_title(self, lang):
        locale = self.get_locale(lang)
        return locale.get_title() if locale else ""

    def get_locale(self, lang):
        """
        :param lang: fr, en, de ...

        :return: String, or None if locale does not exists in this lang
        """

        if "locales" not in self:
            return None

        for locale in self.locales:
            if locale.lang == lang:
                return locale

    def search(self, patterns, lang):
        """
        Search a pattern (regular expression)

        :param lang: fr, de, en...

        :return: True if pattern is found, False otherwise
        """

        locale = self.get_locale(lang)

        for field in locale.get_locale_fields():
            if field in locale and locale[field]:
                for pattern in patterns:
                    if re.search(pattern, locale[field]):
                        return True
        return False

    def print_diff(self):
        report = get_diff_report(self._data, self)

        for l in report:
            print(l)

    def _build_payload(self, message):
        return {"document": self, "message": message}

    def save(self, message, ask_before_saving=True):
        """
        Save object to camptocamp.org. Bot must be authentified.

        :param message: Modification comment
        :param ask_before_saving: Boolean, ask user before saing document

        :return: raw request response, useless.
        """

        self.print_diff()

        if ask_before_saving:
            if _input("Save {} : {}, y/[n] ?\n".format(self.get_url(), message)) != "y":
                return None
        else:
            logging.info(f"Saving {self.get_url()} : {message}")

        return self._campbot.wiki.put(
            "/{}/{}".format(self.url_path, self.document_id),
            self._build_payload(message),
        )

    def is_valid(self):
        """
        :return: True if document can be saved
        """

        return self.get_invalidity_reason() is None

    def is_personal(self):
        return False

    def get_invalidity_reason(self):
        return None

class ShortWikiUser(BotObject):
    def get_contributions_url(self):
        return "{}/whatsnew#u={}".format(self._campbot.wiki.ui_url, self.user_id)

    def is_newbie(self):
        contribs = self._campbot.wiki.get(
            "/documents/changes?limit=50&u={}".format(self.user_id)
        )
        return len(contribs["feed"]) < 50

    def get_wiki_user(self):
        return self._campbot.wiki.get_user(user_id=self.user_id)

class WikiUser(WikiObject):
    url_path = "profiles"

    def get_contributions(self, oldest_date=None, newest_date=None):
        return self._campbot.wiki.get_contributions(
            user_id=self.document_id, oldest_date=oldest_date, newest_date=newest_date
        )

    def get_last_contribution(self, oldest_date=None, newest_date=None):
        for contribution in self.get_contributions(
            oldest_date=oldest_date, newest_date=newest_date
        ):
            return contribution

        return None

    def is_personal(self):
        return True

class Route(WikiObject):
    """Route object : https://www.camptocamp.org/routes"""

    url_path = "routes"

class Article(WikiObject):
    """Article object : https://www.camptocamp.org/articles"""

    url_path = "articles"

    def is_personal(self):
        return self.article_type == "personal"

class Image(WikiObject):
    """Image object : https://www.camptocamp.org/images"""

    url_path = "images"

    def is_personal(self):
        return self.image_type in ("personal", "copyright")

class Book(WikiObject):
    """Book object : https://www.camptocamp.org/books"""

    url_path = "books"

class Xreport(WikiObject):
    """Xreport object : https://www.camptocamp.org/xreports"""

    url_path = "xreports"

    def is_personal(self):
        return True

class Waypoint(WikiObject):
    """Waypoint object : https://www.camptocamp.org/waypoints"""

    url_path = "waypoints"

    def get_invalidity_reason(self):
        if self.waypoint_type in ("hut", "gite") and self.custodianship is None:
            return "custodianship is missing"

        if self.elevation is None and self.waypoint_type not in ("climbing_indoor",):
            return "elevation is missing"

        return None

class Area(WikiObject):
    """Area object : https://www.camptocamp.org/areas"""

    url_path = "areas"

    def _build_payload(self, message):
        payload = super(Area, self)._build_payload(message)

        #  Geometry info must not be present in payload, otherwise, save actions fails
        del payload["document"]["geometry"]

        return payload

class Map(WikiObject):
    """Map object : https://www.camptocamp.org/maps"""

    url_path = "maps"

class Outing(WikiObject):
    """Outings object : https://www.camptocamp.org/outings"""

    url_path = "outings"

    def is_personal(self):
        return True

class ForumUser(BotObject):
    def get_admin_view(self):
        return self._campbot.forum.get(
            "/admin/users/{}/{}.json".format(self.id, self.username)
        )

    def anonymise(self):
        return self._campbot.forum.put(
            "/admin/users/{}/anonymize.json".format(self.id), data=None
        )

class Post(BotObject):
    def __init__(self, campbot, data):
        super(Post, self).__init__(campbot, data)
        self._convert_list("polls", Poll)

class Poll(BotObject):
    def __init__(self, campbot, data):
        super(Poll, self).__init__(campbot, data)


#!/usr/bin/env python3

from itertools import product, permutations, combinations, combinations_with_replacement
import heapq
from collections import deque, defaultdict, Counter
from bisect import bisect
import sys

def input(): return sys.stdin.readline().rstrip()
def list_int(): return list(map(int, input().split()))

def is_ok(mid):
    if mid > 0:
        return True
    else:
        return False

def binary_search(ok, ng):
    # 複雑な二部探索用
    while ng - ok > 1:
        mid = (ok + ng) // 2
        if is_ok(mid):
            ok = mid
        else:
            ng = mid
    return ok

INF = float('inf')

# mod = 1000000007
# mod = 998244353
DIR = [(0, 1), (0, -1), (1, 0), (-1, 0)]
sys.setrecursionlimit(10**9)

#  -----------------------  #
N = int(input())

points = [[]]
Array = defaultdict(int)

for i in range(N):
    x, y = map(int, input().split())
    points.append([x, y])
    Array[(x, y)] += 1

ans = 0

for x in range(1, N+1):
    for y in range(x+1, N+1):
        x1, y1 = points[x]
        x2, y2 = points[y]
        if x1 == x2 or y1 == y2:
            continue
        if Array[(x2, y1)] and Array[(x1, y2)]:
            ans += 1

print(ans // 2)


from django.contrib import admin
from .forms import CustomUserCreationForm, AdminUserChangeForm
from .models import CustomUser
from django.contrib.auth.admin import UserAdmin

# Register your models here.

class CustomUserAdmin(UserAdmin):
    add_form = CustomUserCreationForm
    form = AdminUserChangeForm
    model = CustomUser
    list_display = (
        "email",
        "username",
    )
    fieldsets = (
        (
            None,
            {
                "fields": (
                    "username",
                    "password",
                    "email",
                    "organizations",
                )
            },
        ),
    )

admin.site.register(CustomUser, CustomUserAdmin)


nombre = "Carlos"
apellido = "Juan Perez"
edad = 37
direccionDeCorreo = "CarlosJuanPerez@hotmail.com"
telefono = 342105832
casado = False
hijos = True
listaAmigos = ["Marco", "Daniel", "Hernesto"]
peliculasFav = {"1": "Pulp Fiction", "2": "Anaconda", "3": "El Club De La Pelea"}

datosDeCarlos = [nombre, apellido, edad, direccionDeCorreo, telefono, casado, hijos, listaAmigos, peliculasFav]
print(datosDeCarlos)


# -*- coding: utf-8 -*-
'''
Created on 13th may 2016
@author: Daniel Durrenberger
daniel.durrenberger@amaris.com

Python 2.7.12
Pandas 0.18.1
Numpy 1.11.1
Scipy 0.18.0
'''

import numpy as np
import preprocessing as pp

def window_idx(time_ref, seconds_before, seconds_after, fs):
    start = int(time_ref - seconds_before * fs)
    end = int(time_ref + seconds_after * fs)
    return start, end

def localAutoCorrelate(signal, window):
    '''
    computes local autocorrelation, with a bit of normalization
    to compare results more easily
    '''
    n = len(signal)
    half = (n - window)/2
    m1 = signal[:half].mean()
    m2 = signal.mean()
    cr = 1./(1.*n) * np.correlate(signal-m2, signal[:half]-m1, mode='valid')
    return cr[2:-2]

def is_sticking_on_window(
                        window_start, window_end,
                        sticking_start, sticking_end
                        ):
    return sticking_start<=window_start and window_end<=sticking_end

def rolling_correlation_convolution(signal, fs, beginning=0):
    '''
    signal is the array of amplitude of HHT of the chosen IMF
    window is the size of the window in seconds or tours
    returns two lists of lists
        one list of time index per time window
        one list of correlation per time window
    you can plot them with a for loop on zip(t, corr)
    '''
    n = signal.shape[-1]
    dt = 1./fs
    window = int(1*fs)
    t = []
    corr = []
    time_ref = 1*fs        # tour_ref if normalized
    seconds_before = 1  # tour_before if normalized
    seconds_after = 3   # tour_after if normalized
    while time_ref + seconds_after*fs < n:
        start, end = window_idx(time_ref, seconds_before, seconds_after, fs)
        signal_slice = signal[start:end]
        #print time_ref, start, end, len(signal_slice)
        cr = localAutoCorrelate(signal_slice, window)
        t.append(list(beginning + np.linspace(start*dt, end*dt, len(cr))))
        corr.append(list(cr))
        time_ref += (seconds_before + seconds_after) * fs
    return t, corr

def fft_of_correlation(corr, fs):
    '''
    returns two lists of lists
        one list of frequency index per time window
        one list of fft per time window
    '''
    dt = 1./fs
    freq_list = []
    fft_list = []
    for correlation in corr:
        fft = np.fft.fft(correlation)
        freq = np.fft.fftfreq(len(correlation), 1./fs)
        nTot = len(fft)
        nPoints_to_2Hz = int(nTot*dt*2)
        nPoints_to_10Hz = int(nTot*dt*10)
        idx = np.arange(nPoints_to_2Hz,nPoints_to_10Hz)
        freq_list.append( freq[idx] )
        fft_list.append( np.abs(fft[idx]) )
    return freq_list, fft_list

def peak_coordinates(x, y):
    '''
    returns the coordinates of the maximum of the graph (x, y)
    '''
    imax = np.argmax(y)
    xmax = x[imax]
    ymax = y[imax]
    return xmax, ymax

def peak_list(freq_list, fft_list):
    '''
    returns two lists
        one with the most important frequencies values per time window
        one with the corresponding amplitudes per time window
    '''
    xpeak = []
    ypeak = []
    for freq, fft in zip(freq_list, fft_list):
        x, y = peak_coordinates(freq, fft)
        xpeak.append(x)
        ypeak.append(y)
    return xpeak, ypeak