# %%

from functools import partial
import os
import itertools
from copy import copy, deepcopy
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
from jax.interpreters.ad import JVPTracer

from tabulate import tabulate
import msgpack
import numpy as np
import jax.numpy as jnp
import jax
import matplotlib.pyplot as plt
import plotly.express as px
from attrs import evolve, define, Factory, frozen
import einops

from interp.model.gpt_model import Gpt, partial_gpt_call
from interp.model.grad_modify_fake import ablation_softmax_probs, integrated_gradients_softmax_probs
from interp.model.grad_modify_mul import attn_score_mul_builder, probs_v_mul_builder
from interp.tools.attribution_backend_utils import stop_qkv_except
from interp.tools.grad_modify_query_items import ItemConf, ItemIdx, MulConf, NoneConf, StopGradConf, ModifyInBasisConf
from interp.tools.grad_modify_query_utils import MulBuilder, compose_trees as ct
from interp.tools.interpretability_tools import (
    batch_tokenize,
    losses_runner,
    losses_runner_log,
    print_max_min_by_tok_k,
    single_tokenize,
    LossesRunnerTreeConfig,
    begin_token,
    losses_runner_tree,
    toks_to_string_list,
    run_on_tokens,
    cross_entropy_ablation_log_probs,
    get_cross_entropy_integrated_gradients_log_probs,
)
from interp.tools.jax_util import stack_tree
from interp.model.model_loading import load_model
from interp.tools.grad_modify_query import (
    GradModifierConf,
    ModifierCollectionTreeNodeStack,
    ModifierCollectionTreeNode,
    Query,
    TargetConf,
    run_queries,
    run_query,
)
from interp.tools.indexer import I
from interp.model.grad_modify_output import (
    Attn,
    AttnLayer,
    Embeds,
    FinalOutput,
    OutputConf,
    losses_runner_outputs,
    output_tree,
)
from interp.tools.log import Idxs, KeyIdxs
from interp.ui.attribution_backend import AttributionBackend, AttributionRoot
from interp.tools.data_loading import DATA_DIR
import interp
import interp.tools.optional as op

# os.environ["XLA_PYTHON_CLIENT_ALLOCATOR"] = "platform"

# %%

"""
This notebook does some quick ablations on the induction circuit to see what's
actually important.
"""

# %%

# interp.init()  # if you want interp to work, run in jupyter notebook classic

# %%

models_dir_local = os.path.expanduser("~/interp_models_jax")
model, params, tok = load_model("jan5_attn_only_two_layers/", models_dir=models_dir_local)
model: Gpt = model
params

# %%

text = " \"I don't sleep right,\" Harry said. He waved his hands helplessly. \"My sleep cycle is twenty-six hours long, I always go to sleep two hours later, every day. I can't fall asleep any earlier than that, and then the next day I go to sleep two hours later than that. 10PM, 12AM, 2AM, 4AM, until it goes around the clock. Even if I try to wake up early, it makes no difference and I'm a wreck that whole day. That's why I haven't been going to a normal school up until now.\""
text = " \"I love Mrs. Dursley. I don't sleep right,\" Harry said. He waved his hands helplessly. \"Mrs. Dursley's sleep cycle is twenty-six hours long, I always go to sleep two hours later, every day. I can't fall asleep any earlier than that, and then the next day I go to sleep two hours later than that. 10PM, 12AM, 2AM, 4AM, until it goes around the clock. Mr. Dursley stops me from sleeping two hours everytime. Mr. Dursley dislikes his wife Mrs. Dursley."
# text = " My name is Buck. My name is Buck."
example_seq_no_begin = batch_tokenize([text])
example_seq = jnp.concatenate([jnp.expand_dims(begin_token(), (0, 1)), example_seq_no_begin], axis=1)
example_seq_string_list = toks_to_string_list(example_seq[0])
example_seq_names = [f"{n} ({i})" for i, n in enumerate(example_seq_string_list)]
example_seq.shape

# %%

import torch

fnames = os.listdir(f"{DATA_DIR}/owt_tokens_int16_val")[1:2]
all_tokens = [torch.load(f"{DATA_DIR}/owt_tokens_int16/{f}") for f in fnames]
data_pt = list(itertools.chain(*[torch.split(x["tokens"], x["lens"].tolist()) for x in all_tokens]))

# %%

max_size = 511

# %%

data = torch.stack(
    [data_pt_val[:max_size].to(torch.int64) + 32768 for data_pt_val in data_pt if data_pt_val.size(0) >= max_size],
    dim=0,
).numpy()
data.shape

# %%

data_subset = data[:300]
batch_size = 4

# %%

"""
We'll want to ablate 'through' the log softmax instead of differentiating.

We'll also see what the derivative and integrated derivative look like.
"""

# %%

out_tree = output_tree(model, OutputConf(AttnLayer(1, by_head=True), endpoint=FinalOutput(), item_idx=ItemIdx(I[:, 4])))

extra_tree: List[List[Optional[ModifierCollectionTreeNode]]] = [
    [ct(evolve(out_tree, use_fwd=True), cross_entropy_ablation_log_probs)],
    [ct(evolve(out_tree, use_fwd=True), get_cross_entropy_integrated_gradients_log_probs())],
    [evolve(out_tree, use_fwd=False)],
]

run_split = jax.jit(
    losses_runner_outputs(
        model,
        params,
        config=LossesRunnerTreeConfig(get_loss_no_deriv=True),
        get_extra_tree=lambda runner, seq_len: extra_tree,
    )
)

# %%

out_removed = stack_tree(
    run_on_tokens(
        run_split,
        data_subset,
        batch_size,
        disable_progress=False,
    )
)
out_removed

# %%

loss_removed = out_removed["deriv"]["loss"].mean(axis=0)
loss_orig = out_removed["no_deriv"]["loss"].mean()

ablation_loss = loss_removed[0]
print("ablation:", ablation_loss)
print("integrated gradient", loss_removed[1])
print("normal deriv", loss_removed[2])
print("original loss", loss_orig)

# %%

"""
Now let's start excluding some heads and see how the head does.
"""

# %%

tok_to_layer_1_attn_tree = output_tree(model, OutputConf(Embeds(), endpoint=AttnLayer(1)))
tok_to_layer_1_attn_tree

# %%

def all_except_run(
    q_heads,
    k_heads,
    v_heads,
    exclude_tok_to_k=False,
    ablation_over_ig_attn: Optional[bool] = True,
    ablation_over_ig_log_probs: Optional[bool] = True,
    config: LossesRunnerTreeConfig = LossesRunnerTreeConfig(avg_loss=False),
):
    def get_trees(runner, seq_len) -> ModifierCollectionTreeNodeStack:
        q_tree: ModifierCollectionTreeNodeStack = ModifierCollectionTreeNode(
            MulConf(ItemConf(KeyIdxs.single("blocks.attention.out_by_head", 0), ItemIdx(I[:, q_heads])))
        )
        k_tree: List[Optional[ModifierCollectionTreeNode]] = [
            ModifierCollectionTreeNode(
                MulConf(ItemConf(KeyIdxs.single("blocks.attention.out_by_head", 0), ItemIdx(I[:, k_heads])))
            ),
        ]
        if exclude_tok_to_k:
            k_tree.append(tok_to_layer_1_attn_tree)

        v_tree: ModifierCollectionTreeNodeStack = ModifierCollectionTreeNode(
            MulConf(ItemConf(KeyIdxs.single("blocks.attention.out_by_head", 0), ItemIdx(I[:, v_heads])))
        )

        base_tree = ct(
            ModifierCollectionTreeNode(
                StopGradConf(
                    ItemConf(KeyIdxs.single("blocks.attention.out_by_head", 1), ItemIdx(I[:, 4], except_idx=True))
                )
            ),
            ModifierCollectionTreeNode(StopGradConf(ItemConf(KeyIdxs.single("blocks.inp_res", 1)))),
            op.map(
                ablation_over_ig_attn,
                lambda x: ablation_softmax_probs(Idxs.single(1))
                if x
                else integrated_gradients_softmax_probs(Idxs.single(1)),
            ),
        )

        attn_mul: List[Optional[ModifierCollectionTreeNode]] = list(
            attn_score_mul_builder(
                runner,
                model,
                ct(base_tree, q_tree),
                ct(base_tree, k_tree),
                1,
            ).remove_conjunctive_from_sum()
        )

        all_mulled: List[Optional[ModifierCollectionTreeNode]] = list(
            probs_v_mul_builder(
                runner,
                attn_mul,
                ct(base_tree, v_tree),
                1,
            ).remove_conjunctive_from_sum()
        )

        return ct(
            base_tree,
            all_mulled,
            op.map(
                ablation_over_ig_log_probs,
                lambda x: cross_entropy_ablation_log_probs if x else get_cross_entropy_integrated_gradients_log_probs(),
            ),
        )

    return jax.jit(
        losses_runner_tree(
            partial_gpt_call(model, params, config=Gpt.CallConfig(log_finish=False)),
            get_trees,
            config=evolve(config, use_fwd=True),
        )
    )

# %%

# selected based on some experimentation
v_heads = list(range(1, 8))
base_q_heads = list(range(8))
base_k_heads = list(range(1, 8))

# %%

"""
Let's see how this does.
"""

# %%

run = all_except_run(base_q_heads, base_k_heads, v_heads)

# %%

out_except = stack_tree(run_on_tokens(run, data_subset, 2, disable_progress=False), stack=False)
mean_base = out_except["deriv"]["loss"].squeeze().mean(axis=(0, 1))

print(mean_base)
print("score", (ablation_loss - mean_base) / ablation_loss)

# %%

"""
Now let's try looping over including a single head on both k and q.
"""

# %%

for head_to_include in range(8):
    new_q_heads = copy(base_q_heads)
    new_q_heads.remove(head_to_include)
    new_k_heads = copy(base_k_heads)
    if head_to_include in new_k_heads:
        new_k_heads.remove(head_to_include)

    run = all_except_run(new_q_heads, new_k_heads, v_heads)

    # compile times on this are brutal...
    out_except = stack_tree(run_on_tokens(run, data[:100], 1, disable_progress=False), stack=False)
    out_except

    print("included:", head_to_include)
    mean_l = out_except["deriv"]["loss"].squeeze().mean(axis=(0, 1))
    print(mean_l)
    print("score", (ablation_loss - mean_l) / ablation_loss)

# %%

"""
Note that above we included layer 0 head 0 in v. It's somewhat suprising that
this head makes a reasonable contribution to v (but I claim it does).

Let's figure out exactly what this contribution is.

(This part of the notebook is undocumented and unexplained atm, but will
hopefully be better in the future after I write things up).
"""

# %%

def effect_of_layer_0_0_v(
    ablation_over_ig_log_probs: Optional[bool] = True,
    shape_on_toks=True,
    avg_loss=False,
    log_probs=True,
    fwd=True,
    toks_subset=I[:],
    extra_tree: ModifierCollectionTreeNodeStack = None,
):
    @jax.jit
    def f(toks, targets):
        runner = losses_runner_log(
            partial_gpt_call(model, params, config=Gpt.CallConfig(log_finish=False)), toks, targets
        )

        head_layer_0 = 0
        head_layer_1 = 4

        if not isinstance(toks_subset, slice):
            assert toks.shape[0] == 1

        shape = (
            (toks.shape[0], toks.shape[1] if isinstance(toks_subset, slice) else toks_subset.shape[0], 1)
            if shape_on_toks
            else ()
        )
        tree = ct(
            ModifierCollectionTreeNode(
                MulConf(ItemConf(KeyIdxs.single("blocks.attention.inp", 0), ItemIdx(I[:, toks_subset])), shape=shape)
            ),
            ModifierCollectionTreeNode(
                StopGradConf(
                    ItemConf(
                        KeyIdxs("blocks.attention.out_by_head", Idxs.single(0)),
                        item_idx=ItemIdx(I[:, head_layer_0], except_idx=True),
                    ),
                )
            ),
            ModifierCollectionTreeNode(StopGradConf(ItemConf(KeyIdxs.single("blocks.inp_res", 1)))),
            *stop_qkv_except(Idxs.single(1), "v"),
            ModifierCollectionTreeNode(
                StopGradConf(
                    ItemConf(
                        KeyIdxs.single("blocks.attention.out_by_head", 1),
                        ItemIdx(I[:, head_layer_1], except_idx=True),
                    ),
                )
            ),
            op.map(
                ablation_over_ig_log_probs,
                lambda x: cross_entropy_ablation_log_probs if x else get_cross_entropy_integrated_gradients_log_probs(),
            ),
            extra_tree,
        )

        if avg_loss:
            targets = [TargetConf(KeyIdxs("loss.mean_loss"), "loss")]
        else:
            targets = [TargetConf(KeyIdxs("loss.losses"), "loss")]

        if log_probs:
            targets.append(TargetConf(KeyIdxs("cross_entropy.log_probs"), display_name="log_probs"))

        return run_query(runner, Query(targets, tree, use_fwd=fwd))

    return f

# %%

run_probs_just_v = effect_of_layer_0_0_v(ablation_over_ig_log_probs=None)

# %%

out_except = stack_tree(
    run_on_tokens(run_probs_just_v, example_seq, 1, disable_progress=True, prepend_begin=False), stack=False
)
loss_diff = out_except["loss"].squeeze((1, 3))  # get 1 beyond to remove begin tok
summed_over = loss_diff.sum(-1)
_, idxs = jax.lax.top_k(jnp.abs(summed_over), k=10)
loss_diff[idxs]
np.array(example_seq_names)[np.array(idxs)], summed_over[idxs], summed_over

# %%

print(example_seq_names[97])
print_max_min_by_tok_k(loss_diff[97], k=20, get_tok=lambda idxs: np.array(example_seq_names)[np.array(idxs)])

# %%

print(example_seq_names[85])
print_max_min_by_tok_k(loss_diff[85], k=20, get_tok=lambda idxs: np.array(example_seq_names)[np.array(idxs)])

# %%

b_model = model.bind(params)

tok_embeds = b_model.embedding.token_embedding.embedding

all_ov_combined = b_model.get_ov_combined_mats_all_layers()
ov_0_0_to_1_4 = jnp.einsum(" o m, m i -> o i", all_ov_combined[1, 4], all_ov_combined[0, 0])
ov_0_0_to_1_4_to_out = jnp.einsum("v o, o i -> v i", tok_embeds, ov_0_0_to_1_4)
ov_0_0_to_out = jnp.einsum("v m, m i -> v i", tok_embeds, all_ov_combined[0, 0])
ov_0_0_to_1_4_to_out.shape, ov_0_0_to_out.shape

# %%

def print_for_tok(t: Union[str, int], k=50):
    if isinstance(t, str):
        t = single_tokenize(t)

    print(f'"{tok.decode(t)}"')

    preds_full = jnp.einsum("v i, i -> v", ov_0_0_to_1_4_to_out, tok_embeds[t])
    preds_0_0 = jnp.einsum("v i, i -> v", ov_0_0_to_out, tok_embeds[t])

    print("0_0_to_1_4")
    print("logit variance", preds_full.var())
    print_max_min_by_tok_k(preds_full, k=k)
    print()

    # print("0_0")
    # print("logit variance", preds_0_0.var())
    # print_max_min_by_tok_k(preds_0_0, k=k)

# %%

# print more toks as desired
print_for_tok(" dog", k=10)

# %%

# I think this isn't very meaningful due to general log softmax issues when averaging loss?
# Maybe not though.
run_probs_just_v_mean_loss = effect_of_layer_0_0_v(
    ablation_over_ig_log_probs=None,
    avg_loss=True,
    log_probs=False,
    fwd=False,
)

# %%

out_v_path = stack_tree(run_on_tokens(run_probs_just_v_mean_loss, data_subset, 2, disable_progress=False), stack=False)
losses_by_from_tok = out_v_path["loss"].squeeze(-1)
losses_by_from_tok.shape

# %%

n_tokens = tok_embeds.shape[0]

# %%

# offset by begin!
data_subset[:-1].shape, losses_by_from_tok[1:].shape

# %%

loss_deriv_by_tok = jnp.zeros(n_tokens).at[data_subset[:-1]].add(losses_by_from_tok[1:], unique_indices=False)

# %%

print_max_min_by_tok_k(loss_deriv_by_tok, k=30, normalize=False)

# %%

loss_deriv_by_tok.mean()

# %%

run_probs_from_current_loss = effect_of_layer_0_0_v(
    ablation_over_ig_log_probs=True, shape_on_toks=False, avg_loss=False, log_probs=False, fwd=True
)

# %%

batch_size = 2

# %%

out_v_path_q = stack_tree(
    run_on_tokens(run_probs_from_current_loss, data_subset, batch_size, disable_progress=False),
    stack=True,
)
losses_by_q_tok = einops.rearrange(out_v_path_q["loss"], "n (b s) -> (n b) s", b=batch_size)
losses_by_q_tok.shape

# %%

net_loss_by_q_tok = jnp.zeros(n_tokens).at[data_subset[:-1]].add(losses_by_q_tok[1:], unique_indices=False)
net_loss_by_q_tok.mean()

# %%

print_max_min_by_tok_k(net_loss_by_q_tok, k=30, normalize=False)

# %%

most_helpful = losses_by_q_tok.mean(axis=1).argmin()
most_helpful

# %%

print_max_min_by_tok_k(
    losses_by_q_tok[most_helpful][1:],
    k=20,
    get_tok=lambda idxs: toks_to_string_list(data_subset[most_helpful][(idxs,)]),
)

# %%

g_idxs = jnp.array((np.array(toks_to_string_list(data_subset[most_helpful])) == "g").nonzero()[0])
g_idxs

# %%

print(tok.decode(data_subset[most_helpful]))

# %%

run_probs_ablate_a_few_toks = effect_of_layer_0_0_v(
    ablation_over_ig_log_probs=True,
    shape_on_toks=True,
    avg_loss=False,
    log_probs=True,
    fwd=True,
    toks_subset=g_idxs + 2,  # + 1 for [BEGIN] and + 1 again for next tok
)

# %%

out_v_path_from_selected_toks = stack_tree(
    run_on_tokens(
        run_probs_ablate_a_few_toks,
        jnp.expand_dims(data_subset[most_helpful], 0),
        1,
        disable_progress=False,
    ),
    stack=True,
)
losses = out_v_path_from_selected_toks["loss"].squeeze((0, 2, -1))

# %%

losses[419]

# %%

all_virtual_head_outputs = jnp.einsum("o i, n i -> n o", ov_0_0_to_1_4, tok_embeds)
all_virtual_head_outputs.shape

# %%

import torch

x = torch.tensor(np.array(all_virtual_head_outputs))
u, s, v = torch.pca_lowrank(x, q=32)
u, s, v = u.numpy(), s.numpy(), v.numpy()
u.shape, s.shape, v.shape

# %%

# all_virtual_head_outputs[0]
idx = 21
jnp.abs(all_virtual_head_outputs[idx] - (v @ jnp.transpose(v)) @ all_virtual_head_outputs[idx]).max()

# %%

# inp_to_dims
inp_to_dims = jnp.einsum("o i, o d -> d i", ov_0_0_to_1_4, v)
tok_input_to_dims_values = jnp.einsum("d i, t i -> d t", inp_to_dims, tok_embeds)
inp_to_dims.shape, tok_input_to_dims_values.shape

# %%

plt.plot(s)

# %%

s

# %%

direc_to_toks = jnp.einsum("v o, o d -> d v", tok_embeds, v)

# %%

for dim in range(5):
    print("dim", dim)
    print_max_min_by_tok_k(direc_to_toks[dim], k=20)

# %%

print(print_max_min_by_tok_k(tok_input_to_dims_values[0], k=20))

# %%

def get_run_probs_ablate_some_dims(up_to_n):
    return effect_of_layer_0_0_v(
        ablation_over_ig_log_probs=True,
        shape_on_toks=False,
        avg_loss=True,
        log_probs=False,
        fwd=True,
        extra_tree=ModifierCollectionTreeNode(
            ModifyInBasisConf(
                ItemConf(KeyIdxs.single("blocks.attention.out_by_head", 1), ItemIdx(I[:, 4])),
                proj_to=jnp.transpose(v),
                proj_back=v,
                sub_conf=StopGradConf(ItemConf(KeyIdxs(""), ItemIdx(I[..., :up_to_n]))),
            )
        ),
    )

# %%

run_ablate_after_1 = get_run_probs_ablate_some_dims(1)

# %%

out_some_dims = stack_tree(run_on_tokens(run_ablate_after_1, data_subset, 2, disable_progress=True), stack=True)
out_some_dims["loss"], out_some_dims["loss"].shape, out_some_dims["loss"].mean()

# %%

losses = [
    stack_tree(run_on_tokens(get_run_probs_ablate_some_dims(i), data_subset, 2, disable_progress=False), stack=True)[
        "loss"
    ].mean()
    for i in range(0, 10)
]

# %%

plt.plot(-np.array(losses))


import socket
import colorama
import sequence

#def count_bases(arg):
    #d = {"A": 0, "C": 0, "G": 0, "T": 0}
    #for b in arg:
        #d[b] += 1
    #return d

#def covert_message(base_count):
    #message = ""
    #for k,v in base_count.items():
        #message += k + ":" + str(v) + "\n"
    #return message

# Configure the Server's IP and PORT
PORT = 8080 #6123 always work
IP = "127.0.0.1" #you could just write localhost

# -- Step 1: create the socket
ls = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# -- Optional: This is for avoiding the problem of Port already in use
ls.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)  #addresses can be reused
#run two servers with the same port --> avoid adress already used error.

# -- Step 2: Bind the socket to server's IP and PORT
ls.bind((IP, PORT))

# -- Step 3: Configure the socket for listening
ls.listen()  #if we dont specified the operating system will handle by itself how any clients can be at the same time

print("SEQ Server is configured!")

while True:
    # -- Waits for a client to connect
    print("Waiting for Clients to connect...")
    try:
        (cs, client_ip_port) = ls.accept()
        # -- Server stopped manually
    except KeyboardInterrupt:
        print("Server stopped by the user")

        # -- Close the listening socket
        ls.close()

        # -- Exit!
        exit()

    # -- Execute this part if there are no errors

    else:

        print("A client has connected to the server!")

        # -- Read the message from the client
        # -- The received message is in raw bytes
        msg_raw = cs.recv(2048)

        # -- We decode it for converting it
        # -- into a human-redeable string
        msg = msg_raw.decode().replace("\n", "").strip() #strip para quitar los espacios de la izq y de la derecha
        splitted_command = msg.split(" ")
        cmd = splitted_command[0]
        #arg = msg.split(' ')[1] #if we are not using the msg again we can just do the slipt after the strip
        if cmd != "PING":
            arg = splitted_command[1]
            #print(arg)
        #print(cmd)

        # -- Print the received message
        #print(f"Message received: {msg}")

        # -- Send a response message to the client
        if msg == "PING":
            color_txt = "PING command! "
            #print(color_txt)
            colorama.init()
            print(colorama.Fore.GREEN + color_txt + colorama.Fore.WHITE)
            response = "OK!\n"

        elif cmd == "GET":
            color_txt = "GET"
            colorama.init()
            print(colorama.Fore.GREEN + color_txt + colorama.Fore.WHITE)
            list_seq = ["U5", "FRAT1", "ADA", "FXN", "RNU6_269P"]
            i = int(arg)
            if i <= 4:
                choosen = list_seq[i]
                seq = sequence.Seq()
                filename = f"../P0/sequences/" + choosen + ".txt"
                full_f = str(seq.read_fasta(filename))
                response = full_f

        elif cmd == "INFO":
            color_txt = "INFO"
            colorama.init()
            print(colorama.Fore.GREEN + color_txt + colorama.Fore.WHITE)
            seq = sequence.Seq(arg)
            response, count = seq.bases(arg)
            #response = seq.convert_message(bases)
            response = "Sequence: " + arg + "\n" + "Total Length: " + str(len(arg)) + "\n" + response

            #response += " " + "(" + str(round((count*100)/len(arg), 3)) + "%)"

        elif cmd == "COMP":
            color_txt = "COMP"
            colorama.init()
            print(colorama.Fore.GREEN + color_txt + colorama.Fore.WHITE)
            seq = sequence.Seq(arg)
            compl = seq.complement(arg)
            response = compl + " "

        elif cmd == "REV":
            color_txt = "REV "
            colorama.init()
            print(colorama.Fore.GREEN + color_txt + colorama.Fore.WHITE)
            response = arg[::-1] + " "
            print("New sequence created!")

        elif cmd == "GENE":
            color_txt = "GENE "
            colorama.init()
            print(colorama.Fore.GREEN + color_txt + colorama.Fore.WHITE)

            seq = sequence.Seq(arg)
            #s = "../P0/sequences/" + arg + ".txt"
            #filename = seq.read_fasta(arg)
            filename = f"../P0/sequences/" + arg + ".txt"
            full_f = seq.read_fasta(filename)
            response =  str(full_f)

        else:
            response = "HELLO. I am the Server :-)\n"

        print(response)

        # -- The message has to be encoded into bytes
        cs.send(response.encode())

        # -- Close the data socket
        cs.close()

# " " + "(" + str(round((count*100)/total_count, 3)) + "%)"

#P3.sequence


import numpy
import scipy.interpolate
from scipy.ndimage import filters
import scipy.stats.mstats
import logging

logger = logging.getLogger(__name__)
formatter = logging.Formatter(
    "%(asctime)s - %(name)s - %(levelname)s - %(message)s")

def lat_weights(lats):
    """ Returns cos(lats)

    Required as a weighting for calculating areas on
    a regular cartesian lat-lon grid in which meridians
    converge.

    Arguments:
       lats (numpy ndarray): in degree

    Returns:
       numpy ndarray
    """
    wgts = numpy.cos(numpy.radians(lats))
    return wgts

def runave(data, N, axis=0, step=None):
    """ Compute a smooth box-car running average and masked
    edge values using scipy.ndimage.filters.convolve1d

    Args:
       data (numpy array)
       N (int) : width of the box car
       axis (int): axis along which the average is computed
                (See scipy.ndimage.filters.convolve1d)
       step (optional, int): skips between the box car samples

    Returns:
       numpy masked array (same shape as data)
    """
    if isinstance(data, numpy.ma.core.MaskedArray):
        data = data.filled(numpy.nan)
    if step is None:
        weights = numpy.ones((N,))
    else:
        if type(step) is not int:
            raise Exception("step should be an integer")
        weights = numpy.array(([1.]+[0.]*(step-1))*(N-1)+[1.])
    weights /= weights.sum()
    return numpy.ma.masked_invalid(filters.convolve1d(
        data, weights, mode='constant', cval=numpy.nan, axis=axis))

def skewness(data):
    """ Essentially the same as scipy.stats.skew or
        scipy.stats.mstats.skew depending on the input

    Input:
        data (numpy array or masked array)
    Returns:
        numpy array or masked array
    """
    if isinstance(data,numpy.ma.core.MaskedArray):
        npmod = numpy.ma
    else:
        npmod = numpy
    return npmod.power(data-npmod.mean(data), 3).mean()/\
        npmod.power(npmod.var(data), 3./2.)

def resample_xy(x, y, xnew, nx, ny, y_always_valid=True):
    """ Given paired values of (x,y), randomly sample a set of
    values for ynew given an array xnew such that the joint distribution
    of (xnew, ynew) resembles that of (x,y)

    Arguments:
        x (numpy 1d array)
        y (numpy 1d array): shape should match x
        xnew (numpy 1d array)
        nx (int) : the number of bins applied to x
        ny (int) : the number of bins applied to y
        y_always_valid (bool): keep drawingn from the cdf until y is valid

    Returns:
        ynew (numpy 1d array): length = len(xnew)
    """
    if y.shape != x.shape:
        raise ValueError("shape of y should match the shape of x")

    xedges = numpy.linspace(x.min(), x.max(), nx+1)
    xnew_ibin = numpy.digitize(xnew, xedges)
    y_cdf = {}
    y_mids = {}
    ynew = numpy.empty_like(xnew, dtype=y.dtype)
    for ix, ibin in enumerate(xnew_ibin):
        # If the xnew is out of bound, return numpy.nan for that entry
        if xnew[ix] >= xedges[-1] or xnew[ix] < xedges[0]:
            logger.warn("Out of bound for x={:.2e}. "+\
                        "Return numpy.nan.".format(float(xnew[ix])))
            ynew[ix] = numpy.nan
            continue

        # Don't compute the cdf twice for the same bin
        # Compute it once and save it
        if ibin not in y_cdf:
            ys = y[(x > xedges[ibin-1]) & (x <= xedges[ibin])]
            y_cdf[ibin], y_mids[ibin] = cdf(ys,bins=ny)

        # Keep drawing a new y until the drawn value is valid
        while True:
            rand_num = numpy.random.uniform()
            try:
                ynew[ix] = scipy.interpolate.interp1d(
                    y_cdf[ibin], y_mids[ibin])(rand_num)
                break
            except ValueError:
                if y_always_valid:
                    continue
                else:
                    ynew[ix] = numpy.nan
                    break
    return ynew

def cdf(data, **kwargs):
    """ Compute a histogram and the corresponding cumulative
    frequency polygon.

    Args:
        data (numpy 1d array)
        option (str): currently accepts "hist" only

    Keyword arguments currently are passed to numpy.histogram

    Returns:
        bins, cdf : mid values of bins and the cumulative frequencies
    """
    # Use histogram for cdf
    h, x = numpy.histogram(data,**kwargs)
    x_mid = 0.5*(x[1:]+x[:-1])
    h_cum = numpy.cumsum(h)
    return h_cum/float(h_cum[-1]), x_mid


'''
https://leetcode.com/problems/maximum-product-of-word-lengths/
Given a string array words, find the maximum value of length(word[i]) * length(word[j]) where the two words do not share common letters.
You may assume that each word will contain only lower case letters. If no such two words exist, return 0.

Example 1:
Input: ["abcw","baz","foo","bar","xtfn","abcdef"]
Output: 16
Explanation: The two words can be "abcw", "xtfn".
'''
'''
Time:O(n**2+L)
Space:O(n)
'''
class Solution:
    def maxProduct(self, words: List[str]) -> int:
        n = len(words)
        mask = [0 for _ in range(n)]
        bit = lambda x:ord(x)-ord('a')

        for i in range(n):
            bitmask = 0
            for ch in words[i]:
                bitmask |= 1<<bit(ch)
            mask[i] = bitmask

        res = 0
        for i in range(n):
            for j in range(i+1, n):
                if mask[i] & mask[j]==0:
                    res = max(res, len(words[i])*len(words[j]))

        return res
'''
Time:
Space:O(1)
'''
class Solution:
    def maxProduct(self, words: List[str]) -> int:
        def nocommon(s1,s2):
            for i in s1:
                if i in s2:
                    return False
            return True

        product = 0
        for i in range(len(words)):
            for j in range(i+1, len(words)):
                if nocommon(words[i], words[j]):
                    product = max(product, len(words[i])*len(words[j]))
        return product


# -*- coding: utf-8 -*-

from odoo import models, fields, api
from odoo.exceptions import ValidationError, UserError
from datetime import datetime, timedelta
from odoo.tools import float_is_zero, float_compare, DEFAULT_SERVER_DATETIME_FORMAT

class B2bSaleResendWizard(models.TransientModel):
    _name = 'b2b.sale.resend.wizard'
    _description = 're-send products claimed by end customer'

    origin = fields.Many2one('sale.order', u'销售单号')
    order_line = fields.One2many('b2b.sale.resend.wizard.line', 'order_id', u'订单明细')
    amount_total = fields.Float(u'合计金额', compute='_get_total_amount')
    # procurement_ids = fields.One2many('procurement.order', 'sale_line_id', string='Procurements')
    procurement_group_id = fields.Many2one('procurement.group', 'Procurement Group', copy=False)

    @api.onchange('origin')
    def onchange_origin(self):
        freight_product = self.env.ref('b2b_platform.default_b2b_delivery_carrier')
        list = []
        if self.origin:
            for line in self.origin.order_line:
                if line.product_id != freight_product:
                    list.append([0,0,{'order_id': self.id,
                                      'product_id': line.product_id,
                                      'qty': line.product_uom_qty,
                                      'price_unit': line.price_unit,
                                      'sale_line': line.id,
                                    }])
            self.order_line = list

    @api.one
    def _get_total_amount(self):
        amount = 0
        for line in self.order_line:
            amount += line.qty * line.price_unit
        self.amount_total = amount

    def _prepare_procurement_group(self):
        return {'name': self.origin.name}

    @api.multi
    def btn_confirm(self):
        if not self.order_line:
            raise UserError(u'没有需要采购的产品')

        p_amount = 0
        freight_product = self.env.ref('b2b_platform.default_b2b_delivery_carrier')
        freight_obj = self.env['b2b.product.freight.line'].sudo()

        for p_line in self.order_line:
            p_amount += p_line.price_unit * p_line.qty

        if p_amount > self.origin.partner_id.deposit_avail_amt:
            raise UserError(u'在平台预存的余额不足，请先充值')

        # 计算运费
        if not self.origin.partner_shipping_id.country_id:
            raise UserError(u'发货地址未明确所在国家，无法核算运费')
        dest_country = self.origin.partner_shipping_id.country_id
        amount = 0
        freight_line_id = False
        for line in self.order_line:
            if line.qty <= 0:
                line.unlink()
            else:
                freight = freight_obj.search([('product_id', '=', line.product_id.product_tmpl_id.id),
                                              ('country_id', '=', dest_country.id)], limit=1)
                if not freight:
                    raise UserError(u'未找到%s产品的运费' % line.product_id.name)
                amount += freight.cny_amount * line.qty
                if line.product_id == freight_product:
                    freight_line_id = line
        if freight_line_id:
            freight_line_id.price_unit = amount
        else:
            self.order_line.create({
                'order_id': self.id,
                'price_unit': amount,
                'qty': 1,
                'product_id': freight_product.id,
                'is_delivery': True,
            })

        proc_order_line = self.order_line.search([('order_id', '=', self.id), ('is_delivery', '!=', True)])

        # 创建发货单
        purchase_obj = self.env['purchase.order'].sudo()
        picking_obj = self.env['stock.picking']
        warehouse_obj = self.env['stock.warehouse'].sudo()
        warehouse = warehouse_obj.search([('partner_id', '=', 1)], limit=1)
        if not warehouse:
            raise UserError(u'未找到平台仓库编码，请咨询平台管理人员')

        stock_loc_obj = self.env['stock.location'].sudo()
        cust_loc_id = stock_loc_obj.search([('usage', '=', 'customer')], limit=1)
        if not cust_loc_id:
            raise UserError(u'未找到客户所属库位，请咨询平台管理人员')

        picking_type_obj = self.env['stock.picking.type'].sudo()
        picking_type = picking_type_obj.search([('id', '=', self.env.ref('stock.picking_type_out').id), ('warehouse_id', '=', warehouse.id)], limit=1)
        # picking_type = picking_type_obj.search([('name', '=', u'发货'), ('warehouse_id', '=', warehouse.id)], limit=1)
        if not picking_type:
            raise UserError(u'未找到捡货类型，请咨询平台管理人员')

        # 生成procurement order
        new_procs = self.env['procurement.order']  # Empty recordset
        for line in proc_order_line:
            if line.product_id != freight_product:
                new_proc = new_procs.search([('origin','=',self.origin.name),
                                             ('sale_line_id','=',line.sale_line),
                                             ('state','=','done')],limit=1).copy()
                new_proc.write({'state':'confirmed', 'product_qty':line.qty, 'product_uom_qty':line.qty, 'ordered_qty':line.qty})
                new_procs += new_proc
        # 如果平台有库存，则从平台自有仓库发货：判断是否需从平台发货
        purchase = False
        platform = False
        for line in proc_order_line:
            if line.qty > line.product_id.own_stock:
                purchase = True
                line.is_procure = True
            else:
                platform = True
                line.is_procure = False
        # 如果需从平台库存发货
        if platform:
            # 生成拣货单草稿
            picking = picking_obj.sudo().create({'location_id': self.env.ref('b2b_platform.stock_location_wh_own').id,
                                          'picking_type_id': picking_type.id,
                                          'location_dest_id': cust_loc_id.id,
                                          'ship_address': self.origin.partner_shipping_id.id,
                                          'state': 'draft',
                                          'move_type': 'one',
                                          # 'carrier_id': self.carrier_id.id,
                                          'origin': self.origin.name,
                                          'partner_id': self.origin.partner_id.id,
                                          'owner_id': 1,
                                          'company_id_id': 1,
                                          'min_date': datetime.today(),
                                          })
            for pl_line in proc_order_line:
                if not pl_line.is_procure:
                    move_line = self.env['stock.move'].search([('origin', '=', self.origin.name),
                                                               ('product_id', '=', pl_line.product_id.id),
                                                               ('product_qty', '=', pl_line.qty),
                                                               ('state', '!=', 'done'),
                                                               ], limit=1)
                    if move_line:
                        move_line.sudo().write({'picking_id': picking.id})
        # 如果需从经销商采购
        if purchase:
            # 拆分采购单
            supp_list = proc_order_line.mapped('prod_supplier')
            for supplier in supp_list:
                supp_loc_id = stock_loc_obj.search([('location_id.name', '=', u'供应商库存'),
                                                    ('partner_id', '=', supplier.id)], limit=1)
                if not supp_loc_id:
                    raise UserError(u'未找到供应商所属库位，请咨询平台管理人员')

                # 生成拣货单草稿
                picking = picking_obj.sudo().create({'location_id': supp_loc_id.id,
                                              'picking_type_id': picking_type.id,
                                              'location_dest_id': cust_loc_id.id,
                                              'ship_address': self.origin.partner_shipping_id.id,
                                              'state': 'draft',
                                              'move_type': 'one',
                                              # 'carrier_id': self.carrier_id.id,
                                              'origin': self.origin.name,
                                              'partner_id': self.origin.partner_id.id,
                                              'owner_id': supplier.id,
                                              'company_id_id': 1,
                                              'min_date': datetime.today(),
                                              })
                move_lines = self.env['stock.move'].sudo().search([('origin', '=', self.origin.name),
                                                                    ('product_id.product_owner', '=', supplier.id),
                                                                    ('state', '!=', 'done')])
                if move_lines:
                    move_lines.write({'state': 'draft'})
                    move_lines.write({'picking_id': picking.id})
                    for m_line in move_lines:
                        ord_qty = m_line.procurement_id.product_qty
                        m_line.write({'product_uom_qty': ord_qty,'ordered_qty': ord_qty})

                # 根据库存情况决定供货路线
                # 如果供应商是经销商自己，则不需要生成采购单
                if supplier != self.origin.shop_id.parent_id:
                    supp_order_line = self.order_line.search([('order_id', '=', self.id),
                                                              ('need_procure', '=', True),
                                                              ('is_procure', '=', True),
                                                              ('is_delivery', '!=', True),
                                                              ('prod_supplier', '=', supplier.id)])
                    if supp_order_line:
                        # 创建并确认采购订单
                        amount = 0
                        po_list = []
                        for line in supp_order_line:
                            if line.product_id != freight_product:
                                freight = freight_obj.search([('product_id', '=', line.product_id.product_tmpl_id.id),
                                                              ('country_id', '=', dest_country.id)], limit=1)
                                amount += freight.cny_amount * line.qty
                                po_list.append([0, 0, {'product_id': line.product_id.id,
                                                       'name': line.product_id.name,
                                                       'date_planned': datetime.today(),
                                                       'product_qty': line.qty,
                                                       'product_uom': line.product_id.product_tmpl_id.uom_id.id,
                                                       'price_unit': line.price_unit,
                                                       'taxs_id': False,
                                                       }])
                        if amount:
                            po_list.append([0, 0, {'product_id': freight_product.id,
                                                   'name': freight_product.name,
                                                   'date_planned': datetime.today(),
                                                   'product_qty': 1,
                                                   'product_uom': freight_product.id,
                                                   'price_unit': amount,
                                                   'taxs_id': False,
                                                   }])
                        purchase_id = purchase_obj.create({
                            'partner_id': supplier.id,
                            'currency_id': self.env.user.company_id.currency_id.id,
                            'origin': self.origin.name,
                            'origin_so': self.origin.id,
                            'distributor': self.origin.partner_id.id,
                            'date_order': datetime.today(),
                            'date_planned': datetime.today(),
                            'picking_type_id': self.env['stock.picking.type'].sudo().search([('name', '=', u'直运')],
                                                                                limit=1).id,
                            'dest_address_id': self.origin.partner_shipping_id.id,
                            'order_line': po_list
                        })
                        purchase_id.state = 'purchase'

        # 创建并确认客户发票
        cust_inv_obj = self.env['account.invoice'].sudo()
        cust_inv_line_obj = self.env['account.invoice.line'].sudo()
        cust_inv_id = cust_inv_obj.create({
            'journal_id': self.env['account.journal'].sudo().search([('type', '=', 'sale')],
                                                                    limit=1).id or 1,
            'type': 'out_invoice',
            'partner_id': self.origin.partner_id.id,
            'reference': self.origin.name,
            'origin': self.origin.name,
            'amount_total_company_signed': self.amount_total,
            'residual': self.amount_total,
            'amount_untaxed': self.amount_total,
            'residual_company_signed': self.amount_total,
            'amount_total_signed': self.amount_total,
            'residual_signed': self.amount_total,
            'amount_total': self.amount_total,
            'amount_untaxed_signed': self.amount_total,
            'state': 'draft',
            'account_id': 4,
            'currency_id': 8,
            'name': self.origin.name,
            'commercial_partner_id': self.origin.partner_id.id,
            # 'partner_invoice_id': self.partner_invoice_id.id,
            'partner_shipping_id': self.origin.partner_id.id,
        })
        for line in self.order_line:
            cust_inv_line_obj.create({
                'origin': self.origin.name,
                'price_unit': line.price_unit,
                'price_subtotal': line.amount_subtotal,
                'currency_id': self.env.user.company_id.currency_id.id,
                'uom_id': line.product_id.product_tmpl_id.uom_id.id,
                'partner_id': self.origin.partner_id.id,
                'company_id': 1,
                'account_id': 52,
                'price_subtotal_signed': line.amount_subtotal,
                'name': u'100% 的采购货款',
                'product_id': line.product_id.id,
                'invoice_id': cust_inv_id.id,
                'quantity': line.qty,
                })
            line.write({'invoice_status': 'invoiced'})
        cust_inv_id.action_invoice_open()

        self.origin.write({'qdoo_state': 'part-deliver', 'invoice_status': 'invoiced'})

        # return {'name': u'产品清单',
        #         'type': 'ir.actions.act_window',
        #         'res_model': 'product.product',
        #         'view_type': 'form',
        #         'view_mode': 'tree',
        #         # 'view_id': 'view_tree_b2b_distributor_product_list',
        #         # 'search_view_id': 'search_b2b_distributor_product_list',
        #         'domain': [('my_products', '=', True)],
        #         'context': {'create': False},
        #         }

class B2bSaleResendWizardLine(models.TransientModel):
    _name = 'b2b.sale.resend.wizard.line'

    order_id = fields.Many2one('b2b.sale.resend.wizard', u'订单号')
    product_id = fields.Many2one('product.product', u'产品')
    qty = fields.Float(u'数量')
    price_unit = fields.Float(u'单价')
    amount_subtotal = fields.Float(u'金额', compute='_get_subtotal')
    need_procure = fields.Boolean(u'走平台', default=True)
    is_procure = fields.Boolean(u'走供应商')
    own_product = fields.Boolean(u'自有', compute="_if_own_product")
    prod_supplier = fields.Many2one('res.partner', u'供应商', related='product_id.product_owner', readonly=True)
    sale_line = fields.Integer(u'订单行ID')
    is_delivery = fields.Boolean(u'运费')

    @api.one
    def _get_subtotal(self):
        self.amount_subtotal = self.qty * self.price_unit

    @api.multi
    def _prepare_order_line_procurement(self, group_id=False):
        self.ensure_one()
        return {
            'name': self.product_id.name,
            'origin': self.order_id.origin.name,
            'date_planned': datetime.strptime(self.order_id.origin.date_order, DEFAULT_SERVER_DATETIME_FORMAT),
            'product_id': self.product_id.id,
            'product_qty': self.qty,
            'product_uom': self.product_id.product_tmpl_id.uom_id.id,
            'company_id': self.order_id.origin.company_id.id,
            'group_id': group_id,
            'sale_line_id': self.sale_line,
        }


from typing import Any

class QueueEmpty(IndexError):
    pass

class InvalidFixedLength(ValueError):
    pass

class Queue(object):

    """A queue (FIFO) using a set of fixed-length `arrays`.

    Supports `enqueue`, `dequeue`, and `get_size` operations.
    """

    def __init__(self, fixed_length: int):
        """
        :param fixed_length: Length of internal fixed-length arrays
        """

        try:
            fixed_length = int(fixed_length)
        except ValueError:
            raise InvalidFixedLength(f"Queue could not covert {fixed_length} to valid fixed-length")
        except TypeError:
            raise InvalidFixedLength(f"Queue does not accept {type(fixed_length)}")

        if fixed_length <= 0:
            raise InvalidFixedLength(f"Fixed-length must be greater than 0, got: {fixed_length}")

        self._arrays = []
        self._fixed_length = fixed_length

        # Index to enqueue on
        self._enqueue_idx = fixed_length

        # Index to dequeue from
        self._dequeue_idx = 0

        # Number of elements
        self._count = 0

    @property
    def fixed_length(self):
        """Fixed length used by arrays"""
        return self._fixed_length

    @property
    def n_arrays(self):
        """Number of fixed length arrays in-use"""
        return len(self._arrays)

    def enqueue(self, item: Any):
        """Puts an object onto end of queue"""

        # Allocate new array if needed
        if self._enqueue_idx >= self._fixed_length:
            self._enqueue_idx = 0
            self._arrays.append([None] * self._fixed_length)

        # Enqueue data
        self._arrays[-1][self._enqueue_idx] = item
        self._enqueue_idx += 1
        self._count += 1

    def dequeue(self) -> Any:
        """Pops the object from start of queue"""

        # Size guard
        if self._count <= 0:
            raise QueueEmpty("Cannot dequeue while empty")

        # Remove dequeue array if exhausted
        if self._dequeue_idx >= self._fixed_length:
            self._dequeue_idx = 0
            self._arrays = self._arrays[1:]

        # Get first item
        item = self._arrays[0][self._dequeue_idx]
        self._dequeue_idx += 1
        self._count -= 1

        # Reset original array if empty
        if self._count == 0:
            self._enqueue_idx = 0
            self._dequeue_idx = 0
            self._arrays = self._arrays[:1]

        return item

    def get_size(self) -> int:
        """Returns current size of queue"""
        return self._count


import argparse

from vinfraclient.cmd.base import ShowOne

class ShowSettings(ShowOne):
    _description = 'Show NVMe performance settings.'

    def do_action(self, parsed_args):
        return self.app.vinfra.multiple_cses.settings.show_params()

class ChangeSettings(ShowOne):
    _description = 'Enabling/Disabling NVMe performance.'

    __known_tiers = 0, 1, 2, 3

    @classmethod
    def number_of_cses_per_disk(cls, number):
        try:
            number = int(number)
        except ValueError:
            raise argparse.ArgumentTypeError('1 or more CSes per disk must be specified')

        if number < 1:
            raise argparse.ArgumentTypeError("Minimum number of CSes per disk is 1")

        return number

    def configure_parser(self, parser):
        for tier in self.__known_tiers:
            parser.add_argument(
                '--tier%d' % tier,
                type=self.number_of_cses_per_disk,
                metavar='number',
                default=None,
                help='Set number of CSes per disk for tier %d' % tier,
            )

    def do_action(self, parsed_args):
        kwargs = {}
        for tier in self.__known_tiers:
            number = getattr(parsed_args, 'tier%d' % tier)
            if number is None:
                continue
            kwargs['tier%d' % tier] = number

        return self.app.vinfra.multiple_cses.settings.change_params(**kwargs)


import numpy as np

from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gradients_impl
from tensorflow.python.ops import math_ops
from tensorflow.python.platform import test

_TEST_DTYPES = (
    dtypes.int8,
    dtypes.float32,
    dtypes.float64,
    dtypes.complex64,
    dtypes.complex128,
    dtypes.bfloat16,
    dtypes.float8_e5m2,
    dtypes.float8_e4m3fn,
)

if not test_util.is_xla_enabled():
  # TODO(b/183567451): Support int4 in XLA
  _TEST_DTYPES += (dtypes.int4,)

class SplitOpTest(test.TestCase):

  def _makeData(self, shape, dtype):
    data = np.random.rand(*shape).astype(dtype.as_numpy_dtype)
    if dtype.is_complex:
      data -= 1j * data
    return data

  @test_util.run_deprecated_v1
  def testShapeInference(self):
    model_input = array_ops.placeholder(dtypes.float32, shape=(1, 10))

    # check that we fail during static shape inference if sizes are known
    with self.assertRaises(ValueError):
      # pylint: disable=expression-not-assigned
      array_ops.split(model_input, [4], axis=1)[0]
      # pylint: enable=expression-not-assigned

    model_input = array_ops.placeholder(dtypes.float32)
    inp = np.zeros((1, 10))
    # check that we still fail at runtime if the shapes were unknown
    with self.cached_session() as sess:
      with self.assertRaises(errors_impl.InvalidArgumentError):
        sess.run(array_ops.split(model_input, [4]), {model_input: inp})

    # scalar Tensors are not permitted as num_splits
    for axis in [0, -2]:
      with self.cached_session() as sess:
        with self.assertRaises(ValueError):
          # pylint: disable=expression-not-assigned
          sess.run(
              array_ops.split(
                  array_ops.ones([4, 4]),
                  num_or_size_splits=constant_op.constant(2),
                  axis=axis))
          # pylint: enable=expression-not-assigned

    # test that none split dimensions remain, even if we don't know how
    # the split_dim will be split, but we do know the axis
    result = array_ops.split(
        array_ops.ones([5, 2]), array_ops.constant([2, 1, 2]) * 1, axis=0)

    self.assertEqual(result[0].shape[1], 2)
    self.assertEqual(result[1].shape[1], 2)
    self.assertEqual(result[2].shape[1], 2)

    model_input2 = array_ops.placeholder(dtypes.float32, shape=[None, 2])
    result = array_ops.split(model_input2, [2, 2], axis=0)[0]

    with self.cached_session() as sess:
      sess.run(result, feed_dict={model_input2: np.ones([4, 2])})

  @test_util.run_deprecated_v1
  def testFailWithoutExplicitNum(self):
    size_splits = array_ops.placeholder(dtype=dtypes.int32, shape=[None])

    value = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]

    with self.session() as sess:
      with self.assertRaises(ValueError) as context:
        sess.run(array_ops.split(value, size_splits), {size_splits: [2, 2, 6]})
      self.assertIn("Cannot infer argument `num` from shape",
                    str(context.exception))

  @test_util.run_in_graph_and_eager_modes
  def testExplicitNum(self):
    size_splits = array_ops.constant([2, 2, 6], dtype=dtypes.int32)
    value = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]

    # Eager and Graph modes raise different exceptions
    with self.assertRaises((errors_impl.InvalidArgumentError, ValueError)):
      array_ops.split(value, size_splits, num=4)

    r = self.evaluate(array_ops.split(value, size_splits, num=3))
    self.assertAllEqual(r[0], value[0:2])
    self.assertAllEqual(r[1], value[2:4])
    self.assertAllEqual(r[2], value[4:])

  @test_util.run_in_graph_and_eager_modes
  def testListOfScalarTensors(self):
    a = math_ops.cast(5, dtypes.int32)
    b = math_ops.cast(6, dtypes.int32)

    value = np.random.rand(11, 11)

    with test_util.device(use_gpu=True):
      result = self.evaluate(array_ops.split(value, [a, b]))

    self.assertAllEqual(result[0], value[0:5, :])
    self.assertAllEqual(result[1], value[5:, :])

  def _RunAndVerifyVariable(self, dtype, large_num_splits=False):
    # Random dims of rank 5
    shape = np.random.randint(1, 5, size=5)
    split_dim = np.random.randint(-5, 5)
    if large_num_splits:
      num_split = np.random.randint(16, 25)
    else:
      num_split = np.random.randint(2, 8)
    size_splits = np.random.randint(2, 8, num_split, dtype=np.int32)
    shape[split_dim] = np.sum(size_splits)
    inp = self._makeData(shape, dtype)
    with test_util.device(use_gpu=True):
      result = self.evaluate(array_ops.split(inp, size_splits, split_dim))
    slices = [slice(0, x) for x in shape]
    offset = 0
    for i in range(num_split):
      slices[split_dim] = slice(offset, offset + size_splits[i])
      offset += size_splits[i]
      self.assertAllEqual(result[i], inp[tuple(slices)])

  def _testSpecialCasesVariable(self):
    inp = np.random.rand(4, 4).astype("f")

    with test_util.device(use_gpu=True):
      result = self.evaluate(array_ops.split(inp, [4], 0))
      self.assertAllEqual(result[0], inp)

      result = self.evaluate(array_ops.split(inp, [-1, 3], 0))
      self.assertAllEqual(result[0], inp[0:1, :])
      self.assertAllEqual(result[1], inp[1:4, :])

  def _testHugeNumberOfTensorsVariable(self, dtype):
    num_split = 1000
    size_splits = np.random.randint(1, 3, num_split, dtype=np.int32)
    shape = [3, np.sum(size_splits)]
    split_dim = 1
    inp = self._makeData(shape, dtype)
    with test_util.device(use_gpu=True):
      result = self.evaluate(array_ops.split(inp, size_splits, split_dim))
    slices = [slice(0, x) for x in shape]
    offset = 0
    for i in range(num_split):
      slices[split_dim] = slice(offset, offset + size_splits[i])
      offset += size_splits[i]
      self.assertAllEqual(result[i], inp[tuple(slices)])

  @test_util.run_in_graph_and_eager_modes
  def testSpecialCasesVariable(self):
    self._testSpecialCasesVariable()
    for dtype in _TEST_DTYPES:
      self._testHugeNumberOfTensorsVariable(dtype)

  @test_util.run_in_graph_and_eager_modes
  def testDegenerateVariable(self):
    inp = np.random.rand(4, 4).astype("f")
    with test_util.device(use_gpu=True):
      result = self.evaluate(array_ops.split(inp, [-1, 4], 0))
      self.assertAllEqual(result[0], inp[0:0, :])
      self.assertAllEqual(result[1], inp[0:4, :])

      result = self.evaluate(array_ops.split(inp, [4, -1], 0))
      self.assertAllEqual(result[0], inp[0:4, :])
      self.assertAllEqual(result[1], inp[4:4, :])

      result = self.evaluate(array_ops.split(inp, [-1, 4], 1))
      self.assertAllEqual(result[0], inp[:, 0:0])
      self.assertAllEqual(result[1], inp[:, 0:4])

      result = self.evaluate(array_ops.split(inp, [4, -1], 1))
      self.assertAllEqual(result[0], inp[:, 0:4])
      self.assertAllEqual(result[1], inp[:, 4:4])

  def _testGradientsSimpleVariable(self, dtype):
    inp = self._makeData((4, 4), dtype)
    with test_util.device(use_gpu=True):
      inp_tensor = ops.convert_to_tensor(inp)
      s = array_ops.split(inp_tensor, [1, 3], 1)
      inp_grads = [
          self._makeData((4, 1), dtype), self._makeData((4, 3), dtype)
      ]
      grad_tensors = [constant_op.constant(x) for x in inp_grads]
      grad = gradients_impl.gradients(s, [inp_tensor], grad_tensors)[-1]
      result = self.evaluate(grad)

    self.assertAllEqual(result[:, 0:1], inp_grads[0])
    self.assertAllEqual(result[:, 1:4], inp_grads[1])

  @test_util.run_deprecated_v1
  def testOutputShape(self):
    for axis in [1, -1]:
      with self.cached_session():
        tensor = array_ops.placeholder(dtypes.float32, shape=[None, 12])
        size_splits = [3, 7, 2]
        outputs = array_ops.split(tensor, size_splits, axis)
        for i, output in enumerate(outputs):
          self.assertEqual(output.get_shape().as_list(), [None, size_splits[i]])

  def _compare(self, x, dim, num):
    np_ans = np.split(x, num, dim)
    with test_util.device(use_gpu=True):
      tf_ans = array_ops.split(value=x, num_or_size_splits=num, axis=dim)
      out = self.evaluate(tf_ans)
    self.assertEqual(num, len(np_ans))
    self.assertEqual(num, len(tf_ans))
    self.assertEqual(num, len(out))
    for i in range(num):
      self.assertAllEqual(np_ans[i], out[i])
      self.assertShapeEqual(np_ans[i], tf_ans[i])

  @test_util.run_in_graph_and_eager_modes
  def testSplitRows(self):
    for dtype in _TEST_DTYPES:
      inp = self._makeData((4, 4), dtype)
      self._compare(inp, 0, 4)

  @test_util.run_in_graph_and_eager_modes
  def testSplitCols(self):
    for dtype in _TEST_DTYPES:
      inp = self._makeData((4, 4), dtype)
      self._compare(inp, 1, 4)

  def _testEmpty(self, x, dim, num, expected_shape):
    with test_util.device(use_gpu=True):
      tf_ans = array_ops.split(value=x, num_or_size_splits=num, axis=dim)
      out = self.evaluate(tf_ans)
    self.assertEqual(x.size, 0)
    self.assertEqual(len(out), num)
    for i in range(num):
      self.assertEqual(out[i].shape, expected_shape)
      self.assertEqual(expected_shape, tf_ans[i].get_shape())

  @test_util.run_in_graph_and_eager_modes
  def testEmpty(self):
    # Note: np.split returns a rank-0 empty ndarray
    # if the input ndarray is empty.
    for dtype in _TEST_DTYPES:
      inp = self._makeData((8, 0, 21), dtype)
      self._testEmpty(inp, 0, 2, (4, 0, 21))
      self._testEmpty(inp, 0, 4, (2, 0, 21))
      self._testEmpty(inp, 1, 4, (8, 0, 21))
      self._testEmpty(inp, 2, 3, (8, 0, 7))
      self._testEmpty(inp, 2, 7, (8, 0, 3))

  @test_util.run_in_graph_and_eager_modes
  def testIdentity(self):
    for dtype in _TEST_DTYPES:
      inp = self._makeData((2, 2, 2), dtype)
      self._compare(inp, 0, 1)
      self._compare(inp, 1, 1)
      self._compare(inp, 2, 1)

  @test_util.run_in_graph_and_eager_modes
  def testSplitDim0(self):
    for dtype in _TEST_DTYPES:
      self._compare(self._makeData((6, 10, 18), dtype), 0, 3)
      self._compare(self._makeData((6, 7, 18), dtype), 0, 3)
      self._compare(self._makeData((6, 7, 9), dtype), 0, 3)

  def _RunAndVerify(self, dtype, large_num_splits=False):
    # Random dims of rank 5
    shape = np.random.randint(0, 5, size=5)
    split_dim = np.random.randint(-5, 5)
    if large_num_splits:
      num_split = np.random.randint(9, 15)
    else:
      num_split = np.random.randint(2, 8)
    shape[split_dim] = np.random.randint(2, 5) * num_split
    inp = self._makeData(shape, dtype)
    with test_util.device(use_gpu=True):
      result = self.evaluate(
          array_ops.split(
              value=inp, num_or_size_splits=num_split, axis=split_dim))
    slices = [slice(0, x) for x in shape]
    offset = 0
    length = shape[split_dim] // num_split
    for i in range(num_split):
      slices[split_dim] = slice(offset, offset + length)
      offset += length
      self.assertAllEqual(result[i], inp[tuple(slices)])

  @test_util.run_in_graph_and_eager_modes
  def testRandom(self):
    for dtype in _TEST_DTYPES:
      for _ in range(5):
        self._RunAndVerify(dtype)
        self._RunAndVerify(dtype, large_num_splits=True)
        self._RunAndVerifyVariable(dtype)
        self._RunAndVerifyVariable(dtype, large_num_splits=True)

  def _testGradientsSimple(self, dtype):
    inp = self._makeData((4, 4), dtype)
    with self.cached_session():
      inp_tensor = ops.convert_to_tensor(inp)
      s = array_ops.split(value=inp_tensor, num_or_size_splits=4, axis=1)
      inp_grads = [self._makeData((4, 1), dtype)for _ in range(4)]
      grad_tensors = [constant_op.constant(x) for x in inp_grads]
      grad = gradients_impl.gradients(s, [inp_tensor], grad_tensors)[0]
      result = self.evaluate(grad)
    for i in range(4):
      self.assertAllEqual(result[:, i:i + 1], inp_grads[i])

  @test_util.run_deprecated_v1
  def testGradientsAll(self):
    for dtype in _TEST_DTYPES:
      if not dtype.is_integer and dtype not in [
          dtypes.float8_e5m2,
          dtypes.float8_e4m3fn,
      ]:
        self._testGradientsSimple(dtype)
        self._testGradientsSimpleVariable(dtype)

  @test_util.run_deprecated_v1
  def testShapeFunctionEdgeCases(self):
    # split_dim greater than rank of input.
    with self.assertRaises(ValueError):
      array_ops.split(value=[[0, 1], [2, 3]], num_or_size_splits=4, axis=2)

    # split dim less than -(rank of input)
    with self.assertRaises(ValueError):
      array_ops.split(value=[[0, 1], [2, 3]], num_or_size_splits=4, axis=-3)

    # num_split does not evenly divide the size in split_dim.
    with self.assertRaisesRegex(ValueError, "should evenly divide"):
      array_ops.split(value=[0, 1, 2, 3], num_or_size_splits=3, axis=0)

    # Unknown split_dim.
    splits = array_ops.split(
        value=[[0, 1, 2, 3]],
        num_or_size_splits=4,
        axis=array_ops.placeholder(dtypes.int32))
    for s in splits:
      self.assertEqual([None, None], s.get_shape().as_list())

    # Unknown split_dim and input shape.
    splits = array_ops.split(
        value=array_ops.placeholder(dtypes.float32),
        num_or_size_splits=4,
        axis=array_ops.placeholder(dtypes.int32))
    for s in splits:
      self.assertEqual(None, s.get_shape().ndims)

  @test_util.run_deprecated_v1
  def testVariableShapeFunction(self):
    # size_splits too big
    with self.assertRaises(ValueError):
      array_ops.split([0, 1], [3, -1], axis=0)

    # Correct inference of variable dimension
    s0, s1 = array_ops.split([0, 1, 2], [2, -1], axis=0)
    assert s0.shape.as_list() == [2]
    assert s1.shape.as_list() == [1]

  @test_util.run_deprecated_v1
  def testNonexistentDimTensor(self):
    x = array_ops.placeholder(dtypes.int32)
    values = np.zeros([5, 30])
    splits = array_ops.placeholder(dtypes.int32)
    with self.assertRaisesRegex(ValueError, "Cannot infer"):
      y = array_ops.split(values, splits, axis=x)

    splits = array_ops.placeholder(dtypes.int32, [3])
    y = array_ops.split(values, splits, axis=x)
    with self.session() as sess:
      with self.assertRaisesRegex(errors_impl.InvalidArgumentError,
                                  "must have exactly one element"):
        sess.run(y, {x: np.array([], dtype=np.int32), splits: [4, 11, 15]})

  @test_util.run_in_graph_and_eager_modes
  def testNegativeSizes(self):
    x = constant_op.constant([1, 2, 3], dtypes.float32)
    # A size of -1 signifies to determine size based on sum of other splits.
    with self.assertRaisesRegex((ValueError, errors_impl.InvalidArgumentError),
                                "Split size at index 1 must be >= .*. Got: -2"):
      splits = [-1, -2]
      self.evaluate(array_ops.split(x, splits, axis=0))

  @test_util.run_in_graph_and_eager_modes
  def testBadSplitSizes(self):
    x = constant_op.constant([1, 2], dtypes.float32)
    with self.assertRaisesRegex((ValueError, errors_impl.InvalidArgumentError),
                                "Determined shape must either match input"
                                "|can't split axis"):
      splits = [1, 2]
      self.evaluate(array_ops.split(x, splits, axis=0))

  @test_util.run_in_graph_and_eager_modes
  def testSplitVBigTensors(self):
    input_shape = [1, 64, 32768]
    x = np.linspace(
        start=1,
        stop=np.prod(input_shape),
        num=np.prod(input_shape),
        dtype=np.float32).reshape(input_shape)
    split_axis = 1
    size_splits = [1] * input_shape[split_axis]

    y = array_ops.split(x, num_or_size_splits=size_splits, axis=split_axis)

    for i in range(input_shape[split_axis]):
      result = y[i]
      expected = x[:, i:i + 1, :]
      self.assertAllEqual(result, expected)

  @test_util.run_in_graph_and_eager_modes
  def testSplitVBigTensorsWithIrregularSplits(self):
    input_shape = [1, 64, 32768]
    x = np.linspace(start=1,
                    stop=np.prod(input_shape),
                    num=np.prod(input_shape),
                    dtype=np.float32).reshape(input_shape)
    split_axis = 1
    size_splits = [32, 16, 8, 4, 2, 1, 1]

    y = array_ops.split(x, num_or_size_splits=size_splits, axis=split_axis)

    start = 0
    for i in range(len(size_splits)):
      result = y[i]
      split_size = size_splits[i]
      expected = x[:, start:start+split_size, :]
      start += split_size
      self.assertAllEqual(result, expected)

if __name__ == "__main__":
  test.main()


import asyncio
import datetime
from pathlib import Path

import discord
from discord.ext import commands, tasks

class Locks:
    def __init__(self):
        self.locks = {}
        self.main_lock = asyncio.Lock()

    async def get_lock(self, user_id):
        async with self.main_lock:
            if not user_id in self.locks:
                self.locks[user_id] = asyncio.Lock()

            return self.locks[user_id]

lock_manager = Locks()

class ReactionRolesEvents(commands.Cog):
    def __init__(self, bot):
        self.bot = bot
        self.db = bot.db
        self.base_dir = Path(__file__).resolve().parent

    async def getchannel(self, channel_id):
        channel = self.bot.get_channel(channel_id)

        if not channel:
            channel = await self.bot.fetch_channel(channel_id)

        return channel

    async def getguild(self, guild_id):
        guild = self.bot.get_guild(guild_id)

        if not guild:
            guild = await self.bot.fetch_guild(guild_id)

        return guild

    @commands.Cog.listener()
    async def on_ready(self):
        print("Reaction Light ready!")
        self.cleandb.start()
        self.check_cleanup_queued_guilds.start()

    @tasks.loop(hours=6)
    async def check_cleanup_queued_guilds(self):
        cleanup_guild_ids = self.db.fetch_cleanup_guilds(guild_ids_only=True)
        for guild_id in cleanup_guild_ids:
            try:
                await self.bot.fetch_guild(guild_id)
                self.db.remove_cleanup_guild(guild_id)

            except discord.Forbidden:
                continue

    async def system_notification(self, guild_id, text, embed=None):
        # Send a message to the system channel (if set)
        system_channel1 = await self.getguild(guild_id)
        system_channel = system_channel1.system_channel
        if guild_id:
            server_channel = self.db.fetch_systemchannel(guild_id)

            if isinstance(server_channel, Exception):
                await self.system_notification(
                    None,
                    "Database error when fetching guild system"
                    f" channels:\n```\n{server_channel}\n```\n\n{text}",
                )
                return

            if server_channel:
                server_channel = server_channel[0][0]

            if server_channel:
                try:
                    target_channel = await self.getchannel(server_channel)
                    if embed:
                        await target_channel.send(text, embed=embed)
                    else:
                        await target_channel.send(text)

                except discord.Forbidden:
                    await self.system_notification(None, text)

            else:
                if embed:
                    await self.system_notification(None, text, embed=embed)
                else:
                    await self.system_notification(None, text)

        elif system_channel:
            try:
                target_channel = await self.getchannel(system_channel)
                if embed:
                    await target_channel.send(text, embed=embed)
                else:
                    await target_channel.send(text)

            except discord.NotFound:
                print("I cannot find the system channel.")

            except discord.Forbidden:
                print("I cannot send messages to the system channel.")

        else:
            print(text)

    @tasks.loop(hours=24)
    async def cleandb(self):
        # Cleans the database by deleting rows of reaction role messages that don't exist anymore
        messages = self.db.fetch_all_messages()
        guilds = self.db.fetch_all_guilds()
        # Get the cleanup queued guilds
        cleanup_guild_ids = self.db.fetch_cleanup_guilds(guild_ids_only=True)

        if isinstance(messages, Exception):
            await self.system_notification(
                None,
                "Database error when fetching messages during database"
                f" cleaning:\n```\n{messages}\n```",
            )
            return

        for message in messages:
            try:
                channel_id = message[1]
                channel = await self.bot.fetch_channel(channel_id)

                await channel.fetch_message(message[0])

            except discord.NotFound as e:
                # If unknown channel or unknown message
                if e.code in (10003, 10008):
                    delete = self.db.delete(message[0], message[3])

                    if isinstance(delete, Exception):
                        await self.system_notification(
                            channel.guild.id,
                            "Database error when deleting messages during database"
                            f" cleaning:\n```\n{delete}\n```",
                        )
                        return

                    await self.system_notification(
                        channel.guild.id,
                        "I deleted the database entries of a message that was removed."
                        f"\n\nID: {message} in {channel.mention}",
                    )

            except discord.Forbidden:
                # If we can't fetch the channel due to the bot not being in the guild or permissions we usually cant mention it or get the guilds id using the channels object
                await self.system_notification(
                    message[3],
                    "I do not have access to a message I have created anymore. "
                    "I cannot manage the roles of users reacting to it."
                    f"\n\nID: {message[0]} in channel {message[1]}",
                )

        if isinstance(guilds, Exception):
            await self.system_notification(
                None,
                "Database error when fetching guilds during database"
                f" cleaning:\n```\n{guilds}\n```",
            )
            return

        for guild_id in guilds:
            try:
                await self.bot.fetch_guild(guild_id)
                if guild_id in cleanup_guild_ids:
                    self.db.remove_cleanup_guild(guild_id)

            except discord.Forbidden:
                # If unknown guild
                if guild_id in cleanup_guild_ids:
                    continue
                else:
                    self.db.add_cleanup_guild(
                        guild_id, round(datetime.datetime.utcnow().timestamp())
                    )

        cleanup_guilds = self.db.fetch_cleanup_guilds()

        if isinstance(cleanup_guilds, Exception):
            await self.system_notification(
                None,
                "Database error when fetching cleanup guilds during"
                f" cleaning:\n```\n{cleanup_guilds}\n```",
            )
            return

        current_timestamp = round(datetime.datetime.utcnow().timestamp())
        for guild in cleanup_guilds:
            if int(guild[1]) - current_timestamp <= -86400:
                # The guild has been invalid / unreachable for more than 24 hrs, try one more fetch then give up and purge the guilds database entries
                try:
                    await self.bot.fetch_guild(guild[0])
                    self.db.remove_cleanup_guild(guild[0])
                    continue

                except discord.Forbidden:
                    delete = self.db.remove_guild(guild[0])
                    delete2 = self.db.remove_cleanup_guild(guild[0])
                    if isinstance(delete, Exception):
                        await self.system_notification(
                            None,
                            "Database error when deleting a guilds datebase entries during"
                            f" database cleaning:\n```\n{delete}\n```",
                        )
                        return

                    elif isinstance(delete2, Exception):
                        await self.system_notification(
                            None,
                            "Database error when deleting a guilds datebase entries during"
                            f" database cleaning:\n```\n{delete2}\n```",
                        )
                        return

    @commands.Cog.listener()
    async def on_guild_remove(self, guild):
        self.db.remove_guild(guild.id)

    @commands.Cog.listener()
    async def on_raw_reaction_add(self, payload):
        reaction = str(payload.emoji)
        msg_id = payload.message_id
        ch_id = payload.channel_id
        user_id = payload.user_id
        guild_id = payload.guild_id
        exists = self.db.exists(msg_id)

        async with await lock_manager.get_lock(user_id):
            if isinstance(exists, Exception):
                await self.system_notification(
                    guild_id,
                    f"Database error after a user added a reaction:\n```\n{exists}\n```",
                )

            elif exists:
                # Checks that the message that was reacted to is a reaction-role message managed by the bot
                reactions = self.db.get_reactions(msg_id)

                if isinstance(reactions, Exception):
                    await self.system_notification(
                        guild_id,
                        f"Database error when getting reactions:\n```\n{reactions}\n```",
                    )
                    return

                ch = self.bot.get_channel(ch_id)
                msg = await ch.fetch_message(msg_id)
                user = self.bot.get_user(user_id)
                if reaction not in reactions:
                    # Removes reactions added to the reaction-role message that are not connected to any role
                    await msg.remove_reaction(reaction, user)

                else:
                    # Gives role if it has permissions, else 403 error is raised
                    role_id = reactions[reaction]
                    server = await self.getguild(guild_id)
                    member = server.get_member(user_id)
                    role = discord.utils.get(server.roles, id=role_id)
                    if user_id != self.bot.user.id:
                        unique = self.db.isunique(msg_id)
                        if unique:
                            for existing_reaction in msg.reactions:
                                if str(existing_reaction.emoji) == reaction:
                                    continue
                                async for reaction_user in existing_reaction.users():
                                    if reaction_user.id == user_id:
                                        await msg.remove_reaction(
                                            existing_reaction, user
                                        )
                                        # We can safely break since a user can only have one reaction at once
                                        break

                        try:
                            await member.add_roles(role)
                            notify = self.db.notify(guild_id)
                            if isinstance(notify, Exception):
                                await self.system_notification(
                                    guild_id,
                                    f"Database error when checking if role notifications are turned on:\n```\n{notify}\n```",
                                )
                                return

                            if notify:
                                await user.send(
                                    f"You now have the following role: **{role.name}**"
                                )

                        except discord.Forbidden:
                            await self.system_notification(
                                guild_id,
                                "Someone tried to add a role to themselves but I do not have"
                                " permissions to add it. Ensure that I have a role that is"
                                " hierarchically higher than the role I have to assign, and"
                                " that I have the `Manage Roles` permission.",
                            )

    @commands.Cog.listener()
    async def on_raw_reaction_remove(self, payload):
        reaction = str(payload.emoji)
        msg_id = payload.message_id
        user_id = payload.user_id
        guild_id = payload.guild_id
        exists = self.db.exists(msg_id)

        if isinstance(exists, Exception):
            await self.system_notification(
                guild_id,
                f"Database error after a user removed a reaction:\n```\n{exists}\n```",
            )

        elif exists:
            # Checks that the message that was unreacted to is a reaction-role message managed by the bot
            reactions = self.db.get_reactions(msg_id)

            if isinstance(reactions, Exception):
                await self.system_notification(
                    guild_id,
                    f"Database error when getting reactions:\n```\n{reactions}\n```",
                )

            elif reaction in reactions:
                role_id = reactions[reaction]
                # Removes role if it has permissions, else 403 error is raised
                server = await self.getguild(guild_id)
                member = server.get_member(user_id)

                if not member:
                    member = await server.fetch_member(user_id)

                role = discord.utils.get(server.roles, id=role_id)
                try:
                    await member.remove_roles(role)
                    notify = self.db.notify(guild_id)
                    if isinstance(notify, Exception):
                        await self.system_notification(
                            guild_id,
                            f"Database error when checking if role notifications are turned on:\n```\n{notify}\n```",
                        )
                        return

                    if notify:
                        await member.send(
                            f"You do not have the following role anymore: **{role.name}**"
                        )

                except discord.Forbidden:
                    await self.system_notification(
                        guild_id,
                        "Someone tried to remove a role from themselves but I do not have"
                        " permissions to remove it. Ensure that I have a role that is"
                        " hierarchically higher than the role I have to remove, and that I"
                        " have the `Manage Roles` permission.",
                    )

def setup(bot):
    bot.add_cog(ReactionRolesEvents(bot))


def word_count_distribution(text):
    word_counts = count_words_fast(text)
    count_distribution = dict(Counter(word_counts.values()))
    return count_distribution

distribution = word_count_distribution(text)

def more_frequent(distribution):
    word_counts = count_words_fast(text)
    count_distribution = dict(Counter(word_counts.values()))
    return count_distribution

more_frequent(distribution)

colors = ["crimson", "forestgreen", "blueviolet"]
handles, hamlet_languages = [], []
for index in range(hamlets.shape[0]):
    language, distribution = hamlets.language[index+1], hamlets.distribution[index+1]
    dist = more_frequent(distribution)
    plot, = plt.loglog(sorted(list(dist.keys())),sorted(list(dist.values()),
        reverse = True), color = colors[index], linewidth = 2)
    handles.append(plot)
    hamlet_languages.append(language)
plt.title("Word Frequencies in Hamlet Translations")
xlim    = [0, 2e3]
xlabel  = "Frequency of Word $W$"
ylabel  = "Fraction of Words\nWith Greater Frequency than $W$"
plt.xlim(xlim); plt.xlabel(xlabel); plt.ylabel(ylabel)
plt.legend(handles, hamlet_languages, loc = "upper right", numpoints = 1)
plt.show()


#!/usr/bin/env python3

import sys
import asyncio
import aiohttp
import json

namespaces = {}
broker = "tcp://localhost:1883"

###############################################################################
####################################################################### helpers

def valid_python_version ():
    v = sys.version_info
    if v[0] != 3: return False
    if v[1] < 5: return False
    return True

###############################################################################
#################################################################### processing

async def attach_mqtt_dao (prefix, entity, topic):
    topic = '/building1/'+topic

    MqttLiveData    = await model_ensure_instance(prefix, 'dao:MqttLiveData' , escape('data'+topic))
    hasBroker       = await model_ensure_relationship(MqttLiveData , 'dao:hasBroker', '"%s"' % broker)
    hasTopic        = await model_ensure_relationship(MqttLiveData , 'dao:hasTopic', '"%s"' % topic)
    hasMqttLiveData = await model_ensure_relationship(entity , 'dao:hasMqttLiveData', MqttLiveData)

async def process ():
    # instances
    prefix = 'n:'
    dh   = await model_ensure_instance(prefix, 'gfb:District_Heat' , escape('district_heat'))
    hx   = await model_ensure_instance(prefix, 'gfb:Heat_Exchanger', escape('heat_exchanger'))
    tank      = await model_ensure_instance(prefix, 'gfb:Hot_Water_Tank'          , escape('hot_water_tank'))
    tank_temp = await model_ensure_instance(prefix, 'gfb:Water_Temperature_Sensor', escape('hot_water_tank_temperature'))
    d_ret_flow  = await model_ensure_instance(prefix, 'gfb:Water_Flow_Sensor'                 , escape('district_flow'))
    d_ret_temp  = await model_ensure_instance(prefix, 'gfb:Water_Temperature_Sensor'          , escape('district_return_temp'))
    d_sup_temp  = await model_ensure_instance(prefix, 'gfb:Water_Temperature_Sensor'          , escape('district_supply_temp'))
    d_ret_valve = await model_ensure_instance(prefix, 'gfb:Valve'                             , escape('district_return_valve'))
    d_ret_pres  = await model_ensure_instance(prefix, 'gfb:Water_Pressure_Sensor'             , escape('district_return_pressure'))
    d_sup_pres  = await model_ensure_instance(prefix, 'gfb:Water_Pressure_Sensor'             , escape('district_supply_pressure'))
    d_dif_pres  = await model_ensure_instance(prefix, 'gfb:Water_Differential_Pressure_Sensor', escape('district_differential_pressure'))
    d_heat      = await model_ensure_instance(prefix, 'gfb:Heat_Meter'                        , escape('district_heat_meter'))
    h_sup_temp = await model_ensure_instance(prefix, 'gfb:Water_Temperature_Sensor', escape('heated_supply_temp'))
    h_l1_sup_valve = await model_ensure_instance(prefix, 'gfb:Valve'                   , escape('heated_loop1_supply_valve'))
    h_l1_sup_temp  = await model_ensure_instance(prefix, 'gfb:Water_Temperature_Sensor', escape('heated_loop1_supply_temp'))
    h_l1_sup_pump  = await model_ensure_instance(prefix, 'gfb:Pump'                    , escape('heated_loop1_supply_pump'))
    h_l1_radiator  = await model_ensure_instance(prefix, 'gfb:Radiator'                , escape('heated_loop1_radiator'))
    h_l2_sup_bvalve    = await model_ensure_instance(prefix, 'gfb:Bypass_Valve'            , escape('heated_loop2_supply_bypass_valve'))
    h_l2_sup_temp      = await model_ensure_instance(prefix, 'gfb:Water_Temperature_Sensor', escape('heated_loop2_supply_temp'))
    h_l2_ret_pump      = await model_ensure_instance(prefix, 'gfb:Pump'                    , escape('heated_loop2_return_pump'))
    h_l2_ret_temp_pre  = await model_ensure_instance(prefix, 'gfb:Water_Temperature_Sensor', escape('heated_loop2_return_prebypass_temp'))
    h_l2_ret_temp_post = await model_ensure_instance(prefix, 'gfb:Water_Temperature_Sensor', escape('heated_loop2_return_postbypass_temp'))
    h_l2_radiator      = await model_ensure_instance(prefix, 'gfb:Radiator'                , escape('heated_loop2_radiator'))
    h_l3_ret_valve = await model_ensure_instance(prefix, 'gfb:Valve', escape('heated_loop3_return_valve'))

    # attach data sources
    await attach_mqtt_dao(prefix, tank_temp  , 'tank/temp')
    await attach_mqtt_dao(prefix, d_ret_flow , 'district/return/flow')
    await attach_mqtt_dao(prefix, d_ret_temp , 'district/return/temp')
    await attach_mqtt_dao(prefix, d_sup_temp , 'district/supply/temp')
    await attach_mqtt_dao(prefix, d_ret_valve, 'district/return/valve/position')
    await attach_mqtt_dao(prefix, d_ret_pres , 'district/return/pressure')
    await attach_mqtt_dao(prefix, d_sup_pres , 'district/supply/pressure')
    await attach_mqtt_dao(prefix, d_dif_pres , 'district/differential/pressure')
    await attach_mqtt_dao(prefix, d_heat     , 'district/heat')
    await attach_mqtt_dao(prefix, h_sup_temp, 'heated/supply/temp')
    await attach_mqtt_dao(prefix, h_l1_sup_valve, 'heated/loop1/supply/valve/position') # TODO: should be attached to a sensor on the valve
    await attach_mqtt_dao(prefix, h_l1_sup_temp , 'heated/loop1/supply/temp')
    await attach_mqtt_dao(prefix, h_l1_sup_pump , 'heated/loop1/supply/pump/rpm') # TODO: should be attached to a sensor on the pump
    await attach_mqtt_dao(prefix, h_l2_sup_bvalve   , 'heated/loop2/supply/bypass-valve/position') # TODO: should be attached to a sensor on the bypass valve
    await attach_mqtt_dao(prefix, h_l2_sup_temp     , 'heated/loop2/supply/temp')
    await attach_mqtt_dao(prefix, h_l2_ret_pump     , 'heated/loop2/return/pump/rpm') # TODO: should be attached to a sensor on the pump
    await attach_mqtt_dao(prefix, h_l2_ret_temp_pre , 'heated/loop2/return/temp/pre')
    await attach_mqtt_dao(prefix, h_l2_ret_temp_post, 'heated/loop2/return/temp/post')
    await attach_mqtt_dao(prefix, h_l3_ret_valve, 'heated/loop3/return/valve/position') # TODO: should be attached to a sensor on the valve

    # relations: district heated water
    await model_ensure_relationship(dh         , 'gfb:feedsSupplyDistrictHeatedWater', d_sup_temp)
    await model_ensure_relationship(d_sup_temp , 'gfb:feedsSupplyDistrictHeatedWater', d_sup_pres)
    await model_ensure_relationship(d_sup_pres , 'gfb:feedsSupplyDistrictHeatedWater', hx)
    await model_ensure_relationship(hx         , 'gfb:feedsReturnDistrictHeatedWater', d_ret_pres)
    await model_ensure_relationship(d_ret_pres , 'gfb:feedsReturnDistrictHeatedWater', d_ret_valve)
    await model_ensure_relationship(d_ret_valve, 'gfb:feedsReturnDistrictHeatedWater', d_ret_temp)
    await model_ensure_relationship(d_ret_temp , 'gfb:feedsReturnDistrictHeatedWater', d_ret_flow)
    await model_ensure_relationship(d_sup_temp, 'gfb:controls', d_heat)
    await model_ensure_relationship(d_ret_temp, 'gfb:controls', d_heat)
    await model_ensure_relationship(d_ret_flow, 'gfb:controls', d_heat)
    await model_ensure_relationship(d_sup_pres, 'gfb:controls', d_dif_pres)
    await model_ensure_relationship(d_ret_pres, 'gfb:controls', d_dif_pres)

    # relations: heated
    await model_ensure_relationship(hx, 'gfb:feedsSupplyHeatedWater', h_sup_temp)

    # relations: heated loop 1
    await model_ensure_relationship(h_sup_temp    , 'gfb:feedsSupplyHeatedWater', h_l1_sup_valve)
    await model_ensure_relationship(h_l1_sup_valve, 'gfb:feedsSupplyHeatedWater', h_l1_sup_temp)
    await model_ensure_relationship(h_l1_sup_temp , 'gfb:feedsSupplyHeatedWater', h_l1_sup_pump)
    await model_ensure_relationship(h_l1_sup_pump , 'gfb:feedsSupplyHeatedWater', h_l1_radiator)
    await model_ensure_relationship(h_l1_radiator , 'gfb:feedsReturnHeatedWater', hx)

    # relations: heated loop 2
    await model_ensure_relationship(h_sup_temp        , 'gfb:feedsSupplyHeatedWater', h_l2_sup_bvalve)
    await model_ensure_relationship(h_l2_sup_bvalve   , 'gfb:feedsSupplyHeatedWater', h_l2_sup_temp)
    await model_ensure_relationship(h_l2_sup_temp     , 'gfb:feedsSupplyHeatedWater', h_l2_radiator)
    await model_ensure_relationship(h_l2_radiator     , 'gfb:feedsReturnHeatedWater', h_l2_ret_pump)
    await model_ensure_relationship(h_l2_ret_pump     , 'gfb:feedsReturnHeatedWater', h_l2_ret_temp_pre)
    await model_ensure_relationship(h_l2_ret_temp_pre , 'gfb:feedsReturnHeatedWater', h_l2_sup_bvalve)
    await model_ensure_relationship(h_l2_ret_temp_pre , 'gfb:feedsReturnHeatedWater', h_l2_ret_temp_post)
    await model_ensure_relationship(h_l2_ret_temp_post, 'gfb:feedsReturnHeatedWater', hx)

    # relations: heated loop 3
    await model_ensure_relationship(h_sup_temp    , 'gfb:feedsSupplyHeatedWater', tank)
    await model_ensure_relationship(tank_temp     , 'brick:isPointOf'           , tank)
    await model_ensure_relationship(tank          , 'gfb:feedsReturnHeatedWater', h_l3_ret_valve)
    await model_ensure_relationship(h_l3_ret_valve, 'gfb:feedsReturnHeatedWater', hx)

    # store resulting model
    await rdf_store()

def escape (value: str):
    value = value.replace(' ', '_space_')
    value = value.replace(':', '_colon_')
    value = value.replace('/', '_slash_')
    value = value.replace('æ', '_ae_')
    value = value.replace('ø', '_ao_')
    value = value.replace('å', '_aa_')
    value = value.replace('Æ', '_AE_')
    value = value.replace('Ø', '_AO_')
    value = value.replace('Å', '_AA_')
    return value

###############################################################################
######################################################################### model

async def model_ensure_instance (prefix: str, datatype: str, label: str):
    q = '''
    SELECT ?name
    WHERE {
        ?name rdf:type %s .
        ?name rdf:label "%s" .
    }
    ''' % (datatype, label)
    success, rs = await rdf_query(q)

    if success and len(rs)>0:
        name = rs[0][0]
        if '#' in name:
            parts = name.split('#')
            if len(parts)!=2:
                print('ERROR: Unknown format of entity "%s".' % name)
            parts[0] += '#'
            if parts[0] in namespaces:
                name = '%s:%s' % (namespaces[parts[0]], parts[1])
            else:
                print('ERROR: Namespace not defined for entity "%s".' % name)
    else:
        name = '%s_%s' % (prefix, label)
        await rdf_update_split(insert_clause=['%s rdf:type %s' % (name, datatype),
                                              '%s rdf:label "%s"' % (name, label)])
    return name

async def model_ensure_relationship (sub: str, pred: str, obj: str, onlyobj=False):
    if onlyobj:
        await rdf_update_split(delete_clause=['%s %s ?obj' % (sub, pred)],
                               where_clause=['%s %s ?obj' % (sub, pred)])

    await rdf_update_split(insert_clause=['%s %s %s' % (sub, pred, obj)])

###############################################################################
########################################################################### rdf

async def rdf_namespaces ():
    url = 'http://%s:%u/namespaces' % (host, port)
    async with aiohttp.ClientSession() as session:
        async with session.post(url, data='"dummy"') as response:
            text = await response.text()

            if response.status != 200:
                print('ERROR: Unable to lookup namespaces')
                print(text)
                return False, {}

            try:
                return True, json.loads(text)
            except Exception as e:
                print('ERROR: Exception while trying to parse result of namespace lookup')
                print(text)
                print(str(e))
                return False, {}

async def rdf_store ():
    url = 'http://%s:%u/store' % (host, port)
    async with aiohttp.ClientSession() as session:
        async with session.post(url, data='"dummy"') as response:
            text = await response.text()

            if response.status != 200:
                print('ERROR: Unable to store model')
                print(text)
                return False

            print('NOTICE: Successfully stored model')
            return True

async def rdf_query (query: str):
    query = json.dumps(query, sort_keys=True, indent=4, separators=(',', ': '))
    url = 'http://%s:%u/query' % (host, port)
    async with aiohttp.ClientSession() as session:
        async with session.put(url, data=query) as response:
            text = await response.text()

            if response.status != 200:
                print('ERROR: Unable to query model')
                print(query)
                print(text)
                1/0
                return False, None

            try:
                return True, json.loads(text)['resultset']
            except Exception as e:
                print('ERROR: Exception while trying to parse result model query')
                print(text)
                print(str(e))
                return False, None

async def rdf_update (query: str):
    query = json.dumps(query, sort_keys=True, indent=4, separators=(',', ': '))
    url = 'http://%s:%u/update' % (host, port)
    async with aiohttp.ClientSession() as session:
        async with session.post(url, data=query) as response:
            text = await response.text()

            if response.status != 200:
                print('ERROR: Unable to update model with status "%d"' % response.status)
                print(text)
                1/0
                return False

            try:
                r = json.loads(text)
                if not 'success' in r or type(r['success'])!=bool:
                    print('ERROR: Unable to parse result of model update')
                    print(text)
                    return False

                print('NOTICE: Success of model update: '+str(r['success']))
                return r['success']
            except:
                print('ERROR: Exception while trying to parse result of model update')
                print(text)
                print(str(e))
                return False

async def rdf_update_split (ns : list = None, insert_clause : str = None, delete_clause : str = None, where_clause : str = None):
    if ns == None: ns = []

    q = []

    for entry in ns:
        q.append('%s' % entry)
    q.append('')

    if insert_clause:
        q.append('INSERT {')
        for entry in insert_clause:
            q.append('    %s .' % entry)
        q.append('}')

    if delete_clause:
        q.append('DELETE {')
        for entry in delete_clause:
            q.append('    %s .' % entry)
        q.append('}')

    q.append('WHERE {')
    if where_clause:
        for entry in where_clause:
            q.append('    %s .' % entry)
    q.append('}')

    q = ''.join(map(lambda line: '%s\n' % line, q))
    print(q)

    r = await rdf_update(q)
    print('response:', r)

###############################################################################
########################################################################## main

async def main():
    global namespaces

    # load namespaces
    print('STATUS: Loading namespaces:')
    success, ns = await rdf_namespaces()
    if not success or not 'success' in ns or not ns['success']:
        print('ERROR: Unable to fetch namespaces:')
        print(success)
        print(ns)
        sys.exit(3)
    for key in ns['namespaces']:
        prefix = ns['namespaces'][key]
        print('STATUS: - %s : %s' % (prefix, key))
        namespaces[prefix] = key

    await process()

# guard: python version
if not valid_python_version():
    print('ERROR: Invalid python version (%s), bust be 3.(5+).' % str(sys.version_info))
    sys.exit(1)

# guard: commandline arguments
if len(sys.argv) != 4:
    print('Syntax: %s NAMESPACE RDF_SERVER_HOST RDF_SERVER_PORT' % sys.argv[0])
    print('        %s http://ss.sdu.dk/test/grundstrup-bidi/20200727/# 127.0.0.1 8001' % sys.argv[0])
    sys.exit(2)

# extract parameters
namespace   =     sys.argv[1]
host        =     sys.argv[2]
port        = int(sys.argv[3])

loop = asyncio.get_event_loop()

# enter service loop
try:
    loop.run_until_complete(main())
except KeyboardInterrupt:
    print('')
    print('STATUS: Exiting ...')
    loop.close()
    exit(0)

########################################################################### EOF
###############################################################################


import turtle

shelly = turtle.Turtle()

for i in range(12):
    shelly.color('red')
    shelly.circle(100)
    shelly.left(360/12)
    print(i)

square_length = 400

shelly.penup()
shelly.goto(-(square_length/2), (square_length/2))
shelly.pendown()

shelly.color('green')

square_width = 400

for i in range(4):
    shelly.forward(square_length)
    shelly.left(-90)

# shelly.forward(100)

# shelly.home()

turtle.mainloop()