import argparse
import os

import torch
import torch.backends.cudnn as cudnn
import torch.distributed as dist
import torch.multiprocessing as mp
from timm.data.constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD
from torch import nn
from torchvision import datasets
from torchvision import transforms
from torchvision import transforms as pth_transforms

from config.knn.knn import knn
from module.vits import ViT
from utils import misc

class ImageFolderInstance(datasets.ImageFolder):
    def __getitem__(self, index):
        img, target = super(ImageFolderInstance, self).__getitem__(index)
        return img, target, index

def build_dataset(is_train, args):
    transform = build_transform(args)
    dataset = ImageFolderInstance(
        root=os.path.join(args.data_root, 'train' if is_train else 'val'), transform=transform)
    return dataset

def build_transform(args):
    return transforms.Compose([
        pth_transforms.Resize(int(args.input_size / 224 * 256), interpolation=3),
        pth_transforms.CenterCrop(args.input_size),
        pth_transforms.ToTensor(),
        pth_transforms.Normalize(IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD),
    ])

def get_model_from_frame(checkpoint, args):
    encoder = args.encoder
    state_dict = checkpoint['state_dict']
    encoder = ('module.' if 'module' in list(
        state_dict.keys())[0] else '') + encoder
    for k in list(state_dict.keys()):
        if k.startswith(encoder) and not k.startswith(encoder + '.head'):
            state_dict[k[len(encoder + "."):]] = state_dict[k]
        del state_dict[k]
    return state_dict

def eval_knn(rank, args):
    args.rank = rank
    dist.init_process_group(
        backend="nccl",
        init_method=args.dist_url,
        world_size=args.world_size,
        rank=args.rank,
    )

    misc.fix_random_seeds(args.seed)

    cudnn.benchmark = True

    if args.load_features:
        try:
            print("loading features...")
            train_features = torch.load(os.path.join(
                args.load_features, "trainfeat.pth"))
            test_features = torch.load(os.path.join(
                args.load_features, "testfeat.pth"))
            train_labels = torch.load(os.path.join(
                args.load_features, "trainlabels.pth"))
            test_labels = torch.load(os.path.join(
                args.load_features, "testlabels.pth"))
        except:
            train_features, test_features, train_labels, test_labels = extract_feature_pipeline(
                args)
    else:
        train_features, test_features, train_labels, test_labels = extract_feature_pipeline(
            args)

    if args.rank == 0:
        if args.use_cuda:
            train_features = train_features.cuda()
            test_features = test_features.cuda()
            train_labels = train_labels.cuda()
            test_labels = test_labels.cuda()

        print("Features are ready!\nStart the k-NN classification.")
        for k in args.nb_knn:
            top1, top5 = knn_classifier(train_features, train_labels,
                                        test_features, test_labels, k, args.temperature, args.use_cuda)
            print(f"{k}-NN classifier result: Top1: {top1}, Top5: {top5}")
    dist.barrier()

def extract_feature_pipeline(args):
    dataset_train = build_dataset(is_train=True, args=args)
    dataset_val = build_dataset(is_train=False, args=args)

    sampler = torch.utils.data.DistributedSampler(dataset_train, shuffle=False)

    data_loader_train = torch.utils.data.DataLoader(
        dataset_train,
        sampler=sampler,
        batch_size=args.batch_size_per_gpu,
        num_workers=args.num_workers,
        pin_memory=True,
        drop_last=False,
    )
    data_loader_val = torch.utils.data.DataLoader(
        dataset_val,
        batch_size=args.batch_size_per_gpu,
        num_workers=args.num_workers,
        pin_memory=True,
        drop_last=False,
    )
    print(
        f"Data loaded with {len(dataset_train)} train and {len(dataset_val)} val imgs.")

    if args.arch == 'vit-tiny':
        model = ViT(patch_size=args.patch_size, img_size=args.input_size,
                    embed_dim=192, depth=12, num_heads=3, mlp_ratio=4)
    elif args.arch == 'vit-small':
        model = ViT(patch_size=args.patch_size, img_size=args.input_size,
                    embed_dim=384, depth=12, num_heads=12, mlp_ratio=4)

    elif args.arch == 'vit-base':
        model = ViT(patch_size=args.patch_size, img_size=args.input_size,
                    embed_dim=768, depth=12, num_heads=12, mlp_ratio=4)

    print(f"Model {args.arch} {args.patch_size}x{args.patch_size} built.")
    model.cuda()

    if args.pretrained_weights:
        if os.path.isfile(args.pretrained_weights):
            print("=> loading checkpoint '{}'".format(args.pretrained_weights))
            checkpoint = torch.load(
                args.pretrained_weights, map_location=torch.device(args.rank))

            state_dict = get_model_from_frame(checkpoint, args)

            args.start_epoch = 0
            msg = model.load_state_dict(state_dict, strict=False)
            assert set(msg.missing_keys) == {"head.weight", "head.bias"}

            print("=> loaded pre-trained model '{}'".format(args.pretrained_weights))
        else:
            print("=> no checkpoint found at '{}'".format(
                args.pretrained_weights))

    model.eval()

    print("Extracting features for train set...")
    train_features, train_labels = extract_features(
        model, data_loader_train, args)
    print("Extracting features for val set...")
    test_features, test_labels = extract_features(
        model, data_loader_val, args)

    if args.rank == 0:
        train_features = nn.functional.normalize(train_features, dim=1, p=2)
        test_features = nn.functional.normalize(test_features, dim=1, p=2)

    if args.dump_features and args.get_rank() == 0:
        print("Dumping features ...")
        torch.save(train_features.cpu(), os.path.join(
            args.dump_features, "trainfeat.pth"))
        torch.save(test_features.cpu(), os.path.join(
            args.dump_features, "testfeat.pth"))
        torch.save(train_labels.cpu(), os.path.join(
            args.dump_features, "trainlabels.pth"))
        torch.save(test_labels.cpu(), os.path.join(
            args.dump_features, "testlabels.pth"))
    return train_features, test_features, train_labels, test_labels

@torch.no_grad()
def extract_features(model, data_loader, args, multiscale=False):
    metric_logger = misc.MetricLogger(delimiter="  ")
    features = None
    labels = None
    for samples, labs, index in metric_logger.log_every(data_loader, 10):
        samples = samples.cuda(non_blocking=True)
        labs = labs.cuda(non_blocking=True)
        index = index.cuda(non_blocking=True)

        def forward_single(samples):
            output = model(samples)
            return output

        if multiscale:
            v = None
            for s in [1, 1 / 2 ** (1 / 2), 1 / 2]:
                if s == 1:
                    inp = samples.clone()
                else:
                    inp = nn.functional.interpolate(
                        samples, scale_factor=s, mode='bilinear', align_corners=False)
                feats = forward_single(inp)
                if v is None:
                    v = feats
                else:
                    v += feats
            v /= 3
            v /= v.norm()
            feats = v
        else:
            feats = forward_single(samples)

        if args.rank == 0 and features is None:
            features = torch.zeros(
                len(data_loader.dataset), feats.shape[-1]).to(feats.dtype)
            labels = torch.zeros(len(data_loader.dataset)).to(labs.dtype)
            if args.use_cuda:
                features = features.cuda(non_blocking=True)
                labels = labels.cuda(non_blocking=True)
            print(f"Storing features into tensor of shape {features.shape}")
            print(f"Storing labels into tensor of shape {labels.shape}")

        y_all = torch.empty(args.world_size, index.size(
            0), dtype=index.dtype, device=index.device)
        y_l = list(y_all.unbind(0))
        y_all_reduce = torch.distributed.all_gather(y_l, index, async_op=True)
        y_all_reduce.wait()
        index_all = torch.cat(y_l)

        feats_all = torch.empty(
            args.world_size,
            feats.size(0),
            feats.size(1),
            dtype=feats.dtype,
            device=feats.device,
        )
        output_l = list(feats_all.unbind(0))
        output_all_reduce = torch.distributed.all_gather(
            output_l, feats, async_op=True)
        output_all_reduce.wait()

        labels_all = torch.empty(
            args.world_size,
            labs.size(0),
            dtype=labs.dtype,
            device=labs.device,
        )
        label_l = list(labels_all.unbind(0))
        label_all_reduce = torch.distributed.all_gather(
            label_l, labs, async_op=True)
        label_all_reduce.wait()

        if args.rank == 0:
            if args.use_cuda:
                features.index_copy_(0, index_all, torch.cat(output_l))
                labels.index_copy_(0, index_all, torch.cat(label_l))
            else:
                features.index_copy_(0, index_all.cpu(),
                                     torch.cat(output_l).cpu())
                labels.index_copy_(0, index_all.cpu(),
                                   torch.cat(label_l).cpu())
    return features, labels

@torch.no_grad()
def knn_classifier(train_features, train_labels, test_features, test_labels, k, T, use_cuda=True, num_classes=1000):
    top1, top5, total = 0.0, 0.0, 0
    train_features = train_features.t()
    num_test_images, num_chunks = test_labels.shape[0], 100
    imgs_per_chunk = num_test_images // num_chunks
    retrieval_one_hot = torch.zeros(k, num_classes)
    if use_cuda:
        retrieval_one_hot = retrieval_one_hot.cuda()
    for idx in range(0, num_test_images, imgs_per_chunk):
        features = test_features[
                   idx: min((idx + imgs_per_chunk), num_test_images), :
                   ]
        targets = test_labels[idx: min(
            (idx + imgs_per_chunk), num_test_images)]
        batch_size = targets.shape[0]

        similarity = torch.mm(features, train_features)
        distances, indices = similarity.topk(k, largest=True, sorted=True)
        candidates = train_labels.view(1, -1).expand(batch_size, -1)
        retrieved_neighbors = torch.gather(candidates, 1, indices)

        retrieval_one_hot.resize_(batch_size * k, num_classes).zero_()
        retrieval_one_hot.scatter_(1, retrieved_neighbors.view(-1, 1), 1)
        distances_transform = distances.clone().div_(T).exp_()
        probs = torch.sum(
            torch.mul(
                retrieval_one_hot.view(batch_size, -1, num_classes),
                distances_transform.view(batch_size, -1, 1),
            ),
            1,
        )
        _, predictions = probs.sort(1, True)

        correct = predictions.eq(targets.data.view(-1, 1))
        top1 = top1 + correct.narrow(1, 0, 1).sum().item()
        top5 = top5 + correct.narrow(1, 0, 5).sum().item()
        total += targets.size(0)
    top1 = top1 * 100.0 / total
    top5 = top5 * 100.0 / total
    return top1, top5

def main_ddp(args):
    ngpus_per_node = torch.cuda.device_count()
    args.world_size = args.world_size * ngpus_per_node
    mp.spawn(eval_knn, args=(args,), nprocs=args.world_size)

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--arch", type=str, default='vit-small',
                        choices=['vit-tiny', 'vit-small', 'vit-base'])
    parser.add_argument("--pretrained-weights", type=str,
                        default='')
    return parser

if __name__ == '__main__':
    parser = parse_args()
    _args = parser.parse_args()
    args = knn()
    args.pretrained_weights = _args.pretrained_weights
    args.arch = _args.arch
    main_ddp(args)


# %% [markdown]
# # Process human eval result files
# This script reads the files and merge eval scores into one csv. The result csv
# contains scores for each model and each sample.
# 1. count human-eval scores for every model, including overall score and each metric.
# Save the results to jsonl and csv files. Each row are scores for one example.
# 2. Correlation between style score and other metrics.
# 3. pick some best examples.
# %%
import os
import json
import pandas as pd
import numpy as np
import argparse
import pprint
from matplotlib import pyplot as plt
import seaborn as sns
from typing import Callable, List, Dict

from utils import (
    Evaluator,
    fast_kendalls_tau,
    isnotebook,
    kendalls_tau,
    rouge_1f,
    rouge_lf,
    rouge_score,
    nltk_bleu,
    meteor_score,
    bertscore,
    codebertscore,
    fact_scores,
)

sns.set_theme(context="notebook", font_scale=1.2)
# %%
parser = argparse.ArgumentParser()
parser.add_argument(
    "-d",
    "--data-dir",
    type=str,
    required=False,
    default="/home/v-haotiancui/NL2Code/Copilot-2/dataset/human-eval/results",
    help="Path to the data directory human eval results.",
)
parser.add_argument(
    "-s",
    "--save-dir",
    type=str,
    required=False,
    default="/home/v-haotiancui/NL2Code/Copilot-2/dataset/human-eval/",
    help="Directory to save the processed human eval results, in jsonl and csv format.",
)
parser.add_argument(
    "--internal-dir",
    type=str,
    required=False,
    default="/home/v-haotiancui/NL2Code/Copilot-2/dataset/human-eval/internal",
    help="Directory containing the internal mapping files. Those files have the"
    "testset_ids to mapp human-eval examples to test set",
)
parser.add_argument(
    "--replace",
    action="store_true",
    default=False,
    help="Replace the saved file with newly generated file",
)

# list of models used in make_human_eval_set.py. The order is kept.
models = [
    "reference",
    "ft12_codeT5",
    "ft12_gpt-neo-27",
    "ft12_gpt-neo",
    "ft12_gpt2",
    "ft1_codeT5",
    "ft2_codeT5",
    "codex_Py2Doc",
    "codex_Py2NL",
]

if isnotebook():
    args = parser.parse_args(args=[])
else:
    args = parser.parse_args()

# %%
# make dataframe by metric steps
def _to_gens_df(results: List[Dict]) -> pd.DataFrame:
    """
    Convert a list of results to a dataframe of generated texts of all models.
    """
    gens_df = pd.DataFrame(
        columns=[
            "human_eval_num",
            "testset_id",
            "code",
            "reference",
            "model",
            "gen",
            "s1",
            "s2",
            "s3",
            "s4",
            "s5",
            "avg_s1-4",
            "avg_s1-5",
        ]
    )
    for res in results:
        human_eval_num = res["human_eval_num"]
        testset_id = res["testset_id"]
        code = res["code"]
        reference = res["reference"]["docstring"]
        for m in models:
            if m != "reference":
                gen = res[m]["docstring"]
                s1, s2, s3, s4, s5 = res[m]["step 1-5 scores"]
                avg_s1_4 = res[m]["avg step 1-4 scores"]
                avg_s1_5 = res[m]["avg step 1-5 scores"]
                gens_df.loc[len(gens_df)] = [
                    human_eval_num,
                    testset_id,
                    code,
                    reference,
                    m,
                    gen,
                    s1,
                    s2,
                    s3,
                    s4,
                    s5,
                    avg_s1_4,
                    avg_s1_5,
                ]
    return gens_df

def res2csvrow(res: Dict) -> Dict:
    """
    Convert a single result to a row in csv file.
    """
    row = {}
    row["human_eval_num"] = res["human_eval_num"]
    row["testset_id"] = res["testset_id"]
    row["code"] = res["code"]
    for m in models:
        row[f"{m}_docstring"] = res[m]["docstring"]
        row[f"{m}_s1"] = res[m]["step 1-5 scores"][0]
        row[f"{m}_s2"] = res[m]["step 1-5 scores"][1]
        row[f"{m}_s3"] = res[m]["step 1-5 scores"][2]
        row[f"{m}_s4"] = res[m]["step 1-5 scores"][3]
        row[f"{m}_s5"] = res[m]["step 1-5 scores"][4]
        row[f"{m}_avg_s1-4"] = res[m]["avg step 1-4 scores"]
        row[f"{m}_avg_s1-5"] = res[m]["avg step 1-5 scores"]
    return row

# score stats for one model
def model_stats(results: List[Dict], model_name: str) -> Dict:
    """
    Compute the stats for one model. including mean and std.
    """
    stats = {}
    all_scores = {
        "s1": [],
        "s2": [],
        "s3": [],
        "s4": [],
        "s5": [],
        "avg_s1-4": [],
        "avg_s1-5": [],
    }
    for res in results:
        model_res = res[model_name]
        all_scores["s1"].append(model_res["step 1-5 scores"][0])
        all_scores["s2"].append(model_res["step 1-5 scores"][1])
        all_scores["s3"].append(model_res["step 1-5 scores"][2])
        all_scores["s4"].append(model_res["step 1-5 scores"][3])
        all_scores["s5"].append(model_res["step 1-5 scores"][4])
        all_scores["avg_s1-4"].append(model_res["avg step 1-4 scores"])
        all_scores["avg_s1-5"].append(model_res["avg step 1-5 scores"])
    for key in all_scores.keys():
        stats[key] = {
            "mean": np.mean(all_scores[key]),
            "std": np.std(all_scores[key]),
        }
    print(f"{model_name}:")
    # format dict for printing, float to 3 decimal places
    pprint.pprint(
        {
            k: {"mean": round(v["mean"], 3), "std": round(v["std"], 4)}
            for k, v in stats.items()
        }
    )
    return stats, all_scores

# %% [markdown]
# # Read human eval results
files = os.listdir(args.data_dir)
data = {}
for file in files:
    try:
        data[int(file.split(".")[0])] = pd.read_csv(os.path.join(args.data_dir, file))
        assert "code" in data[int(file.split(".")[0])].columns
    except:
        print(file)

# %%
all_results = []
for k, v in data.items():
    # k is the file number
    example_results = {}
    example_results["human_eval_num"] = int(k)
    _tmp = pd.read_csv(os.path.join(args.internal_dir, f"{k}.csv"))
    example_results["testset_id"] = int(_tmp["testset_id"].values[0])
    example_results["code"] = _tmp["code"].values[0]
    for i, m in enumerate(models):
        example_results[m] = {}
        example_results[m]["docstring"] = v["docstring"].values[i]
        example_results[m]["step 1-5 scores"] = (
            int(v["step 1. explaining"].values[i]),
            int(v["step 2. informative/coverage"].values[i]),
            int(v["step 3. coherence/correctness"].values[i]),
            int(v["step 4. readability/fluency"].values[i]),
            int(v["step 5. format/style"].values[i]),
        )
        example_results[m]["avg step 1-4 scores"] = np.mean(
            example_results[m]["step 1-5 scores"][:4]
        )
        example_results[m]["avg step 1-5 scores"] = np.mean(
            example_results[m]["step 1-5 scores"]
        )
    # make sure the reference is the original
    example_results["reference"]["docstring"] = _tmp[_tmp["model"] == "reference"][
        "docstring"
    ].values[0]

    all_results.append(example_results)
all_results.sort(key=lambda x: x["human_eval_num"])

# %% [markdown]
# ## Save human eval results
output_jsonl = os.path.join(args.save_dir, "human_eval_results.jsonl")
output_csv = os.path.join(args.save_dir, "human_eval_results.csv")
if args.replace or not os.path.exists(output_jsonl):
    with open(output_jsonl, "w") as f:
        for r in all_results:
            f.write(json.dumps(r) + "\n")

all_gens_df = _to_gens_df(all_results)
all_results_df = pd.DataFrame([res2csvrow(res) for res in all_results])
if args.replace or not os.path.exists(output_csv):
    all_results_df.to_csv(output_csv, index="human_eval_num")
# %% [markdown]
# # Results stats
all_stats = {}
all_scores = {}
for m in models:
    all_stats[m], all_scores[m] = model_stats(all_results, m)

# %% [markdown]
# ## Histograms (w/ kde) and violin plots

# make dataframe of avg step1-4 scores for all models
plot_df = pd.DataFrame({m: all_scores[m]["avg_s1-4"] for m in models})

# violinplot of avg step 1-4 scores for all models
fig, ax = plt.subplots(figsize=(10, 10))
sns.violinplot(
    data=plot_df,
    ax=ax,
    bw=0.15,
    cut=0,
)
ax.set_title("Average step 1-4 scores for all models")
ax.set_xlabel("Model")
ax.set_ylabel("Average step 1-4 scores")
ax.set_xticklabels(models)
# plt.savefig(os.path.join(args.save_dir, "avg_step1-4_violin.png"))

# %%
# kdeplot of avg step 1-4 scores for all models
fig, ax = plt.subplots(figsize=(10, 6), dpi=300)
model_kde_kws = {
    "reference": {"linewidth": 2, "alpha": 0.6, "label": "Reference"},
    "ft12_codeT5": {"linewidth": 3, "label": "codeT5-(r+r)"},
    "ft12_gpt-neo-27": {
        "linewidth": 2,
        "linestyle": "--",
        "alpha": 0.3,
        "label": "GPT-Neo27-(r+r)",
    },
    "ft12_gpt-neo": {
        "linewidth": 2,
        "linestyle": "--",
        "alpha": 0.3,
        "label": "GPT-Neo13-(r+r)",
    },
    "ft12_gpt2": {
        "linewidth": 2,
        "linestyle": "--",
        "alpha": 0.3,
        "label": "GPT-2-base-(r+r)",
    },
    "ft1_codeT5": {
        "linewidth": 2,
        "linestyle": "--",
        "alpha": 0.3,
        "label": "CodeT5-(raw)",
    },
    "ft2_codeT5": {"linewidth": 3, "label": "CodeT5-(refined)"},
    "codex_Py2Doc": {"linewidth": 2, "label": "Codex-Py2Doc"},
    "codex_Py2NL": {"linewidth": 2, "label": "Codex-Py2NL"},
}
for m in models:
    sns.kdeplot(
        all_scores[m]["avg_s1-4"],
        ax=ax,
        cut=0,  # clip=(0, 4), fill=True,
        bw_adjust=0.7,
        **model_kde_kws.get(m, {}),
    )
ax.set_xlabel("Overall scores")
# ax.set_xlim(0, 4)
ax.set_ylabel("Density")
ax.set_title("Distribution of overall scores for all human-evaluated models, w/ KDE")
ax.annotate(
    "",
    xy=(0.78, 0.84),
    xycoords="axes fraction",
    xytext=(-42, 18),
    textcoords="offset points",
    size=20,
    arrowprops=dict(arrowstyle="fancy", fc="0.6", ec="none"),
)
ax.legend(loc="upper left")
plt.tight_layout()
plt.savefig(os.path.join(args.save_dir, "avg_step_1-4_scores_kde.png"))
plt.show()

# %% [markdown]
# ## Kendall's Tau between auto metrics and human evaluations
# 4+1 human eval aspects and 6+1 auto metrics, that should generate a 5x6 table
# of scores.

# %% make the dataframe wheree each coloumn is a metric
# update all_results with auto metric scores
auto_metrics: List[Callable] = [
    rouge_1f,
    rouge_lf,
    nltk_bleu,
    meteor_score,
    bertscore,
    codebertscore,
    fact_scores,
]
for sent_metric in auto_metrics:
    Evaluator(sent_metric).eval_df(
        all_gens_df,
        code_col="code",
        ref_col="reference",
        gen_col="gen",
        out_col=sent_metric.__name__,
    )

# %% using code as the reference
for sent_metric in auto_metrics:
    if sent_metric is not fact_scores:
        Evaluator(sent_metric).eval_df(
            all_gens_df,
            code_col="code",
            ref_col="code",
            gen_col="gen",
            out_col=sent_metric.__name__ + "_to_code",
        )

# %% save all gens dataframe to csv
if args.replace or not os.path.exists(os.path.join(args.save_dir, "all_scores.csv")):
    all_gens_df.to_csv(os.path.join(args.save_dir, "all_scores.csv"), index=False)

# %% compute kendalls tau
human_aspect_cols = ["s1", "s2", "s3", "s4", "avg_s1-4"]
auto_metrics_cols = [m.__name__ for m in auto_metrics] + [
    m.__name__ + "_to_code" for m in auto_metrics if m is not fact_scores
]
tau_df = pd.DataFrame(columns=auto_metrics_cols)
info_df = pd.DataFrame(columns=auto_metrics_cols)
for h_col in human_aspect_cols:
    print(f"computing kendall's tau for {h_col} ...")
    tau_row = []
    info_row = []
    for i, a_col in enumerate(auto_metrics_cols):
        print(f"\t{i+1}/{len(auto_metrics_cols)}: {a_col}")
        tau, concordant, discordant, ties = fast_kendalls_tau(
            all_gens_df,
            human_col=h_col,
            metric_col=a_col,
        )
        tau_row.append(tau)
        info_row.append((concordant, discordant, ties))
    tau_df.loc[h_col] = tau_row
    info_df.loc[h_col] = info_row

# %% save kendall's tau dataframe to csv
if args.replace or not os.path.exists(os.path.join(args.save_dir, "kendalls_tau.csv")):
    tau_df.to_csv(os.path.join(args.save_dir, "kendalls_tau.csv"), index=True)

# %% [markdown]
# # Pick best examples

print("best CodeT5-(m) examples:")
all_results_df[all_results_df["ft2_codeT5_avg_s1-4"] == 4.0][
    all_results_df["codex_Py2Doc_avg_s1-4"] < 3.0
]
# %%
print("best CodeT5-(l,m) examples:")
all_results_df[all_results_df["ft12_codeT5_avg_s1-4"] == 4.0][
    all_results_df["codex_Py2Doc_avg_s1-4"] < 3.0
]

# %%


def sequence(n):
    list1 = []
    list = [list1.append(i) for i in range(-n, n+1)]
    return list1

def enterPosition():
    position = input('Введите через пробел номера позиций, значения которых нужно перемножить: ').split()
    positionList = [int(i) for i in position]
    return positionList

def multPosition(position):
    multiples = 1
    for i in range(0, len(positionList)):
        multiples *= list1[positionList[i]]
    return multiples

n = (int(input('Задайте количество элементов : ')))
list1 = sequence(n)
print(list1)
positionList = enterPosition()
print(positionList)
mult = multPosition(positionList)
print('Произведение чисел равно:', mult)


from merchantapi.abstract import Model
from .order_item import OrderItem

class SplitOrderItem(Model):
	def __init__(self, data: dict = None):
		"""
		SplitOrderItem Constructor

		:param data: dict
		"""

		super().__init__(data)
		if self.has_field('original_orderitem'):
			value = self.get_field('original_orderitem')
			if isinstance(value, dict):
				if not isinstance(value, OrderItem):
					self.set_field('original_orderitem', OrderItem(value))
			else:
				raise Exception('Expected OrderItem or a dict')

		if self.has_field('split_orderitem'):
			value = self.get_field('split_orderitem')
			if isinstance(value, dict):
				if not isinstance(value, OrderItem):
					self.set_field('split_orderitem', OrderItem(value))
			else:
				raise Exception('Expected OrderItem or a dict')

	def get_original_order_item(self):
		"""
		Get original_orderitem.

		:returns: OrderItem|None
		"""

		return self.get_field('original_orderitem', None)

	def get_split_order_item(self):
		"""
		Get split_orderitem.

		:returns: OrderItem|None
		"""

		return self.get_field('split_orderitem', None)

	def to_dict(self) -> dict:
		"""
		Reduce the model to a dict.
		"""

		ret = self.copy()

		if 'original_orderitem' in ret and isinstance(ret['original_orderitem'], OrderItem):
			ret['original_orderitem'] = ret['original_orderitem'].to_dict()

		if 'split_orderitem' in ret and isinstance(ret['split_orderitem'], OrderItem):
			ret['split_orderitem'] = ret['split_orderitem'].to_dict()

		return ret


# -*- coding: utf-8 -*-
"""
Created on Mon Apr 22 14:02:19 2019

@author: Mahnoor Ali
"""
import numpy as np
#Input array
X=np.array([[1,1,1],
           [1,-1,1],
           [1,1,-1],
           [1,-1,-1]])

# Output array
y=np.array([1,-1,-1,-1])

#print(y.shape[0])
learnRate=0.1
print("Shape of X is: ",X.shape)

#weight=np.random.uniform(size=(np.array(X.shape[1])))
weight=np.array([0.1,0.1,0.1])
print( "weight is: ",weight)
print("First value of X is: ",X[0])
print("Trasnpose of matrix weight is: ",weight.T.shape,"\n")#giving transpose of this matrix

def activationFunc(x):#Takes some dot product as parameter
    return 1 if x >= 0 else -1

def predict(x):
    net=np.dot(x,weight)#dot product of weights and inputs
    print("Dot product of x and weight is: ",net)
    out = activationFunc(net)
    print ("Predicted Output = ",out)
    return out

for epoch in range(0,3):#3 epochs
    print('----------\nEPOCH ',epoch,'\n----------\n')
    for i in range(len(y)):
        print ("BATCH: ",i)
        print("\nInput X at ",epoch," is: ",X[i])
        print("Actual Output y at ",epoch," is: ",y[i])
        x=X[i]
        tOut=y[i]
        pOut =predict(x)

        error = tOut - pOut
        print("Error: ",error)
        for j in range(len(x)):
            print('\nIteration ',j,':\n')
            if (pOut!=tOut):
                new_weight = weight[j] + learnRate * (error * x[j])
                print('New weight is: ',new_weight)
                print ("--")
            else:
                print ("--Do Nothing--")


"""
FILE STRUCTURE:
1.genuines : genuine signatures of 30 people, 5 sample each.
2.forged : forged signature of the same 30 people as in genuine folder, 5 sample each.

Naming of files : NFI-XXXYYZZZ
explaination    XXX - ID number of a person who has done the signature.
		YY - Image smaple number.
		ZZZ - ID number of person whose signature is in photo.

for example: NFI-00602023 is an image of signature of person number 023 done by person 006.
             This is a forged signature.
	         NFI-02103021 is an image of signature of person number 021 done by person 021.
             This is a genuine signature.

"""

import os
from os import listdir
from os.path import isfile, join
from pathlib import Path
from PIL import Image
import progressbar
import gc

import numpy as np
import pandas as pd

def get_ownername(x):
    # NFI-01005010.png
    x = x[9:12]
    return int(x)

def get_signed_owner(x):
    # NFI-01005010.png
    x = x[4:7]
    return int(x)

def get_sample_number(x):
    # NFI-01005010.png
    x = x[7:9]
    return int(x)

def preprocess(dataset, dtype):
    if dtype == "train":
        dataset["owner_id"] = dataset["file"].apply(get_ownername)
        dataset["genuine_sample_number"] = dataset["file"].apply(get_sample_number)
        dataset["signature_owner_id"] = dataset["file"].apply(get_signed_owner)
        dataset["signed_file"] = "genuine/" + dataset["file"]
    else:
        dataset["owner_id"] = dataset["forged_file"].apply(get_ownername)
        dataset["sample_number"] = dataset["forged_file"].apply(get_sample_number)
        dataset["signature_owner_id"] = dataset["forged_file"].apply(get_signed_owner)
        dataset["signed_file"] = "forged/" + dataset["forged_file"]
    return dataset

gc.collect()

genuine_path = Path('sample_Signature/genuine')
forged_path = Path('sample_Signature/forged')

size = 224

genuine_files = [f for f in listdir(genuine_path) if isfile(join(genuine_path, f))]
forged_files = [f for f in listdir(forged_path) if isfile(join(forged_path, f))]

train_csv = pd.DataFrame(genuine_files, columns =['file'])
test_csv = pd.DataFrame(forged_files, columns =['forged_file'])

train_csv = preprocess(train_csv, "train")
test_csv = preprocess(test_csv, "test")

genuine_owner_id = train_csv["owner_id"].unique()

print(genuine_owner_id)

row_count = len(genuine_owner_id)
#row_count = 1
print(row_count)
bar = progressbar.ProgressBar(maxval=row_count, widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()])
bar.start()

final_data = pd.DataFrame()

for i in range(row_count):
    #id = genuine_owner_id[i]
    genuine_block = train_csv.loc[train_csv['owner_id'] == genuine_owner_id[i]]
    genuine_block_head = genuine_block[["file", "owner_id", "genuine_sample_number"]]
    genuine_block = genuine_block.drop(columns=["file"])

    genuine_block = genuine_block.rename(columns={
    "genuine_sample_number": "sample_number"}, errors="raise")

    forged_block = test_csv.loc[test_csv['owner_id'] == genuine_owner_id[i]]

    #print(forged_block['owner_id'])
    genuine_data = pd.merge(genuine_block_head, genuine_block, on=['owner_id'], how='left').drop_duplicates().reset_index(drop=True)
    genuine_data["status"] = 0
    forged_data = pd.merge(genuine_block_head, forged_block, on=['owner_id'], how='inner').drop_duplicates().reset_index(drop=True)
    forged_data["status"] = 1

    #genuine_data = genuine_data.query('owner_id == signature_owner_id and genuine_sample_number != sample_number')

    #forged_block["signed_file"] = forged_block["forged_file"].values
    #forged_block = pd.merge(genuine_block, forged_block, on=['signed_owner'], how='left').drop_duplicates().reset_index(drop=True)

    #genuine_block["signed_file"] = genuine_block["file"].values
    forged_data = forged_data.drop(columns=["forged_file"])

    final_data = pd.concat([forged_data, genuine_data, final_data], ignore_index=True)
    bar.update(i+1)
    gc.collect()

#train_csv = pd.merge(train_csv, test_csv, on=['signed_owner'], how='inner').drop_duplicates().reset_index(drop=True)
final_data = final_data.sort_values(by=['owner_id', 'genuine_sample_number', 'signature_owner_id','sample_number'])

#final_data = final_data[(final_data.owner_id != final_data.signature_owner_id) and (final_data.genuine_sample_number != final_data.sample_number) ]

#final_data = final_data.query('owner_id == signature_owner_id and genuine_sample_number != sample_number')

bar.finish()
print(final_data.head())
#print(test_csv.head())

final_data.to_csv("sample_Signature/train_final.csv", index=False)

#train_csv.to_csv("sample_Signature/train.csv", index=False)
#test_csv.to_csv("sample_Signature/test.csv", index=False)

gc.collect()


import os
import hashlib
import re
from collections import OrderedDict

from annotation_manager.importer import AnnotationImporterFactory, AnnotationImporter
from annotation_manager.common_representation import AnnotatedDocument, DocumentLibrary, AnnotationSet
from pdfloc_converter.pdfloc import PDFLocPair

class AnnotationStorage(object):
    _annotations_dir = None

    def __init__(self, annotations_dir):
        super(AnnotationStorage, self).__init__()

        self._annotations_dir = annotations_dir

    def get_annotation_file(self, document_file):
        assert isinstance(document_file, PocketbookAnnotatedDocument)

        return "%s%s%s_A_%s.html" % (self._annotations_dir, os.sep, document_file.filename, document_file.file_hash)

    def get_annotation_dir(self, document_file):
        assert isinstance(document_file, PocketbookAnnotatedDocument)

        return "%s%s%s_A_%s.files" % (self._annotations_dir, os.sep, document_file.filename, document_file.file_hash)

class PocketbookDocumentLibrary(DocumentLibrary):

    _system_drive_path = None
    _external_drive_path = None
    _annotations_dir = None

    _annotation_storage = None
    _document_roots = []

    def __init__(self, system_drive_path, annotations_dir, external_drive_path=None):
        super(PocketbookDocumentLibrary, self).__init__()

        self._system_drive_path = system_drive_path
        self._external_drive_path = external_drive_path
        self._annotations_dir = annotations_dir

        self._annotation_storage = AnnotationStorage(self._annotations_dir)

        self._document_roots.append(self._system_drive_path)
        if self._external_drive_path is not None:
            self._document_roots.append(self._external_drive_path)

        self._seek_for_documents()

    def _seek_for_documents(self):
        documents = OrderedDict()

        for root in self._document_roots:
            for dirpath, _, filenames in os.walk(root):
                for filename in filenames:
                    if filename.endswith(".pdf") or filename.endswith(".PDF"):
                        document = PocketbookAnnotatedDocument(filename, dirpath, self._annotation_storage)
                        documents[document.full_path] = document

        self.add_documents(**documents)

class PocketbookAnnotatedDocument(AnnotatedDocument):

    _dir = None
    _annotation_storage = None

    _file_hash = None

    _pdfloc_regex = r'


#%% 12- Desenvolvedor de Tabuada

"""
Desenvolva um gerador de tabuada, capaz de gerar a tabuada de qualquer
número inteiro entre 1 a 10. O usuário deve informar de qual numero ele
deseja ver a tabuada. A saída deve ser conforme o exemplo abaixo:

Tabuada de 5:
5 X 1 = 5
5 X 2 = 10
...
5 X 10 = 50
"""

while True:
    try:
        numero_tabuada = int(input('Insira um numero entre 1 a 10: '))
    except ValueError:
        print('Valor inserido não é um numero inteiro')
    else:
        if 1 <= numero_tabuada <= 10:
            break
        print('Numero não está entre 1 a 10')

for i in range(1,11):
    mult = numero_tabuada*i
    print(f'{numero_tabuada} X {i} = {mult}')


#! /usr/bin/env python3
import importlib
import inspect
import signal
import socket
import traceback
from typing import Dict, List, Tuple

import os
import sys

import dbus
import dbus.mainloop.glib
import dbus.service
import yaml

from PyQt5 import QtWidgets, QtGui, QtCore, QtDBus
from PyQt5.QtWidgets import QTabWidget

from lib.common import AbstractView, set_dashboard_instance, get_dashboard_instance, AbstractPlugin

modules = {}

class DBusHandler(dbus.service.Object):
    def __init__(self, dashboard_instance: "Dashboard", session_bus: dbus.Bus):
        dbus.service.Object.__init__(self, session_bus, "/")

        self.dashboard_instance = dashboard_instance

    @dbus.service.method("com.selfcoders.Dashboard", in_signature="b", out_signature="")
    def fake_screensaver(self, state: bool):
        self.dashboard_instance.screensaver_active_changed(state)

class Dashboard(QtWidgets.QMainWindow, AbstractView):
    instance: "Dashboard" = None
    screensaver_state_changed = QtCore.pyqtSignal(bool)
    window_state_changed = QtCore.pyqtSignal(QtCore.Qt.WindowState)
    screensaver_active = False
    window_active = False

    def __init__(self, config_filepath: str, screens: List[QtGui.QScreen], session_dbus: dbus.Bus):
        super().__init__()

        set_dashboard_instance(self)

        self.overlay_widget = None

        self.session_dbus = session_dbus
        self.tabs: Dict[str, Tuple[QTabWidget, int]] = {}
        self.widget_instances: Dict[str, List[AbstractView]] = {}
        self.plugin_instances: Dict[str, List[AbstractPlugin]] = {}
        self.error_message = QtWidgets.QErrorMessage(self)

        # Instance name to have a unique name if running multiple instances on the same machine or on multiple machines
        self.instance_name = "{}/{}".format(socket.gethostname(), os.getpid())

        self.splash_screen = QtWidgets.QSplashScreen(QtGui.QPixmap(os.path.join(os.path.dirname(os.path.realpath(__file__)), "images", "splash.png")))
        self.splash_screen.setWindowFlag(QtCore.Qt.WindowStaysOnTopHint)
        self.splash_screen.show()
        self.update_splash_screen("Loading...")

        with open(config_filepath, "r") as config_file:
            config = yaml.safe_load(config_file)

            self.show_error_messages = config.get("show_error_messages", True)

            window_options = config.get("window_options", {})

            screen = window_options.get("screen", None)
            if screen is not None:
                screen = screens[screen]

                screen_geometry: QtCore.QRect = screen.geometry()
                self.move(screen_geometry.x(), screen_geometry.y())

            size = window_options.get("size")

            window_flags = QtCore.Qt.Window

            if window_options.get("stay_on_bottom", False):
                window_flags |= QtCore.Qt.WindowStaysOnBottomHint

            if window_options.get("frameless", True):
                window_flags |= QtCore.Qt.FramelessWindowHint
            else:
                window_flags |= QtCore.Qt.WindowMinimizeButtonHint | QtCore.Qt.WindowCloseButtonHint

            if window_options.get("tool", False):
                window_flags |= QtCore.Qt.Tool

            if window_options.get("allow_resize", False):
                window_flags |= QtCore.Qt.WindowMaximizeButtonHint

                if size:
                    self.resize(size[0], size[1])
            else:
                if size:
                    self.setFixedSize(size[0], size[1])

            self.setWindowFlags(window_flags)

            font_options = config.get("font", {})
            font = self.font()

            if "family" in font_options:
                font.setFamily(font_options.get("family"))

            if "size" in font_options:
                font.setPointSize(font_options.get("size"))

            self.setFont(font)

            if window_options.get("maximize", True):
                self.showMaximized()

            self.setCentralWidget(self.create_widget(config["central_widget"]))

            if "overlay_widget" in config:
                self.overlay_widget: QtWidgets.QWidget = self.create_widget(config["overlay_widget"])
                self.overlay_widget.setParent(self)
                self.overlay_widget.size_changed.connect(self.move_overlay_widget)
                self.move_overlay_widget()

            if "plugins" in config:
                for plugin in config["plugins"]:
                    self.init_plugin(plugin)

            widget_instance: AbstractView
            for widget_instances in self.widget_instances.values():
                for widget_instance in widget_instances:
                    if hasattr(widget_instance, "start_view"):
                        widget_instance.start_view()

            if self.overlay_widget and hasattr(self.overlay_widget, "start_view"):
                self.overlay_widget.start_view()

            plugin_instance: AbstractPlugin
            for plugin_instances in self.plugin_instances.values():
                for plugin_instance in plugin_instances:
                    if hasattr(plugin_instance, "start_plugin"):
                        plugin_instance.start_plugin()

        self.size_changed.connect(self.move_overlay_widget)

        self.register_screensaver_events()

        self.splash_screen.hide()

        self.window_active = not bool(self.windowState() & QtCore.Qt.WindowMinimized)
        self.window_state_changed.emit(self.windowState())

        DBusHandler(self, self.session_dbus)

    def move_overlay_widget(self):
        if self.overlay_widget is None:
            return

        size = self.overlay_widget.size()

        x = int(self.width() / 2 - size.width() / 2)
        y = int(self.height() / 2 - size.height() / 2)

        if x < 0:
            x = 0

        if y < 0:
            y = 0

        self.overlay_widget.move(x, y)

    def create_widget(self, config):
        widget_type = config["type"]

        self.update_splash_screen("Creating widget '{}'".format(widget_type))

        module = modules[widget_type]

        options = dict(config)
        del options["type"]

        if "size" in options:
            widget_size = QtCore.QSize(options["size"][0], options["size"][1])
            del options["size"]
        else:
            widget_size = None

        if "fixed_size" in options:
            fixed_widget_size = QtCore.QSize(options["fixed_size"][0], options["fixed_size"][1])
            del options["fixed_size"]
        else:
            fixed_widget_size = None

        parameters = inspect.signature(module.View.__init__).parameters
        if "dashboard_instance" in parameters:
            options["dashboard_instance"] = self

        widget = module.View(**options)

        if widget_size:
            widget.resize(widget_size)

        if fixed_widget_size:
            widget.setFixedSize(fixed_widget_size)

        if widget_type not in self.widget_instances:
            self.widget_instances[widget_type] = []

        self.widget_instances[widget_type].append(widget)

        return widget

    def init_plugin(self, config):
        plugin_type = config["type"]

        self.update_splash_screen("Loading plugin '{}'".format(plugin_type))

        module = modules[plugin_type]

        options = dict(config)
        del options["type"]

        parameters = inspect.signature(module.Plugin.__init__).parameters
        if "dashboard_instance" in parameters:
            options["dashboard_instance"] = self

        plugin = module.Plugin(**options)

        if plugin_type not in self.plugin_instances:
            self.plugin_instances[plugin_type] = []

        self.plugin_instances[plugin_type].append(plugin)

    def tab_by_id(self, tab_id):
        return self.tabs.get(tab_id, (None, 0))

    def get_module(self, name):
        return modules.get(name)

    def get_widget_instance(self, name, index=0):
        instances = self.widget_instances.get(name)

        if instances is None or len(instances) - 1 < index:
            return None

        return instances[index]

    def get_plugin_instance(self, name, index=0):
        instances = self.plugin_instances.get(name)

        if instances is None or len(instances) - 1 < index:
            return None

        return instances[index]

    def show_error(self, message: str):
        if not self.show_error_messages:
            return

        self.error_message.showMessage(message, message)

    def update_splash_screen(self, message: str):
        self.splash_screen.showMessage(message, color=QtCore.Qt.white)
        QtCore.QCoreApplication.processEvents()

    def register_screensaver_events(self):
        session_bus = QtDBus.QDBusConnection.sessionBus()

        session_bus.connect("org.freedesktop.ScreenSaver", "/org/freedesktop/ScreenSaver", "org.freedesktop.ScreenSaver", "ActiveChanged", self.screensaver_active_changed)
        session_bus.connect("org.gnome.ScreenSaver", "/org/gnome/ScreenSaver", "org.gnome.ScreenSaver", "ActiveChanged", self.screensaver_active_changed)
        session_bus.connect("org.cinnamon.ScreenSaver", "/org/cinnamon/ScreenSaver", "org.cinnamon.ScreenSaver", "ActiveChanged", self.screensaver_active_changed)
        session_bus.connect("org.kde.ScreenSaver", "/org/kde/ScreenSaver", "org.kde.ScreenSaver", "ActiveChanged", self.screensaver_active_changed)
        session_bus.connect("org.mate.ScreenSaver", "/org/mate/ScreenSaver", "org.mate.ScreenSaver", "ActiveChanged", self.screensaver_active_changed)
        session_bus.connect("org.xfce.ScreenSaver", "/org/xfce/ScreenSaver", "org.xfce.ScreenSaver", "ActiveChanged", self.screensaver_active_changed)

    @QtCore.pyqtSlot(bool)
    def screensaver_active_changed(self, state):
        self.screensaver_active = state
        self.screensaver_state_changed.emit(state)

    def closeEvent(self, event: QtGui.QCloseEvent):
        QtWidgets.QApplication.quit()

    def changeEvent(self, event):
        if event.type() == QtCore.QEvent.WindowStateChange:
            self.window_active = not bool(self.windowState() & QtCore.Qt.WindowMinimized)
            self.window_state_changed.emit(self.windowState())

def import_modules(config, children_property=None):
    module_type = config["type"]

    if children_property is not None and children_property in config:
        for child_module in config[children_property]:
            import_modules(child_module, children_property)

    if module_type not in modules:
        modules[module_type] = importlib.import_module("modules.{}".format(module_type))

def exception_hook(exception_type, exception_value, exception_traceback):
    traceback_string = "".join(traceback.format_exception(exception_type, exception_value, exception_traceback))
    print(traceback_string, file=sys.stderr)

    dashboard_instance = get_dashboard_instance()
    if dashboard_instance:
        dashboard_instance.show_error(traceback_string)
    else:
        error_message = QtWidgets.QErrorMessage()
        error_message.showMessage(traceback_string, traceback_string)
        error_message.exec()

def main():
    if len(sys.argv) > 1:
        config_filepath = sys.argv[1]
    else:
        config_filepath = "~/.config/dashboard.yml"

    config_filepath = os.path.expanduser(config_filepath)

    with open(config_filepath, "r") as config_file:
        config = yaml.safe_load(config_file)

        pid_file = config.get("pid_file", "~/.cache/dashboard.pid")

        if isinstance(pid_file, str):
            pid_file = os.path.expanduser(pid_file)
        else:
            pid_file = None

        import_paths = []

        for path in config.get("import_paths", []):
            path = os.path.expanduser(path)

            import_paths.append(path)

        sys.path = import_paths + sys.path

        import_modules(config["central_widget"], "widgets")

        if "overlay_widget" in config:
            import_modules(config["overlay_widget"], "widgets")

        if "plugins" in config:
            for plugin in config["plugins"]:
                import_modules(plugin)

    app = QtWidgets.QApplication(sys.argv)
    app.setApplicationName("Dashboard")

    # Prevent closing main window if it is a tool window and a sub window is closed
    app.setQuitOnLastWindowClosed(False)

    sys.excepthook = exception_hook

    try:
        if pid_file is not None:
            if os.path.exists(pid_file):
                try:
                    with open(pid_file, "r") as pid_file_stream:
                        pid = int(pid_file_stream.readline().strip())
                        if pid:
                            os.kill(pid, signal.SIGTERM)
                except:
                    pass

            with open(pid_file, "w") as pid_file_stream:
                pid_file_stream.write(str(app.applicationPid()))

        dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
        session_bus = dbus.SessionBus()

        # Returned value must be stored in a variable even if not used?!
        bus = dbus.service.BusName("com.selfcoders.Dashboard", session_bus)

        dashboard = Dashboard(config_filepath, app.screens(), session_bus)
        dashboard.show()

        exit_code = app.exec()

        for plugin_type, plugin_instances in dashboard.plugin_instances.items():
            for plugin_instance in plugin_instances:
                plugin_instance.stop_plugin()
    finally:
        if pid_file is not None and os.path.exists(pid_file):
            os.remove(pid_file)

    sys.exit(exit_code)

if __name__ == "__main__":
    main()


# Standard library imports
import pathlib
import sys
import re
import math
import numpy as np
from numpy.linalg import matrix_power

def parse(puzzle_input):
    """Parse input"""
    return [[parse_line(line) for line in section.splitlines()[1:]] for section in puzzle_input.split("\n\n")]

def parse_line(line):
    return np.array([int(i) for i in line.split(",")])

def rotate_vector_in_all_directions(vector):
    all_directions = dict()
    for xy in range(0, 4):
        for zx in range(0, 2):
            for zxy in range(0, 3):
                all_directions[(xy, zx, zxy)] = rotate_vector(vector, xy, zx, zxy)
    return all_directions

# each direction at most  4   2   3
def rotate_vector(vector, xy, zx, zxy):
    xy_mat = np.array([[0, 1, 0],
                       [-1, 0, 0],
                       [0, 0, 1]])
    zx_mat = np.array([[-1, 0, 0],
                       [0, 1, 0],
                       [0, 0, -1]])
    zxy_mat = np.array([[0, 1, 0],
                        [0, 0, 1],
                        [1, 0, 0]])

    return np.matmul(np.matmul(matrix_power(zxy_mat, zxy), matrix_power(zx_mat, zx)),
                     matrix_power(xy_mat, xy)).dot(vector)

def invert_rotate_vector(vector, xy, zx, zxy):
    xy_mat = np.array([[0, -1, 0],
                       [1, 0, 0],
                       [0, 0, 1]])
    zx_mat = np.array([[-1, 0, 0],
                       [0, 1, 0],
                       [0, 0, -1]])
    zxy_mat = np.array([[0, 0, 1],
                        [1, 0, 0],
                        [0, 1, 0]])

    return np.matmul(np.matmul(matrix_power(xy_mat, xy), matrix_power(zx_mat, zx)),
                     matrix_power(zxy_mat, zxy)).dot(vector)

def part1(data):
    """Solve part 1"""

    vector = np.array([1, 2, 3])
    for xy in range(0, 4):
        for zx in range(0, 2):
            for zxy in range(0, 3):
                new_vector = invert_rotate_vector(rotate_vector(vector, xy, zx, zxy), xy, zx, zxy)
                print(xy, zx, zxy, new_vector)

    differences_per_scanner_per_orientation = calc_differences_per_scanner_per_orientation(data)

    scanner_to_scanner = dict()
    scanner_to_scanner_inverse = dict()

    for scanner in range(0, len(data)):
        for other_scanner in range(scanner + 1, len(data)):
            scanner_set = differences_per_scanner_per_orientation[scanner][(0, 0, 0)]
            for key in differences_per_scanner_per_orientation[other_scanner].keys():
                other_scanner_set = differences_per_scanner_per_orientation[other_scanner][key]
                number_of_differences = len(scanner_set.intersection(other_scanner_set))
                if number_of_differences >= 132:
                    print(scanner, other_scanner, key, number_of_differences)
                    if scanner not in scanner_to_scanner:
                        connections = list()
                        connections.append((other_scanner, key))
                        scanner_to_scanner[scanner] = connections
                    else:
                        scanner_to_scanner[scanner].append((other_scanner, key))

                    if other_scanner not in scanner_to_scanner_inverse:
                        connections = list()
                        connections.append((scanner, key))
                        scanner_to_scanner_inverse[other_scanner] = connections
                    else:
                        scanner_to_scanner_inverse[other_scanner].append((scanner, key))

    indexed_scanners = set()
    indexed_scanners.add(0)

    beacons, scanners = index_connected_scanners(0, indexed_scanners, scanner_to_scanner, scanner_to_scanner_inverse, data)
    print("part 1:", len(beacons))

    max_distance = 0
    for scanner in scanners:
        for scanner1 in scanners:
            distance = abs(scanner[0] - scanner1[0]) + abs(scanner[1] - scanner1[1]) + abs(scanner[2] - scanner1[2])
            if distance > max_distance:
                max_distance = distance
                # print(max_distance, scanner, scanner1)
    print("part 2:", max_distance)
    return 0

def index_connected_scanners(scanner, indexed_scanners, scanner_to_scanner,
                             scanner_to_scanner_inverse, data):
    beacons = set()
    scanners = set()

    for beacon in data[scanner]:
        beacons.add(tuple(beacon))
    scanners.add((0, 0, 0))

    if scanner in scanner_to_scanner:
        for other_scanner, rotate in scanner_to_scanner[scanner]:
            if other_scanner not in indexed_scanners:
                print(scanner, other_scanner)
                indexed_scanners.add(other_scanner)

                normalized_rotation = [rotate_vector(beacon, rotate[0], rotate[1], rotate[2]) for beacon in
                                       data[other_scanner]]
                difference = find_difference(data[scanner], normalized_rotation)

                moved_rotated_tuple_beacons, moved_rotated_tuple_scanners = index_connected_scanners(other_scanner,
                                                                                indexed_scanners,
                                                                                scanner_to_scanner,
                                                                                scanner_to_scanner_inverse,
                                                                                data)
                other_beacons = normalize_other_points(moved_rotated_tuple_beacons, rotate, difference, False)
                other_scanners = normalize_other_points(moved_rotated_tuple_scanners, rotate, difference, False)

                for beacon in other_beacons:
                    beacons.add(tuple(beacon))
                for other_scanner_in_list in other_scanners:
                    scanners.add(tuple(other_scanner_in_list))
                print(len(data[scanner]), len(other_beacons), len(beacons))

    if scanner in scanner_to_scanner_inverse:
        for other_scanner, rotate in scanner_to_scanner_inverse[scanner]:
            if other_scanner not in indexed_scanners:
                print(scanner, other_scanner)
                indexed_scanners.add(other_scanner)

                normalized_rotation = [invert_rotate_vector(beacon, rotate[0], rotate[1], rotate[2]) for beacon in
                                       data[other_scanner]]
                difference = find_difference(data[scanner], normalized_rotation)

                moved_rotated_tuple_beacons, moved_rotated_tuple_scanners = index_connected_scanners(other_scanner,
                                                                                indexed_scanners,
                                                                                scanner_to_scanner,
                                                                                scanner_to_scanner_inverse,
                                                                                data)

                other_beacons = normalize_other_points(moved_rotated_tuple_beacons, rotate, difference, True)
                other_scanners = normalize_other_points(moved_rotated_tuple_scanners, rotate, difference, True)

                for beacon in other_beacons:
                    beacons.add(tuple(beacon))
                for other_scanner_in_list in other_scanners:
                    scanners.add(tuple(other_scanner_in_list))
                print(len(data[scanner]), len(other_beacons), len(beacons))
    return beacons, scanners

def normalize_other_points(moved_rotated_tuple_points, rotate, difference, inverted):
    moved_rotated_points = [np.array(tuple_beacon) for tuple_beacon in moved_rotated_tuple_points]

    if inverted:
        moved_points = [invert_rotate_vector(beacon, rotate[0], rotate[1], rotate[2]) for beacon in
                        moved_rotated_points]
    else:
        moved_points = [rotate_vector(beacon, rotate[0], rotate[1], rotate[2]) for beacon in
                        moved_rotated_points]

    other_points = [np.add(vector, difference) for vector in moved_points]

    return other_points

def find_difference(beacons_scanner, beacons_other_scanner):
    for beacon1 in beacons_scanner:
        for beacon2 in beacons_scanner:
            if not np.array_equal(beacon1, beacon2):
                difference_1_and_2 = np.subtract(beacon1, beacon2)

                for other_beacon1 in beacons_other_scanner:
                    for other_beacon2 in beacons_other_scanner:
                        if not np.array_equal(other_beacon1, other_beacon2):

                            if np.array_equal(np.subtract(other_beacon1, other_beacon2), difference_1_and_2):
                                return np.subtract(beacon1, other_beacon1)
    print("stuff is not good")

def calc_differences_per_scanner_per_orientation(data):
    differences_per_scanner_per_orientation = dict()
    for scanner in range(0, len(data)):
        scanner_data = data[scanner]
        differences_per_orientation = dict()
        for xy in range(0, 4):
            for zx in range(0, 2):
                for zxy in range(0, 3):
                    rotated_scanner_data = [rotate_vector(vector, xy, zx, zxy) for vector in scanner_data]
                    differences = set()
                    for i in range(0, len(scanner_data)):
                        for j in range(0, len(scanner_data)):
                            if i != j:
                                difference = np.subtract(rotated_scanner_data[i], rotated_scanner_data[j])
                                if tuple(difference) in differences:
                                    print("doubles")
                                differences.add(tuple(difference))
                    differences_per_orientation[(xy, zx, zxy)] = differences
        differences_per_scanner_per_orientation[scanner] = differences_per_orientation
    return differences_per_scanner_per_orientation

def attempt_one(data):
    differences_per_scanner = list()
    for scanner in range(0, len(data)):
        scanner_data = data[scanner]
        differences = set()
        for i in range(0, len(scanner_data)):
            for j in range(0, len(scanner_data)):
                if i != j:
                    difference = np.subtract(scanner_data[i], scanner_data[j])
                    differences.add(tuple(difference))
        differences_per_scanner.append(differences)

    for scanner in range(0, len(data)):
        for other_scanner in range(scanner + 1, len(data)):
            number_of_differences = len(
                differences_per_scanner[scanner].intersection(differences_per_scanner[other_scanner]))
            if number_of_differences >= 132:
                print(scanner, other_scanner, number_of_differences)

def part2(data):
    """Solve part 2"""

def solve(path):
    """Solve the puzzle for the given input"""
    puzzle_input = pathlib.Path(path).read_text().strip()
    data = parse(puzzle_input)
    solution1 = part1(data)

    puzzle_input = pathlib.Path(path).read_text().strip()
    data = parse(puzzle_input)
    solution2 = part2(data)

    return solution1, solution2

if __name__ == "__main__":
    for path in sys.argv[1:]:
        print(f"{path}:")
        solutions = solve(path)
        print("\n".join(str(solution) for solution in solutions))


'''
https://leetcode.com/problems/strobogrammatic-number-ii/
A strobogrammatic number is a number that looks the same when rotated 180 degrees (looked at upside down).
Find all strobogrammatic numbers that are of length = n.

Example:
Input:  n = 2
Output: ["11","69","88","96"]
'''
'''
Time:O(n)
Space:O(n)
'''
class Solution:
    def findStrobogrammatic(self, n: int) -> List[str]:
        odd = ['0', '1', '8']
        even = ['11','69','88','96','00']
        def helper(n):
            if n==1:
                return odd
            if n==2:
                return even
            else:
                res = []
                for base in helper(n-2):
                    res.append('6' + base + '9')
                    res.append('9' + base + '6')
                    res.append('1' + base + '1')
                    res.append('8' + base + '8')
                    res.append('0' + base + '0')
                return res

        if n == 1:
            return odd
        return [num for num in helper(n) if num[0]!='0' and num[-1]!='0']


import json
import operator
import re

import sqlparse
from datamanage import pizza_settings as settings
from datamanage.pro import exceptions as dm_pro_errors
from datamanage.pro.datamodel.models.model_dict import InnerField, SchedulingEngineType
from datamanage.pro.datamodel.utils import (
    get_sql_fields_alias_mapping,
    get_sql_source_columns_mapping,
)
from datamanage.utils.api import DataflowApi
from django.core.cache import cache

from common.log import logger

from . import time

python_string_type = str

class SQLVerifier(object):

    sql_template = """SELECT {fields} FROM {table_name} {conditions} {appends};"""
    default_map = dict(table_name="table_name", fields="*", conditions="", appends="")
    cast_type_mapping = dict(
        int="int",
        float="float",
        double="double",
        string={
            SchedulingEngineType.FLINK.value: "varchar",
            SchedulingEngineType.SPARK.value: "string",
        },
    )
    requests_clients = {
        SchedulingEngineType.FLINK.value: DataflowApi.flink_sql_check,
        SchedulingEngineType.SPARK.value: DataflowApi.spark_sql_check,
    }

    def __init__(self, bk_biz_id=1, sql=None, sql_parts=None, scopes=None, auto_format=True):
        """
        初始化sql校验模块，传入sql语句或局部sql以及必要参数
        SELECT [fields] FROM [table] WHERE [conditions] [appends]

        :param bk_biz_id: int 业务id
        :param sql: string 完整sql语句 "SELECT * FROM table WHERE 1"
        :param sql_parts: dict 局部sql，传入后需要对sql进行补全(sql参数若不为空，优先于sql_parts)
            {
                "fields": {"field_alias1": "field_name", "field_alias2":"concat(field_name, '_', 'ex')"},
                # fields中，key是输出的字段名称，value是输入表达式(或输入字段名称)
                "conditions": "a >= 1 and b < 2",
                "appends": {"group_by": ["b", "c"]} # OR "GROUP BY b, c"
                "table_name": "591_durant1119"
            }
        :param scopes: dict sql执行可引用的表范围
            {"591_durant1119": [
                    {
                        "field_type": "string",
                        "field_name": "a"
                    }
                ]
            }
        }
        :param auto_format: boolean 是否自动格式化sql内容
        """
        self.bk_biz_id = bk_biz_id
        self.auto_format = auto_format
        self.sql = sql
        self.sql_parts = sql_parts if sql_parts is not None else {}
        self.scopes = dict()
        self.source_table_mapping = dict()
        self._set_scope_table_name(scopes)
        self.sql_alias_exp_mapping = dict()
        self._set_sql_alias_exp_mapping()
        self.logger = logger
        self.err_trace = dict()

    @staticmethod
    def _pre_deal_scope_list(scope_field_list):
        """
        预处理scope字段列表
        1.去除无效字段
        2.添加默认字段

        :param scope_field_list: list
        :return: list
        """
        processed_scope_field_list = list()
        if isinstance(scope_field_list, list):
            processed_scope_field_list = [
                field_item
                for field_item in scope_field_list
                if isinstance(field_item, dict)
                and field_item.get("field_type", "")
                and field_item.get("field_name", "")
            ]
        processed_scope_field_list.append(dict(field_type="string", field_name="verify__bk_data_sql__default"))
        return processed_scope_field_list

    def _set_scope_table_name(self, scopes):
        """
        对scope表的表名称进行预处理

        :param scopes: dict sql执行可引用的表范围
        :return: None
        """

        raw_scopes = scopes if scopes is not None else {}
        # Todo: 因防止缓存竞争冲突，主表名称会增加随机后缀，所以需要对scope中的表名进行替换，当期因需要只替换了主表表名, 完备方案有两套:
        #   1.首选, 推动flink去掉缓存中转的方案，直接支持参数传递schema, 即可取消随机后缀
        #   2.对多输入scope情况中主表外其他所有表进行随机后缀替换，且需要替换sql中对scope表的引用表名, 较复杂，暂未实现
        if self.sql is None and self.sql_parts:
            raw_main_table_name = self.sql_parts.get("table_name", "")
            self.sql_parts["table_name"] = self._gen_table_name(self.bk_biz_id, raw_main_table_name)
            for scope_table_name, scope_table_info in list(raw_scopes.items()):
                tb_name = self.sql_parts["table_name"] if scope_table_name == raw_main_table_name else scope_table_name
                scope_table_info = self._pre_deal_scope_list(scope_table_info)
                self.scopes[tb_name] = scope_table_info
                self.source_table_mapping[tb_name] = scope_table_name
        else:
            self.scopes = raw_scopes

    def _set_sql_alias_exp_mapping(self):
        """
        从输入中获取字段别名和表达式的关系

        :return: None
        """

        fields_part = self.sql_parts.get(str("fields"), {})
        for field_alias, field_exp in list(fields_part.items()):
            if field_alias and field_exp:
                self.sql_alias_exp_mapping[field_alias] = field_exp

    @staticmethod
    def get_func_list(expr, prefix=None):
        """
        从表达式中捕获使用的函数

        :param expr: string 表达式
        :param prefix: string 指定函数名的前缀
        :return: list 捕获函数列表
        """
        func_list = list(set(re.findall(r"([\S]+?)\(", expr)))
        if prefix and isinstance(prefix, str):
            prefix = prefix.lower()
            return list({func_name for func_name in func_list if func_name.lower().startswith(prefix)})
        return func_list

    @classmethod
    def gen_simple_sql(cls, table_name=None, fields=None, conditions=None, appends=None):
        """
        生成一个基础sql

        :param table_name: string 表名
        :param fields: string 字段表达式
        :param conditions: string 条件表达式(包含where)
        :param appends: string 聚合表达式
        :return: string sql表达式
        """
        raw_sql = cls.sql_template.format(
            table_name=table_name if table_name else cls.default_map["table_name"],
            fields=fields if fields else cls.default_map["fields"],
            conditions=conditions if conditions else cls.default_map["conditions"],
            appends=appends if appends else cls.default_map["appends"],
        )
        return sqlparse.format(raw_sql, strip_comments=True, strip_whitespace=True).strip()

    def gen_sql_expression(self, require_schema, engine):
        """
        根据局部sql和参数构建校验用的sql表达式

        :param require_schema: dict 希望sql返回的schema
        :param engine: string 执行sql的引擎
        :return: 返回str的sql语句
        """

        require_schema_fields = list(require_schema.values())[0]
        field_type_mapping = {
            str(field_item["field_name"]): str(field_item["field_type"]) for field_item in require_schema_fields
        }
        raw_sql = self.sql_template.format(
            table_name=self.sql_parts["table_name"],
            fields=self._gen_fields_exp(field_type_mapping, engine),
            conditions=self._gen_conditions_exp(),
            appends=self._gen_appends_exp(),
        )
        self.sql = sqlparse.format(raw_sql, strip_comments=self.auto_format, strip_whitespace=self.auto_format).strip()
        return self.sql

    @staticmethod
    def _gen_table_name(prefix, table_name):
        """
        生成表名

        :param prefix: 业务id前缀(兼容flink接口)
        :param table_name: 表名
        :return: string 字符串表名
        """
        # 默认带上时间戳后缀，避免缓存竞争
        table_name = "{}_{}".format(table_name, int(round(time.time() * 1000)))
        return "{}_{}".format(prefix, table_name) if not table_name.startswith("{}_".format(prefix)) else table_name

    def _gen_fields_exp(self, field_type_mapping, engine):
        """
        SQL片段 fields部分
        {"key1": "field_1", "key2": "sum(a) as field_2"}
        变量解释: field_name: 模型要求的输出字段 field_alias: 从表达式解析出来的别名 field_expr: sql字段表达式

        :param field_type_mapping: dict 字段类型映射字典
        :param engine: string 执行sql的引擎
        :return: string 字段表达式
        """

        field_exp_list = []
        fields_alias_mapping = get_sql_fields_alias_mapping(
            self.sql_alias_exp_mapping,
            strip_comments=self.auto_format,
            strip_whitespace=self.auto_format,
        )
        for field_name, field_alias_item in list(fields_alias_mapping.items()):
            field_alias = field_alias_item["alias"]
            field_expr = field_alias_item["expr"]
            target_field_type = self.get_target_parse_type(field_type_mapping[field_name], engine)
            # 表达式和模型字段名一致(没做处理)
            if field_expr == field_name:
                field_exp_list.append("cast(`{}` as {}) as `{}`".format(field_expr, target_field_type, field_name))
            elif field_alias:
                # 表达式中设置了指定别名
                if field_alias == field_name:
                    field_exp_list.append("cast({} as {}) as `{}`".format(field_expr, target_field_type, field_name))
                # 表达式中没有设置指定别名
                else:
                    raise dm_pro_errors.AliasSetError(message_kv={"alias": field_name, "err_alias": field_alias})
            else:
                # 表达式没有设置别名，自动补充
                field_exp_list.append("cast({} as {}) as `{}`".format(field_expr, target_field_type, field_name))
        return ", ".join(field_exp_list)

    def _gen_conditions_exp(self):
        """
        SQL片段 conditions部分
        "a >= 1 and b < 2" OR dict(暂未实现)

        :return: string 条件表达式
        """

        conditions_part = self.sql_parts.get(str("conditions"), None)
        if conditions_part and isinstance(conditions_part, dict):
            # TODO:暂未实现字典类型传入
            return ""
        conditions_expr = format(conditions_part) if conditions_part else self.default_map["conditions"]
        return (
            "WHERE {}".format(conditions_expr)
            if conditions_expr and not conditions_expr.strip().lower().startswith("where")
            else conditions_expr
        )

    def _gen_appends_exp(self):
        """
        SQL片段 appends部分

        {"group_by": ["b", "c"]} # OR "GROUP BY b, c"
        :return: string 聚合表达式
        """

        appends_part = self.sql_parts.get(str("appends"), None)
        if appends_part and isinstance(appends_part, dict):
            appends_exp_template = "{group_by_exp}"
            group_by_exp_str = ""
            group_by_exp_list = appends_part.get("group_by", [])
            if group_by_exp_list:
                group_by_exp_str = "GROUP BY {}".format(", ".join(group_by_exp_list))
            return appends_exp_template.format(group_by_exp=group_by_exp_str)
        return appends_part if appends_part else self.default_map["appends"]

    @classmethod
    def get_target_parse_type(cls, require_type, engine):
        """
        根据类型和engine获取实际cast的目标类型

        :param require_type: 用户需要的通用字段类型
        :param engine: 执行引擎
        :return: 实际有效的执行字段类型
        """

        require_type = require_type if require_type in cls.cast_type_mapping else "string"
        target_type_item = cls.cast_type_mapping.get(require_type)
        return str(target_type_item[engine]) if isinstance(target_type_item, dict) else str(target_type_item)

    def _get_runner_properties(self, engine, udf_list, ret_table_name_list, agg):
        """
        根据校验引擎构造参数

        :param engine: string 校验引擎
        :param udf_list: list udf列表
        :param ret_table_name_list: list 期望获得的表名列表
        :param agg: boolean 是否需要聚合
        :return: dict 校验参数
        """
        # Todo: 配置化
        param_dict = {
            "flink_sql": {
                "flink.env": "product",
                "flink.bk_biz_id": self.bk_biz_id,
                "flink.source_data": list(self.scopes.keys()),
                "flink.function_name": ", ".join(udf_list),
                "flink.result_table_name": ", ".join(ret_table_name_list),
                "flink.lateness_time": 1,
                "flink.system_fields": [],
                "flink.expired_time": 0,
                "flink.allowed_lateness": False,
                "flink.session_gap": 0,
                "flink.window_length": 0,
                "flink.count_freq": 1,
            },
            "spark_sql": {
                "spark.dataflow_udf_env": "product",
                "spark.bk_biz_id": self.bk_biz_id,
                "spark.input_result_table": [],
                "spark.input_result_table_with_schema": self.scopes,
                "spark.dataflow_udf_function_name": ", ".join(udf_list),
                "spark.result_table_name": ", ".join(ret_table_name_list),
                "spark.self_dependency_config": {},
            },
        }
        properties = param_dict.get(engine, {})
        if agg and engine == SchedulingEngineType.FLINK.value:
            properties["flink.window_type"] = "tumbling"
            properties["flink.window_length"] = 30
        return properties

    @classmethod
    def _pre_deal_validation_message(cls, message):
        """
        对校验api返回的错误信息进行处理后再输出

        :param message: 原始错误信息
        :return: 处理后的错误信息
        """
        match_obj = re.match(
            r'ParseException: Encountered "[^\w]*([\w]+)" at(.+)Was expecting one of',
            message,
            re.S,
        )
        if match_obj:
            message = '"{0}"是一个保留关键字,请在"字段加工逻辑"中用"`{0}`"替换"{0}"来进行转义(仅替换AS之前的部分)'.format(match_obj.group(1))
        return message

    def _check_sql_validation(self, require_schema, engine, agg):
        """
        提取sql中的udf等信息，组装参数，请求sql校验api

        :param require_schema: dict 期望得到的结果schema字典
        :param engine: string sql检查引擎
        :param agg: boolean 是否需要聚合
        :return: tuple (boolean, dict) check后得到的结果和schema
            CASE 成功: (True, {message: succeed, schema: table_name: [
                {field_name: YYY, field_type: int},
                {field_name: XXX, field_type: string}
            ]})
        """

        # if self.sql is None:
        self.gen_sql_expression(require_schema, engine)
        udf_list = self.get_func_list(self.sql, prefix="udf_")
        ret_table_name_list = list(require_schema.keys())
        check_sql = self.sql.encode("utf-8") if isinstance(self.sql, str) else self.sql
        resp = self.requests_clients[engine](
            {
                "sql": check_sql,
                "properties": self._get_runner_properties(engine, udf_list, ret_table_name_list, agg),
            },
            raise_exception=False,
        )
        if not resp.is_success() or not resp.data:
            self.err_trace = dict(
                message=self._pre_deal_validation_message(resp.message),
                code=resp.code,
                sql=check_sql,
            )
            return False, self.err_trace
        # 对返回数据进行格式化清洗
        ret_data = resp.data
        ret_fields_dict = dict()
        for schema_item in ret_data:
            for field_item in schema_item.get("fields", []):
                if schema_item["name"] not in ret_fields_dict:
                    ret_fields_dict[schema_item["name"]] = list()
                if "field_name" not in field_item and "field" in field_item:
                    field_item["field_name"] = field_item["field"]
                if "field_type" not in field_item and "type" in field_item:
                    field_item["field_type"] = field_item["type"]
                if field_item["field_name"] in InnerField.INNER_FIELD_LIST:
                    continue
                ret_fields_dict[schema_item["name"]].append(
                    dict(
                        field_name=field_item["field_name"],
                        field_type=field_item["field_type"],
                    )
                )
        return True, dict(message="check succeed", schema=ret_fields_dict)

    def get_err_trace(self):
        return self.err_trace

    @staticmethod
    def check_schema_consistency(ret_schema, require_schema):
        """
        校验每个返回的表及其字段是否符合预期

        :param ret_schema: 校验返回结果schema
        :param require_schema: 期望得到schema
        :return:  tuple (boolean, None/string)
            CASE 校验成功: (True, None)
            CASE 校验失败: (False, 错误Message)
        """

        matched = False
        match_info = None
        # 校验返回表名列表是否匹配
        if operator.ne(set(require_schema.keys()), set(ret_schema.keys())):
            return False, "require tables: {}, but return tables: {}".format(
                ", ".join(list(require_schema.keys())),
                ", ".join(list(ret_schema.keys())),
            )
        # 校验返回字段信息是否匹配
        for tb_name, fields in list(require_schema.items()):
            require_fields_dict = {str(field["field_name"]): field for field in fields}
            return_fields_dict = {str(ret_field["field_name"]): ret_field for ret_field in ret_schema[tb_name]}
            # 校验字段名列表
            if operator.ne(set(require_fields_dict.keys()), set(return_fields_dict.keys())):
                match_info = "require fields: {}, but return fields: {}".format(
                    ", ".join(list(require_fields_dict.keys())),
                    ", ".join(list(return_fields_dict.keys())),
                )
                break
            # 校验字段类型
            for field in fields:
                require_type = field["field_type"]
                if require_type == "numeric":
                    require_type_list = settings.VALID_NUMERIC_TYPE_LIST
                elif require_type == "float":
                    require_type_list = ["float", "double"]
                else:
                    require_type_list = [require_type]
                ret_field_type = return_fields_dict[field["field_name"]]["field_type"]
                if ret_field_type not in require_type_list:
                    match_info = '{} should be "{}" type, given "{}"'.format(
                        field["field_name"], require_type, ret_field_type
                    )
                    break
            if match_info is not None:
                break
        else:
            matched = True
        return matched, match_info

    def check_res_schema(
        self,
        require_schema,
        engines=SchedulingEngineType.FLINK.value,
        agg=True,
        extract_source_column=False,
    ):
        """
        验证sql是否能得到期望的schema

        :param require_schema: dict 期望得到的结果schema字典
        numeric 指数值类型，int，float，bigint等任一数值类型均可
        {
            "require_result_table_name": [
                {"field_name": "field_name_1", "field_type": "int"},
                {"field_name": "field_name_2", "field_type": "numeric"},
                {"field_name": "field_name_3", "field_type": "text"}
            ]
        }
        :param engines: list/string sql检查引擎
        :param agg: boolean 是否需要聚合
        :param extract_source_column: boolean 是否解析提取source_column
        :return: tuple (boolean, dict/string)
            CASE 校验成功: (True, 校验相关信息)
            CASE 校验失败: (False, 错误Message)
        """
        if not isinstance(engines, list):
            engines = [engines]
        matched = False
        ret_info = dict()
        for engine in engines:
            if engine == SchedulingEngineType.FLINK.value:
                cache_key = None
                try:
                    for scope_table_name, scope_table_info in list(self.scopes.items()):
                        cache_key = settings.SQL_VERIFY_CACHE_KEY.format(table=scope_table_name)
                        cache_info = {
                            "bk_biz_id": self.bk_biz_id,
                            "result_table_name": scope_table_name,
                            "fields": scope_table_info,
                        }
                        cache.set(cache_key, json.dumps(cache_info), 300)
                except Exception as e:
                    raise dm_pro_errors.SetCacheError(message_kv={"cache_key": cache_key, "error": e})
            check_flag, check_info = self._check_sql_validation(require_schema, engine, agg)
            # 校验sql失败
            if not check_flag:
                return False, check_info["message"]
            ret_schema = check_info["schema"]
            matched, match_info = self.check_schema_consistency(ret_schema, require_schema)
            if not matched:
                return False, match_info
        if extract_source_column:
            source_columns_mapping = get_sql_source_columns_mapping(self.sql, formatted=True)
            ret_info["source_columns"] = source_columns_mapping["fields"]
            ret_info["condition_columns"] = source_columns_mapping["conditions"]
        return matched, ret_info

    @property
    def show_sql(self):
        show_sql = self.sql
        for processed_table_name in list(self.scopes.keys()):
            show_sql = show_sql.replace(processed_table_name, self.source_table_mapping[processed_table_name])
        return show_sql

def validate_calculation_content(
    table_name,
    calculation_atom_name,
    field_type,
    calculation_formula,
    schema_list,
    filter_formula=None,
    aggregation_fields=None,
):
    """
    统计口径和指标SQL校验
    :param table_name: {String} 主表名称
    :param calculation_atom_name: {String} 统计口径名称
    :param field_type: {String} 数据类型
    :param calculation_formula: {String} 统计SQL
    :param schema_list: {List} 主表字段列表
    :param filter_formula: {String} 过滤条件
    :param aggregation_fields: {List} 聚合字段列表
    :return:
    """
    schema_dict = {table_name: schema_list}
    calculation_content_dict = {
        "scopes": schema_dict,
        "sql_parts": {
            "fields": {calculation_atom_name: calculation_formula},
            "conditions": filter_formula,
            "appends": {"group_by": aggregation_fields},
            "table_name": table_name,
        },
    }
    require_schema_dict = {table_name: [{"field_type": field_type, "field_name": calculation_atom_name}]}

    verifier = SQLVerifier(**calculation_content_dict)
    matched, match_info = verifier.check_res_schema(
        require_schema_dict,
        [SchedulingEngineType.FLINK.value, SchedulingEngineType.SPARK.value],
        extract_source_column=True,
    )
    if not matched:
        # matched为False时，match_info是错误message
        raise dm_pro_errors.SqlValidateError(
            message_kv={"error_info": "{} [ERROR_SQL: {}]".format(match_info, verifier.show_sql)}
        )
    # matched为True时, match_info是Dict, 例如{'source_columns': {'item_sales_amt': ['channel_id', 'price']}}
    return match_info["source_columns"], match_info["condition_columns"]

def strip_comment(orig_sql):
    """
    删除sql中的注释语句
    :param orig_sql: {String} 原始sql片段
    :return: {String} 删除注释后的sql片段
    """
    return sqlparse.format(orig_sql, strip_comments=True).strip()


from collections import deque
T = int(input())
for tc in range(1, T+1):
    F, N = map(int, input().split())
    arr = list(map(int, input().split()))
    n = len(arr)
    arr = deque(arr)
    cnt = 0
    cnt_turn = 0
    flag = -1
    dic = {}
    for i in range(n):
        if arr[i] in dic:
            dic[arr[i]] += 1
        else:
            dic[arr[i]] = 1
    max_num = sorted(dic.items(), key=lambda x:x[1], reverse=True)[0][0]
    while cnt != F-1:
        if cnt_turn % n == 0:
            flag = 0
            cnt_turn = 0
            n = len(arr)
        if dic[arr[0]] == dic[max_num]:
            if flag == 0:
                flag = 1
                dic[arr[0]] -= 1
                sorted_arr = sorted(dic.items(), key=lambda x:x[1], reverse=True)
                max_num = sorted_arr[0][0]
                for j in range(1, len(sorted_arr)):
                    if sorted_arr[j][1] == 0:
                        continue
                    else:
                        break
                else:
                    arr.popleft()
                    break
                arr.popleft()
            else:
                arr.append(arr.popleft())
        else:
            arr.append(arr.popleft())
        cnt += 1
        cnt_turn += 1
    print(f'#{tc} {arr[0]}')


import logging
import sys

logger = logging.getLogger(__name__)

def log_exceptions(cls):
    """
    This decorator is meant to decorate management commands.
    Any exceptions raised in the command's handle method will be logged and re-raised.
    Can be replaced if https://code.djangoproject.com/ticket/27877 gets implemented.
    """
    class NewClass(cls):
        def handle(self, *args, **options):
            try:
                super().handle(args, options)
            except Exception:
                logger.exception("Management command '{}' failed. Traceback follows: ".format(sys.argv[1]))
                raise
    return NewClass


# 첫 번째는 문제를 제대로 읽지 않아서 접근 방법이 잘못되었고, 두 번째는 조건을 구별하지 않아서 틀림
# 두 번째 시도 후 5개의 테스크케이스와 1개의 효율성 검사에서 통과하지 못해서\
# 블로그에서 접근방법을 확인 후 풀이
# 그냥 쉽게 첫 번째 스티커를 뗄 때와 떼지 않을 때를 구별하면 됨
# 해당 문제는 다음 주에 다시 풀어볼 예정
def solution(sticker):
    if len(sticker) == 1:
        return sticker[0]

    # 맨 앞 스티커를 떼는 경우
    board = [0 for _ in sticker]
    board[0] = sticker[0]
    board[1] = board[0]
    for i in range(2, len(sticker) - 1):
        board[i] = max(board[i-2] + sticker[i], board[i-1])
    answer1 = max(board)

    # 맨 앞 스티커를 떼지 않는 경우 (테스트 케이스는 둘다 떼지않는 경우)
    board2 = [0 for _ in sticker]
    board2[0] = 0
    board2[1] = sticker[1]
    for i in range(2, len(sticker)):
        board2[i] = max(sticker[i] + board2[i-2], board2[i-1])
    answer2 = max(board2)

    return max(answer1, answer2)
    # return max(dp)

스티커모으기(2).py


import os
import pathlib
import re

import psutil

class FilesystemGeneric:
    @staticmethod
    def escape_filename(filename):
        replacements = [
            (re.compile(r"[/:]"), r"_"),
            (re.compile(r"[\?]"), r"!"),
        ]

        for regexp_replace, regexp_substit in replacements:
            filename = re.sub(regexp_replace, regexp_substit, filename)

        return filename

class FilesystemExFAT(FilesystemGeneric):
    ...

_fsname_klass = {
    "exfat": FilesystemExFAT,
}

def get_fs_class(path):
    path = pathlib.Path(path).resolve(strict=True)

    partitions = {}

    for part in psutil.disk_partitions():
        partitions[pathlib.Path(part.mountpoint)] = part.fstype

    if path in partitions:
        if partitions[path] in _fsname_klass:
            return _fsname_klass[partitions[path]]
        else:
            return FilesystemGeneric

    while path != pathlib.Path("/"):
        path = path.parent

        if path in partitions:
            if partitions[path] in _fsname_klass:
                return _fsname_klass[partitions[path]]
            else:
                return FilesystemGeneric

    return FilesystemGeneric


import socket
import datetime

SERVER_PORT = 9965

PROTOCOL_VER = "001"

SERVER_MESSAGE = b"HeartBeatSenderHere" + PROTOCOL_VER
CLIENT_MESSAGE = b"HeartBeatRecHere" + PROTOCOL_VER

############ find server ip and port ###############
sock = socket.socket(socket.AF_INET, # Internet
                     socket.SOCK_DGRAM) # UDP
sock.bind(("0.0.0.0", SERVER_PORT))
(msg, (server_ip, server_port)) = sock.recvfrom(1024)
sock.close()

if msg != SERVER_MESSAGE:
    print("not a valid server package:", server_ip, server_port)
    print(msg)
    exit(-1)

print("server at ", server_ip, server_port)

############# access data from server ###############
sock = socket.socket(socket.AF_INET, # Internet
                     socket.SOCK_DGRAM) # UDP

dev_infos = dict()
heart_beat_package_sent_time = datetime.datetime.now() - datetime.timedelta(seconds=1000)

def print_infos():
    import os
    os.system("cls")
    print("last heartbeat package: ", heart_beat_package_sent_time.isoformat())
    for (name, addr, heartbeat, time) in dev_infos.values():
        print("%20s" % name, "|", addr, "|", "% 3d" % heartbeat, "|", time.isoformat())

while True:
    ##### tell server where are you, you need repeat it no more than one minutes again #####
    if (datetime.datetime.now() - heart_beat_package_sent_time).total_seconds() > 20:
        sock.sendto(CLIENT_MESSAGE, (server_ip, server_port))
        heart_beat_package_sent_time = datetime.datetime.now()
        # print("send alive package")

    ##### receive and parse data from server #####
    (msg, (server_ip, server_port)) = sock.recvfrom(1024)
    ####################### msg format ##########################
    # name '\0' bluetooth_device_mac_addr '\0' heartbeat(4byte) #
    #############################################################
    i_name = 0
    i_addr = msg.index(b'\0')+1
    i_heartbeat = msg.index(b'\0', i_addr) + 1

    name = msg[:i_addr-1].decode()
    addr = msg[i_addr:i_heartbeat-1].decode()
    heartbeat = int.from_bytes(msg[-4:])

    dev_infos[addr] = (name, addr, heartbeat, datetime.datetime.now())
    print_infos()


#Simulating L&L Course on Theoretical Physics: Classical Mechanics Problem 1.1
import numpy as np
import matplotlib.pyplot as plt

# Var
l1 = 2
l2 = 2
g = 9.8
m1 = 1
m2 = 1

# Function

def dub_pen(theta1_i, theta2_i, ang_vel1_i, ang_vel2_i, tot_time, tau=0.05):
    time_vec = [0]*tot_time
    theta1_vec = [0]*tot_time
    ang_vel1_vec = [0]*tot_time
    theta2_vec = [0]*tot_time
    ang_vel2_vec = [0]*tot_time

    theta1 = theta1_i
    ang_vel1 = ang_vel1_i

    theta2 = theta2_i
    ang_vel2 = ang_vel2_i

    for i in range(0, tot_time):
        ang_vel1_old = ang_vel1
        theta1_old = theta1
        ang_vel2_old = ang_vel2
        theta2_old = theta2

        ang_vel1 = ang_vel1_old+((((m2*g*np.sin(theta2_old)*np.cos(theta1_old-theta2_old))-
                                 (m2*np.sin(theta1_old-theta2_old)*((l1*(ang_vel1_old**2)*np.cos(theta1_old-theta2_old))+l2*(ang_vel2_old**2)))
                                 -((m1+m2)*g*np.sin(theta1_old)))/(l1*(m1+(m2*(np.sin(theta1_old-theta2_old)**2)))))*tau)
        ang_vel2 = ang_vel2_old+(((((m1+m2)*(l1*(ang_vel1_old**2)*np.sin(theta1_old-theta2_old)-
                                           g*np.sin(theta2_old)+g*np.sin(theta1_old)*np.cos(theta1_old-theta2_old)))
                                 +(m2*l2*(ang_vel2_old**2)*np.sin(theta1_old-theta2_old)*np.cos(theta1_old-theta2_old)))/(l2*(m1+(m2*(np.sin(theta1_old-theta2_old)**2)))))*tau)

        theta1 = theta1_old + (ang_vel1_old * tau)
        theta2 = theta2_old + (ang_vel2_old * tau)

        time_vec[i] = i*tau
        theta1_vec[i] = theta1
        theta2_vec[i] = theta2
        ang_vel1_vec[i] = ang_vel1
        ang_vel2_vec[i] = ang_vel2
    return [time_vec, theta1_vec, theta2_vec, ang_vel1_vec, ang_vel2_vec]

pendulum = dub_pen(np.pi/4, np.pi/3, 0, 0, 300)
print(pendulum[1])
plt.plot(pendulum[0],pendulum[1])
plt.plot(pendulum[0],pendulum[2])

# Sim
l = np.arange(0,300,1)

x1 = l1*np.sin(pendulum[1])
y1 = -l1*np.cos(pendulum[1])
x2 = x1 + (l2*np.sin(pendulum[2]))
y2 = y1 - (l2*np.cos(pendulum[2]))
# Division of trail
ns = 20
sec_per_trail = 2
for i in l:
    plt.figure()
    plt.plot(x1[i],y1[i],'bo',markersize=5)
    plt.plot([0,x1[i]],[0,y1[i]])
    plt.plot(x2[i],y2[i],'bo',markersize=5)
    plt.plot([x1[i],x2[i]],[y1[i],y2[i]])
    plt.xlim(-(l1+l2)-0.5,(l1+l2)+0.5)
    plt.ylim(-(l1+l2)-0.5,(l1+l2)+0.5)
    for j in range(ns):
        tau = 0.05
        max_trail = int(sec_per_trail/tau)
        s = max_trail // ns
        imin = i - (ns-j)*s
        if imin < 0:
            continue
        imax = imin + s + 1
        # The fading looks better if we square the fractional length along the
        # trail.
        alpha = (j/ns)**2
        plt.plot(x2[imin:imax], y2[imin:imax], c='r', solid_capstyle='butt',
                lw=2, alpha=alpha)


# Inspired by < https://www.pyimagesearch.com/2018/07/09/face-clustering-with-python/ >

from sklearn.cluster import DBSCAN
from .path import Path
from .util import force_mkdir
import logging
import numpy as np
import shutil

class Clusterer:
    def cluster_encodings(self, encodings):
        logging.debug(f"Clustering {len(encodings)} encodings")

        clt = DBSCAN(
            metric = "euclidean"
        )

        # Generate a list so we also have an index
        faces = [ { "file" : k, "encoding" : v } for k, v in encodings.items()]
        faces_encodings = [ i["encoding"] for i in faces ]

        clt.fit(np.array(faces_encodings))
        output = []

        for fid in np.unique(clt.labels_):
            if fid < 0:
                # Skip outliers
                continue

            fid_items = np.where(clt.labels_ == fid)

            output.append({
                "count" : len(fid_items[0]),
                "files" : [ faces[index]["file"] for index in fid_items[0]],
                "id" : int(fid)
            })

        return output

    def move_files(self, clusters, directory):
        for cluster in clusters:
            cluster_id = cluster["id"]
            cluster_path = Path(directory) / str(cluster_id)
            force_mkdir(cluster_path)

            for path in cluster["files"]:
                new_path = Path(cluster_path) / Path(path).name
                logging.debug(f"Copying '{path}' to '{new_path}'")
                shutil.copy(path, new_path)


class Solution:
    def licenseKeyFormatting(self, s: str, k: int) -> str:
        s = s.replace('-', '')
        r = len(s) % k
        strs = []
        if r != 0:
            strs = [s[:r].upper()]

        while r < len(s):
            strs.append(s[r:r+k].upper())
            r += k

        return '-'.join(strs)


# Aufgabe 6.2
# ===========

from copy import deepcopy
from typing import Dict

def nextranking(ranking: Dict[str, float],
                network: Dict[str, set]) -> Dict[str, float]:
    """Returns next ranking for all websites based on network.

    Args:
        ranking (dict[str, float]): The current Ranking of the websites.
        network (dict[str, set[str]): Websites internal linkage.

    Returns:
        dict[str, float]: the new ranking of the current network

    """

    # 85%: Verlinkte Website.
    # 15%: Beliebige Website in network.

    # j,i: Webseiten
    # n seiten von i verlinkt.
    # m: Anzahl aller Seiten im Netzwerk
    # pij = 0.85/n + 0.15/m
    # xij = ranking[i] * pij(i, network)
    # neues ranking (Seite j): Σxij über alle i

    def p(i, j):
        n = len(network[i])
        m = len(network)
        if n == 0:
            return 1/m
        elif j in network[i]:
            return 0.85/n + 0.15/m
        else:
            return 0.15/m

    def x(i, j):
        return ranking[i] * p(i, j)

    nranking = {}

    for j in network:
        for i in network:
            nranking[j] = nranking.get(j, 0) + x(i, j)

    return nranking

def ranking(network: Dict[str, set]) -> Dict[str, float]:
    """Iteratively returns the ranking for the given interlinked network.

    Args:
        network (Dict[str, set]): The known network on which the user will be

    Returns:
        rating (Dict[str, float]): The probabilities of the user being
                                    on each of the sites.

    """

    def isdiff(r1, r2):
        for k in r1.keys():
            if abs(r1[k] - r2[k]) > 1e-6:
                return True
        return False

    frank = {e: 1/len(network) for e in network.keys()}
    srank = nextranking(frank, network)

    while isdiff(frank, srank):
        frank = srank
        srank = nextranking(frank, network)
    return srank


import os

eye_centre_folders_raw = [os.path.join(os.getcwd(), "data", "raw", "centres"),
                          os.path.join(os.getcwd(), "data", "raw", "labels_eme2")]

eye_centre_columns = ["filename", "lx", "ly", "rx", "ry"]
bounding_columns = ["filename", "relative_LE_left", "relative_LE_top", "relative_LE_right", "relative_LE_bottom", "relative_RE_left", "relative_RE_top", "relative_RE_right", "relative_RE_bottom"]

bounding_box_folder = os.path.join(os.getcwd(), "data", "raw", "predictions")
combined_eye_centre_folder = os.path.join(os.getcwd(), "data", "raw", "combined_centres")
final_data_folder = os.path.join(os.getcwd(), "data", "raw", "final_dataset")


import numpy as np
import cv2
from math import ceil, floor, sqrt
import time
from eq_opencl import clHistEq
import os
import matplotlib.pyplot as plt
import time

def calc_transfer_func(hist, alpha, punch, clip):
    X = np.arange(0, len(hist))

    # CDF
    cdf = np.cumsum(hist) / np.sum(hist)

    # apply punch
    dark_punch = np.where(cdf >= punch)[0][0]
    bright_punch = np.where(cdf >= 1-punch)[0][0]
    hist_punched = hist[dark_punch:bright_punch]
    cdf[:dark_punch] = 0
    cdf[bright_punch:] = 1
    cdf[dark_punch:bright_punch] = np.cumsum(hist_punched) / np.sum(hist_punched)

    # linear blending with alpha
    # dark region
    mapping = np.zeros((256), dtype=np.float32)
    # bright region
    mapping[bright_punch:] = 255
    # blended region
    I = X
    mapping = alpha * cdf * 255 + (1-alpha) * I
    # clipping
    mapping = np.clip(mapping, 0, 255)

    # gain limiting with clip
    mapping = np.clip(mapping, I / clip, I * clip)
    return mapping

def histeq_global(gray, alpha=1, punch=0.05, clip=2, use_gpu=True):
    cleq = clHistEq.getInstance()

    if use_gpu:
        histGrid, elapsed = cleq.histGrid(gray)
        hist = histGrid.sum(axis=0).sum(axis=0)
        histElapsed = elapsed / 1000
    else:
        hist, _ = np.histogram(gray, bins=256, range=(0, 256))

    t1 = time.time()
    mapping = calc_transfer_func(hist, alpha, punch, clip).astype(np.uint8)
    mapElapsed = time.time() - t1

    if use_gpu:
        gray, elapsed = cleq.histeqGlobal(gray, mapping)
        eqElapsed = elapsed / 1000
        print('global histogram equalization took GPU: {:.3f} + {:.3f} ms, CPU: {:.3f} ms'.format(histElapsed*1000, eqElapsed*1000, mapElapsed*1000))
    else:
        eq = lambda t: mapping[t]
        vfunc = np.vectorize(eq)
        gray = vfunc(gray)

    return gray