from forum.models import Post, Comment
from django import forms
from tinymce.widgets import TinyMCE

class PostForm(forms.ModelForm):
    class Meta:
        model = Post
        exclude = ['author', 'slug', 'course']

    title = forms.CharField(
        label='Title',
        max_length=50,
        widget=forms.TextInput(
            attrs={
                'required': 'True',
                'placeholder': 'Enter post title'
            }))

    content = forms.CharField(widget=TinyMCE())

class CommentForm(forms.ModelForm):
    class Meta:
        model = Comment
        exclude = ['author', 'post', 'course']

    content = forms.CharField(label='Message', widget=TinyMCE())


import argparse
import os
from distutils.util import strtobool
import random
import time

import numpy as np
import torch
import gym
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.tensorboard import SummaryWriter
import torch.optim as optim
from stable_baselines3.common.buffers import ReplayBuffer

def parse_args():
    parser = argparse.ArgumentParser()
    # Experiment settings
    parser.add_argument("--exp-name", type=str, default=os.path.basename(__file__).rstrip(".py"),
                        help="the name of this experiment")
    parser.add_argument("--seed", type=int, default=1,
                        help="seed of the experiment")
    parser.add_argument("--torch-deterministic", type=lambda x: bool(strtobool(x)), default=True, nargs="?", const=True,
                        help="if toggled, `torch.backends.cudnn.deterministic=False`")
    parser.add_argument("--cuda", type=lambda x: bool(strtobool(x)), default=True, nargs="?", const=True,
                        help="if toggled, cuda will be enabled by default")
    parser.add_argument("--track", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
                        help="if toggled, this experiment will be tracked with Weights and Biases")
    parser.add_argument("--wandb-project-name", type=str, default="cfrl",
                        help="the wandb's project name")
    parser.add_argument("--wandb-entity", type=str, default=None,
                        help="the entity (team) of wandb's project")
    parser.add_argument("--capture-video", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
                        help="whether to capture videos of the agent performances (check out `videos` folder)")
    parser.add_argument("--env-id", type=str, default="CartPole-v1",
                        help="the id of the environment")
    # Hyperparameters
    parser.add_argument("--v-max", type=float, default=100,
                        help="the number of atoms")
    parser.add_argument("--v-min", type=float, default=-100,
                        help="the number of atoms")
    parser.add_argument("--n-atoms", type=int, default=101,
                        help="the number of atoms")
    parser.add_argument("--replay-memory-size", type=int, default=1000000,
                        help="SGD updates are sampled from this number of most recent frames")
    parser.add_argument("--agent-history-length", type=int, default=4,
                        help="The number of most recent frames experienced by the agent that are given as input to the Q network")
    parser.add_argument("--action-repeat", type=int, default=4,
                        help="repeate each action selected by the agent this many times")
    parser.add_argument("--minibatch-size", type=int, default=128,
                        help="the batch size of sample from the reply memory")
    parser.add_argument("--update-frequency", type=int, default=4,
                        help="the number of actions selected by the agent between successive SGD updates")
    parser.add_argument("--gamma", type=float, default=0.99,
                        help="the discount factor gamma")
    parser.add_argument("--learning-rate", type=float, default=2.5e-4,
                        help="the learning rate of the optimizer")
    parser.add_argument("--target-network-frequency", type=int, default=10000,
                        help="the timesteps it takes to update the target network")
    parser.add_argument("--initial-exploration", type=float, default=1,
                        help="the starting epsilon for exploration")
    parser.add_argument("--final-exploration", type=float, default=0.1,
                        help="the ending epsilon for exploration")
    parser.add_argument("--final-exploration-frame", type=int, default=1000000,
                        help="the number of frames over which the initial value of epsilon is linearly annealed to its final value")
    parser.add_argument("--replay-start-size", type=int, default=50000,
                        help="A uniform random  policy is run for this number of frames before learning starts and the resulting experience is used to populate the replay memory")
    parser.add_argument("--noop-max", type=int, default=30,
                        help="maximum number of doing nothing action to be performed by the agent at the start of an episode")
    parser.add_argument("--total-timesteps", type=int, default=10000000,
                        help="total timesteps of the experiments")

    args = parser.parse_args()
    # fmt: on
    return args

def make_env(env_id, seed, idx, capture_video, run_name):
    def thunk():
        env = gym.make(env_id)
        env = gym.wrappers.RecordEpisodeStatistics(env)
        if capture_video:
            if idx == 0:
                env = gym.wrappers.RecordVideo(env, f"videos/{run_name}")
        env.seed(seed)
        env.action_space.seed(seed)
        env.observation_space.seed(seed)
        return env

    return thunk

class QNetwork(nn.Module):
    def __init__(self, env, n_atoms=101, v_min=-100, v_max=100):
        super().__init__()
        self.env = env
        self.n_atoms = n_atoms
        self.register_buffer("atoms", torch.linspace(
            v_min, v_max, steps=n_atoms))
        self.n = env.single_action_space.n
        self.network = nn.Sequential(
            nn.Linear(np.array(env.single_observation_space.shape).prod(), 120),
            nn.ReLU(),
            nn.Linear(120, 84),
            nn.ReLU(),
            nn.Linear(84, self.n * n_atoms),
        )

    def forward(self, x):
        return self.network(x)

    def get_action(self, x, action=None):
        logits = self.network(x)
        # probability mass function for each action
        pmfs = torch.softmax(logits.view(len(x), self.n, self.n_atoms), dim=2)
        q_values = (pmfs * self.atoms).sum(2)
        if action is None:
            action = torch.argmax(q_values, 1)
        return action, pmfs[torch.arange(len(x)), action]

def linear_schedule(start_e: float, end_e: float, duration: int, t: int):
    slope = (end_e - start_e) / duration
    return max(slope * t + start_e, end_e)

if __name__ == "__main__":
    args = parse_args()
    run_name = f"{args.env_id}__{args.exp_name}__{args.seed}__{int(time.time())}"
    if args.track:
        import wandb

        wandb.init(
            project=args.wandb_project_name,
            entity=args.wandb_entity,
            sync_tensorboard=True,
            config=vars(args),
            name=run_name,
            monitor_gym=True,
            save_code=True,
        )
    writer = SummaryWriter(f"runs/{run_name}")
    writer.add_text(
        "hyperparameters",
        "|param|value|\n|-|-|\n%s" % (
            "\n".join([f"|{key}|{value}|" for key, value in vars(args).items()])),
    )

    # TRY NOT TO MODIFY: seeding
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    torch.backends.cudnn.deterministic = args.torch_deterministic

    device = torch.device(
        "cuda" if torch.cuda.is_available() and args.cuda else "cpu")

    # env setup
    envs = gym.vector.SyncVectorEnv(
        [make_env(args.env_id, args.seed, 0, args.capture_video, run_name)])
    assert isinstance(envs.single_action_space,
                      gym.spaces.Discrete), "only discrete action space is supported"

    q_network = QNetwork(envs, n_atoms=args.n_atoms,
                         v_min=args.v_min, v_max=args.v_max).to(device)
    optimizer = optim.Adam(q_network.parameters(
    ), lr=args.learning_rate, eps=0.01 / args.minibatch_size)
    target_network = QNetwork(envs).to(device)
    target_network.load_state_dict(q_network.state_dict())

    rb = ReplayBuffer(
        args.replay_memory_size,
        envs.single_observation_space,
        envs.single_action_space,
        device,
        optimize_memory_usage=True,
        handle_timeout_termination=True,
    )
    start_time = time.time()

    # TRY NOT TO MODIFY: start the game
    obs = envs.reset()
    for global_step in range(args.total_timesteps):
        # ALGO LOGIC: put action logic here
        epsilon = linear_schedule(
            args.initial_exploration, args.final_exploration, args.final_exploration_frame, global_step)
        if random.random() < epsilon:
            actions = np.array([envs.single_action_space.sample()
                               for _ in range(envs.num_envs)])
        else:
            actions, pmf = q_network.get_action(torch.Tensor(obs).to(device))
            actions = actions.cpu().numpy()

        # TRY NOT TO MODIFY: execute the game and log data.
        next_obs, rewards, dones, infos = envs.step(actions)

        # TRY NOT TO MODIFY: record rewards for plotting purposes
        for info in infos:
            if "episode" in info.keys():
                print(
                    f"global_step={global_step}, episodic_return={info['episode']['r']}")
                writer.add_scalar("charts/episodic_return",
                                  info["episode"]["r"], global_step)
                writer.add_scalar("charts/episodic_length",
                                  info["episode"]["l"], global_step)
                writer.add_scalar("charts/epsilon", epsilon, global_step)
                break

        # TRY NOT TO MODIFY: save data to reply buffer; handle `terminal_observation`
        real_next_obs = next_obs.copy()
        for idx, d in enumerate(dones):
            if d:
                real_next_obs[idx] = infos[idx]["terminal_observation"]
        rb.add(obs, real_next_obs, actions, rewards, dones, infos)

        # TRY NOT TO MODIFY: CRUCIAL step easy to overlook
        obs = next_obs

        # ALGO LOGIC: training.
        if global_step > args.replay_start_size and global_step % args.update_frequency == 0:
            data = rb.sample(args.minibatch_size)
            with torch.no_grad():
                _, next_pmfs = target_network.get_action(
                    data.next_observations)
                next_atoms = data.rewards + args.gamma * \
                    target_network.atoms * (1 - data.dones)
                # projection
                delta_z = target_network.atoms[1] - target_network.atoms[0]
                tz = next_atoms.clamp(args.v_min, args.v_max)

                b = (tz - args.v_min) / delta_z
                l = b.floor().clamp(0, args.n_atoms - 1)
                u = b.ceil().clamp(0, args.n_atoms - 1)
                # (l == u).float() handles the case where bj is exactly an integer
                # example bj = 1, then the upper ceiling should be uj= 2, and lj= 1
                d_m_l = (u + (l == u).float() - b) * next_pmfs
                d_m_u = (b - l) * next_pmfs
                target_pmfs = torch.zeros_like(next_pmfs)
                for i in range(target_pmfs.size(0)):
                    target_pmfs[i].index_add_(0, l[i].long(), d_m_l[i])
                    target_pmfs[i].index_add_(0, u[i].long(), d_m_u[i])

            _, old_pmfs = q_network.get_action(
                data.observations, data.actions.flatten())
            loss = (-(target_pmfs * old_pmfs.clamp(min=1e-5,
                    max=1 - 1e-5).log()).sum(-1)).mean()

            if global_step % 100 == 0:
                writer.add_scalar("losses/td_loss", loss, global_step)
                old_val = (old_pmfs * q_network.atoms).sum(1)
                writer.add_scalar("losses/q_values",
                                  old_val.mean().item(), global_step)
                print("SPS:", int(global_step / (time.time() - start_time)))
                writer.add_scalar(
                    "charts/SPS", int(global_step / (time.time() - start_time)), global_step)

            # optimize the model
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            # update the target network
            if global_step % args.target_network_frequency == 0:
                target_network.load_state_dict(q_network.state_dict())

    envs.close()
    writer.close()


#!/usr/bin/env python3
import argparse
import os
import subprocess
import sys
from utilities import functional_annotation
from utilities import toGFF3
from utilities import clustering
from utilities import mapping
from utilities import mergeAll_to_gff
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-p", "--faa", required = True, help="The location of the directory containing all .faa files")
    parser.add_argument("-n", "--fna", required = True, help = "The location of the directory containing all .fna files")
    parser.add_argument("-a", "--asem", required = True, help="The location of the directory containing all assembly files")
    parser.add_argument("-oa", "--annotationFolder",required = True, help="name of output containing annotation results")
    parser.add_argument("-og", "--gffFolder",required = True, help="name of output containing  gff files of each annotation tool")
    parser.add_argument("-om", "--mergeFolder",required = True, help="name of output containing all merged gff files")
    args = parser.parse_args()

    """ In order to run DOOR2 and VFDB, we need a database, so have to check if the database is provided in current working folder """
    operonDB="operonDB.fasta"
    vfdbDB="VFDBdb.fasta"
    if not os.path.exists(operonDB) or not os.path.exists(vfdbDB):
        raise SystemExit("missing database for DOOR2 or VFDB, cannot running DOOR2/VFDB.Exit")
    """ if there is no missing required files, then run annotation """
    faa_dir=args.faa
    """ cluster for eggNOG """
    mergeFile_faa="merge.txt"     #file that merge all 50 genomes
    clusterFolder="cluster_CDHIT"
    clusterFile="clusterFile.txt"
    if not os.path.exists(clusterFolder):
        clustering.main(faa_dir,mergeFile_faa,clusterFolder,clusterFile,0.95)
    """ name the output folder for each tool """
    outputFolder=args.annotationFolder
    tmhmm_out="{}/tmhmm_result".format(outputFolder)
    singalP_out="{}/signalP_result".format(outputFolder)
    door2_out="{}/door2_result".format(outputFolder)
    vfdb_out ="{}/VFDB_result".format(outputFolder)
    card_out="{}/card_result".format(outputFolder)
    piler_seq="{}/pilercr_seq".format(outputFolder)
    piler_out="{}/pilercr_result".format(outputFolder)
    eggNOG_out="{}/eggnog_result_oneFile".format(outputFolder) #contain 1 output , from annotated 1 cluster with eggnog
    eggNOG_map_out="{}/eggnog_result_allFiles".format(outputFolder)

    if not outputFolder in os.listdir():
        subprocess.call(["mkdir",outputFolder])
    try:
        if not door2_out.split("/")[1] in os.listdir(outputFolder):
            #functional_annotation.door2blast_local_all50(faa_dir,operonDB,"blast_operon_db",door2_out)
            functional_annotation.door2blast_server_all50(faa_dir,operonDB,"blast_operon_db",door2_out)
        if not vfdb_out.split("/")[1] in os.listdir(outputFolder):
            #functional_annotation.vfdbblast_local_all50(args.fna,vfdbDB,"blast_vfdb_db",vfdb_out)
            functional_annotation.vfdbblast_server_all50(args.fna,vfdbDB,"blast_vfdb_db",vfdb_out)
        if not card_out.split("/")[1] in os.listdir(outputFolder):
            functional_annotation.rgi_all50(faa_dir,card_out)
        if not tmhmm_out.split("/")[1] in os.listdir(outputFolder):
            functional_annotation.tmhmm_all50(faa_dir,tmhmm_out)
        if not singalP_out.split("/")[1] in os.listdir(outputFolder):
            functional_annotation.signalP_all50(faa_dir,singalP_out)
        if not piler_out.split("/")[1] in os.listdir(outputFolder):
            functional_annotation.piler_all50(args.asem,piler_seq,piler_out)
        if not eggNOG_out.split("/")[1] in os.listdir(outputFolder):
            subprocess.call(["python2","utilities/eggnog.py",eggNOG_out])
    except Exception as e:
        print(e)
        raise SystemExit("please fix the error. Pipeline is terminated")

    """ EGGNOG mapping: from one output of EGGNOG, map to 50 output files for 50 faa files"""
    try:
        if not eggNOG_map_out.split("/")[1] in os.listdir(outputFolder):
            names_file="allfiles.txt" #contains 50 id number of each genome
            id_file_faa="title.txt" # contains header line in merge file that concatenate all 50 genomes faa/fna
            mapping.main(faa_dir,names_file,id_file_faa,mergeFile_faa,eggNOG_out,clusterFolder,eggNOG_map_out)
    except Exception as e:
        print(e)
        sys.exit(-1)

    """ after annotation all 50 files, then convert to gff for each annotation tool"""
    try:
        gff_out_folder=args.gffFolder #folder containing gff files for each annotation tool
        if not gff_out_folder in os.listdir():
            subprocess.call(["mkdir",gff_out_folder])

        tmhmm_gff,signalP_gff,piler_gff,card_gff,door2_gff,vfdb_gff,eggnog_gff="{}/tmhmm_to_gff".format(gff_out_folder),"{}/signalP_to_gff".format(gff_out_folder)\
            ,"{}/pilercf_to_gff".format(gff_out_folder),"{}/card_to_gff".format(gff_out_folder),\
             "{}/door2_to_gff".format(gff_out_folder),"{}/vfdb_to_gff".format(gff_out_folder),"{}/eggnog_to_gff".format(gff_out_folder)

        toGFF3.main(tmhmm_out,tmhmm_gff,singalP_out,signalP_gff,door2_out,door2_gff,vfdb_out,vfdb_gff,card_out,card_gff,piler_out,piler_gff,eggNOG_map_out,eggnog_gff,outputFolder,gff_out_folder)
    except Exception as e:
        print(e)
        sys.exit(-1)

    #if there is exist all gff folders for all tool, we can begin to merge
    try:
        all_gff_faa=args.mergeFolder
        if all_gff_faa not in os.listdir():
            mergeAll_to_gff.main(tmhmm_gff,signalP_gff,piler_gff,card_gff,door2_gff,vfdb_gff,eggnog_gff,all_gff_faa)
    except Exception as e:
        print(e)
        sys.exit(-1)
if __name__ == "__main__":
    main()


import xml.etree.ElementTree as Tree
import unittest
import os

# Clear Screen
def clear_screen():
    if os.name == 'posix':  # Linux
        os.system('clear')
    elif os.name in ('nt', 'dos', 'ce'):    # Windows
        os.system('CLS')

class ETLTool:
    # Constructor
    def __init__(self):
        self.tree = None

    # Function to read an XML file and load the elements.
    # Since the presence of the file is tested here, we don"t test it elsewhere.
    def parse_xml(self, file_path):
        try:
            self.tree = Tree.parse(file_path)
        except Exception as e:
            print(f"Error: {e}")

    # Function to save the XML file after data manipulation.
    def save_xml(self, save_path):
        if self.tree is not None:
            self.tree.write(save_path)

    # Modified price of items in a category.
    def modify_price(self, category, increase):
        root = self.tree.getroot()
        for prod in root.findall(f"./product[@category='{category}']"):
            price = float(prod.find("price").text)
            new_price = price * (1 + increase/100)
            prod.find("price").text = str(new_price)

    # Rename a category to another one.
    def rename_category(self, old_name, new_name):
        root = self.tree.getroot()
        for prod in root.findall(f"./product[@category='{old_name}']"):
            prod.set("category", new_name)

    # Removes products below a certain rating.
    def remove_products(self, category, min_rating):
        root = self.tree.getroot()
        for prod in root.findall(f"./product[@category='{category}']"):
            if float(prod.find("rating").text) < min_rating:
                root.remove(prod)

    # Outputs the report on the CLI
    def generate_report(self):
        report = {}
        root = self.tree.getroot()

        for prod in root.findall("product"):
            category = prod.attrib["category"]
            price = float(prod.find("price").text)
            report[category] = report.get(category, {"count": 0, "total_price": 0})
            report[category]["count"] += 1
            report[category]["total_price"] += price

        for k, v in report.items():
            print(f"\nCategory: {k}")
            print(f"Total Product Count: {v['count']}")
            print(f"Total Price: {v['total_price']}")

    # Runs the UI for the tool as a menu based interface.
    def run_tool(self):
        while True:
            clear_screen()
            print("\nWelcome to the Product ETL Tool")
            print("1: Load XML File")
            print("2: Increase Price By Percent")
            print("3: Rename Category")
            print("4: Remove Products Below Minimum Rating")
            print("5: Save Changes to New File")
            print("6: Generate Report on CLI")
            print("7: Exit")

            select = input("Enter your choice here (Number): ")
            LOAD_ERR = "Please load an XML file first by selecting option 1."

            # Menu based CLI
            if select == "1":
                file_path = input("Enter the relative path to the XML file: ")
                self.parse_xml(file_path)

            elif select == "2":
                # Check if tree is filled with a loaded XML. If not, we first need to load it (1)
                if self.tree is None:
                    print(LOAD_ERR)
                    break
                category = input("Enter the category name: ")
                percentage = float(input("Enter the percentage increase (number only): "))
                self.modify_price(category, percentage)

            elif select == "3":
                if self.tree is None:
                    print(LOAD_ERR)
                    break
                old_name = input("Enter the current category name: ")
                new_name = input("Enter the new category name: ")
                self.rename_category(old_name, new_name)

            elif select == "4":
                if self.tree is None:
                    print(LOAD_ERR)
                    break
                category = input("Enter the category name: ")
                min_rating = float(input("Enter the minimum rating: "))
                self.remove_products(category, min_rating)

            elif select == "5":
                if self.tree is None:
                    print(LOAD_ERR)
                    break
                save_path = input("Enter the path to save the XML file: ")
                self.save_xml(save_path)

            elif select == "6":
                if self.tree is None:
                    print(LOAD_ERR)
                    break
                self.generate_report()

            elif select == "7":
                print("\nGoodbye!")
                break

            input("\nClick any key to proceed")     # Breakpoint before next menu appears

if __name__ == "__main__":
    # Init new tool
    etl_tool = ETLTool()

    # Run the tool
    etl_tool.run_tool()


import pandas as pd
from simulation_model.create_params_grid import default_params, path_experiment_table
from utilities.small_functions import mkDir_if_not
import os
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
from simulation_model.gather_results import bilinear_name

def get_results_experiment_name_path(experiment_name):
    results_dir = mkDir_if_not("simulation_model/results")
    return os.path.join(results_dir, "{}_results_cv.pkl".format(experiment_name))

def min_of_gt_0(x):
    return np.min(x[x > 0])

acronyms_dict = {"optirank": "optirank",
                 "logistic_regression_on_ranks": "rank-lr",
                 "logistic_regression": "lr"}

def acronym(serie):
    """df is a dataframe with column classifier_name. Add column classifier_acronym"""
    fun = lambda x: acronyms_dict[x]
    return serie.apply(fun)

dict_param_name_to_latex = {"d": "$d$",  # correspondences to the paper
                            "n_perturbing": "$d_{P}$",
                            "n_samples": "$n_{samples}$",
                            "tau": "$\\tau$",
                            "sigma": "$\sigma$"}

if __name__ == "__main__":
    experiment_table = pd.read_pickle(path_experiment_table)
    sns.set(font_scale=1.5)  # was 1.5
    sns.set_style("whitegrid")

    for i_row, row in experiment_table.iterrows():

        experiment_name = row["experiment_name"]
        n_params = row["n_params"]
        params_grid = list(row["param_grid"])
        results = pd.read_pickle(get_results_experiment_name_path(experiment_name))
        results["classifier"] = acronym(results["classifier_name"])

        for with_legend in [True, False]:
            output_dir = "simulation_model/results"
            if with_legend:
                legend = "full"
                location = "upper left"
                output_dir_plots = mkDir_if_not(os.path.join(output_dir, "plots", "legend_on"))

            else:
                legend = False
                location = "best"  # should have no effect
                output_dir_plots = mkDir_if_not(os.path.join(output_dir, "plots", "legend_off"))

            plt.figure()
            p = sns.lineplot(data=results.reset_index(), x="param_value", y="test_balanced_accuracy", hue="classifier",
                             legend=legend)
            lgd = plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.2), fancybox=True, shadow=False, ncol=3)
            p.axes.set_xlabel(dict_param_name_to_latex[pd.unique(results["param_name"]).item()], )
            p.axes.set_ylabel("test balanced accuracy (%)")
            # if pd.unique(results["param_name"]).item() == "sigma_m":
            #     p.axes.set_xscale("symlog", linthresh=min_of_gt_0(results[["param_value"]].values))
            figpath = os.path.join(output_dir_plots, "results_{}.pdf".format(experiment_name))
            plt.tight_layout()
            if with_legend:
                lgd.set_visible(True)
            else:
                lgd.set_visible(False)
            plt.savefig(figpath)
            plt.close()

            # overlap figure
            plt.figure()
            results_of_interest = results.loc[results.classifier_name == bilinear_name]
            p = sns.lineplot(data=results_of_interest.reset_index(), x="param_value", y="overlap", legend=legend)

            lgd = plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.2), fancybox=True, shadow=False, ncol=3)
            p.axes.set_xlabel(dict_param_name_to_latex[pd.unique(results["param_name"]).item()], )
            p.axes.set_ylabel("overlap")
            p.axes.set_ylim([0, 1])
            # if pd.unique(results["param_name"]).item() == "sigma_m":
            #     p.axes.set_xscale("symlog", linthresh=min_of_gt_0(results[["param_value"]].values))
            figpath = os.path.join(output_dir_plots, "results_overlap_{}.pdf".format(
                experiment_name))
            plt.tight_layout()
            if with_legend:
                lgd.set_visible(True)
            else:
                lgd.set_visible(False)
            plt.savefig(figpath)
            plt.close()

    # results for default parameters (taken from the experiment on d for instance)
    experiment_name = "different_d"
    results = pd.read_pickle(get_results_experiment_name_path(experiment_name))
    results_per_classifier = results.loc[
        results.param_value == default_params["d"], ["test_balanced_accuracy", "classifier_name", "overlap"]].groupby(
        ["classifier_name"]).agg(['mean', 'sem'])
    results_per_classifier.to_csv("simulation_model/results/results_default_parameters.csv")

    #output file for markdown
    outfile_md = "simulation_model/results/results_default_parameters.md"
    out_md = (100*results_per_classifier[("test_balanced_accuracy", "mean")]).map('{:,.0f}'.format) + " ± " + (100*results_per_classifier[("test_balanced_accuracy", "sem")]).map('{:,.0f}'.format)
    out_md = out_md.to_frame(name="test balanced accuracy")
    markdown_table = out_md.to_markdown()
    f = open(outfile_md, "w")
    f.write(markdown_table)
    f.close()


def lengthOfLIS(nums):
	dp = [1]*len(nums)
	for i in range(1,len(nums)):
		longestSoFar = 1
		for j in range(i):
			if nums[i] > nums[j]:
				longestSoFar = max(longestSoFar,1+dp[j])
		dp[i] = longestSoFar
	return max(dp)

nums = [10,9,2,5,3,7,101,18]
print(lengthOfLIS(nums))


from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.wait import WebDriverWait
from selenium.common import NoSuchElementException
from pages.utils import write_file

class BasePage:

    def __init__(self, driver):
        self.driver = driver
        self.waiter = WebDriverWait(driver=driver, timeout=5)

    def wait_until_displayed(self, by, xpath):
        """Waits until element displayed and return it, else raise an exception"""
        return self.waiter.until(
            method=expected_conditions.visibility_of_element_located(
                (by, xpath)
            )
        )

    def wait_until_clickable(self, by, xpath):
        """Waits until element clickable and return it, else raise an exception"""
        return self.waiter.until(
            method=expected_conditions.element_to_be_clickable((by, xpath)))

    def is_element_exist(self, xpath):
        """Waits until element exist, else raise an exception"""
        try:
            self.driver.find_element(by=By.XPATH, value=xpath)
            return True
        except (TimeoutError, NoSuchElementException):
            return False

    def is_element_visible(self, xpath):
        """Waits until element exist, else raise an exception"""
        try:
            self.wait_until_displayed(by=By.XPATH, xpath=xpath)
            return True
        except (TimeoutError, NoSuchElementException):
            return False

    def fill_field(self, xpath, value):
        """Fill field using provided value"""
        element = self.wait_until_clickable(by=By.XPATH, xpath=xpath)
        element.clear()
        element.send_keys(value)

    def fill_field_with_submit(self, xpath, value):
        """Fill field using provided value"""
        element = self.wait_until_clickable(by=By.XPATH, xpath=xpath)
        element.clear()
        element.send_keys(value)
        element.submit()

    def click(self, xpath):
        """Find and click on the element by providing xpath"""
        # self.wait_until_displayed(by=By.XPATH, xpath=xpath).click()
        self.driver.find_element(by=By.XPATH, value=xpath).click()

    def move_mouse_on_element(self, xpath):
        """Moves mouse on provided element"""
        try:
            action = webdriver.ActionChains(self.driver)
            element = self.driver.find_element(by=By.XPATH, value=xpath)
            action.move_to_element(element)
            action.perform()
        except (BaseException, Exception) as ex:
            write_file('move_mouse_on_element() Exception = ', ex)

    def switch_to_alert(self, alert_accept_dismiss):
        """Moves focus to Alert window"""
        self.waiter.until(
            method=expected_conditions.alert_is_present()
        )
        if alert_accept_dismiss:
            self.driver.switch_to.alert.accept()
        else:
            self.driver.switch_to.alert.dismiss()

    def get_element_value(self, xpath):
        """Get element attribute value"""
        if self.is_element_exist(xpath=xpath):
            element = self.driver.find_element(By.XPATH, xpath).get_attribute('value')
            return element

    def compare_element_text(self, text, xpath):
        """Compare element's text with provided text """
        element = self.wait_until_displayed(by=By.XPATH, xpath=xpath)
        return element.text == text


from .method import get_json_ret, json_response_zh

class AuthMiddleware(object):
    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request, *args, **kwargs):
        if "application/json" in request.headers.get("Content-Type"):
            import json
            request.json = json.loads(request.body)
        else:
            request.json = request.POST

        if not request.path.startswith('/negotiate_key'):
            shared_secret = request.session.get("shared_secret")
            if shared_secret is None:
                return json_response_zh(get_json_ret(42, msg="请先协商密钥"))
            from Crypto.Util.number import long_to_bytes
            request.DH_key = long_to_bytes(shared_secret)[:16].ljust(16, b'\x00')

        if not request.path.startswith('/negotiate_key') and not request.path == '/dynamicauth_api3/':
            request.data = request.json.get("data")
            if request.data is None:
                return json_response_zh(get_json_ret(40, msg="请传递 data 参数"))

        user_name = request.session.get("user_name")
        if user_name:
            from UserModel.models import UserModel
            request.user = UserModel.objects.get(user_name=user_name)
            if request.user is None:
                request.session["user_name"] = None

        response = self.get_response(request)
        return response


import numpy as np
import matplotlib
#matplotlib.use('TkAgg')
import matplotlib.pyplot as plt

from skimage import data, img_as_float
from skimage.metrics import structural_similarity as ssim
from skimage.metrics import mean_squared_error

from skimage.transform import rescale, resize, downscale_local_mean
from skimage import io

im1 = io.imread('frac3_run2_sp5_map16.pgm')
im2 = io.imread('frac2_run4_sp5_map2.pgm')
im3 = io.imread('frac1A_run2_map5.pgm')
badim = io.imread('frac9A_run1_map2.pgm')
imfull = io.imread('frac3A_run2_map8.pgm')

plt.imshow(im1, plt.cm.gray)
plt.savefig("mygraph.png")

rows, cols = im1.shape
print("Size of im1: ",rows,cols)
print("Size of im2: ", im2.shape)
print("Size of im3: ", im3.shape)

img1 = resize(im1, (520, 500) )
img2 = resize(im2, (520, 500) )
img3 = resize(im3, (520, 500) )
imgFull = resize(imfull, (520, 500) )
badimg = resize(badim, (520, 500) )

plt.imshow(img1, plt.cm.gray)
plt.savefig("mygraph1.png")

plt.imshow(img2, plt.cm.gray)
plt.savefig("mygraph2.png")

plt.imshow(img3, plt.cm.gray)
plt.savefig("mygraph3.png")

plt.imshow(imgFull, plt.cm.gray)
plt.savefig("mygraph.png")

mse_11 = mean_squared_error(img1,img1)
mse_12 = mean_squared_error(img1,img2)
mse_22 = mean_squared_error(img2,img2)
mse_23 = mean_squared_error(img2, img3)
mse_13 = mean_squared_error(img1, img3)

print("MSE Error 11: %f, 12: %f, 22: %f, 23: %f, 13: %f"%(mse_11, mse_12, mse_22, mse_23, mse_13))

ssim11 = ssim(img1, img1, data_range=img1.max()-img1.min())
ssim22 = ssim(img2, img2, data_range=img2.max()-img2.min())
ssim12 = ssim(img1, img2, data_range=img1.max()-img1.min())
ssim23 = ssim(img2, img3, data_range=img1.max()-img1.min())
ssim13 = ssim(img1, img3, data_range=img1.max()-img1.min())
print("SSIM 12: ", ssim12, "ssim11 %f, ssim22 %f, ssim23 %f, ssim13: %f"%(ssim11, ssim22, ssim23, ssim13) )

print("Comparing with FULL map:")
print("MSE 1full: %f, 2full: %f, 3full: %f, badimg-full: %f"%(mean_squared_error(img1, imgFull) ,mean_squared_error(img2, imgFull) ,mean_squared_error(img3, imgFull), mean_squared_error(badimg, imgFull) ) )
print("SSIM with full: 1f: %f, 2f: %f, 3f: %f, badimg-full %f"%( ssim(img1, imgFull, data_range=imgFull.max()-imgFull.min()), ssim(img2, imgFull, data_range=imgFull.max()-imgFull.min()), ssim(img3, imgFull, data_range=imgFull.max()-imgFull.min()), ssim(badimg, imgFull, data_range=imgFull.max()-imgFull.min()) ) )


from pybricks.hubs import CityHub
from pybricks.pupdevices import Motor, ColorDistanceSensor
from pybricks.parameters import Port, Stop, Color
from pybricks.tools import wait

# Initialize devices.
hub = CityHub()
motor = Motor(Port.B)
colorSensor = ColorDistanceSensor(Port.A)

while True:
    colors = []
    angles = []
    #Mode 1 - Train
    hub.light.on(Color.WHITE)
    wait(500)
    hub.light.off()
    wait(500)
    hub.light.on(Color.WHITE)
    wait(500)
    hub.light.off()
    wait(500)
    hub.light.on(Color.WHITE)
    wait(500)
    #Train for 4 color cases
    for case in range(4):
        #5 trainings for each case
        for i in range(5):
            try:
                while True:
                    color = colorSensor.color()
                    hub.light.on(color)
                    wait(100)
            except SystemExit:
                pass
            #Get color
            color = colorSensor.color()
            print(color)
            #Turn hub light color of sensor reading
            hub.light.on(color)
            #Get motor angle
            angle = motor.angle()
            print(angle)
            #Add the sensor reading and motor angle to list
            colors.append(color)
            angles.append(angle)
            wait(100)
        hub.light.on(Color.WHITE)
        wait(2000)

    print(colors)
    print(angles)

    #Mode 2 - Run
    try:
        while True:
            counter = 0
            avgPos = 0
            sumPos = 0
            #Get color sensor reading
            color = colorSensor.color()
            hub.light.on(color)
            if color != Color.NONE:
                for i in range(len(colors)):
                    #Find average position reading for sensed color
                    if colors[i] == color:
                        counter = counter + 1
                        sumPos = (sumPos+angles[i])
                avgPos = sumPos/counter
                print(avgPos)
                #Move motor to average position at 400 (can change speed)
                motor.run_target(400, avgPos)
                motor.stop()
            wait(500)

    except SystemExit:
        try:
            wait(1000)
            break
        except SystemExit:
            pass


#!/usr/bin/env python3

import asyncio
import unittest

from click.testing import CliRunner

from base_cli import _handle_debug, async_main, main

class TestCLI(unittest.TestCase):
    def test_async_main(self) -> None:
        self.assertEqual(0, asyncio.run(async_main(True)))

    def test_debug_output(self) -> None:
        self.assertTrue(_handle_debug(None, None, True))

    def test_help(self) -> None:
        runner = CliRunner()
        result = runner.invoke(main, ["--help"])
        assert result.exit_code == 0

if __name__ == "__main__":  # pragma: no cover
    unittest.main()


import pytest

from src.guess_number_higher_or_lower import Solution

@pytest.mark.parametrize(
    "n,pick,expected",
    [
        (10, 6, 6),
        (1, 1, 1),
        (2, 1, 1),
    ],
)
def test_solution(n, pick, expected, monkeypatch):
    monkeypatch.setenv("SECRET", str(pick))
    assert Solution().guessNumber(n) == expected


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Python version: 3.6
import os
import copy
import time
import pickle
import numpy as np
from tqdm import tqdm
import yaml
import argparse

import torch
from tensorboardX import SummaryWriter

# from src.options import args_parser
from update import LocalUpdate
from utils import get_dataset, average_weights, exp_details
from deepctr_torch.inputs import get_feature_names
from sklearn.metrics import log_loss, roc_auc_score

from deepctr_torch.models import DeepFM

if __name__ == '__main__':
    start_time = time.time()

    # define paths
    path_project = os.path.abspath('..')
    logger = SummaryWriter('../logs')

    parser = argparse.ArgumentParser(description='PPDL')
    parser.add_argument('--params', dest='params', default='utils/params.yaml')
    params = parser.parse_args()

    with open(f'../{params.params}', 'r') as f:
        args = yaml.load(f)

    exp_details(args)
    exit(0)
    if args['gpu']:
        torch.cuda.set_device(args['gpu'])
    device = 'cuda' if args['gpu'] else 'cpu'

    # load dataset and user groups  # prepare feature for model
    (train_dataset, test_dataset, user_groups),fixlen_feature_columns = get_dataset(args)

    # count #unique features for each sparse field,and record dense feature field name

    dnn_feature_columns = fixlen_feature_columns
    linear_feature_columns = fixlen_feature_columns

    feature_names = get_feature_names(linear_feature_columns + dnn_feature_columns)

    # BUILD MODEL
    if args['model'].lower() == 'deepfm':
        # 4.Define Model,train,predict and evaluate
        global_model = DeepFM(linear_feature_columns, dnn_feature_columns, task='binary')
        global_model.compile("adam", "binary_crossentropy",
                             metrics=['binary_crossentropy'], )
    else:
        exit('Error: unrecognized model')

    # # Set the model to train and send it to device.
    # global_model.to(device)
    # global_model.train() # torch claim

    # print(global_model)

    # copy weights
    global_weights = global_model.state_dict()
    # print(global_weights.keys())

    # Training
    # train_loss, train_accuracy = [], []
    # val_acc_list, net_list = [], []
    # cv_loss, cv_acc = [], []
    # print_every = 2
    # val_loss_pre, counter = 0, 0

    # temp test data

    test_model_input = {name: test_dataset[name] for name in feature_names}

    # for comparsion
    # best_model = copy.deepcopy(global_model)
    min_loss = 1000.0
    max_auc = -1.0

    for epoch in tqdm(range(args['epochs'])):
        local_weights= [] #, local_losses , []
        print(f'\n | Global Training Round : {epoch+1} |\n')
        # frac default 0.1; num_users default 100
        m = max(int(args['frac'] * args['num_users']), 1)
        # 100 randomly select 10 as training client
        idxs_users = np.random.choice(range(args['num_users']), m, replace=False)

        for idx in idxs_users: # 10 random users
            local_model = LocalUpdate(args=args, dataset=train_dataset,
                                      idxs=user_groups[idx], logger=logger)
            w = local_model.update_weights(
                model=copy.deepcopy(global_model), features=feature_names)

            local_weights.append(copy.deepcopy(w))
            # local_losses.append(copy.deepcopy(loss))

        # update global weights
        global_weights = average_weights(local_weights)
        global_model.load_state_dict(global_weights)

        # temp test
        pred_ans = global_model.predict(test_model_input, batch_size=256)
        logloss = log_loss(test_dataset['label'].values, pred_ans)
        aucscore = roc_auc_score(test_dataset['label'].values, pred_ans)
        print("test LogLoss", round(logloss, 4))
        print("test AUC", round(aucscore, 4))
        if aucscore > max_auc:
            # best_model = copy.deepcopy(global_model)
            min_loss = logloss
            max_auc = aucscore

    print("|---- Min log loss: {:.4f}%".format(min_loss))
    print("|---- Best AUC: {:.4f}%".format(max_auc))
    print("test done")


import collections
import re

import furl
from django.core.urlresolvers import resolve, reverse, NoReverseMatch
from django.core.exceptions import ImproperlyConfigured
from django.http.request import QueryDict
from rest_framework import exceptions
from rest_framework import serializers as ser
from rest_framework.fields import SkipField
from rest_framework.fields import get_attribute as get_nested_attributes

from api.base import utils
from api.base.exceptions import InvalidQueryStringError
from api.base.exceptions import Conflict
from api.base.exceptions import JSONAPIException
from api.base.exceptions import TargetNotSupportedError
from api.base.exceptions import RelationshipPostMakesNoChanges
from api.base.settings import BULK_SETTINGS
from api.base.utils import absolute_reverse, extend_querystring_params, get_user_auth, extend_querystring_if_key_exists
from framework.auth import core as auth_core
from website import settings
from website import util as website_utils
from website.models import Node
from website.util.sanitize import strip_html

def format_relationship_links(related_link=None, self_link=None, rel_meta=None, self_meta=None):
    """
    Properly handles formatting of self and related links according to JSON API.

    Removes related or self link, if none.
    """

    ret = {'links': {}}

    if related_link:
        ret['links'].update({
            'related': {
                'href': related_link or {},
                'meta': rel_meta or {}
            }
        })

    if self_link:
        ret['links'].update({
            'self': {
                'href': self_link or {},
                'meta': self_meta or {}
            }
        })

    return ret

def is_anonymized(request):
    private_key = request.query_params.get('view_only', None)
    return website_utils.check_private_key_for_anonymized_link(private_key)

class HideIfRegistration(ser.Field):
    """
    If node is a registration, this field will return None.
    """

    def __init__(self, field, **kwargs):
        super(HideIfRegistration, self).__init__(**kwargs)
        self.field = field
        self.source = field.source
        self.required = field.required
        self.read_only = field.read_only

    def get_attribute(self, instance):
        if instance.is_registration:
            if isinstance(self.field, RelationshipField):
                raise SkipField
            else:
                return None
        return self.field.get_attribute(instance)

    def bind(self, field_name, parent):
        super(HideIfRegistration, self).bind(field_name, parent)
        self.field.bind(field_name, self)

    def to_internal_value(self, data):
        return self.field.to_internal_value(data)

    def to_representation(self, value):
        if getattr(self.field.root, 'child', None):
            self.field.parent = self.field.root.child
        else:
            self.field.parent = self.field.root
        return self.field.to_representation(value)

    def to_esi_representation(self, value, envelope='data'):
        if getattr(self.field.root, 'child', None):
            self.field.parent = self.field.root.child
        else:
            self.field.parent = self.field.root
        return self.field.to_esi_representation(value, envelope)

class HideIfDisabled(ser.Field):
    """
    If the user is disabled, returns None for attribute fields, or skips
    if a RelationshipField.
    """

    def __init__(self, field, **kwargs):
        super(HideIfDisabled, self).__init__(**kwargs)
        self.field = field
        self.source = field.source
        self.required = field.required
        self.read_only = field.read_only

    def get_attribute(self, instance):
        if instance.is_disabled:
            if isinstance(self.field, RelationshipField):
                raise SkipField
            else:
                return None
        return self.field.get_attribute(instance)

    def bind(self, field_name, parent):
        super(HideIfDisabled, self).bind(field_name, parent)
        self.field.bind(field_name, self)

    def to_internal_value(self, data):
        return self.field.to_internal_value(data)

    def to_representation(self, value):
        if getattr(self.field.root, 'child', None):
            self.field.parent = self.field.root.child
        else:
            self.field.parent = self.field.root
        return self.field.to_representation(value)

    def to_esi_representation(self, value, envelope='data'):
        if getattr(self.field.root, 'child', None):
            self.field.parent = self.field.root.child
        else:
            self.field.parent = self.field.root
        return self.field.to_esi_representation(value, envelope)

class HideIfWithdrawal(HideIfRegistration):
    """
    If registration is withdrawn, this field will return None.
    """

    def get_attribute(self, instance):
        if instance.is_retracted:
            if isinstance(self.field, RelationshipField):
                raise SkipField
            else:
                return None
        return self.field.get_attribute(instance)

class AllowMissing(ser.Field):
    def __init__(self, field, **kwargs):
        super(AllowMissing, self).__init__(**kwargs)
        self.field = field

    def to_representation(self, value):
        return self.field.to_representation(value)

    def bind(self, field_name, parent):
        super(AllowMissing, self).bind(field_name, parent)
        self.field.bind(field_name, self)

    def get_attribute(self, instance):
        """
        Overwrite the error message to return a blank value is if there is no existing value.
        This allows the display of keys that do not exist in the DB (gitHub on a new OSF account for example.)
        """
        try:
            return self.field.get_attribute(instance)
        except SkipField:
            return ''

    def to_internal_value(self, data):
        return self.field.to_internal_value(data)

def _url_val(val, obj, serializer, **kwargs):
    """Function applied by `HyperlinksField` to get the correct value in the
    schema.
    """
    url = None
    if isinstance(val, Link):  # If a Link is passed, get the url value
        url = val.resolve_url(obj, **kwargs)
    elif isinstance(val, basestring):  # if a string is passed, it's a method of the serializer
        if getattr(serializer, 'field', None):
            serializer = serializer.parent
        url = getattr(serializer, val)(obj) if obj is not None else None
    else:
        url = val

    if not url and url != 0:
        raise SkipField
    else:
        return url

class IDField(ser.CharField):
    """
    ID field that validates that 'id' in the request body is the same as the instance 'id' for single requests.
    """

    def __init__(self, **kwargs):
        kwargs['label'] = 'ID'
        super(IDField, self).__init__(**kwargs)

    # Overrides CharField
    def to_internal_value(self, data):
        request = self.context.get('request')
        if request:
            if request.method in utils.UPDATE_METHODS and not utils.is_bulk_request(request):
                id_field = self.get_id(self.root.instance)
                if id_field != data:
                    raise Conflict(detail=('The id you used in the URL, "{}", does not match the id you used in the json body\'s id field, "{}". The object "{}" exists, otherwise you\'d get a 404, so most likely you need to change the id field to match.'.format(id_field, data, id_field)))
        return super(IDField, self).to_internal_value(data)

    def get_id(self, obj):
        return getattr(obj, self.source, '_id')

class TypeField(ser.CharField):
    """
    Type field that validates that 'type' in the request body is the same as the Meta type.

    Also ensures that type is write-only and required.
    """

    def __init__(self, **kwargs):
        kwargs['write_only'] = True
        kwargs['required'] = True
        super(TypeField, self).__init__(**kwargs)

    # Overrides CharField
    def to_internal_value(self, data):
        if isinstance(self.root, JSONAPIListSerializer):
            type_ = self.root.child.Meta.type_
        else:
            type_ = self.root.Meta.type_

        if type_ != data:
            raise Conflict(detail=('This resource has a type of "{}", but you set the json body\'s type field to "{}". You probably need to change the type field to match the resource\'s type.'.format(type_, data)))
        return super(TypeField, self).to_internal_value(data)

class TargetTypeField(ser.CharField):
    """
    Enforces that the related resource has the correct type
    """

    def __init__(self, **kwargs):
        kwargs['write_only'] = True
        kwargs['required'] = True
        self.target_type = kwargs.pop('target_type')
        super(TargetTypeField, self).__init__(**kwargs)

    def to_internal_value(self, data):
        if self.target_type != data:
            raise Conflict(detail=('The target resource has a type of "{}", but you set the json body\'s type field to "{}".  You probably need to change the type field to match the target resource\'s type.'.format(self.target_type, data)))
        return super(TargetTypeField, self).to_internal_value(data)

class JSONAPIListField(ser.ListField):
    def to_internal_value(self, data):
        if not isinstance(data, list):
            self.fail('not_a_list', input_type=type(data).__name__)

        return super(JSONAPIListField, self).to_internal_value(data)

class AuthorizedCharField(ser.CharField):
    """
    Passes auth of the logged-in user to the object's method
    defined as the field source.

    Example:
        content = AuthorizedCharField(source='get_content')
    """

    def __init__(self, source=None, **kwargs):
        assert source is not None, 'The `source` argument is required.'
        self.source = source
        super(AuthorizedCharField, self).__init__(source=self.source, **kwargs)

    def get_attribute(self, obj):
        user = self.context['request'].user
        auth = auth_core.Auth(user)
        field_source_method = getattr(obj, self.source)
        return field_source_method(auth=auth)

class RelationshipField(ser.HyperlinkedIdentityField):
    """
    RelationshipField that permits the return of both self and related links, along with optional
    meta information. ::

        children = RelationshipField(
            related_view='nodes:node-children',
            related_view_kwargs={'node_id': '<pk>'},
            self_view='nodes:node-node-children-relationship',
            self_view_kwargs={'node_id': '<pk>'},
            related_meta={'count': 'get_node_count'}
        )

    The lookup field must be surrounded in angular brackets to find the attribute on the target. Otherwise, the lookup
    field will be returned verbatim. ::

        wiki_home = RelationshipField(
            related_view='addon:addon-detail',
            related_view_kwargs={'node_id': '<_id>', 'provider': 'wiki'},
        )

    '_id' is enclosed in angular brackets, but 'wiki' is not. 'id' will be looked up on the target, but 'wiki' will not.
     The serialized result would be '/nodes/abc12/addons/wiki'.

    Field can handle nested attributes: ::

        wiki_home = RelationshipField(
            related_view='wiki:wiki-detail',
            related_view_kwargs={'node_id': '<_id>', 'wiki_id': '<wiki_pages_current.home>'}
        )

    Field can handle a filter_key, which operates as the source field (but
    is named differently to not interfere with HyperLinkedIdentifyField's source

    The ``filter_key`` argument defines the Mongo key (or ODM field name) to filter on
    when using the ``FilterMixin`` on a view. ::

        parent = RelationshipField(
            related_view='nodes:node-detail',
            related_view_kwargs={'node_id': '<parent_node._id>'},
            filter_key='parent_node'
        )

    Field can include optional filters:

    Example:
    replies = RelationshipField(
        self_view='nodes:node-comments',
        self_view_kwargs={'node_id': '<node._id>'},
        filter={'target': '<pk>'})
    )
    """
    json_api_link = True  # serializes to a links object

    def __init__(self, related_view=None, related_view_kwargs=None, self_view=None, self_view_kwargs=None,
                 self_meta=None, related_meta=None, always_embed=False, filter=None, filter_key=None, **kwargs):
        related_view = related_view
        self_view = self_view
        related_kwargs = related_view_kwargs
        self_kwargs = self_view_kwargs
        self.views = {'related': related_view, 'self': self_view}
        self.view_kwargs = {'related': related_kwargs, 'self': self_kwargs}
        self.related_meta = related_meta
        self.self_meta = self_meta
        self.always_embed = always_embed
        self.filter = filter
        self.filter_key = filter_key

        assert (related_view is not None or self_view is not None), 'Self or related view must be specified.'
        if related_view:
            assert related_kwargs is not None, 'Must provide related view kwargs.'
            if not callable(related_kwargs):
                assert isinstance(related_kwargs,
                                  dict), "Related view kwargs must have format {'lookup_url_kwarg: lookup_field}."
        if self_view:
            assert self_kwargs is not None, 'Must provide self view kwargs.'
            assert isinstance(self_kwargs, dict), "Self view kwargs must have format {'lookup_url_kwarg: lookup_field}."

        view_name = related_view
        if view_name:
            lookup_kwargs = related_kwargs
        else:
            view_name = self_view
            lookup_kwargs = self_kwargs
        if kwargs.get('lookup_url_kwarg', None):
            lookup_kwargs = kwargs.pop('lookup_url_kwarg')
        super(RelationshipField, self).__init__(view_name, lookup_url_kwarg=lookup_kwargs, **kwargs)

        # Allow a RelationshipField to be modified if explicitly set so
        if kwargs.get('read_only') is not None:
            self.read_only = kwargs['read_only']

    def resolve(self, resource, field_name):
        """
        Resolves the view when embedding.
        """
        lookup_url_kwarg = self.lookup_url_kwarg
        if callable(lookup_url_kwarg):
            lookup_url_kwarg = lookup_url_kwarg(getattr(resource, field_name))

        kwargs = {attr_name: self.lookup_attribute(resource, attr) for (attr_name, attr) in
                  lookup_url_kwarg.items()}
        view = self.view_name
        if callable(self.view_name):
            view = view(getattr(resource, field_name))
        return resolve(
            reverse(
                view,
                kwargs=kwargs
            )
        )

    def process_related_counts_parameters(self, params, value):
        """
        Processes related_counts parameter.

        Can either be a True/False value for fetching counts on all fields, or a comma-separated list for specifying
        individual fields.  Ensures field for which we are requesting counts is a relationship field.
        """
        if utils.is_truthy(params) or utils.is_falsy(params):
            return params

        field_counts_requested = [val for val in params.split(',')]

        countable_fields = {field for field in self.parent.fields if
                            getattr(self.parent.fields[field], 'json_api_link', False) or
                            getattr(getattr(self.parent.fields[field], 'field', None), 'json_api_link', None)}
        for count_field in field_counts_requested:
            # Some fields will hide relationships, e.g. HideIfWithdrawal
            # Ignore related_counts for these fields
            fetched_field = self.parent.fields.get(count_field)

            hidden = fetched_field and isinstance(fetched_field, HideIfWithdrawal) and getattr(value, 'is_retracted', False)

            if not hidden and count_field not in countable_fields:
                raise InvalidQueryStringError(
                    detail="Acceptable values for the related_counts query param are 'true', 'false', or any of the relationship fields; got '{0}'".format(
                        params),
                    parameter='related_counts'
                )
        return field_counts_requested

    def get_meta_information(self, meta_data, value):
        """
        For retrieving meta values, otherwise returns {}
        """
        meta = {}
        for key in meta_data or {}:
            if key == 'count' or key == 'unread':
                show_related_counts = self.context['request'].query_params.get('related_counts', False)
                if self.context['request'].parser_context.get('kwargs'):
                    if self.context['request'].parser_context['kwargs'].get('is_embedded'):
                        show_related_counts = False
                field_counts_requested = self.process_related_counts_parameters(show_related_counts, value)

                if utils.is_truthy(show_related_counts):
                    meta[key] = website_utils.rapply(meta_data[key], _url_val, obj=value, serializer=self.parent)
                elif utils.is_falsy(show_related_counts):
                    continue
                elif self.field_name in field_counts_requested:
                    meta[key] = website_utils.rapply(meta_data[key], _url_val, obj=value, serializer=self.parent)
                else:
                    continue
            elif key == 'projects_in_common':
                if not get_user_auth(self.context['request']).user:
                    continue
                if not self.context['request'].query_params.get('show_projects_in_common', False):
                    continue
                meta[key] = website_utils.rapply(meta_data[key], _url_val, obj=value, serializer=self.parent)
            else:
                meta[key] = website_utils.rapply(meta_data[key], _url_val, obj=value, serializer=self.parent)
        return meta

    def lookup_attribute(self, obj, lookup_field):
        """
        Returns attribute from target object unless attribute surrounded in angular brackets where it returns the lookup field.

        Also handles the lookup of nested attributes.
        """
        bracket_check = _tpl(lookup_field)
        if bracket_check:
            source_attrs = bracket_check.split('.')
            # If you are using a nested attribute for lookup, and you get the attribute wrong, you will not get an
            # error message, you will just not see that field. This allows us to have slightly more dynamic use of
            # nested attributes in relationship fields.
            try:
                return_val = get_nested_attributes(obj, source_attrs)
            except KeyError:
                return None
            return return_val

        return lookup_field

    def kwargs_lookup(self, obj, kwargs_dict):
        """
        For returning kwargs dictionary of format {"lookup_url_kwarg": lookup_value}
        """
        if callable(kwargs_dict):
            kwargs_dict = kwargs_dict(obj)

        kwargs_retrieval = {}
        for lookup_url_kwarg, lookup_field in kwargs_dict.items():
            try:
                lookup_value = self.lookup_attribute(obj, lookup_field)
            except AttributeError as exc:
                raise AssertionError(exc)
            if lookup_value is None:
                return None
            kwargs_retrieval[lookup_url_kwarg] = lookup_value
        return kwargs_retrieval

    # Overrides HyperlinkedIdentityField
    def get_url(self, obj, view_name, request, format):
        urls = {}
        for view_name, view in self.views.items():
            if view is None:
                urls[view_name] = {}
            else:
                kwargs = self.kwargs_lookup(obj, self.view_kwargs[view_name])
                if kwargs is None:
                    urls[view_name] = {}
                else:
                    if callable(view):
                        view = view(getattr(obj, self.field_name))
                    url = self.reverse(view, kwargs=kwargs, request=request, format=format)
                    if self.filter:
                        formatted_filter = self.format_filter(obj)
                        if formatted_filter:
                            url = extend_querystring_params(url, {'filter': formatted_filter})
                        else:
                            url = None

                    url = extend_querystring_if_key_exists(url, self.context['request'], 'view_only')
                    urls[view_name] = url

        if not urls['self'] and not urls['related']:
            urls = None
        return urls

    def to_esi_representation(self, value, envelope='data'):
        relationships = self.to_representation(value)
        try:
            href = relationships['links']['related']['href']
        except KeyError:
            raise SkipField
        else:
            if href and not href == '{}':
                if self.always_embed:
                    envelope = 'data'
                query_dict = dict(format=['jsonapi', ], envelope=[envelope, ])
                if 'view_only' in self.parent.context['request'].query_params.keys():
                    query_dict.update(view_only=[self.parent.context['request'].query_params['view_only']])
                esi_url = extend_querystring_params(href, query_dict)
                return '<esi:include src="{}"/>'.format(esi_url)

    def format_filter(self, obj):
        qd = QueryDict(mutable=True)
        filter_fields = self.filter.keys()
        for field_name in filter_fields:
            try:
                # check if serializer method passed in
                serializer_method = getattr(self.parent, self.filter[field_name])
            except AttributeError:
                value = self.lookup_attribute(obj, self.filter[field_name])
            else:
                value = serializer_method(obj)
            if not value:
                continue
            qd.update({'[{}]'.format(field_name): value})
        if not qd.keys():
            return None
        return qd.urlencode(safe=['[', ']'])

    # Overrides HyperlinkedIdentityField
    def to_representation(self, value):
        request = self.context.get('request', None)
        format = self.context.get('format', None)

        assert request is not None, (
            '`%s` requires the request in the serializer'
            " context. Add `context={'request': request}` when instantiating "
            'the serializer.' % self.__class__.__name__
        )

        # By default use whatever format is given for the current context
        # unless the target is a different type to the source.
        #
        # Eg. Consider a HyperlinkedIdentityField pointing from a json
        # representation to an html property of that representation...
        #
        # '/snippets/1/' should link to '/snippets/1/highlight/'
        # ...but...
        # '/snippets/1/.json' should link to '/snippets/1/highlight/.html'
        if format and self.format and self.format != format:
            format = self.format

        # Return the hyperlink, or error if incorrectly configured.
        try:
            url = self.get_url(value, self.view_name, request, format)
        except NoReverseMatch:
            msg = (
                'Could not resolve URL for hyperlinked relationship using '
                'view name "%s". You may have failed to include the related '
                'model in your API, or incorrectly configured the '
                '`lookup_field` attribute on this field.'
            )
            if value in ('', None):
                value_string = {'': 'the empty string', None: 'None'}[value]
                msg += (
                    ' WARNING: The value of the field on the model instance '
                    "was %s, which may be why it didn't match any "
                    'entries in your URL conf.' % value_string
                )
            raise ImproperlyConfigured(msg % self.view_name)

        if url is None:
            raise SkipField

        related_url = url['related']
        related_meta = self.get_meta_information(self.related_meta, value)
        self_url = url['self']
        self_meta = self.get_meta_information(self.self_meta, value)
        return format_relationship_links(related_url, self_url, related_meta, self_meta)

class FileCommentRelationshipField(RelationshipField):
    def get_url(self, obj, view_name, request, format):
        if obj.kind == 'folder':
            raise SkipField
        return super(FileCommentRelationshipField, self).get_url(obj, view_name, request, format)

class TargetField(ser.Field):
    """
    Field that returns a nested dict with the url (constructed based
    on the object's type), optional meta information, and link_type.

    Example:

        target = TargetField(link_type='related', meta={'type': 'get_target_type'})

    """
    json_api_link = True  # serializes to a links object
    view_map = {
        'node': {
            'view': 'nodes:node-detail',
            'lookup_kwarg': 'node_id'
        },
        'comment': {
            'view': 'comments:comment-detail',
            'lookup_kwarg': 'comment_id'
        },
        'nodewikipage': {
            'view': None,
            'lookup_kwarg': None
        }
    }

    def __init__(self, **kwargs):
        self.meta = kwargs.pop('meta', {})
        self.link_type = kwargs.pop('link_type', 'url')
        super(TargetField, self).__init__(read_only=True, **kwargs)

    def resolve(self, resource, field_name):
        """
        Resolves the view for target node or target comment when embedding.
        """
        view_info = self.view_map.get(resource.target.referent._name, None)
        if not view_info:
            raise TargetNotSupportedError('{} is not a supported target type'.format(
                resource.target._name
            ))
        if not view_info['view']:
            return None, None, None
        embed_value = resource.target._id

        kwargs = {view_info['lookup_kwarg']: embed_value}
        return resolve(
            reverse(
                view_info['view'],
                kwargs=kwargs
            )
        )

    def to_esi_representation(self, value, envelope='data'):
        href = value.get_absolute_url()

        if href:
            esi_url = extend_querystring_params(href, dict(envelope=[envelope, ], format=['jsonapi', ]))
            return '<esi:include src="{}"/>'.format(esi_url)
        return self.to_representation(value)

    def to_representation(self, value):
        """
        Returns nested dictionary in format {'links': {'self.link_type': ... }

        If no meta information, self.link_type is equal to a string containing link's URL.  Otherwise,
        the link is represented as a links object with 'href' and 'meta' members.
        """
        meta = website_utils.rapply(self.meta, _url_val, obj=value, serializer=self.parent)
        return {'links': {self.link_type: {'href': value.referent.get_absolute_url(), 'meta': meta}}}

class LinksField(ser.Field):
    """Links field that resolves to a links object. Used in conjunction with `Link`.
    If the object to be serialized implements `get_absolute_url`, then the return value
    of that method is used for the `self` link.

    Example: ::

        links = LinksField({
            'html': 'absolute_url',
            'children': {
                'related': Link('nodes:node-children', node_id='<pk>'),
                'count': 'get_node_count'
            },
            'contributors': {
                'related': Link('nodes:node-contributors', node_id='<pk>'),
                'count': 'get_contrib_count'
            },
            'registrations': {
                'related': Link('nodes:node-registrations', node_id='<pk>'),
                'count': 'get_registration_count'
            },
        })
    """

    def __init__(self, links, *args, **kwargs):
        ser.Field.__init__(self, read_only=True, *args, **kwargs)
        self.links = links

    def get_attribute(self, obj):
        # We pass the object instance onto `to_representation`,
        # not just the field attribute.
        return obj

    def extend_absolute_url(self, obj):
        return extend_querystring_if_key_exists(obj.get_absolute_url(), self.context['request'], 'view_only')

    def to_representation(self, obj):
        ret = {}
        for name, value in self.links.iteritems():
            try:
                url = _url_val(value, obj=obj, serializer=self.parent)
            except SkipField:
                continue
            else:
                ret[name] = url
        if hasattr(obj, 'get_absolute_url') and 'self' not in self.links:
            ret['self'] = self.extend_absolute_url(obj)
        return ret

_tpl_pattern = re.compile(r'\s*<\s*(\S*)\s*>\s*')

def _tpl(val):
    """Return value within ``< >`` if possible, else return ``None``."""
    match = _tpl_pattern.match(val)
    if match:
        return match.groups()[0]
    return None

def _get_attr_from_tpl(attr_tpl, obj):
    attr_name = _tpl(str(attr_tpl))
    if attr_name:
        attribute_value = obj
        for attr_segment in attr_name.split('.'):
            attribute_value = getattr(attribute_value, attr_segment, ser.empty)
        if attribute_value is not ser.empty:
            return attribute_value
        elif attr_name in obj:
            return obj[attr_name]
        else:
            raise AttributeError(
                '{attr_name!r} is not a valid '
                'attribute of {obj!r}'.format(
                    attr_name=attr_name, obj=obj,
                ))
    else:
        return attr_tpl

# TODO: Make this a Field that is usable on its own?
class Link(object):
    """Link object to use in conjunction with Links field. Does reverse lookup of
    URLs given an endpoint name and attributed enclosed in `<>`. This includes
    complex key strings like 'user.id'
    """

    def __init__(self, endpoint, args=None, kwargs=None, query_kwargs=None, **kw):
        self.endpoint = endpoint
        self.kwargs = kwargs or {}
        self.args = args or tuple()
        self.reverse_kwargs = kw
        self.query_kwargs = query_kwargs or {}

    def resolve_url(self, obj):
        kwarg_values = {key: _get_attr_from_tpl(attr_tpl, obj) for key, attr_tpl in self.kwargs.items()}
        arg_values = [_get_attr_from_tpl(attr_tpl, obj) for attr_tpl in self.args]
        query_kwarg_values = {key: _get_attr_from_tpl(attr_tpl, obj) for key, attr_tpl in self.query_kwargs.items()}
        # Presumably, if you have are expecting a value but the value is empty, then the link is invalid.
        for item in kwarg_values:
            if kwarg_values[item] is None:
                raise SkipField
        return utils.absolute_reverse(
            self.endpoint,
            args=arg_values,
            kwargs=kwarg_values,
            query_kwargs=query_kwarg_values,
            **self.reverse_kwargs
        )

class WaterbutlerLink(Link):
    """Link object to use in conjunction with Links field. Builds a Waterbutler URL for files.
    """

    def __init__(self, must_be_file=None, must_be_folder=None, **kwargs):
        self.kwargs = kwargs
        self.must_be_file = must_be_file
        self.must_be_folder = must_be_folder

    def resolve_url(self, obj):
        """Reverse URL lookup for WaterButler routes
        """
        if self.must_be_folder is True and not obj.path.endswith('/'):
            raise SkipField
        if self.must_be_file is True and obj.path.endswith('/'):
            raise SkipField
        url = website_utils.waterbutler_api_url_for(obj.node._id, obj.provider, obj.path, **self.kwargs)
        if not url:
            raise SkipField
        else:
            return url

class NodeFileHyperLinkField(RelationshipField):
    def __init__(self, kind=None, never_embed=False, **kws):
        self.kind = kind
        self.never_embed = never_embed
        super(NodeFileHyperLinkField, self).__init__(**kws)

    def get_url(self, obj, view_name, request, format):
        if self.kind and obj.kind != self.kind:
            raise SkipField
        return super(NodeFileHyperLinkField, self).get_url(obj, view_name, request, format)

class JSONAPIListSerializer(ser.ListSerializer):
    def to_representation(self, data):
        enable_esi = self.context.get('enable_esi', False)
        envelope = self.context.update({'envelope': None})
        # Don't envelope when serializing collection
        errors = {}
        bulk_skip_uneditable = utils.is_truthy(self.context['request'].query_params.get('skip_uneditable', False))

        if isinstance(data, collections.Mapping):
            errors = data.get('errors', None)
            data = data.get('data', None)
        if enable_esi:
            ret = [
                self.child.to_esi_representation(item, envelope=None) for item in data
            ]
        else:
            ret = [
                self.child.to_representation(item, envelope=envelope) for item in data
            ]

        if errors and bulk_skip_uneditable:
            ret.append({'errors': errors})

        return ret

    # Overrides ListSerializer which doesn't support multiple update by default
    def update(self, instance, validated_data):

        # avoiding circular import
        from api.nodes.serializers import ContributorIDField

        # if PATCH request, the child serializer's partial attribute needs to be True
        if self.context['request'].method == 'PATCH':
            self.child.partial = True

        bulk_skip_uneditable = utils.is_truthy(self.context['request'].query_params.get('skip_uneditable', False))
        if not bulk_skip_uneditable:
            if len(instance) != len(validated_data):
                raise exceptions.ValidationError({'non_field_errors': 'Could not find all objects to update.'})

        id_lookup = self.child.fields['id'].source
        data_mapping = {item.get(id_lookup): item for item in validated_data}

        if isinstance(self.child.fields['id'], ContributorIDField):
            instance_mapping = {self.child.fields['id'].get_id(item): item for item in instance}
        else:
            instance_mapping = {getattr(item, id_lookup): item for item in instance}

        ret = {'data': []}

        for resource_id, resource in instance_mapping.items():
            data = data_mapping.pop(resource_id, None)
            ret['data'].append(self.child.update(resource, data))

        # If skip_uneditable in request, add validated_data for nodes in which the user did not have edit permissions to errors
        if data_mapping and bulk_skip_uneditable:
            ret.update({'errors': data_mapping.values()})
        return ret

    # overrides ListSerializer
    def run_validation(self, data):
        meta = getattr(self, 'Meta', None)
        bulk_limit = getattr(meta, 'bulk_limit', BULK_SETTINGS['DEFAULT_BULK_LIMIT'])

        num_items = len(data)

        if num_items > bulk_limit:
            raise JSONAPIException(source={'pointer': '/data'},
                                   detail='Bulk operation limit is {}, got {}.'.format(bulk_limit, num_items))

        return super(JSONAPIListSerializer, self).run_validation(data)

    # overrides ListSerializer: Add HTML-sanitization similar to that used by APIv1 front-end views
    def is_valid(self, clean_html=True, **kwargs):
        """
        After validation, scrub HTML from validated_data prior to saving (for create and update views)

        Exclude 'type' from validated_data.

        """
        ret = super(JSONAPIListSerializer, self).is_valid(**kwargs)

        if clean_html is True:
            self._validated_data = website_utils.rapply(self.validated_data, strip_html)

        for data in self._validated_data:
            data.pop('type', None)

        return ret

class JSONAPISerializer(ser.Serializer):
    """Base serializer. Requires that a `type_` option is set on `class Meta`. Also
    allows for enveloping of both single resources and collections.  Looks to nest fields
    according to JSON API spec. Relational fields must set json_api_link=True flag.
    Self/html links must be nested under "links".
    """

    # Don't serialize relationships that use these views
    # when viewing thru an anonymous VOL
    views_to_hide_if_anonymous = {
        'users:user-detail',
        'nodes:node-registrations',
    }

    # overrides Serializer
    @classmethod
    def many_init(cls, *args, **kwargs):
        kwargs['child'] = cls()
        return JSONAPIListSerializer(*args, **kwargs)

    def invalid_embeds(self, fields, embeds):
        fields_check = fields[:]
        for index, field in enumerate(fields_check):
            if getattr(field, 'field', None):
                fields_check[index] = field.field
        invalid_embeds = set(embeds.keys()) - set(
            [f.field_name for f in fields_check if getattr(f, 'json_api_link', False)])
        return invalid_embeds

    def to_esi_representation(self, data, envelope='data'):
        href = None
        query_params_blacklist = ['page[size]']
        href = self.get_absolute_url(data)
        if href and href != '{}':
            esi_url = furl.furl(href).add(args=dict(self.context['request'].query_params)).remove(
                args=query_params_blacklist).remove(args=['envelope']).add(args={'envelope': envelope}).url
            return '<esi:include src="{}"/>'.format(esi_url)
        # failsafe, let python do it if something bad happened in the ESI construction
        return super(JSONAPISerializer, self).to_representation(data)

    # overrides Serializer
    def to_representation(self, obj, envelope='data'):
        """Serialize to final representation.

        :param obj: Object to be serialized.
        :param envelope: Key for resource object.
        """
        ret = {}
        meta = getattr(self, 'Meta', None)
        type_ = getattr(meta, 'type_', None)
        assert type_ is not None, 'Must define Meta.type_'

        data = {
            'id': '',
            'type': type_,
            'attributes': {},
            'relationships': {},
            'embeds': {},
            'links': {},
        }

        embeds = self.context.get('embed', {})
        context_envelope = self.context.get('envelope', envelope)
        if context_envelope == 'None':
            context_envelope = None
        enable_esi = self.context.get('enable_esi', False)
        is_anonymous = is_anonymized(self.context['request'])
        to_be_removed = set()
        if is_anonymous and hasattr(self, 'non_anonymized_fields'):
            # Drop any fields that are not specified in the `non_anonymized_fields` variable.
            allowed = set(self.non_anonymized_fields)
            existing = set(self.fields.keys())
            to_be_removed = existing - allowed

        fields = [field for field in self.fields.values() if
                  not field.write_only and field.field_name not in to_be_removed]

        invalid_embeds = self.invalid_embeds(fields, embeds)
        invalid_embeds = invalid_embeds - to_be_removed
        if invalid_embeds:
            raise InvalidQueryStringError(parameter='embed',
                                          detail='The following fields are not embeddable: {}'.format(
                                              ', '.join(invalid_embeds)))

        for field in fields:
            try:
                attribute = field.get_attribute(obj)
            except SkipField:
                continue

            nested_field = getattr(field, 'field', None)
            if attribute is None:
                # We skip `to_representation` for `None` values so that
                # fields do not have to explicitly deal with that case.
                data['attributes'][field.field_name] = None
            else:
                try:
                    representation = field.to_representation(attribute)
                except SkipField:
                    continue
                if getattr(field, 'json_api_link', False) or getattr(nested_field, 'json_api_link', False):
                    # If embed=field_name is appended to the query string or 'always_embed' flag is True, directly embed the
                    # results in addition to adding a relationship link
                    if embeds and (field.field_name in embeds or getattr(field, 'always_embed', None)):
                        if enable_esi:
                            try:
                                result = field.to_esi_representation(attribute, envelope=envelope)
                            except SkipField:
                                continue
                        else:
                            try:
                                # If a field has an empty representation, it should not be embedded.
                                result = self.context['embed'][field.field_name](obj)
                            except SkipField:
                                result = None

                        if result:
                            data['embeds'][field.field_name] = result
                        else:
                            data['embeds'][field.field_name] = {'error': 'This field is not embeddable.'}
                    try:
                        if not (is_anonymous and
                                    hasattr(field, 'view_name') and
                                        field.view_name in self.views_to_hide_if_anonymous):
                            data['relationships'][field.field_name] = representation
                    except SkipField:
                        continue
                elif field.field_name == 'id':
                    data['id'] = representation
                elif field.field_name == 'links':
                    data['links'] = representation
                else:
                    data['attributes'][field.field_name] = representation

        if not data['relationships']:
            del data['relationships']

        if not data['embeds']:
            del data['embeds']

        if context_envelope:
            ret[context_envelope] = data
            if is_anonymous:
                ret['meta'] = {'anonymous': True}
        else:
            ret = data
        return ret

    def get_absolute_url(self, obj):
        raise NotImplementedError()

    def get_absolute_html_url(self, obj):
        return extend_querystring_if_key_exists(obj.absolute_url, self.context['request'], 'view_only')

    # overrides Serializer: Add HTML-sanitization similar to that used by APIv1 front-end views
    def is_valid(self, clean_html=True, **kwargs):
        """
        After validation, scrub HTML from validated_data prior to saving (for create and update views)

        Exclude 'type' and '_id' from validated_data.

        """
        ret = super(JSONAPISerializer, self).is_valid(**kwargs)

        if clean_html is True:
            self._validated_data = self.sanitize_data()

        self._validated_data.pop('type', None)
        self._validated_data.pop('target_type', None)

        if self.context['request'].method in utils.UPDATE_METHODS:
            self._validated_data.pop('_id', None)

        return ret

    def sanitize_data(self):
        return website_utils.rapply(self.validated_data, strip_html)

class JSONAPIRelationshipSerializer(ser.Serializer):
    """Base Relationship serializer. Requires that a `type_` option is set on `class Meta`.
    Provides a simplified serialization of the relationship, allowing for simple update request
    bodies.
    """
    id = ser.CharField(required=False, allow_null=True)
    type = TypeField(required=False, allow_null=True)

    def to_representation(self, obj):
        meta = getattr(self, 'Meta', None)
        type_ = getattr(meta, 'type_', None)
        assert type_ is not None, 'Must define Meta.type_'
        relation_id_field = self.fields['id']
        attribute = relation_id_field.get_attribute(obj)
        relationship = relation_id_field.to_representation(attribute)

        data = {'type': type_, 'id': relationship} if relationship else None

        return data

def DevOnly(field):
    """Make a field only active in ``DEV_MODE``. ::

        experimental_field = DevMode(CharField(required=False))
    """
    return field if settings.DEV_MODE else None

class RestrictedDictSerializer(ser.Serializer):
    def to_representation(self, obj):
        data = {}
        fields = [field for field in self.fields.values() if not field.write_only]

        for field in fields:
            try:
                attribute = field.get_attribute(obj)
            except ser.SkipField:
                continue

            if attribute is None:
                # We skip `to_representation` for `None` values so that
                # fields do not have to explicitly deal with that case.
                data[field.field_name] = None
            else:
                data[field.field_name] = field.to_representation(attribute)
        return data

def relationship_diff(current_items, new_items):
    """
    To be used in POST and PUT/PATCH relationship requests, as, by JSON API specs,
    in update requests, the 'remove' items' relationships would be deleted, and the
    'add' would be added, while for create requests, only the 'add' would be added.

    :param current_items: The current items in the relationship
    :param new_items: The items passed in the request
    :return:
    """

    return {
        'add': {k: new_items[k] for k in (set(new_items.keys()) - set(current_items.keys()))},
        'remove': {k: current_items[k] for k in (set(current_items.keys()) - set(new_items.keys()))}
    }

class AddonAccountSerializer(JSONAPISerializer):
    id = ser.CharField(source='_id', read_only=True)
    provider = ser.CharField(read_only=True)
    profile_url = ser.CharField(required=False, read_only=True)
    display_name = ser.CharField(required=False, read_only=True)

    links = links = LinksField({
        'self': 'get_absolute_url',
    })

    class Meta:
        type_ = 'external_accounts'

    def get_absolute_url(self, obj):
        kwargs = self.context['request'].parser_context['kwargs']
        kwargs.update({'account_id': obj._id})
        return absolute_reverse(
            'users:user-external_account-detail',
            kwargs=kwargs
        )
        return obj.get_absolute_url()

class LinkedNode(JSONAPIRelationshipSerializer):
    id = ser.CharField(source='node._id', required=False, allow_null=True)

    class Meta:
        type_ = 'linked_nodes'

class LinkedNodesRelationshipSerializer(ser.Serializer):

    data = ser.ListField(child=LinkedNode())
    links = LinksField({'self': 'get_self_url',
                        'html': 'get_related_url'})

    def get_self_url(self, obj):
        return obj['self'].linked_nodes_self_url

    def get_related_url(self, obj):
        return obj['self'].linked_nodes_related_url

    class Meta:
        type_ = 'linked_nodes'

    def get_pointers_to_add_remove(self, pointers, new_pointers):
        diff = relationship_diff(
            current_items={pointer.node._id: pointer for pointer in pointers},
            new_items={val['node']['_id']: val for val in new_pointers}
        )

        nodes_to_add = []
        for node_id in diff['add']:
            node = Node.load(node_id)
            if not node:
                raise exceptions.NotFound(detail='Node with id "{}" was not found'.format(node_id))
            nodes_to_add.append(node)

        return nodes_to_add, diff['remove'].values()

    def make_instance_obj(self, obj):
        # Convenience method to format instance based on view's get_object
        return {'data': [
            pointer for pointer in
            obj.nodes_pointer
            if not pointer.node.is_deleted and not pointer.node.is_collection
        ], 'self': obj}

    def update(self, instance, validated_data):
        collection = instance['self']
        auth = utils.get_user_auth(self.context['request'])

        add, remove = self.get_pointers_to_add_remove(pointers=instance['data'], new_pointers=validated_data['data'])

        for pointer in remove:
            collection.rm_pointer(pointer, auth)
        for node in add:
            collection.add_pointer(node, auth)

        return self.make_instance_obj(collection)

    def create(self, validated_data):
        instance = self.context['view'].get_object()
        auth = utils.get_user_auth(self.context['request'])
        collection = instance['self']

        add, remove = self.get_pointers_to_add_remove(pointers=instance['data'], new_pointers=validated_data['data'])

        if not len(add):
            raise RelationshipPostMakesNoChanges

        for node in add:
            collection.add_pointer(node, auth)

        return self.make_instance_obj(collection)


#   !/usr/bin/env python3

#   -*- coding: utf-8 -*-

#   Created on 2019-05-25  10:00

import re

"""
    验证邮箱地址
"""

def verification(str):
    re_str = re.compile('^[\w-]+(\.[\w-]+)*@[\w-]+(\.[\w-]+)+$')
    result = re.match(re_str, str)
    print(result)

    if result is None:
        return "error"
    else:
        return 'ok'

if __name__ == '__main__':
    s = verification('wpf1011467276@gmail.com')
    print(s)


import random
from collections import defaultdict

import torch
from ltp import LTP

from .base_func import BaseFunc

class NerFunc(BaseFunc):
    def __init__(self, config):
        super(NerFunc, self).__init__(config)
        self.augment_num = config.ner_func.augment_num
        self.combine_dict = self.load_ner_files()
        self.model = LTP(config.ner_func.ner_tool_name)
        if torch.cuda.is_available():
            self.model.to("cuda")

    @staticmethod
    def load_ner_files():
        combine_dict = defaultdict(set)
        # Nh file
        for line in open("files/ner/people_name.txt", "r", encoding="utf-8"):
            combine_dict["Nh"].add(line.strip())
        # Ns file
        for line in open("files/ner/place_name.txt", "r", encoding="utf-8"):
            combine_dict["Ns"].add(line.strip())
        # Ni file
        for line in open("files/ner/company_name.txt", "r", encoding="utf-8"):
            combine_dict["Ni"].add(line.strip())
        return combine_dict

    def process(self, sentence):
        final_augment_sentence = []
        seg_list = self.cut_words(sentence)
        result = self.model.pipeline(seg_list, tasks=["ner"])
        if len(result.ner) == 0:
            return final_augment_sentence
        for _ in range(self.augment_num):
            n, word = random.choice(result.ner)
            if n in self.combine_dict.keys():
                new_word = random.choice(list(self.combine_dict[n]))
                old_index = seg_list.index(word)
                seg_list[old_index] = new_word
                new_sentence = ''.join(seg_list)
                final_augment_sentence.append(new_sentence)

        return final_augment_sentence