#!/usr/bin/env python3

import sys
import datetime
import re
import os
import time
import json
import csv
import tempfile
import zipfile
import argparse

import matplotlib.pyplot as plt
import matplotlib.dates as mdates

hmms_pattern = re.compile(
r"""
^(
(?P<L>L)|
(?P<yup>yup|yep)|
(?P<heh>he(h+))|
(?P<huh>hu(h+))|
(?P<why>wh(y+))|
(?P<what>wh(a+)(t*))|
(?P<mhm>(m+)h(m+))|
(?P<hmm>h(m+))|
(?P<lol>l(o+)l)|
(?P<lmao>lm(f?)(a+)(o+))|
(?P<yes>ye(s*))|
(?P<no>n(o+))|
(?P<oh>(o+)(h*))
)$
""", re.IGNORECASE | re.MULTILINE | re.VERBOSE)

def err(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)
    sys.exit(1)

def find_hmms(pattern, string, hmms_dict):
    results = pattern.finditer(string)
    for found in results:
        hmm_found = None
        for name, value in found.groupdict().items():
            if value is not None:
                hmm_found = name

        if hmm_found is None:
            continue

        if hmm_found not in hmms_dict:
            hmms_dict[hmm_found] = 0
        hmms_dict[hmm_found] += 1

def clean(string):
    return "".join([i if i.isalnum() else "_" for i in string])

def word_clean(word, prefix):
    return "i_%s_%s" % (clean(prefix.upper()), clean(word))

def compile_words(words, prefix):
    buf = ""
    for i in words:
        print(i)
        buf += "(?P<%s>%s)|" % (word_clean(i, prefix), i)
    buf = buf[:-1]
    print(buf)
    return re.compile(buf, re.IGNORECASE | re.MULTILINE | re.VERBOSE)

def get_dms(path):
    try:
        with open(os.path.join(path, "index.json"), encoding="utf-8") as f:
            names = json.load(f)
    except Exception:
        err("Could not find %s" % os.path.join(path, "index.json"))

    # generate list of dms
    #     channel id : name
    print("Collecting Names")
    dms = {}
    for i in os.listdir(path):
        if os.path.isdir(os.path.join(path, i)):
            with open(os.path.join(path, i, "channel.json"), encoding="utf-8") as file:
                if json.load(file)["type"] == 1:
                    name = names[i[1:]]
                    startstr = "Direct Message with "
                    dms[i] = name[len(startstr):] if name.startswith(startstr) else name
    return dms

def read(path, args):
    path = os.path.join(path, "messages/")
    print("Finding DMs")
    dms = get_dms(path)
    if args.users: # if we are selecting by users find them
        args.users = [i.lower() for i in args.users] # to make it not case sensetive
        selected = dict(filter(lambda u: any_in(args.users, u[1].lower()), dms.items()))
        if selected:
            dms = selected
        else:
            print(f"Can't find a user named {args.users}, showing all", file=sys.stderr)

    # Read contents of DMs
    print("Reading DMs")
    leaders = {}

    for directory, username in dms.items():
        with open(os.path.join(path, directory, "messages.csv"), encoding="utf-8") as f:
            timestamp = []
            hmms = []
            msgs = csv.reader(f)
            next(msgs) # skip the header
            msgs = reversed(list(msgs))
            pattern = hmms_pattern if args.words is None else compile_words(args.words, username.split('#')[0])
            hmms_dict = {}

            for line in msgs:
                timestamp.append(datetime.datetime.fromisoformat(line[1]))
                msg_content = line[2]
                find_hmms(pattern, msg_content, hmms_dict)
                hmms.append(hmms_dict.copy())

            msg_total = [i for i in range(len(timestamp))]
            if len(msg_total) != 0:
                leaders[username] = [timestamp, msg_total, hmms]

    # sort leaders
    leaders = dict(sorted(leaders.items(), key=lambda i: i[1][1]))

    if args.list:
        total_total = 0
        for name, v in leaders.items():
            total_messages = v[1][-1]
            total_total += total_messages
            print(f"{name:<40} {total_messages:>10}")
        name = "Total"
        print(f"{name:<40} {total_total:>10}")
        return

    # use -n and -s args to determine how many users to show assumes the leaders have been sorted
    leaders = {k: leaders[k] for k in list(reversed(list(leaders)))[args.startafter:args.numlines]}

    # find the maximum timestamp and add it to all of the lists to make lines reach the end
    max_timestamp = max([max(v[0]) for _, v in leaders.items() if len(v[0]) != 0])
    for k, v in leaders.items():
        v[0].append(max_timestamp)
        v[1].append(v[1][-1])

    # TODO clean up
    # transpose hmms into lists of counts
    for k, v in leaders.items():
        transposed_hmms = []
        hmms_list_of_dicts = v[2]
        hmm_types = hmms_list_of_dicts[-1].keys()
        for name in hmm_types:
             values = [hmm_dict.get(name, 0) for hmm_dict in hmms_list_of_dicts]
             values.append(values[-1])
             transposed_hmms.append((name, values))

        leaders[k][2] = transposed_hmms

    startafter = args.startafter
    if startafter > len(leaders):
        print(f"Can't start after {startafter} users, you only have {len(leaders)}, starting at 0", file=sys.stderr)
        startafter = 0

    leaders = dict(sorted(leaders.items(), key=lambda i: i[1][1], reverse=True))

    names = [user for user in leaders.keys()]
    print(f"Showing data for user(s): {', '.join(names)}")

    if args.hmms or args.words != None:
        print("AHHHHHHHHH")
        leaders = dict(sorted(leaders.items(), key=lambda i: hack(i), reverse=True))
    for name, data in leaders.items():
        if args.hmms or args.words != None:
#            if len(leaders.items()) > 1:
#                err("Can't show hmms for more than one user, please make your constraints more specific,\nRun with --list to see all users")
            for name, values in sorted(data[2], key=lambda i: i[1][-1], reverse=True):
                plt.plot(data[0], values, "-", label=name)
        else:
            plt.plot(data[0], data[1], "-", label=name)

    # Graph stuff
    print("Plotting")
    plt.legend(loc="upper left")
    #plt.title("Top %d most messaged users over time" % args.numlines)
    plt.ylabel("Messages")
    plt.xlabel("Date")
    plt.xticks(rotation=75)
    plt.tight_layout()
    plt.show()

def hack(i):
    try:
        x = i[1][2][-1][-1][-1]
        return x
    except Exception:
        return 0

def uint(value):
    ivalue = int(value)
    if ivalue <= 0:
        raise argparse.ArgumentTypeError("%s is an invalid, need a positive int value" % value)
    return ivalue

def any_in(lst1, lst2):
    for i in lst1:
        if i in lst2:
            return True
    return False

parser = argparse.ArgumentParser(description="Graph discord messages over time")
# REQUIRED ARGS
parser.add_argument("path", metavar="FILE", type=str, nargs=None, help="The path to your data package in its .zip format")
# SELECTION ARGS
parser.add_argument("-n", dest="numlines", metavar="numlines", type=uint, default=10, help="The top n users to graph")
parser.add_argument("-s", "--skip", dest="startafter", metavar="startafter", type=uint, default=0, help="Skip the first n top users")
parser.add_argument("-u", "--user" , dest="users", metavar="users", type=str, nargs="+", default=None, help="Show only the given users data. Works with --hmms and ---words")
# PROGRAMS
parser.add_argument("-l", "--list" , dest="list", action="store_true", help="List all DMs to stdout and exit")
parser.add_argument("--hmms" , dest="hmms", action="store_true", help="Show statistics for words like hmm, huh, or lol")
parser.add_argument("-w", "--words", nargs="+", dest="words", default=None, help="Show statistics for words of your choice. Supports regex (buggy)")

args = parser.parse_args()
if args.hmms and args.words:
    raise argparse.ArgumentTypeError("-w/--words cannot be used with --hmms")

path = args.path
if not os.path.isfile(path):
    err("No file %s" % path)

zf = zipfile.ZipFile(path, "r")

with tempfile.TemporaryDirectory() as tf:
    print("Extracting")
    zf.extractall(tf)
    # ensure we have ma messages dir
    if "messages" not in os.listdir(tf):
        err("No \"messages\" directory found in file")
    read(tf, args)


"""
Given a binary tree, find its minimum depth.

The minimum depth is the number of nodes along the shortest path from the root node down to the nearest leaf node.

Note: A leaf is a node with no children.

Example:

Given binary tree [3,9,20,null,null,15,7],

    3
   / \
  9  20
    /  \
   15   7
return its minimum depth = 2.
"""
"""
Comments
与直接找最大不同的地方是，有可能出现0的情况，所以要先看是不是为0，
为0就用 left+right+1 因为0不影响结果
不为0就用min

Fast 使用的是非递归的广度优先搜索，先判断（是否非空）再入队列
"""
"""
My
"""
# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, x):
#         self.val = x
#         self.left = None
#         self.right = None

class Solution:

    def minDepth(self, root: TreeNode) -> int:
        if root is None:
            return 0
        left = self.minDepth(root.left)
        right = self.minDepth(root.right)
        return (left * right == 0 and left + right + 1 or min(left, right) + 1)
"""
Fast
"""

class Solution:

    def minDepth(self, root: 'TreeNode') -> 'int':
        if not root:
            return 0
        depth = 0
        queue = [root]
        while queue:
            depth += 1
            for i in range(len(queue)):
                temp = queue.pop(0)
                if temp.left is None and temp.right is None:
                    return depth
                if temp.left:
                    queue.append(temp.left)
                if temp.right:
                    queue.append(temp.right)
        return depth


# Palindrome Check: Write a Python function that checks whether a given word or phrase is a palindrome.

word = "fdfd"

rev=""

for x in word:
  rev = x + rev

# print(word,rev)
if(word==rev):
  print("is this is palindrom")
else :
  print("not palindrome")


from django.urls import path
from .views import *

urlpatterns = [
    path('room/add', room_add),
    path('reserve/add', reserve_add),
    path('room/delete/<int:room_id>', room_delete),
    path('reserve/delete/<int:booking_id>', reserve_delete),
    path('rooms', get_all_room),
    path('reserve', get_all_reserve),
]


import os
import tqdm
import yaml
import torch
import inspect
import datetime
import itertools
import numpy as np
import torch.nn as nn
import torch.optim as optim

from copy import deepcopy
from gym_unity.envs import UnityToGymWrapper
from torch.utils.tensorboard import SummaryWriter
from mlagents_envs.environment import UnityEnvironment
from mlagents_envs.side_channel.engine_configuration_channel import (
    EngineConfigurationChannel
)

from core import TD3Buffer, get_action
from agents import TD3ActorCritic

class TD3:
    def __init__(
        self, ac, env, epochs=100, steps_per_epoch=4000, batch_size=32,
        lr=1e-4, polyak=0.995, start_steps=10000, act_noise=0.1, target_noise=0.2,
        noise_clip=0.5, gamma=0.99, max_ep_len=1000, update_after=1000, update_every=50,
        policy_delay=2, model_path=None, config=None
    ):
        self.ac = ac
        self.env = env
        self.epochs = epochs
        self.steps_per_epoch = steps_per_epoch
        self.batch_size = batch_size
        self.lr = lr
        self.polyak = polyak
        self.start_steps = start_steps
        self.act_noise = act_noise
        self.target_noise = target_noise
        self.noise_clip = noise_clip
        self.gamma = gamma
        self.policy_delay = policy_delay
        self.max_ep_len = max_ep_len
        self.update_after = update_after
        self.update_every = update_every

        self.ac_targ = deepcopy(self.ac)
        for p in self.ac_targ.parameters():
            p.requires_grad = False

        if model_path is None:
            # New Model
            now = datetime.datetime.now()
            now_str = now.strftime("%Y%m%d_%H:%M:%S")
            self.model_path = f"experiments/{now_str}"
        else:
            # Existing model
            self.model_path = model_path
            self.load_model(self.model_path)

        self.writer = SummaryWriter(log_dir=self.model_path)
        self.pi_optimizer = optim.Adam(ac.pi.parameters(), lr=self.lr)
        self.q_params = itertools.chain(self.ac.q1.parameters(), self.ac.q2.parameters())
        self.q_optimizer = optim.Adam(self.q_params, lr=self.lr)

    def compute_loss_q(self, data):
        s, a, r, s2, d = data["state"], data["action"], data["reward"], data["next_state"], data["done"]
        s = s.float()
        a = a.float()
        r = r.float()
        s2 = s2.float()
        d = d.float()

        q1 = self.ac.q1(s, a)
        q2 = self.ac.q2(s, a)

        with torch.no_grad():
            pi_targ = self.ac_targ.pi(s2)

            # Target policy smoothing
            epsilon = torch.rand_like(pi_targ) * self.target_noise
            epsilon = torch.clamp(epsilon, -self.noise_clip, self.noise_clip)
            a2 = pi_targ + epsilon
            a2 = torch.clamp(a2, -self.ac.act_limit, self.ac.act_limit)

            # Target Q-values
            q1_pi_targ = self.ac_targ.q1(s2, a2)
            q2_pi_targ = self.ac_targ.q2(s2, a2)
            q_pi_targ = torch.min(q1_pi_targ, q2_pi_targ)
            backup = r + self.gamma * (1 - d) * q_pi_targ

        loss_q1 = ((q1 - backup)**2).mean()
        loss_q2 = ((q1 - backup)**2).mean()
        loss_q = loss_q1 + loss_q2

        return loss_q

    def compute_loss_pi(self, data):
        s = data["state"]
        s = s.float()
        q1_pi = self.ac.q1(s, self.ac.pi(s))
        return -q1_pi.mean()

    def update(self, data, timer):
        self.q_optimizer.zero_grad()
        loss_q = self.compute_loss_q(data)
        loss_q.backward()
        self.q_optimizer.step()

        if timer % self.policy_delay == 0:

            for p in self.q_params:
                p.requires_grad = False

            self.pi_optimizer.zero_grad()
            loss_pi = self.compute_loss_pi(data)
            loss_pi.backward()
            self.pi_optimizer.step()

            for p in self.q_params:
                p.requires_grad = True

            with torch.no_grad():
                for p, p_targ in zip(self.ac.parameters(), self.ac_targ.parameters()):
                    p_targ.data.mul_(self.polyak)
                    p_targ.data.add_((1 - self.polyak) * p.data)

            return loss_pi.detach().numpy(), loss_q.detach().numpy()

        return None, None

    def log_params(self):
        os.makedirs(self.model_path, exist_ok=True)
        attributes = inspect.getmembers(self, lambda a: not(inspect.isroutine(a)) and type(a) in (int, str, float))
        config = [a for a in attributes if not(a[0].startswith("__") and a[0].endswith("__"))]
        config = dict(config)
        config["algorithm"] = "td3"
        with open(f"{self.model_path}/config.yaml", "w") as f:
            f.write(yaml.dump(config))

    def train(self):
        # setup_pytorch_for_mpi()
        self.log_params()
        ep_len = 0
        ep_ret = 0
        s = self.env.reset()
        replay_buffer = TD3Buffer()

        episode_lengths = []
        episode_rewards = []
        pi_losses = []
        q_losses = []

        pbar = tqdm.tqdm(range(self.epochs * self.steps_per_epoch), ncols=100)
        for t in pbar:
            # Encourage exploration
            if t < self.start_steps:
                a = self.env.action_space.sample()
            else:
                a = get_action(self.ac, s, self.act_noise)

            s2, r, d, info = self.env.step(a)
            if hasattr(self.env, "render"):
                self.env.render()
            ep_len += 1
            ep_ret += r
            d = False if ep_len == self.max_ep_len else d

            record = dict(
                state=s,
                action=a,
                reward=np.array([r]),
                next_state=s2,
                done=np.array([d], dtype=np.float32),
            )
            replay_buffer.push(record)

            s = s2

            if d or (ep_len == self.max_ep_len):
                episode_lengths.append(ep_len)
                episode_rewards.append(ep_ret)
                o, ep_ret, ep_len = self.env.reset(), 0, 0

            if t >= self.update_after and t % self.update_every == 0:
                for j in range(self.update_every):
                    batch = replay_buffer.sample_batch(self.batch_size)
                    loss_pi, loss_q = self.update(batch, j)

                    if loss_q:
                        q_losses.append(loss_q)
                    if loss_pi:
                        pi_losses.append(loss_pi)

            if (t + 1) % self.steps_per_epoch == 0:
                epoch = (t + 1) // self.steps_per_epoch
                pbar.set_postfix(
                    dict(avg_epsiode_length=f"{np.mean(episode_lengths): .2f}")
                )
                metrics = {
                    "Environment/Episode Length": np.mean(episode_lengths),
                    "Environment/Cumulative Reward": np.mean(episode_rewards),
                    "Losses/Policy": np.mean(pi_losses),
                    "Losses/Value": np.mean(q_losses),
                }
                episode_lengths = []
                episode_rewards = []
                pi_losses = []
                q_losses = []
                self.log_summary(epoch, metrics)

                if (epoch + 1) % 10 == 0:
                    self.save_model()

    def save_model(self):
        torch.save(self.ac.state_dict(), f"{self.model_path}/actor_critic")
        torch.save(self.ac_targ.state_dict(), f"{self.model_path}/actor_critic_targ")

    def log_summary(self, epoch, metrics):
        for name, value in metrics.items():
            self.writer.add_scalar(name, value, epoch)

    def load_model(self, path):
        self.ac.load_state_dict(torch.load(f"{path}/actor_critic"))
        self.ac_targ.load_state_dict(torch.load(f"{path}/actor_critic_targ"))

    def test_agent(self, test_episodes):
        for j in range(test_episodes):
            s, d, ep_ret, ep_len = self.env.reset(), False, 0, 0
            while not(d or (ep_len == self.max_ep_len)):
                # Take deterministic actions at test time (noise_scale=0)
                s, r, d, _ = self.env.step(get_action(self.ac, s, 0))
                ep_ret += r
                ep_len += 1

def main():
    agent_file = "3DBall_single/3DBall_single.x86_64"
    no_graphics = True
    channel = EngineConfigurationChannel()
    unity_env = UnityEnvironment(
        file_name=agent_file,
        seed=1,
        no_graphics=no_graphics,
        side_channels=[channel]
    )
    channel.set_configuration_parameters(
        time_scale=50.,
    )
    env = UnityToGymWrapper(unity_env)
    l1, l2 = 64, 64
    activation = nn.ReLU
    output_activation = nn.Tanh
    ac = TD3ActorCritic(
        env.observation_space,
        env.action_space,
        l1,
        l2,
        activation=activation
    )

    params = dict(
        gamma=0.99,
        polyak=0.995,
        act_noise=0.1,
        target_noise=0.2,
        epochs=100,
        steps_per_epoch=4000,
        start_steps=10000,
        batch_size=256,
        update_after=10000,
        update_every=50,
        policy_delay=2,
        lr=1e-3,
    )

    model = TD3(ac=ac, env=env, **params)
    model.train()

if __name__ == "__main__":
    main()


import copy
import json
import logging
import random
import re
import shutil
from argparse import ArgumentParser
from pathlib import Path
from typing import Dict, List, Set

from jinf import Jinf
from rhoknp import Jumanpp, Morpheme, Sentence
from rhoknp.utils.reader import chunk_by_sentence
from tqdm import tqdm

from kwja.utils.logging_util import filter_logs

filter_logs(environment="production")
logging.basicConfig(format="")

logger = logging.getLogger("kwja_cli")
logger.setLevel(logging.INFO)

jinf = Jinf()

STOP_SURFS: Set[str] = {'"', "\u3000"}

def is_hiragana(value):
    return re.match(r"^[\u3040-\u309F\u30FC]+$", value) is not None

def get_canons(morpheme: Morpheme) -> List[str]:
    canons: List[str] = []
    if morpheme.canon is not None:
        canons.append(morpheme.canon)
    for feature in morpheme.features:
        if feature[:4] == "ALT-":
            canons.append(feature.split('"')[1].split(" ")[0].replace("代表表記:", ""))
    return canons

def set_other_canons(sentence: Sentence, target_morpheme_indexes: Set[int]) -> None:
    for morpheme in sentence.morphemes:
        if morpheme.index in target_morpheme_indexes:
            continue
        if "代表表記" in morpheme.semantics:
            continue
        if morpheme.conjtype == "*" or morpheme.pos == "特殊":
            canon: str = f"{morpheme.lemma}/{morpheme.reading}"
        elif is_hiragana(morpheme.lemma):
            canon = f"{morpheme.lemma}/{morpheme.lemma}"
        else:
            canon_right: str = jinf(morpheme.reading, morpheme.conjtype, morpheme.conjform, "基本形")
            canon = f"{morpheme.lemma}/{canon_right}"
        morpheme.semantics["代表表記"] = canon

def get_is_excluded(sentence: Sentence) -> bool:
    is_excluded: bool = False
    for morpheme in sentence.morphemes:
        for stop_surf in STOP_SURFS:
            if stop_surf in morpheme.surf:
                is_excluded = True
                break
        if (morpheme.pos != "特殊") and (not is_hiragana(morpheme.reading)):
            is_excluded = True
            break
    return is_excluded

def main():
    parser = ArgumentParser()
    parser.add_argument("-i", "--input-dirs", nargs="*", required=True)
    parser.add_argument("-o", "--output-dir", type=str, required=True)
    parser.add_argument("-max", "--max-samples", type=int, default=3)
    args = parser.parse_args()

    random.seed(42)

    jumanpp = Jumanpp()

    input_paths: List[Path] = []
    for input_dir in args.input_dirs:
        input_paths.extend(list(Path(input_dir).glob("**/*.txt")))

    sentences: List[Sentence] = []
    canon2freq: Dict[str, int] = {}
    excluded_nums: Dict[str, int] = {}
    for input_path in tqdm(input_paths):
        with input_path.open() as f:
            for knp in chunk_by_sentence(f):
                try:
                    sentence: Sentence = Sentence.from_knp(knp)
                except ValueError:
                    excluded_nums["sent_from_knp"] = excluded_nums.get("sent_from_knp", 0) + 1
                    continue

                juman_sentence: Sentence = jumanpp.apply_to_sentence(sentence.text)

                kwja_words: List[str] = [morpheme.text for morpheme in sentence.morphemes]
                juman_words: List[str] = [morpheme.text for morpheme in juman_sentence.morphemes]
                if kwja_words != juman_words:
                    excluded_nums["word_segmentation"] = excluded_nums.get("word_segmentation", 0) + 1
                    continue

                for morpheme in sentence.morphemes:
                    if morpheme.pos not in {"名詞", "動詞", "形容詞", "接頭辞", "接尾辞", "副詞", "連体詞"}:
                        continue
                    canons: List[str] = get_canons(morpheme)
                    if len(canons) > 1:
                        for canon in canons:
                            canon2freq[canon] = canon2freq.get(canon, 0) + 1
                sentences.append(sentence)
    print(f"num_sentences: {len(sentences)}")

    sid2knp_str: Dict[str, str] = {}
    sid2info: Dict[str, Dict[int, Dict[str, str]]] = {}
    sampled_canon2freq: Dict[str, int] = {}
    for sentence in sentences:
        if get_is_excluded(sentence):
            continue
        info: Dict[int, Dict[str, str]] = {}
        target_canons: Set[str] = set()
        target_morpheme_indexes: Set[int] = set()
        for morpheme in sentence.morphemes:
            canons: List[str] = get_canons(morpheme)
            if len(canons) > 1 or sampled_canon2freq.get(morpheme.canon, 0) >= args.max_samples:
                continue

            if canon2freq.get(morpheme.canon, 0) >= 2:
                if morpheme.conjtype == "*":
                    surf: str = morpheme.reading
                    lemma: str = morpheme.reading
                else:
                    try:
                        lemma = jinf(morpheme.reading, morpheme.conjtype, morpheme.conjform, "基本形")
                        surf = jinf(lemma, morpheme.conjtype, "基本形", morpheme.conjform)
                    except (ValueError, NotImplementedError):
                        excluded_nums["jinf"] = excluded_nums.get("jinf", 0) + 1
                        continue
                if surf == morpheme.text and lemma == morpheme.lemma:
                    continue
                surf_before: str = copy.deepcopy(morpheme.text)
                lemma_before: str = copy.deepcopy(morpheme.lemma)
                morpheme.text = surf
                morpheme._text_escaped = surf
                morpheme.lemma = lemma
                info[morpheme.index] = {
                    "partial_annotation_type": "canon",
                    "surf_before": surf_before,
                    "lemma_before": lemma_before,
                    "surf_after": surf,
                    "lemma_after": lemma,
                    "conjtype": morpheme.conjtype,
                    "conjform": morpheme.conjform,
                }
                target_canons.add(morpheme.canon)
                target_morpheme_indexes.add(morpheme.index)
        if len(info) == 0:
            continue

        try:
            set_other_canons(sentence, target_morpheme_indexes)
        except (ValueError, NotImplementedError):
            excluded_nums["set_canon"] = excluded_nums.get("set_canon", 0) + 1
            continue
        try:
            sid2knp_str[sentence.sid] = sentence.to_knp()
        except AttributeError:
            excluded_nums["to_knp"] = excluded_nums.get("to_knp", 0) + 1
            continue
        sid2info[sentence.sid] = info
        for target_canon in target_canons:
            sampled_canon2freq[target_canon] = sampled_canon2freq.get(target_canon, 0) + 1

        if len(sid2knp_str) >= 5000:
            break

    sid2knp_str_list: List[tuple[str, str]] = list(sid2knp_str.items())
    random.shuffle(sid2knp_str_list)
    train_size: int = int(len(sid2knp_str_list) * 0.9)
    valid_size: int = int(len(sid2knp_str_list) * 0.05)
    train_list: List[tuple[str, str]] = sid2knp_str_list[:train_size]
    valid_list: List[tuple[str, str]] = sid2knp_str_list[train_size : train_size + valid_size]
    test_list: List[tuple[str, str]] = sid2knp_str_list[train_size + valid_size :]

    output_dir: Path = Path(args.output_dir)
    output_dir.mkdir(parents=True, exist_ok=True)

    with open(output_dir / "canon2freq.txt", "w") as f:
        for canon, freq in canon2freq.items():
            f.write(f"{canon}\t{freq}\n")
    with open(output_dir / "sampled2freq.txt", "w") as f:
        for canon, freq in sampled_canon2freq.items():
            f.write(f"{canon}\t{freq}\n")
    with open(output_dir / "info.json", "w") as f:
        json.dump(sid2info, f, indent=2, ensure_ascii=False)

    train_dir: Path = output_dir / "train"
    if train_dir.exists():
        shutil.rmtree(str(train_dir))
    train_dir.mkdir(exist_ok=True)
    for name, knp_str in train_list:
        with (train_dir / f"{name}.knp").open("w") as f:
            f.write(f"{knp_str}\n")

    valid_dir: Path = output_dir / "valid"
    if valid_dir.exists():
        shutil.rmtree(str(valid_dir))
    valid_dir.mkdir(exist_ok=True)
    for name, knp_str in valid_list:
        with (valid_dir / f"{name}.knp").open("w") as f:
            f.write(f"{knp_str}\n")

    test_dir: Path = output_dir / "test"
    if test_dir.exists():
        shutil.rmtree(str(test_dir))
    test_dir.mkdir(exist_ok=True)
    for name, knp_str in test_list:
        with (test_dir / f"{name}.knp").open("w") as f:
            f.write(f"{knp_str}\n")

    print(f"train: {len(train_list)}")
    print(f"valid: {len(valid_list)}")
    print(f"test: {len(test_list)}")
    print(f"total: {len(train_list) + len(valid_list) + len(test_list)}")
    print(f"excluded_nums: {excluded_nums}")

if __name__ == "__main__":
    main()


"""
@Author: 邓润庭
@Date:   2021/3/19
"""

from typing import List

# 自底向上
def min_coin_num(coins: List, amount: int):
    dp = [float("inf")] * (amount + 1)
    dp[0] = 0

    for n in range(amount + 1):
        for coin in coins:
            if coin <= n:
                dp[n] = min(dp[n], dp[n - coin] + 1)

    return dp[amount] if dp[amount] != float("inf") else -1

print(min_coin_num([1, 2, 5], 6))


from flask import Flask
from multiprocessing import Pool
from flask.globals import request
from healthcheck import HealthCheck
from utils.decorators import AppDecorator
import gunicorn_conf
import json
from utils import constants
from utils import endpoints
import time
import concurrent.futures
from utils import merge_sort
from utils import scrap

app = Flask(__name__)
content_type = "application/json"

@app.route('/')
def hello_world():
    return 'Hello, World!'

@app.route('/error')
def error():
    return '403-Bad Request'

@app.route(endpoints.ping, methods = ['GET'])
@AppDecorator.content_type_check
def get_ping():
    response = {}
    health = HealthCheck()
    response['message'] = constants.hc_success
    (system_data, status, c_t) = health.run()
    response['code'] = status
    if status != 200:
        response['code'] = constants.failure_code
        response['message'] = constants.hc_failure
    return response

@app.route(endpoints.exec_params, methods = ['POST'])
@AppDecorator.content_type_check
def execution_params():
    response = {}
    try:
        response['code'] = constants.success_code
        response['MaxConcurrentTransforms'] = gunicorn_conf.workers
        response['MaxPayloadInMB'] = gunicorn_conf.max_payload_in_mb
        response['message'] = constants.ep_success
    except:
        response['code'] = constants.failure_code
        response['message'] = constants.ep_failure
    return response
# @app.route('/execution-parameters')

@app.route(endpoints.sort, methods = ['POST'])
@AppDecorator.content_type_check
@AppDecorator.limit_content_length(gunicorn_conf.max_payload_in_mb)
def sort_list():
    request_data = json.loads(request.data)
    response = {}
    if 'input_list' in request_data.keys():
        start_time = time.time()
        input_list = request_data['input_list']
        if request_data['concurrent']:
            response['code'] = constants.success_code
            with concurrent.futures.ThreadPoolExecutor(gunicorn_conf.workers) as executor:
                future = executor.submit(merge_sort.sort, numbers = input_list, executor = executor)
                sorted_list = future.result()
        else:
            sorted_list = sorted(input_list)
        end_time = time.time()
        time_consumed = int(round((end_time - start_time)))
        response['code'] = constants.success_code
        response['message'] = constants.sort_success
        response['time-taken'] = str(time_consumed) + "s"
        response['sorted-list'] = sorted_list
        return response
    else:
        return error()

@app.route(endpoints.web_crawl, methods = ['POST'])
@AppDecorator.content_type_check
@AppDecorator.limit_content_length(gunicorn_conf.max_payload_in_mb)
def web_crawl():
    request_data = json.loads(request.data)
    response = {}
    if 'input_list' in request_data.keys():
        start_time = time.time()
        input_list = request_data['input_list']
        if request_data['parallel']:
            response['code'] = constants.success_code
            p = Pool(gunicorn_conf.max_process_pool)
            sorted_list = p.map(scrap.scrape, input_list)
            p.terminate()
            p.join()
        else:
            sorted_list = scrap.webcrawl(input_list)
            print('sorted_list ',sorted_list)
        end_time = time.time()
        time_consumed = int(round((end_time - start_time)))
        response['code'] = constants.success_code
        response['message'] = constants.sort_success
        response['time-taken'] = str(time_consumed)+"s"
        response['sorted-list'] = sorted_list
        return response
    else:
        return error()
# def init_db():
#     import db
#     database = db.get_db()
#     with open_resource('schema.sql', mode='r') as f:
#         database.cursor().executescript(f.read())
#     database.commit()

if __name__ == '__main__':
    # init_db()
    app.run()


import spacy
import neuralcoref

nlp = spacy.load('en_core_web_sm')
neuralcoref.add_to_pipe(nlp)

def resolve_corefs(text):
    doc = nlp(text)
    text_revised=text

    for i in range(len(doc._.coref_clusters)):             #Coref clusters is a list of cluster objects
        cluster = doc._.coref_clusters[i]
        words_in_cluster=[]
        for l in cluster.mentions:                         #Each cluster objects contains mentions, a list of lists of tokens e.g. [[Jon,Snow],[he]]
            tokens_combined=""
            for text in l:
                tokens_combined+=str(text) + ' '
            words_in_cluster.append(tokens_combined.strip())
        text_revised = text_revised.replace(words_in_cluster[1],words_in_cluster[0])
    return text_revised

#Some sample strings to try

#Working examples:

#text='My sister has a dog. She loves him.'    #Detects sister=she, dog=him
#text='The President today announced a new government program. It would promote things worth promoting, he said.'
#^ Ties The President to He and 'a new government program' to 'it'.

#Kinda working:

#text='Jim is a rich man. He shaves daily with a razor. It is gold-plated.' #Detects Jim=He, but not Razor=It
#text='Johnny was so adjective, even Sarah had to admit Johhny was more adjective than she was.' #Detects Johnny=he, but says 'even Sarah'=she.


import numpy as np

class Perceptron:

    def __int__(self):
        self._theta = None

    def train(self, X, Y, lr):
        _X = np.array(X)
        m, d = X.shape
        _Y = np.array(Y).reshape(m,1)
        _X = np.append(np.ones((m,1)), _X, axis=1)
        d += 1
        lr /= m
        self._theta = np.random.random((d, 1))
        miss = np.logical_xor(_X.dot(self._theta) > 0, Y)
        it = 0
        while miss.sum() > 0:
            it += 1
            diff = lr * _X[miss.flatten()].sum(axis=0)
            self._theta += diff.reshape(d,1)
            miss = np.logical_xor(_X.dot(self._theta) > 0, Y)
            print('iteration', it, ', total mistake:', miss.sum())

    def run(self, X):
        _X = np.array(X)
        return _X.dot(self._theta) > 0

    def print_normalized_weight(self):
        print('Classifier weights:', self._theta.flatten())
        t = self._theta.flatten()
        print('Normalized with threshold:', t[1:] / -t[:1])


from subprocess import PIPE, run
import time
import zmq

def listen():
    ctx = zmq.Context()
    sock = ctx.socket(zmq.REP)
    sock.bind("tcp://*:8085")
    print("Starting loop for OT2 internal listener...")
    i = 1
    while True:
        msg = sock.recv_string()
        print(msg)
        i += 1
        time.sleep(1)
        if msg != None:
            msg_output, msg_error, msg_returncode = execute_command(msg)
            sock.send_string(msg_output + '@' + msg_error + '@' + str(msg_returncode))

    sock.close()

def execute_command(command):
    print("<---------------- Executing the protocol on the OT2 --------------->")
    result = run(command, stdout=PIPE, stderr=PIPE, universal_newlines=True, shell=True)
    print("Return code: ",  result.returncode)
    print("Error Message:\n" + result.stderr)
    print("Output Message:\n" + result.stdout)
    return result.stdout, result.stderr, result.returncode

def main_null():
    print("This function is not meant to have a main function")

if __name__ == "__main__":
    listen()


# BJ 6064 카잉 달력 <수학, 정수론, 중국인의 나머지 정리>
# https://www.acmicpc.net/problem/6064
# 시간 : 60+
# 문제 리뷰 : H,R
# 회고 : 시간 복잡도를 신경 쓰지 않고 알고리즘 자체를 구현한 다음, 해당 알고리즘을 출력하여 패턴을 파악하였다.
#        이후에 수학적 패턴을 발견하여 간단한 수식이나 반복문으로 구현하려 하였다.

for _ in range(int(input())):
    M,N,x,y = map(int, input().split())

    if M < N: M,N,x,y = N,M,y,x

    through = True
    for i in range(M):
        if y == (x+(M-N)*i-1)%N+1:
            print(x+M*i)
            through = False
            break

    if through:print(-1)

problems/202110/[H,R]_[BJ_6064_카잉달력]_[수학,정수론,중국인의나머지정리].py


"""
This module contains various utility functions that are exposed to API users
"""
from __future__ import print_function, absolute_import, division
from future.builtins import *
from future import standard_library
standard_library.install_aliases()

# Copyright 2017 Autodesk Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import moldesign as mdt
from moldesign import units as u

from . import toplevel, __all__ as _pkgall

from moldesign.interfaces.openbabel import add_hydrogen, guess_bond_orders
from moldesign.interfaces.pdbfixer_interface import mutate_residues, add_water
from moldesign.interfaces.tleap_interface import create_ff_parameters
from moldesign.interfaces.ambertools import calc_am1_bcc_charges, calc_gasteiger_charges

_pkgall.extend(('add_hydrogen guess_bond_orders mutate_residues add_water'
                ' create_ff_parameters calc_am1_bcc_charges calc_gasteiger_charges ').split())

ATNUM_VALENCE_CHARGE = {6: {3: -1, 4: 0},
                        7: {2: -1, 3: 0, 4: 1},
                        8: {1: -1, 2: 0, 3: 1},
                        9: {1: 0, 2: 1}}

ION_CHARGE = {1: 1,  # H
              11: 1,  # Na+
              19: 1,  # K+
              12: 2,  # Mg 2+
              20: 2,  # Ca 2+
              9: -1,  # F-
              17: -1,  # Cl-
              35: -1,  # Br-
              53: -1}  # I-

@toplevel
def assign_formal_charges(mol, ignore_nonzero=True):
    """ Assign formal charges to C,N,O,F atoms in this molecule based on valence

    Args:
        mol (moldesign.Molecule): Molecule to assign formal charges to. The formal charges of its
           atoms and its total charge will be adjusted in place.
        ignore_nonzero (bool): If formal charge is already set to a nonzero value, ignore this atom

    Note:
        This method ONLY applies to C,N, O and F, based on a simple valence model.
        These results should be manually inspected for consistency.

    Raises:
        UnhandledValenceError: for cases not handled by the simple valence model

    References:
        These assignments are illustrated by the formal charge patterns in
            http://www.chem.ucla.edu/~harding/tutorials/formalcharge.pdf
    """
    from moldesign.exceptions import UnhandledValenceError

    # TODO: scrape formal charge data from the PDB chem comp dictionary

    # cache these values in case we fail to assign charges
    totalcharge = mol.charge
    oldcharges = [atom.formal_charge for atom in mol.atoms]

    for atom in mol.atoms:
        if ignore_nonzero and atom.formal_charge != 0:
            continue

        v = atom.valence
        newcharge = None

        if atom.atnum in ATNUM_VALENCE_CHARGE:
            if v in ATNUM_VALENCE_CHARGE[atom.atnum]:
                newcharge = ATNUM_VALENCE_CHARGE[atom.atnum][v]
            else:
                for oldcharge, a in zip(oldcharges, mol.atoms):
                    a.oldcharge = oldcharge
                mol.charge = totalcharge
                raise UnhandledValenceError(atom)
        elif atom in ION_CHARGE and v == 0:
            newcharge = ION_CHARGE[atom.atnum]

        if newcharge is not None:
            mol.charge += newcharge * u.q_e - atom.formal_charge
            atom.formal_charge = newcharge * u.q_e

@toplevel
def set_hybridization_and_saturate(mol):
    """ Assign bond orders, saturate with hydrogens, and assign formal charges

    Specifically, this is a convenience function that runs:
    ``mdt.guess_bond_orders``, ``mdt.add_hydrogen``, and ``mdt.assign_formal_charges``

    Note:
        This does NOT add missing residues to biochemical structures. This functionality will be
        available as :meth:`moldesign.add_missing_residues`

    Args:
        mol (moldesign.Molecule): molecule to clean

    Returns:
        moldesign.Molecule: cleaned version of the molecule
    """
    m1 = mdt.guess_bond_orders(mol)
    m2 = mdt.add_hydrogen(m1)
    assign_formal_charges(m2)
    return m2

@toplevel
def guess_histidine_states(mol):
    """ Attempt to assign protonation states to histidine residues.

    Note:
        This function is highly unlikely to give accurate results! It is intended for convenience
        when histidine states can easily be guessed from already-present hydrogens or when they are
        judged to be relatively unimportant.

    This can be done simply by renaming HIS residues:
      1. If HE2 and HD1 are present, the residue is renamed to HIP
      2. If only HE2 is present, the residue is renamed to HIE
      3. Otherwise, the residue is renamed to HID (the most common form)

    Args:
        mol (moldesign.Molecule): molecule to change (in place)
    """
    for residue in mol.residues:
        if residue.resname == 'HIS':
            oldname = str(residue)
            if 'HE2' in residue and 'HD1' in residue:
                residue.resname = 'HIP'
            elif 'HE2' in residue:
                residue.resname = 'HIE'
            else:
                residue.resname = 'HID'
            print('Renaming %s from HIS to %s' % (oldname, residue.resname))

@toplevel
def split_chains(mol, distance_threshold=1.75*u.angstrom):
    """ Split a molecule's chains into unbroken biopolymers and groups of non-polymers

    This function is non-destructive - the passed molecule will not be modified.

    Specifically, this function will:
       - Split any chain with non-contiguous biopolymeric pieces into single, contiguous polymers
       - Remove any solvent molecules from a chain into their own chain
       - Isolate ligands from each chain into their own chains

    Args:
        mol (mdt.Molecule): Input molecule
        distance_threshold (u.Scalar[length]): if not ``None``, the maximum distance between
           adjacent residues for which we consider them "contiguous". For PDB data, values greater
           than 1.4 Angstrom are eminently reasonable; the default threshold of 1.75 Angstrom is
           purposefully set to be extremely cautious (and still much lower than the distance to
           the *next* nearest neighbor, generally around 2.5 Angstrom)

    Returns:
        mdt.Molecule: molecule with separated chains
    """

    tempmol = mol.copy()

    def bonded(r1, r2):
        if r2 not in r1.bonded_residues:
            return False
        if distance_threshold is not None and r1.distance(r2) > distance_threshold:
            return False
        return True

    def addto(chain, res):
        res.chain = None
        chain.add(res)

    allchains = [mdt.Chain(tempmol.chains[0].name)]
    for chain in tempmol.chains:
        chaintype = chain.residues[0].type
        solventchain = mdt.Chain(None)
        ligandchain = mdt.Chain(None)

        for ires, residue in enumerate(chain.residues):
            if residue.type == 'unknown':
                thischain = ligandchain
            elif residue.type in ('water', 'solvent', 'ion'):
                thischain = solventchain
            else:
                assert residue.type == chaintype
                if ires != 0 and not bonded(residue.prev_residue, residue):
                    allchains.append(mdt.Chain(None))
                thischain = allchains[-1]

            addto(thischain, residue)

        for c in (solventchain, ligandchain):
            if c.num_atoms > 0:
                allchains.append(c)

    return mdt.Molecule(allchains)


import debugpy

from detector import *

debugpy.listen(5678);
print("Waiting for debugger to attach");
debugpy.wait_for_client();

#rules = load_definitions("./detector.json");
det=Detector("./detector.json");
rules=det.get_rules();
print(rules);
fields=rules['fields'];
print(fields);
print(fields[1]);


import pandas as pd
import numpy as np
import sys
from LikeEstimator import Estimator
from sklearn.utils import shuffle
from sklearn.metrics import accuracy_score
from sklearn import metrics
CONST_DEBUG = False
CONST_TEST_PATH = '../data/public-test-data/'
CONST_OUTPUT_PATH = 'output/'
CONST_BIG5 = ['ope', 'ext', 'con', 'agr', 'neu']

def main():
    df = createTestLikeDataframe(CONST_TEST_PATH)
    print(df)
    '''
    profile_df = pd.read_csv(CONST_TEST_PATH + "profile/profile.csv")
    test_profile=profile_df.iloc[8001:,:]
    estimator = Estimator.unpickle()

    df = estimator.predictAge(df)
    df = estimator.predictGender(df)
    df = estimator.predictPersonality(df)

    score = [0,0,0,0,0,0,0]
    score[0] =  accuracy_score(test_profile.age, df.age)
    score[1] =  accuracy_score(test_profile.gender, df.gender)
    score[2] = np.sqrt(metrics.mean_squared_error(test_profile.ope, df.ope))
    score[3] = np.sqrt(metrics.mean_squared_error(test_profile.neu, df.neu))
    score[4] = np.sqrt(metrics.mean_squared_error(test_profile.ext, df.ext))
    score[5] = np.sqrt(metrics.mean_squared_error(test_profile.agr, df.agr))
    score[6] = np.sqrt(metrics.mean_squared_error(test_profile.con, df.con))
    print(score)
    #dataframeToXML(testDF)
    '''
def dataframeToXML(dataframe):
    for index in dataframe.index.tolist()[:]:
        userid = dataframe.get_value(index, 'userid')

        age = dataframe.get_value(index, 'age')

        gender = dataframe.get_value(index, 'gender')

        personality = []
        personality.append(dataframe.get_value(index, 'ext'))
        personality.append(dataframe.get_value(index, 'neu'))
        personality.append(dataframe.get_value(index, 'agr'))
        personality.append(dataframe.get_value(index, 'con'))
        personality.append(dataframe.get_value(index, 'ope'))

        dataToXML(userid, age, gender, personality)

def dataToXML(userid, age, gender, personality):
    file = open(CONST_OUTPUT_PATH + userid + ".xml", "w+")
    file.write("<user\n")
    file.write("\tid=\"%s\"\n" % (userid))
    file.write("age_group=\"%s\"\n" % (age))
    file.write("gender=\"%s\"\n" % (gender))
    file.write("extrovert=\"%s\"\n" % (personality[0]))
    file.write("neurotic=\"%s\"\n" % (personality[1]))
    file.write("agreeable=\"%s\"\n" % (personality[2]))
    file.write("conscientious=\"%s\"\n" % (personality[3]))
    file.write("open=\"%s\"\n" % (personality[4]))
    file.write("/>")
    file.close()

# Create dataframe containing merged data from
# profile.csv and relation.csv
def createTestLikeDataframe(path):
    profile_df = pd.read_csv(path + "profile/profile.csv")
    profile_df=profile_df.iloc[8001:,:]
    relation_df = pd.read_csv(path + "relation/relation.csv")
    merge_file = pd.merge(left=profile_df, right=relation_df, left_on='userid', right_on='userid')
    cols = ['age', 'gender','Unnamed: 0_y','Unnamed: 0_x'] + CONST_BIG5
    merge_file.drop(cols, inplace=True, axis=1)

    merge_file['like_id'] = merge_file['like_id'].astype(str)
    pageIDs = createUserAndPagesDf(merge_file)
    training_df = pd.merge(left=profile_df, right=pageIDs, left_on='userid', right_on='userid')
    del training_df['Unnamed: 0']
    print(training_df)
    return training_df

def createUserAndPagesDf(merge_file):
    merge_file['ids'] = merge_file['like_id'] + " "
    df = merge_file.groupby('userid')['ids'].apply(lambda x: x.sum()).reset_index()
    return df

def toBracket(age):
    bracket = ""
    if age <= 24.0:
        bracket = "xx-24"
    elif age >= 25.0 and age  <= 34.0:
        bracket = "25-34"
    elif age  >= 35.0 and age  <= 49.0:
        bracket = "35-49"
    else: # age >= 50
        bracket = "50-xx"
    return bracket

if __name__ == "__main__":
    main()


import logging
import random
from argparse import ArgumentParser
from itertools import chain
from pprint import pformat
import warnings
import tensorflow as tf

import torch
import torch.nn.functional as F

from transformers import OpenAIGPTLMHeadModel, OpenAIGPTTokenizer, GPT2LMHeadModel, GPT2Tokenizer
from train import SPECIAL_TOKENS, build_input_from_segments, add_special_tokens_
from utils import get_dataset, download_pretrained_model

def top_filtering(logits, top_k=0.0, top_p=0.9, threshold=-float("Inf"), filter_value=-float("Inf"), beam=True):
    """Filter a distribution of logits using top-k, top-p (nucleus) and/or threshold filtering
    Args:
        logits: logits distribution shape (vocabulary size)
        top_k: <=0: no filtering, >0: keep only top k tokens with highest probability.
        top_p: <=0.0: no filtering, >0.0: keep only a subset S of candidates, where S is the smallest subset
            whose total probability mass is greater than or equal to the threshold top_p.
            In practice, we select the highest probability tokens whose cumulative probability mass exceeds
            the threshold top_p.
        threshold: a minimal threshold to keep logits
    """
    assert logits.dim() == 1

    if beam == True:
        sorted_logits, sorted_indices = torch.sort(logits, descending=True)
        indices_to_remove = sorted_indices[5:]
        logits[indices_to_remove] = filter_value

    else:
        if top_k > 0:
            # Only work for batch size 1 for now - could update but it would obfuscate a bit the code
            top_k = min(top_k, logits.size(-1))
            # Remove all tokens with a probability less than the last token in the top-k tokens
            indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]
            logits[indices_to_remove] = filter_value

        if top_p > 0.0:
            # Compute cumulative probabilities of sorted tokens
            sorted_logits, sorted_indices = torch.sort(logits, descending=True)
            cumulative_probabilities = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)

            # Remove tokens with cumulative probability above the threshold
            sorted_indices_to_remove = cumulative_probabilities > top_p
            # Shift the indices to the right to keep also the first token above the threshold
            sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
            sorted_indices_to_remove[..., 0] = 0

            # Back to unsorted indices and set them to -infinity
            indices_to_remove = sorted_indices[sorted_indices_to_remove]
            logits[indices_to_remove] = filter_value

    indices_to_remove = logits < threshold
    logits[indices_to_remove] = filter_value

    return logits

def sample_sequence(personality, history, tokenizer, model, args, current_output=None):
    special_tokens_ids = tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS)
    if current_output is None:
        current_output = []

    for i in range(args.max_length):
        instance = build_input_from_segments(personality, history, current_output, tokenizer, with_eos=False)

        input_ids = torch.tensor(instance["input_ids"], device=args.device).unsqueeze(0)

        outputs = model.generate(
            input_ids,
            max_length=input_ids.shape[-1] + 40,
            num_beams=10,
            num_return_sequences=1,
            no_repeat_ngram_size=1,
            remove_invalid_values=True,
        )
        return outputs[0].tolist()
        # print(tokenizer.decode(outputs[0], skip_special_tokens=True))

        # token_type_ids = torch.tensor(instance["token_type_ids"], device=args.device).unsqueeze(0)

        # logits = model(input_ids, token_type_ids=token_type_ids)

        # beam_output = model.generate(input_ids, max_length=40, num_beams=5)

        # return beam_output[0].tolist()

    #     if isinstance(logits, tuple):  # for gpt2 and maybe others
    #         logits = logits[0]

    #     logits = logits[0, -1, :] / args.temperature

    #     logits = top_filtering(logits, top_k=args.top_k, top_p=args.top_p)

    #     probs = F.softmax(logits, dim=-1)

    #     prev = torch.topk(probs, 1)[1] if args.no_sample else torch.multinomial(probs, 1)
    #     if i < args.min_length and prev.item() in special_tokens_ids:
    #         while prev.item() in special_tokens_ids:
    #             if probs.max().item() == 1:
    #                 warnings.warn("Warning: model generating special token with probability 1.")
    #                 break  # avoid infinitely looping over special token
    #             prev = torch.multinomial(probs, num_samples=1)

    #     if prev.item() in special_tokens_ids:
    #         break
    #     current_output.append(prev.item())

    # return current_output

## ADDED CODE
def get_data():
    datapath = (
        "/raid/cs18resch11003/aditi/transfer-learning-conv-ai/ParlAI/data/ConvAI2/test_self_original_no_cands.txt"
    )
    data = open(datapath, "r")
    lines = data.readlines()

    personality = []
    persona = []
    for idx, line in enumerate(lines):
        if "your persona:" in line:
            line = line.strip()
            line = line.replace("your persona:", "")
            line = line[3:]
            persona.append(line)
        else:
            if len(persona) != 0:
                personality.append(persona)
                persona = []

    conversations = []
    conversation = []
    for idx, line in enumerate(lines):
        if "your persona:" not in line:
            line = line.strip()
            line = line[2:]
            conversation.append(line)
        else:
            if len(conversation) != 0:
                conversations.append(conversation)
                conversation = []
    conversations.append(conversation)

    prompts = []
    replies = []

    for conversation in conversations:
        temp_prompt = []
        temp_replies = []
        for turn in conversation:
            utterance1, utterance2 = turn.split("\t")
            utterance1 = utterance1.strip()
            utterance2 = utterance2.strip()

            temp_prompt.append(utterance1)
            temp_replies.append(utterance2)
        prompts.append(temp_prompt)
        replies.append(temp_replies)

    assert len(personality) == (len(prompts) + len(replies)) / 2

    return personality, prompts, replies

def run():
    parser = ArgumentParser()
    parser.add_argument(
        "--dataset_path", type=str, default="", help="Path or url of the dataset. If empty download from S3."
    )
    parser.add_argument(
        "--dataset_cache", type=str, default="./dataset_cache", help="Path or url of the dataset cache"
    )
    parser.add_argument(
        "--model",
        type=str,
        default="openai-gpt",
        help="Model type (openai-gpt or gpt2)",
        choices=["openai-gpt", "gpt2"],
    )  # anything besides gpt2 will load openai-gpt
    parser.add_argument("--model_checkpoint", type=str, default="", help="Path, url or short name of the model")
    parser.add_argument(
        "--max_history", type=int, default=2, help="Number of previous utterances to keep in history"
    )
    parser.add_argument(
        "--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu", help="Device (cuda or cpu)"
    )

    parser.add_argument("--no_sample", action="store_true", help="Set to use greedy decoding instead of sampling")
    parser.add_argument("--max_length", type=int, default=20, help="Maximum length of the output utterances")
    parser.add_argument("--min_length", type=int, default=1, help="Minimum length of the output utterances")
    parser.add_argument("--seed", type=int, default=0, help="Seed")
    parser.add_argument("--temperature", type=float, default=0.7, help="Sampling softmax temperature")
    parser.add_argument(
        "--top_k", type=int, default=0, help="Filter top-k tokens before sampling (<=0: no filtering)"
    )
    parser.add_argument(
        "--top_p", type=float, default=0.9, help="Nucleus filtering (top-p) before sampling (<=0.0: no filtering)"
    )
    args = parser.parse_args()

    logging.basicConfig(level=logging.INFO)
    logger = logging.getLogger(__file__)
    logger.info(pformat(args))

    if args.model_checkpoint == "":
        if args.model == "gpt2":
            raise ValueError("Interacting with GPT2 requires passing a finetuned model_checkpoint")
        else:
            args.model_checkpoint = download_pretrained_model()

    if args.seed != 0:
        random.seed(args.seed)
        torch.random.manual_seed(args.seed)
        torch.cuda.manual_seed(args.seed)

    logger.info("Get pretrained model and tokenizer")
    tokenizer_class, model_class = (
        (GPT2Tokenizer, GPT2LMHeadModel) if args.model == "gpt2" else (OpenAIGPTTokenizer, OpenAIGPTLMHeadModel)
    )
    tokenizer = tokenizer_class.from_pretrained(args.model_checkpoint)
    model = model_class.from_pretrained(args.model_checkpoint)
    model.to(args.device)
    add_special_tokens_(model, tokenizer)

    ####################

    # logger.info("Sample a personality")

    # dataset = get_dataset(tokenizer, args.dataset_path, args.dataset_cache)

    # personalities = [dialog["personality"] for dataset in dataset.values() for dialog in dataset]

    # personality = random.choice(personalities)
    # logger.info("Selected personality: %s", tokenizer.decode(chain(*personality)))

    ## ADDED CODE

    personality, prompts, replies = get_data()

    tokens_personality = []

    for persona in personality:
        temp_p = []
        for p in persona:
            temp_p.append(tokenizer.convert_tokens_to_ids(tokenizer.tokenize(p)))
        tokens_personality.append(temp_p)

    history = []
    count = 0
    for idx, (tp, p, r) in enumerate(zip(tokens_personality, prompts, replies)):
        print(f"Personality: {personality[idx]}")
        for _p, _r in zip(p, r):
            print(f"Prompt: {_p}")
            print(f"Original: {_r}")

            history.append(tokenizer.encode(_p))
            with torch.no_grad():
                out_ids = sample_sequence(tp, history, tokenizer, model, args)
            history.append(out_ids)

            history = history[-(2 * args.max_history + 1) :]
            out_text = tokenizer.decode(out_ids, skip_special_tokens=True)
            print(f"Generated: {out_text}\n")

            count += 1

            if count == 50:
                exit(0)

    # while True:
    #     raw_text = input(">>> ")
    #     while not raw_text:
    #         print("Prompt should not be empty!")
    #         raw_text = input(">>> ")

    #     history.append(tokenizer.encode(raw_text))

    #     with torch.no_grad():
    #         out_ids = sample_sequence(personality, history, tokenizer, model, args)

    #     history.append(out_ids)

    #     history = history[-(2 * args.max_history + 1) :]
    #     out_text = tokenizer.decode(out_ids, skip_special_tokens=True)
    #     print(out_text)

if __name__ == "__main__":
    run()


#!/usr/bin/env python3

import os
import sys
import csv
import time
import json
import pprint
import socket
import argparse

HOSTNAME = "api.vndb.org"
PORT = 19534
BUFFER_SIZE = 2**12

class TCPSession:
    def __init__(self, hostname, port, sock=None):
        if sock is None:
            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        else:
            self.sock = sock
        self.sock.connect((hostname, port))

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.sock.shutdown(socket.SHUT_RDWR)
        self.sock.close()

    def send(self, msg):
        totalsent = 0
        while totalsent < len(msg):
            sent = self.sock.send(msg[totalsent:])
            if sent == 0:
                raise RuntimeError('socket connection broken')
            totalsent = totalsent + sent

    def recv(self, delim=b'\x04', buf_size=BUFFER_SIZE):
        chunks = [b'']
        bytes_recd = 0
        while not chunks[-1].endswith(delim):
            chunk = self.sock.recv(buf_size)
            if chunk == b'':
                raise RuntimeError('socket connection broken')
            chunks.append(chunk)
        return b''.join(chunks).rstrip(delim)

    def query(self, cmd):
        self.send(cmd.encode())
        res = self.recv().decode()
        return res

    def login(self):
        cfg = {"protocol": 1, "client": "test", "clientver": 0.1}
        cmd = f'login {json.dumps(cfg).replace(" ", "")}\x04'
        return self.query(cmd)

    def dbstats(self):
        cmd = f'dbstats\x04'
        buf = self.query(cmd)
        res = json.loads(' '.join(buf.split(' ')[1:]))
        return res

    def get(self, typ, flags, filters, options):
        cmd = f'get {typ} {flags} {filters} {options}\x04'
        buf = self.query(cmd)
        res = json.loads(' '.join(buf.split(' ')[1:]))
        return res

    def set(self, typ, ids, fields):
        cmd = f'set {typ} {ids} {fields}\x04'
        return self.query(cmd)

def parse_opt():
    parser = argparse.ArgumentParser()

    parser.add_argument('typ',
                        help='the type of data to fetch')
    parser.add_argument('flags', nargs='?', default='basic',
                        help='what part of that data to fetch')
    parser.add_argument('--dir', '-d', default='data',
                        help='the target dir to output')
    parser.add_argument('--csv', '-c', action='store_true',
                        help='output the result as csv')

    return parser.parse_args()

def fetch_vndb(typ, flags):
    attr = {
        "vn": ('id', 'vn'),
        "release": ('id', 'releases'),
        "producer": ('id', 'producers'),
        "character": ('id', 'chars'),
        "staff": ('id', 'staff'),
        "user": ('id', 'users'),
        "votelist": ('vn', 'vn'),
        "vnlist": ('vn', 'vn'),
        "wishlist": ('vn', 'vn'),
    }
    attr_name, attr_src = attr[typ]

    items = list()
    with TCPSession(HOSTNAME, PORT) as s:
        print(s.login(), flush=True, file=sys.stderr)
        stats = s.dbstats()

        count = 0
        index = 1
        page = 1
        results = 25 if "list" not in typ else 100
        while count < stats[attr_src]:
            fltr = f'({attr_name} = {repr(list(range(index, index+results)))})'
            cfg = {"page": page, "results": results, "reverse": False}
            res = s.get(typ, flags, fltr, json.dumps(cfg).replace(" ", ""))

            if 'id' in res:
                print(pprint.pformat(res), flush=True, file=sys.stderr)
                if res['id'] == 'throttled':
                    time.sleep(res['fullwait'])
                    continue
                raise RuntimeError("Invalid query was posted to VNDB")

            print(pprint.pformat(res), flush=True)
            items.extend(res['items'])
            count += res['num']
            if res['more']:
                page += 1
            else:
                index += results
                page = 1

    return items

def main():
    args = parse_opt()
    if not os.path.exists(args.dir):
        os.makedirs(args.dir)

    if args.typ == "dbstats":
        with TCPSession(HOSTNAME, PORT) as s:
            print(s.login(), flush=True, file=sys.stderr)
            stats = s.dbstats()
            print(pprint.pformat(stats), flush=True)
        if args.csv:
            with open(f"{args.dir}/dbstats.csv", "w") as f:
                writer = csv.DictWriter(f, stats.keys())
                writer.writeheader()
                writer.writerow(stats)
        else:
            with open(f"{args.dir}/dbstats.json", "w") as f:
                json.dump(stats, f)
    else:
        items = fetch_vndb(args.typ, args.flags)
        if args.csv:
            with open(f"{args.dir}/{args.typ}_{args.flags}.csv", "w") as f:
                writer = csv.DictWriter(f, items[0].keys())
                writer.writeheader()
                for item in items:
                    writer.writerow(item)
        else:
            with open(f"{args.dir}/{args.typ}_{args.flags}.json", "w") as f:
                json.dump(items, f)

if __name__ == "__main__":
    main()


from quart import Blueprint, jsonify
from typing import TYPE_CHECKING

from litecord.auth import admin_check
from litecord.schemas import validate, USER_UPDATE
from litecord.admin_schemas import USER_CREATE
from litecord.errors import BadRequest, NotFound
from litecord.utils import extract_limit
from litecord.blueprints.users import handle_user_update
from litecord.common.users import (
    create_user,
    delete_user,
    user_disconnect,
)

if TYPE_CHECKING:
    from litecord.typing_hax import app, request
else:
    from quart import current_app as app, request

bp = Blueprint("users_admin", __name__)

@bp.route("", methods=["POST"], strict_slashes=False)
async def _create_user():
    await admin_check()
    j = validate(await request.get_json(), USER_CREATE)
    user_id, _ = await create_user(
        j["username"], j["email"], j["password"], j.get("date_of_birth"), id=j.get("id")
    )
    return jsonify(await app.storage.get_user(user_id, True)), 201

def args_try(args: dict, typ, field: str, default):
    """Try to fetch a value from the request arguments,
    given a type."""
    try:
        return typ(args.get(field, default))
    except (TypeError, ValueError):
        raise BadRequest(message=f"invalid {field} value")

@bp.route("", methods=["GET"], strict_slashes=False)
async def query_users():
    await admin_check()

    limit = extract_limit(request, 1, 25, 100)
    j = validate(
        request.args.to_dict(),
        {
            "q": {"coerce": str, "required": False, "maxlength": 32},
            "offset": {"coerce": int, "default": 0},
        },
    )
    query = j.get("q") or ""
    offset = j["offset"]
    extra = ""
    args = (query, limit, offset)

    discriminator = None
    if "#" in query:
        query, _, discriminator = query.rpartition("#")
        try:
            discriminator = "%04d" % int(discriminator)
        except ValueError:
            discriminator = None

    if discriminator:
        extra = "AND discriminator = $4"
        args = (query, limit, offset, discriminator)

    result = await app.storage.get_users(
        secure=True,
        extra_clause=", COUNT(*) OVER() as total_results",
        where_clause=f"""
        WHERE username ILIKE '%'||$1||'%' {extra}
        ORDER BY username
        LIMIT $2 OFFSET $3
        """,
        args=args,
    )

    total_results = result[0]["total_results"] if result else 0
    for user in result:
        user.pop("total_results")
    return jsonify({"users": result, "total_results": total_results})

@bp.route("/@me", methods=["GET"])
async def get_me():
    user_id = await admin_check()
    return jsonify(await app.storage.get_user(user_id, True))

@bp.route("/<int:target_id>", methods=["GET"])
async def get_other(target_id):
    await admin_check()
    other = await app.storage.get_user(target_id, True)
    if not other:
        raise NotFound(10013)
    return jsonify(other)

@bp.route("/<int:user_id>", methods=["DELETE"])
@bp.route("/<int:user_id>/delete", methods=["POST"])
async def _delete_user(user_id: int):
    await admin_check()

    await delete_user(user_id)
    await user_disconnect(user_id)

    new_user = await app.storage.get_user(user_id, True)
    return jsonify(new_user)

@bp.route("/<int:user_id>", methods=["PATCH"])
async def patch_user(user_id: int):
    await admin_check()

    j = validate(await request.get_json(), USER_UPDATE)

    if "flags" in j and j["flags"] is not None:
        await app.db.execute(
            """
        UPDATE users
        SET flags = $1
        WHERE id = $2
        """,
            j["flags"],
            user_id,
        )
        j.pop("flags")

    private_user = await handle_user_update(user_id, False)
    return jsonify(private_user)

@bp.route("/<int:user_id>/channels", methods=["GET"])
async def user_dms(user_id: int):
    await admin_check()
    return jsonify(await app.user_storage.get_dms(user_id))

@bp.route("/<int:user_id>/relationships", methods=["GET"])
async def user_relationships(user_id: int):
    await admin_check()
    return jsonify(await app.user_storage.get_relationships(user_id))


LIST_REPORT_NAME = "Звіт з харчування"

# Column names
NAME_TABLE_COL = ["Клас", "Кількість дітей \n за списком", "Кількість дітей,\n які харчуються",
                  "Дітоднів", "Сума", "Примітка"]

REPORT1_SIGNS_DICT = {"ДИРЕКТОР ХЗОШ № 95": "Н.В.МЕРЗЛЯКОВА",
                      "ЗАСТУПНИК ДИРЕКТОРА З НВР": "Т.О.ЛОБАЧ",
                      "ЗАВ. ВИРОБНИЦТВОМ": "М.В.ДЕГТЯР"}

FORMS_TITLE = ('ПЕРШІ', 'ДРУГІ', 'ТРЕТІ', 'ЧЕТВЕРТІ')

# Class report column names
CLASS_NAME_TABLE_COL = ["№", "ПІБ уч-ся", "Пропуски", "Дітоднів", "Сума"]

CLASS_REPORT_SIGNS_DICT = {"ДИРЕКТОР ХЗОШ № 95": "Н.В.МЕРЗЛЯКОВА",
                           "ЗАСТУПНИК ДИРЕКТОРА З НВР": "Т.О.ЛОБАЧ",
                           "ЗАВ. ВИРОБНИЦТВОМ": "М.В.ДЕГТЯР",
                           }
CLASS_TEACHER = "КЛАСНИЙ КЕРІВНИК"

CLASS_WAS_NOT = float(0.0)
RESULT_NAME = "ВСЬОГО"


#!/usr/bin/env python
import sys
sys.path.append('Sublist3r')
import argparse, requests
import sublist3r
from colorama import Fore
from art import *

status_accepted = [200, 403, 402, 404, 503, 505]
basic_tests =   [
				".htaccess",
				".htaccess~",
				"htaccess",
				"htaccess~",
				"htaccess.txt",
				"robots.txt",
				"readme.txt",
				"config/install.php",
				"config/readme.txt"
				]

def fucking_false_positive(response):
	if response.status_code == 200:
		if response.content == None:
			return True
		if (b"404" or b"Not found" or b"not found") in response.content:
			return False
	return False

def print_header():
		ascii_header = text2art("LEECHER", "rand")
		print(ascii_header)
		print("\t\t\tAutomated Security Audit\n")

def make_request(domain, redirect):
	try:
		response = requests.get("https://"+domain, allow_redirects=redirect, verify=False, timeout=5)
		if not response:
			response = requests.get("http://"+domain, allow_redirects=redirect, verify=False, timeout=5)
		return response
	except:
		return None

class   Leecher:
	def __init__(self):
		print_header()
		parser = argparse.ArgumentParser()
		parser.add_argument("-t", "--target", help="Set the target")
		parser.add_argument("--full-check", help="Make a recursive check for all subdomains for all CMS", action="store_true")
		parser.add_argument("-s", "--silent", help="Disable modules output", default=False, action="store_true")
		parser.add_argument("-v", "--verbose", help="Verbose mod", default=False, action="store_true")
		self.args = parser.parse_args()
		if not self.args.target:
			parser.error("Target needed")

	def launch_scan(self):
		subdomains = self.sublister_wrapper(self.args.target)
		print("\nLet's h4ck\n")
		self.process_scan(subdomains)

	def sublister_wrapper(self, domain):
		print("Launching Sublist3r...")
		try:
			subdomains = sublist3r.main(
					domain,
					40,
					None,
					ports=None,
					silent=self.args.silent, verbose=self.args.verbose, enable_bruteforce=False, engines=None
					)
			subdomains.insert(0, domain)
			if self.args.verbose:
				print("Subdomains founds :")
				print(subdomains)
			return subdomains
		except:
			sys.exit("Error while processing Sublist3r");

	def process_scan(self, subdomains):
		for domain in subdomains:
			response = make_request(domain, True)
			if response:
				if response.status_code in status_accepted:
					print("[" + Fore.YELLOW + "+" + Fore.RESET + "] [%s] is responding" % domain)
					self.process_basic_tests(domain)
				else:
					if self.args.verbose:
						print("[" + Fore.RED + "-" + Fore.RESET + "] [%s] is not responding" % domain)

	def process_basic_tests(self, domain):
		for test in basic_tests:
			response = make_request(domain+'/'+test, False)
			if response is not None:
				if response.status_code == 200 and not fucking_false_positive(response):
					print("[" + Fore.GREEN + "✓" + Fore.RESET + "]\t[%s/%s] succeed" % (domain, test))
				else:
					if self.args.verbose:
						print("[" + Fore.RED + "X" + Fore.RESET + "]\t[%s/%s] failed" % (domain, test))

	def wpscan_wrapper(self):
		print("WPScan processing...")
		#use wpscan and save result with sublist for the current url
		return

def	leecher_launch():
	instance = Leecher()
	instance.launch_scan()

if __name__ == '__main__':
	leecher_launch()


import sqlite3

connection = sqlite3.connect('gfg.db')
cursor = connection.cursor()

#SQL command to create a table in the database
command = """
CREATE TABLE emp(
    staff_number INTEGER PRIMARY KEY,
    fname VARCHAR(20),
    lname VARCHAR(30),
    gender CHAR(1),
    joining DATE
);
"""
cursor.execute(command)

command = """
INSERT INTO emp
VALUES (23, "Rishabh", "Bansal", "M", "2014-03-28");
"""
cursor.execute(command)

command = """
INSERT INTO emp
VALUES (1, "Bill", "Gates", "M", "1980-10-28");
"""
cursor.execute(command)

#geting results
cursor.execute("SELECT * FROM emp")
result = cursor.fetchall()
print(result)

connection.commit()
connection.close()


#!/usr/bin/env python

import sys
import os

# We will use the inkex module with the predefined Effect base class.
import inkex
# The simplestyle module provides functions for style parsing.
from simplestyle import *

def export_to_starling(document):

    # Get access to the main SVG document
    svg = document.getroot()

    # Create folder if not exists
    folder = "C:\\Users\\Andres\\Desktop\\StarlingTest"
    if not os.path.exists(folder):
        os.makedirs(folder)

    # Generate Main.as document
    generate_main(folder, svg)

    # Generate Game.as document
    generate_game(folder, svg)

# Generate Main.as document
def generate_main(folder, svg):

    # Create Main.as file
    filename = os.path.abspath(folder + "\\Main.as")
    f = open(filename, 'w')

    # Start to generate
    width = svg.attrib['width']
    height = svg.attrib['height']
    properties = svg.xpath('//*[@id="base"]', namespaces=inkex.NSS)[0]
    bgcolor = properties.attrib['pagecolor']

    f.write("import flash.display.Sprite;\nimport starling.core.Starling;\n\n[SWF(width=\"" + width + "\", height=\"" + height + "\", frameRate=\"60\", backgroundColor=\"" + bgcolor + "\")]\npublic class Startup extends Sprite\n{\n\tprivate var _starling:Starling;\n\n\tpublic function Startup()\n\t{\n\t\t_starling = new Starling(Game, stage);\n\t\t_starling.start();\n\t}\n}\n");

# Generate Game.as document
def generate_game(folder, svg):

    # Create Game.as file
    filename = os.path.abspath(folder + "\\Game.as")
    f = open(filename, 'w')

    # Write imports
    f.write("import starling.display.Sprite;\n\n")

    # Write Game class
    f.write("public class Game extends Sprite\n{\n")

    # Generate constructor

    f.write("\tpublic function Game()\n\t{\n\t}\n")

    # End of Game class
    f.write("}\n")


import csv
import codecs
import calendar

from datetime import datetime, timedelta
from django.db import connection
from django.db.models import Q, Sum, Case, When, F
from django.utils import timezone
from django.http import HttpResponse
from django.utils.encoding import escape_uri_path
from dynaconf import settings as dynasettings

from common.encryption import AESCipher
from common.models import (
    Merchant,
    Marketer,
    Client,
    Coupon,
    Payment,
)
from config import (
    MERCHANT_STATUS,
    SYSTEM_USER_STATUS,
    PAYMENT_STATUS,
    PAY_CHANNELS,
    WITHDRAW_STATUS,
    WITHDRAW_TYPE,
    WECHAT_PAY_BROKERAGE,
    ALIPAY_PAY_BROKERAGE,
    COUPON_STATUS,
    TRANSACTION_TYPE
)

def fen_to_yuan(fen):
    if fen is None:
        return 0
    return fen / 100.0

def format_yuan(float_yuan):

    return round(float_yuan, 2)

class CsvResponseFactory(object):
    @staticmethod
    def transaction_details_csv_response(result_data):
        csv_response = HttpResponse(content_type='text/csv')
        csv_response['Content-Disposition'] = "attachment; filename*=utf-8''{}".format(escape_uri_path("资金账单.csv"))
        csv_response.write(codecs.BOM_UTF8)
        writer = csv.writer(csv_response)
        head_list = [
            '时间',
            "有效交易笔数",
            '订单总额（元）',
            '交易总额（元）',
            '手续费（元）',
            '普通账单金额（元）',
            '优惠账单金额（元）',
        ]
        writer.writerow(head_list)
        for row in result_data['data']:
            row_list = [
                row['datetime'],
                row['tran_count'],
                row['order_price'],
                "{}\n支付宝：{} | 微信：{}".format(row['paid_price'], row['paid_price_alipay'], row['paid_price_wechat']),
                "{}\n支付宝：{} | 微信：{}".format(row['brokerage'], row['brokerage_alipay'], row['brokerage_wechat']),
                "{}\n支付宝：{} | 微信：{}".format(row['ordinary'], row['ordinary_alipay'], row['ordinary_wechat']),
                "{}\n支付宝：{} | 微信：{}".format(row['preferential'], row['preferential_alipay'],
                                            row['preferential_wechat']),
            ]
            writer.writerow(row_list)
        result = result_data['result']
        result_list = [
            "总计",
            result['all_tran_count'],
            result['all_order_price'],
            result['all_paid_price'],
            result['all_brokerage'],
            result['all_ordinary'],
            result['all_preferential'],
        ]
        writer.writerow(result_list)
        return csv_response

    @staticmethod
    def sharing_details_csv_response(result_data):
        csv_response = HttpResponse(content_type='text/csv')
        csv_response['Content-Disposition'] = "attachment; filename*=utf-8''{}".format(escape_uri_path("资金分成.csv"))
        csv_response.write(codecs.BOM_UTF8)
        writer = csv.writer(csv_response)
        head_list = [
            '时间',
            '优惠账单交易总额（元）',
            '总抽成（元）',
            '引流商户提成（元）',
            '邀请人提成（元）',
            '平台分成（元）',
        ]
        writer.writerow(head_list)
        for row in result_data['data']:
            row_list = [
                row['datetime'],
                row['paid_price'],
                row['share_all'],
                row['share_originator'],
                row['share_inviter'],
                row['share_platform'],
            ]
            writer.writerow(row_list)
        result = result_data['result']
        result_list = [
            "总计",
            result['all_paid_price'],
            result['all_share_all'],
            result['all_share_originator'],
            result['all_share_inviter'],
            result['all_share_platform'],
        ]
        writer.writerow(result_list)
        return csv_response

    @staticmethod
    def withdraw_record_csv_response(result_data):
        csv_response = HttpResponse(content_type='text/csv')
        csv_response['Content-Disposition'] = "attachment; filename*=utf-8''{}".format(escape_uri_path("提现申请记录表.csv"))
        csv_response.write(codecs.BOM_UTF8)
        writer = csv.writer(csv_response)
        head_list = [
            '申请时间',
            '提现商户/个人',
            '标志',
            '交易单号',
            '提现类型',
            '提现金额（元）',
            '状态',
        ]
        writer.writerow(head_list)
        for row in result_data['data']:
            row_list = [
                row['datetime'],
                row['name'],
                row['is_merchant'],
                f"'{row['serial_number']}",
                row['withdraw_type'],
                row['with_draw_money'],
                row['status'],
            ]
            writer.writerow(row_list)
        result = result_data['result']
        result_list = [
            "总计提现成功金额: {}".format(result['with_draw_total_money']),
        ]
        writer.writerow(result_list)
        return csv_response

class TimeTransform(object):
    @staticmethod
    def date_string_to_start_date(data_string):
        if data_string:
            date = datetime.strptime(data_string, "%Y-%m-%d")
            return date
        return None

    @staticmethod
    def date_string_to_end_date(data_string):
        if data_string:
            date = datetime.strptime(data_string, "%Y-%m-%d")
            end_date = date.replace(hour=23, minute=59, second=59, microsecond=999999)
            return end_date
        return None

class FinancialQuery(object):
    @classmethod
    def data_overview(cls):
        merchant_all = Merchant.objects.filter(~Q(status=MERCHANT_STATUS['DISABLED'])).count()
        inviter_all = Marketer.objects.filter(status=SYSTEM_USER_STATUS['USING']).count()
        client_all = Client.objects.filter(Q(status=SYSTEM_USER_STATUS['USING'])).count()
        coupon_all = Coupon.objects.filter(~Q(status=COUPON_STATUS['DESTROYED'])).count()
        coupon_used = Coupon.objects.filter(status=COUPON_STATUS['USED']).count()
        payment_query = Payment.objects.filter(status__in=(PAYMENT_STATUS['FROZEN'], PAYMENT_STATUS['REFUND_REQUESTED'],
                                           PAYMENT_STATUS['REFUND_FAILED'], PAYMENT_STATUS['FINISHED']
                                           )).select_related('coupon').values('pay_channel').\
            annotate(paid_price = Sum(Case(When(coupon__isnull=True, then='order_price'),
                                           default=(F('order_price')-F('coupon__discount')))))
        payment_all = {
            "wechat": 0,
            "alipay": 0,
            "total": 0,
        }
        for item in payment_query:
            payment_all['total'] += item['paid_price']
            if item['pay_channel'] == PAY_CHANNELS['WECHAT']:
                payment_all['wechat'] = item['paid_price']
            elif item['pay_channel'] == PAY_CHANNELS['ALIPAY']:
                payment_all['alipay'] = item['paid_price']
        payment_all = {
            "wechat": fen_to_yuan(payment_all['wechat']),
            "alipay": fen_to_yuan(payment_all['alipay']),
            "total": fen_to_yuan(payment_all['total']),
        }
        return dict(
            merchant=merchant_all,  # 商户总数
            inviter=inviter_all,  # 邀请人数
            client=client_all,  # 用户数
            coupon=coupon_all,  # 优惠券数
            coupon_used=coupon_used,  # 已使用优惠券数
            payment=payment_all  # 入账金额
        )

    @classmethod
    def transaction_details(cls, start_date=None, end_date=None):
        # 除开退款的payment
        query_sql_start = f"""
            select
                (DATE_FORMAT(datetime, '%%Y-%%m-%%d')) as time_str,
                cast( sum(p.order_price) as SIGNED) as order_price ,
                cast( sum((case when p.coupon_id is null then p.order_price else p.order_price - c.discount end)) as SIGNED)as paid_price ,
                cast( sum((case when p.pay_channel = {PAY_CHANNELS['WECHAT']} then (case when p.coupon_id is null then p.order_price else p.order_price - c.discount end)  else 0 end)) as SIGNED)as paid_price_wechat ,
                cast( sum((case when p.pay_channel = {PAY_CHANNELS['ALIPAY']} then (case when p.coupon_id is null then p.order_price else p.order_price - c.discount end)  else 0 end)) as SIGNED)as paid_price_alipay ,
                cast( sum((case when p.coupon_id is null then p.order_price else 0 end)) as SIGNED)as ordinary,
                cast( sum((case when p.coupon_id is null and p.pay_channel = {PAY_CHANNELS['WECHAT']} then p.order_price else 0 end)) as SIGNED)as ordinary_wechat,
                cast( sum((case when p.coupon_id is null and p.pay_channel = {PAY_CHANNELS['ALIPAY']} then p.order_price else 0 end)) as SIGNED)as ordinary_alipay,
                cast( sum((case when p.coupon_id is not null then p.order_price - c.discount else 0 end)) as SIGNED)as preferential,
                cast( sum((case when p.coupon_id is not null and p.pay_channel = {PAY_CHANNELS['WECHAT']} then p.order_price - c.discount else 0 end)) as SIGNED)as preferential_wechat,
                cast( sum((case when p.coupon_id is not null and p.pay_channel = {PAY_CHANNELS['ALIPAY']} then p.order_price - c.discount else 0 end)) as SIGNED)as preferential_alipay,
                cast( sum((case when p.pay_channel = {PAY_CHANNELS['WECHAT']} then (case when p.coupon_id is null then cast( round(p.order_price/100*{WECHAT_PAY_BROKERAGE}, 2)*100 as SIGNED) else cast( round((p.order_price - c.discount)/100 * {WECHAT_PAY_BROKERAGE}, 2)*100 as SIGNED) end) else 0 end) ) as SIGNED) as paid_price_wechat_brokerage ,
                cast( sum((case when p.pay_channel = {PAY_CHANNELS['ALIPAY']} then (case when p.coupon_id is null then cast( round(p.order_price/100*{ALIPAY_PAY_BROKERAGE}, 2)*100 as SIGNED) else cast( round((p.order_price - c.discount)/100 * {ALIPAY_PAY_BROKERAGE}, 2)*100 as SIGNED) end)  else 0 end) ) as SIGNED )as paid_price_alipay_brokerage,
                count(*) as tran_count
            from common_payment as p LEFT JOIN common_coupon as c
            on p.coupon_id = c.id
            where p.status in ({PAYMENT_STATUS['FROZEN']}, {PAYMENT_STATUS['REFUND_REQUESTED']},{PAYMENT_STATUS['REFUND_FAILED']},{PAYMENT_STATUS['FINISHED']}) and 1 = %s
            """
        query_sql_condition = ""
        query_params = [1]
        query_sql_end = "group by time_str order by time_str desc"

        if start_date:
            query_sql_condition = query_sql_condition + " and p.datetime >= %s "
            query_params.append(start_date.strftime('%Y-%m-%d %H:%M:%S.%f'))
        if end_date:
            query_sql_condition = query_sql_condition + " and p.datetime <= %s "
            query_params.append(end_date.strftime('%Y-%m-%d %H:%M:%S.%f'))

        query_sql = query_sql_start + query_sql_end
        if query_sql_condition != "" and len(query_params) != 1:
            query_sql = query_sql_start + query_sql_condition + query_sql_end

        all_order_price = 0
        all_paid_price = 0
        all_paid_price_wechat = 0
        all_paid_price_alipay = 0
        all_ordinary = 0
        all_ordinary_wechat = 0
        all_ordinary_alipay = 0
        all_preferential = 0
        all_preferential_wechat = 0
        all_preferential_alipay = 0
        all_brokerage = 0
        all_brokerage_wechat = 0
        all_brokerate_alipay = 0
        all_tran_count = 0
        with connection.cursor() as cursor:
            cursor.execute(query_sql, query_params)
            row_list = []
            for row in cursor.fetchall():
                all_order_price += row[1]
                all_paid_price += row[2]
                all_paid_price_wechat += row[3]
                all_paid_price_alipay += row[4]
                all_ordinary += row[5]
                all_ordinary_wechat += row[6]
                all_ordinary_alipay += row[7]
                all_preferential += row[8]
                all_preferential_wechat += row[9]
                all_preferential_alipay += row[10]
                all_brokerage_wechat += row[11]
                all_brokerate_alipay += row[12]
                all_brokerage += row[11] + row[12]  # 手续费
                all_tran_count += row[13] # 有效交易

                row_dict = {
                    "datetime": row[0],
                    "order_price": fen_to_yuan(row[1]),
                    "paid_price": fen_to_yuan(row[2]),
                    "paid_price_wechat": fen_to_yuan(row[3]),
                    "paid_price_alipay": fen_to_yuan(row[4]),
                    "ordinary": fen_to_yuan(row[5]),
                    "ordinary_wechat": fen_to_yuan(row[6]),
                    "ordinary_alipay": fen_to_yuan(row[7]),
                    "preferential": fen_to_yuan(row[8]),
                    "preferential_wechat": fen_to_yuan(row[9]),
                    "preferential_alipay": fen_to_yuan(row[10]),
                    "brokerage": fen_to_yuan(row[11] + row[12]),
                    "brokerage_wechat": fen_to_yuan(row[11]),
                    "brokerage_alipay": fen_to_yuan(row[12]),
                    "tran_count": row[13],
                }
                row_list.append(row_dict)
        result = {
            "result": {
                "all_brokerage": format_yuan(fen_to_yuan(all_brokerage)),
                "all_brokerage_wechat": format_yuan(fen_to_yuan(all_brokerage_wechat)),
                "all_brokerate_alipay": format_yuan(fen_to_yuan(all_brokerate_alipay)),
                "all_order_price": format_yuan(fen_to_yuan(all_order_price)),
                "all_paid_price": format_yuan(fen_to_yuan(all_paid_price)),
                "all_paid_price_wechat": format_yuan(fen_to_yuan(all_paid_price_wechat)),
                "all_paid_price_alipay": format_yuan(fen_to_yuan(all_paid_price_alipay)),
                "all_ordinary": format_yuan(fen_to_yuan(all_ordinary)),
                "all_ordinary_alipay": format_yuan(fen_to_yuan(all_ordinary_alipay)),
                "all_ordinary_wechat": format_yuan(fen_to_yuan(all_ordinary_wechat)),
                "all_preferential": format_yuan(fen_to_yuan(all_preferential)),
                "all_preferential_alipay": format_yuan(fen_to_yuan(all_preferential_alipay)),
                "all_preferential_wechat": format_yuan(fen_to_yuan(all_preferential_wechat)),
                "all_tran_count": all_tran_count,
            },
            "data": row_list,
        }
        return result

    @classmethod
    def sharing_details(cls, start_date=None, end_date=None):
        # 除开退款的以及没有使用优惠券的payment
        query = Payment.objects.filter(Q(status__in=(PAYMENT_STATUS['FROZEN'], PAYMENT_STATUS['REFUND_REQUESTED'],
                       PAYMENT_STATUS['REFUND_FAILED'], PAYMENT_STATUS['FINISHED']
                       )), Q(coupon__isnull=False))
        if start_date:
            query = query.filter(datetime__gte=start_date)
        if end_date:
            query = query.filter(datetime__lte=end_date)

        query = query.select_related('coupon').extra(
            select={'datetime': "DATE_FORMAT(datetime, %s)"},
            select_params=("%Y-%m-%d", ), ) \
            .values("datetime") \
            .annotate(
            paid_price=Sum(F('order_price') - F('coupon__discount')),
            share_platform=Sum('platform_share'),
            share_inviter=Sum('inviter_share'),
            share_originator=Sum('originator_share'),
            share_all = Sum(F('platform_share') + F('inviter_share') + F('originator_share')),
        ).order_by("-datetime")

        query_set = list(query)
        all_paid_price = 0
        all_share_all = 0
        all_share_platform = 0
        all_share_inviter = 0
        all_share_originator = 0
        for item in query_set:
            item['paid_price'] = format_yuan(fen_to_yuan(item['paid_price']))
            item['share_platform'] = format_yuan(fen_to_yuan(item['share_platform']))
            item['share_inviter'] = format_yuan(fen_to_yuan(item['share_inviter'] ))
            item['share_originator'] = format_yuan(fen_to_yuan(item['share_originator'] ))
            item['share_all'] = format_yuan(fen_to_yuan(item['share_all'] ))

            all_paid_price = all_paid_price + item['paid_price']
            all_share_all = all_share_all + item['share_all']
            all_share_originator = all_share_originator + item['share_originator']
            all_share_inviter = all_share_inviter + item['share_inviter']
            all_share_platform = all_share_platform + item['share_platform']

        result = {
            "result": {
                "all_paid_price": format_yuan(all_paid_price),
                "all_share_all": format_yuan(all_share_all),
                "all_share_platform": format_yuan(all_share_platform),
                "all_share_inviter": format_yuan(all_share_inviter),
                "all_share_originator": format_yuan(all_share_originator),
            },
            "data": query_set
        }
        return result

    @classmethod
    def withdraw_record(cls, status=None, start_date=None, end_date=None):
        sql_start = """
            select
                a.datetime,
                a.status,
                a.amount,
                (case when a.merchant_name is null then a.inviter_name else a.merchant_name end) as p_name,
                (case when a.merchant_phone is null then a.inviter_phone else a.merchant_phone end ) as p_phone,
                a.serial_number,
                (case when a.merchant_name is null then 0 else 1 end) as is_merchant,
                a.withdraw_type
                from (
                    select *
                    from common_withdraw w  LEFT JOIN  ( select account_id as merchant_account_id, name as merchant_name, contact_phone as merchant_phone from common_merchant ) merchant
                    on  w.account_id = merchant.merchant_account_id
                    left join ( select account_id as inviter_account_id, name as inviter_name, phone as inviter_phone from common_marketer) inviter
                    on w.account_id = inviter.inviter_account_id
                    where 1 = %s """
        sql_condition = ""
        sql_end = """ ) a order by a.datetime desc """
        sql_params = [1]

        if status:
            try:
                status = int(status)
            except ValueError:
                pass
            else:
                sql_condition = sql_condition + " and status = %s "
                sql_params.append(status)
        if start_date:
            sql_condition = sql_condition + " and datetime >= %s "
            sql_params.append(start_date.strftime('%Y-%m-%d %H:%M:%S.%f'))
        if end_date:
            sql_condition = sql_condition + " and datetime <= %s "
            sql_params.append(end_date.strftime('%Y-%m-%d %H:%M:%S.%f'))

        all_withdraw_money = 0
        list_data = []
        query_sql = sql_start + sql_condition + sql_end
        with connection.cursor() as cursor:
            cursor.execute(query_sql, sql_params)
            for row in cursor.fetchall():
                if row[1] == WITHDRAW_STATUS['FINISHED']:
                    all_withdraw_money += row[2]
                list_data.append({
                    'datetime': row[0].strftime("%Y-%m-%d %H:%M:%S"),
                    'name': row[3],
                    'serial_number': row[5],
                    'is_merchant': "个体商户" if row[6] == 1 else "邀请人",
                    'withdraw_type': "微信零钱" if row[7] == WITHDRAW_TYPE['WECHAT'] else "支付宝余额",
                    'with_draw_money': fen_to_yuan(row[2]),
                    'status': WITHDRAW_STATUS[row[1]]['name'],
                    'note': ""
                })
        result = {
            "result": {
                "with_draw_records": len(list_data),
                "with_draw_total_money": format_yuan(fen_to_yuan(all_withdraw_money)),
            },
            "data": list_data
        }

        return result

    @classmethod
    def query_merchant_month_bill(cls):
        """ 查询所有商户月账单 """
        # 根据当前时间求出上个月的第一天和最后一天时间
        curr_time = timezone.now()
        curr_month_first_day = curr_time.replace(day=1,hour=0, minute=0, second=0, microsecond=0)
        end_time = curr_month_first_day - timedelta(days = 1)
        month_days = calendar.monthrange(end_time.year, end_time.month)[1]
        start_time = curr_month_first_day - timedelta(days=month_days)
        end_time = end_time.replace(hour=23, minute=59, second=59, microsecond=999999)

        start_time_str = start_time.strftime('%Y-%m-%d %H:%M:%S.%f')
        end_time_str = end_time.strftime('%Y-%m-%d %H:%M:%S.%f')

        query_sql =f"""
        select a.wechat_openid, b.turnover, b.sharing_earning from (
                select
                    m.account_id,
                    admin.wechat_openid
                from common_merchant m INNER JOIN common_merchantadmin admin
                on m.id = admin.work_merchant_id
                where m.status = {MERCHANT_STATUS['USING']}
          ) a left join (
                select account_id,
                    cast( sum( (case when transaction_type in ({TRANSACTION_TYPE['MERCHANT_RECEIVE']},{TRANSACTION_TYPE['MERCHANT_REFUND']}) then amount else 0 end)) as SIGNED) as turnover,
                    cast( sum( (case when transaction_type = {TRANSACTION_TYPE['MERCHANT_SHARE']} then amount else 0 end)) as SIGNED) as sharing_earning
                from common_transaction
                where datetime >= %s and datetime <= %s and account_id in (select account_id from common_merchant where status = {MERCHANT_STATUS['USING']})
                group by account_id
        ) b
        on a.account_id = b.account_id
        """
        sql_params = [start_time_str, end_time_str]
        result_list = []
        aes = AESCipher(dynasettings.OPENID_AES_KEY)
        with connection.cursor() as cursor:
            cursor.execute(query_sql, sql_params)
            for row in cursor.fetchall():
                result_list.append(
                    {
                        "open_id": aes.decrypt(row[0]),
                        "bill": {
                            "turnover": format_yuan(fen_to_yuan(row[1])),
                            "sharing_earning": format_yuan(fen_to_yuan(row[2])),
                        }
                    })
        return start_time.year,  start_time.month, result_list