from ..dateparser import DateSearchStatus, AbstractDateParser
from .time_parser import TimeParser
from .after_minutes_parser import AfterMinutesParser
from .after_hours_parser import AfterHoursParser
from .relative_day_parser import RelativeDayParser
from .day_month_parser import DayMonthParser
from .week_day_parser import WeekDayParser
from typing import List
from datetime import datetime, timedelta, time

class DateSearcher():
    """
    Класс, содержащий парсеры дат.
    Инкапсулирует конкретные парсеры.
    """
    PARSERS = [
        TimeParser(),
        AfterHoursParser(),
        AfterMinutesParser(),
        RelativeDayParser(),
        WeekDayParser(),
        DayMonthParser()
    ]

    # Время по умолчанию
    FALLBACK_TIME = time(hour=10, minute=0)

    def get_parsers(self) -> List[AbstractDateParser]:
        return self.PARSERS

    def parse(self, msg: str, current_datetime: datetime) -> DateSearchStatus:
        status = DateSearchStatus(current_datetime)
        for parser in self.get_parsers():
            parser.search(msg, status)

        if status.date and not status.time:
            self.add_fallback_time(status)

        if not status.date and status.time:
            self.add_fallback_date(status)

        return status

    def add_fallback_date(self, status: DateSearchStatus) -> None:
        """
        Добавляет дату по умолчанию.
        Используется, когда указано только время.
        Если сегодня больше времени, чем нужно, то напоминание будет установлено на завтра.
        :param status:
        :return:
        """
        if status.current_datetime.time() > status.time:
            status.date = status.current_datetime.date()
        else:
            status.date = status.current_datetime.date() + timedelta(days=1)

    def add_fallback_time(self, status: DateSearchStatus):
        """
        Добавляет время.
        Используется, когда указано только дата.
        Время по умолчанию захардкожено.
        :param status:
        :return:
        """
        status.time = self.FALLBACK_TIME


import sys
sys.stdin = open("cowqueue.in", "r")
sys.stdout = open("cowqueue.out", "w")

n = int(input())
record = []
for i in range(n):
    start, gap = map(int, input().split())
    record.append((start,gap))

def by_time(r):
    return r[0]
record.sort(key=by_time)

cur_time = 0
for i in range(n):
    start, gap = record[i]
    if start > cur_time:
        cur_time = start
    cur_time += gap

print(cur_time)


import os
import csv
import glob
import json
import torch
import warnings
import itertools
import torchaudio
import numpy as np
from pathlib import Path
from tqdm import tqdm
from PIL import Image as PILImage
from itertools import cycle, islice, chain
from einops import rearrange, repeat

import multiprocessing as mp
import torch.utils.data as data
import torch.nn.functional as F

from .audio import (
    make_transform, _extract_kaldi_spectrogram
)
from .image import make_clip_image_transform as make_image_transform
from .audio_text import (
    build_dataloader, build_audiocaps_data_list, AudioTextDatasetSrc
)
from clip import tokenize

class ImageTextDatasetSrc(AudioTextDatasetSrc):
    """ `__getitem__' loads raw file from disk.
    """
    def __init__(self, cfg, data_list, train):
        super().__init__(cfg, data_list, train)
        self.frame_key = cfg.frame_key
        self.transform_image = make_image_transform(cfg.resolution)

    def _image2embed(self, fname):
        try:
            image = np.load(fname)["v"]
        except Exception as e:
            image = np.random.rand(self.cfg.embed_dim).astype("float32")
            warnings.warn(f"use random image instead because `{e}` {fname}.")
        return image

    def _image2numpy(self, fname):
        if fname is not None:
            try:
                if fname.endswith(".npz"):
                    images = np.load(fname)
                    images = [images[key] for key in images.files if len(images[key]) != 0]
                    idx = np.random.choice(len(images), 1)[0] if self.train else int(np.ceil(len(images) / 2)) - 1
                    image = images[idx]
                else:
                    image = PILImage.open(fname)
                    image = self.transform_image(image).cpu().numpy()
            except Exception as e:
                h = w = self.cfg.resolution
                image = PILImage.fromarray(
                    (np.random.rand(h, w, 3) * 256).astype(np.uint8)
                )
                warnings.warn(f"use random image instead because `{e}` {fname}.")
                image = self.transform_image(image).cpu().numpy()
        else:
            image = np.array([[[1]]])
        return image

    def __getitem__(self, index):
        akey = self.aclip_key
        fkey = self.frame_key
        name = self.dataset[index]["id"]
        sub_dir = self.dataset[index]["dir"]
        label_str = self.dataset[index]["label_str"]
        label_int = self.dataset[index]["label_int_bpe"]
        aclip = self.dataset[index][akey][0]
        frame = images = self.dataset[index][fkey]

        sub_dir = "" if len(sub_dir) == 0 else f"{sub_dir}/"
        aclip = aclip if aclip == name else f"{akey}/{name}.{aclip}"
        aclip_file = f"{self.cfg.data_root}/{sub_dir}{aclip}"

        # image
        frame_emb_file = None
        if isinstance(frame, str):
            frame_file = f"{self.cfg.data_root}/{sub_dir}{fkey}/{name}.{frame}"
        else:
            idx = np.random.choice(len(images), 1)[0] if self.train else int(np.ceil(len(images) / 2)) - 1
            frame_file = f"{self.cfg.data_root}/{sub_dir}{fkey}/{name}.{images[idx]}"
            if self.cfg.frame_emb is not None:
                frame_emb_file = f"{self.cfg.data_root}/{self.cfg.frame_emb}/{name}.{images[idx].rsplit('.', 1)[0]}.npz"
        # higher priority for pre-computed frame embeddings
        image = self._image2embed(frame_emb_file) if frame_emb_file is not None else self._image2numpy(frame_file)

        # audio
        audio = self._audio2numpy_cst(aclip_file)

        if not self.cfg.audio.eval_norms and len(self.audio_norms) == 2:
            mean, std = self.audio_norms
            audio = (audio - mean) / std

        #if self.train and self.transform_fbank is not None:
        if not self.cfg.audio.eval_norms and self.train and self.transform_fbank is not None:
            audio = self.transform_fbank(audio)

        if self.train:
            idx = np.random.choice(len(label_int), 1)[0]
            text = label_int[idx]
        else:
            text = label_int

        audio = audio[None]
        image = image[None]
        item = {"image": image, "audio": audio, "text": text, "name": name}
        return item

    def __len__(self):
        return self.length

class ImageTextCollator:
    def __init__(self, device=torch.device("cpu")):
        # RuntimeError: cannot pin 'torch.cuda.FloatTensor' only dense CPU tensors can be pinned
        # when pin_memory is true, the collator has to return CPU tensors
        self.device = device

    def __call__(self, records):
        union = {
            k: [record.get(k) for record in records] for k in set().union(*records)
        }
        name = union["name"]
        text_list = union["text"]
        if isinstance(text_list[0][0], int): # train
            pass
            """ https://stackoverflow.com/a/43149308
            lengths = [len(x) for x in text_list]
            max_len = max(lengths)
            text = np.zeros((len(text_list), max_len), int)
            mask = np.arange(max_len) < np.array(lengths)[:, None]
            text[mask] = np.concatenate(text_list)
            """
        elif isinstance(text_list[0][0], list): # test
            text_list = list(itertools.chain.from_iterable(text_list))
            #name = list(itertools.chain.from_iterable(name))
        else:
            raise ValueError(f"unrecognized `{type(text_list[0][0])}`")
        # https://stackoverflow.com/a/38619333
        text = np.array(list(itertools.zip_longest(*text_list, fillvalue=0))).T
        return (
            np.concatenate(union["image"], axis=0),
            text,
            name,
        )

def build_dataloader_audiocaps(cfg, data_name, shuffle=True, train=True):
    name_list = data_name.split(",")
    dataset = list()
    for name in name_list:
        subset = build_audiocaps_data_list(cfg.running, name)
        dataset.extend(subset)
    return build_dataloader(cfg, dataset, ImageTextDatasetSrc, shuffle=shuffle, train=train, collator_cls=ImageTextCollator)

def build_image_text_dataloader(cfg, data_name, *args, shuffle=True, train=True, **kwargs):
    if data_name.startswith("audiocaps"): # can only do w/ AudioCaps
        return build_dataloader_audiocaps(
            cfg, data_name, shuffle=shuffle, train=train
        )
    else:
        raise ValueError(f"unrecognized dataset `{data_name}`.")


from Orange.misc.utils.embedder_utils import EmbedderCache
from Orange.util import dummy_callback
from orangecontrib.imageanalytics.utils.embedder_utils import ImageLoader

class LocalEmbedder:

    embedder = None

    def __init__(self, model, model_settings):
        self.embedder = model_settings["model"]()

        self._target_image_size = model_settings["target_image_size"]

        self._image_loader = ImageLoader()
        self._cache = EmbedderCache(model)

    def embedd_data(self, file_paths, callback=dummy_callback):
        all_embeddings = []

        for i, row in enumerate(file_paths, start=1):
            all_embeddings.append(self._embed(row))
            callback(i / len(file_paths))

        self._cache.persist_cache()
        return all_embeddings

    def _embed(self, file_path):
        """ Load images and compute cache keys and send requests to
        an http2 server for valid ones.
        """
        image = self._image_loader.load_image_or_none(
            file_path, self._target_image_size
        )
        if image is None:
            return None
        image = self.embedder.preprocess(image)

        cache_key = self._cache.md5_hash(image)
        cached_im = self._cache.get_cached_result_or_none(cache_key)
        if cached_im is not None:
            return cached_im

        embedded_image = self.embedder.predict(image)

        self._cache.add(cache_key, embedded_image)
        return embedded_image


import datetime
import glob
import os
import subprocess

import numpy as np
import pandas as pd
import pose
import poses
import rosbag

# Forward errors so we can recover failures
# even when running commands through multiprocessing
# pooling
def full_traceback(func):
    import functools
    import traceback

    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except Exception as e:
            msg = "{}\n\nOriginal {}".format(e, traceback.format_exc())
            raise type(e)(msg)

    return wrapper

def get_files(directory, file_string):
    return glob.glob(os.path.join(directory, file_string))

def get_files_recursive(directory, file_string):
    subdirectory_csv_files = []
    _, subdirectories, _ = next(os.walk(directory))
    for subdirectory in subdirectories:
        subdirectory_path = os.path.join(directory, subdirectory)
        for subdirectory_csv_file in get_files(subdirectory_path, file_string):
            subdirectory_csv_files.append(subdirectory_csv_file)
    return subdirectory_csv_files

def create_directory(directory=None):
    if directory == None:
        directory = os.path.join(
            os.getcwd(), datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
        )
    if os.path.exists(directory):
        print((directory + " already exists!"))
        exit()
    os.makedirs(directory)
    return directory

def load_dataframe(files):
    dataframes = [pd.read_csv(file) for file in files]
    dataframe = pd.concat(dataframes)
    return dataframe

def run_command_and_save_output(command, output_filename, print_command=True):
    if print_command:
        print(command)
    with open(output_filename, "w") as output_file:
        subprocess.call(command, shell=True, stdout=output_file, stderr=output_file)

def basename(filename):
    return os.path.splitext(os.path.basename(filename))[0]

# TODO(rsoussan): Move these to different utilities file
def get_topic_rates(
    bag_name,
    topic,
    min_time_diff_for_gap,
    use_header_time=True,
    verbose=False,
    ignore_zero_time_diffs=True,
):
    with rosbag.Bag(bag_name, "r") as bag:
        last_time = 0.0
        gaps = 0
        time_diffs = []
        for _, msg, t in bag.read_messages([topic]):
            time = (
                msg.header.stamp.secs + msg.header.stamp.nsecs * 1.0e-9
                if use_header_time
                else t.secs + t.nsecs * 1.0e-9
            )
            time_diff = time - last_time
            if last_time != 0 and time_diff >= min_time_diff_for_gap:
                if verbose:
                    print(
                        (
                            topic
                            + " Gap: time: "
                            + str(time)
                            + ", last_time: "
                            + str(last_time)
                            + ", diff: "
                            + str(time_diff)
                        )
                    )
                gaps += 1
            if last_time != 0 and (time_diff != 0 or not ignore_zero_time_diffs):
                time_diffs.append(time_diff)
            last_time = time

        mean_time_diff = np.mean(time_diffs)
        min_time_diff = np.min(time_diffs)
        max_time_diff = np.max(time_diffs)
        stddev_time_diff = np.std(time_diffs)
        if verbose:
            if use_header_time:
                print("Using Header time.")
            else:
                print("Using Receive time.")
            print(
                (
                    "Found "
                    + str(gaps)
                    + " time diffs >= "
                    + str(min_time_diff_for_gap)
                    + " secs."
                )
            )
            print(("Mean time diff: " + str(mean_time_diff)))
            print(("Min time diff: " + str(min_time_diff)))
            print(("Max time diff: " + str(max_time_diff)))
            print(("Stddev time diff: " + str(stddev_time_diff)))

def make_absolute_poses_from_relative_poses(absolute_poses, relative_poses, name):
    starting_relative_time = relative_poses.times[0]
    np_times = np.array(absolute_poses.times)
    closest_index = np.argmin(np.abs(np_times - starting_relative_time))
    start_pose = absolute_poses.pose(closest_index)
    new_pose = start_pose
    new_poses_list = [start_pose]
    new_poses_times = [absolute_poses.times[closest_index]]
    for index in range(len(relative_poses.times)):
        relative_pose = relative_poses.pose(index)
        new_pose = new_pose * relative_pose
        new_poses_list.append(new_pose)
        new_poses_times.append(relative_poses.times[index])
    new_poses = poses.Poses(name, "")
    new_poses.init_from_poses(new_poses_list, new_poses_times)
    return new_poses

def integrate_velocities(localization_states):
    delta_times = [
        j - i
        for i, j in zip(localization_states.times[:-1], localization_states.times[1:])
    ]
    # Make sure times are same length as velocities, ignore last velocity
    delta_times.append(0)
    # TODO(rsoussan): Integrate angular velocities?
    # TODO(rsoussan): central difference instead?
    x_increments = [
        velocity * delta_t
        for velocity, delta_t in zip(localization_states.velocities.xs, delta_times)
    ]
    y_increments = [
        velocity * delta_t
        for velocity, delta_t in zip(localization_states.velocities.ys, delta_times)
    ]
    z_increments = [
        velocity * delta_t
        for velocity, delta_t in zip(localization_states.velocities.zs, delta_times)
    ]

    return add_increments_to_absolute_pose(
        x_increments,
        y_increments,
        z_increments,
        localization_states.positions.xs[0],
        localization_states.positions.ys[0],
        localization_states.positions.zs[0],
        localization_states.times,
        "Integrated Graph Velocities",
    )

def add_increments_to_absolute_pose(
    x_increments,
    y_increments,
    z_increments,
    starting_x,
    starting_y,
    starting_z,
    times,
    poses_name="Increment Poses",
):
    integrated_positions = poses.Poses(poses_name, "")
    cumulative_x_increments = np.cumsum(x_increments)
    integrated_positions.positions.xs = [
        starting_x + cumulative_x_increment
        for cumulative_x_increment in cumulative_x_increments
    ]
    cumulative_y_increments = np.cumsum(y_increments)
    integrated_positions.positions.ys = [
        starting_y + cumulative_y_increment
        for cumulative_y_increment in cumulative_y_increments
    ]
    cumulative_z_increments = np.cumsum(z_increments)
    integrated_positions.positions.zs = [
        starting_z + cumulative_z_increment
        for cumulative_z_increment in cumulative_z_increments
    ]

    # Add start positions
    integrated_positions.positions.xs.insert(0, starting_x)
    integrated_positions.positions.ys.insert(0, starting_y)
    integrated_positions.positions.zs.insert(0, starting_z)

    # Remove last elements (no timestamp for these)
    del integrated_positions.positions.xs[-1]
    del integrated_positions.positions.ys[-1]
    del integrated_positions.positions.zs[-1]

    integrated_positions.times = times
    return integrated_positions


import os
import tempfile

print(tempfile.gettempdir())
print(tempfile.gettempprefix())

with tempfile.TemporaryFile("w+") as tfp:
    tfp.write("Some temp data")
    tfp.seek(0)
    print(tfp.read())

with tempfile.TemporaryDirectory() as tdp:
    filepath = os.path.join(tdp, "tempfile.txt")
    print(filepath)

    with open(filepath, "w+") as tfp:
        tfp.write("Temp file in temp directory")
        tfp.seek(0)
        print(tfp.read())


#!/usr/bin/python3
"""Networking with Python.

Similar to the "2-post_email.py" task but with
`requests`.

"""
import sys
import requests

if __name__ == "__main__":
    if len(sys.argv) < 3:
        sys.exit(1)
    email = requests.post(
            sys.argv[1],
            data={"email": sys.argv[2]},
            timeout=(6, 9)
            )
    print("{}".format(email.text))


# Code from Chapter 3 of Machine Learning: An Algorithmic Perspective (2nd Edition)
# by Stephen Marsland (http://stephenmonika.net)

# You are free to use, change, or redistribute the code in any way you wish for
# non-commercial purposes, but please maintain the name of the original author.
# This code comes with no warranty of any kind.

# Stephen Marsland, 2008, 2014

# Demonstration of the Perceptron on the Pima Indian dataset
# This dataset is now on kaggle https://www.kaggle.com/datasets/uciml/pima-indians-diabetes-database

# Just press F5 - runs interactively in IDE

# Ref kaggle solution https://www.kaggle.com/code/pradeepgurav/pima-diabetes-98-accuracy/notebook

import os

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.metrics import confusion_matrix, accuracy_score, f1_score
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

from sandbox.ann.np_bare_ann import NeuralNet_By_Numpy

def do_confusion_matrix(y_train, y_pred, plt):
    plt.figure()
    cm = confusion_matrix(y_train, y_pred)
    sns.set(color_codes=True)
    sns.set(font_scale=1)
    sns.heatmap(cm, annot=True, fmt='g')

SMALL_TRAIN_DS = '/diabetes.csv'
BIG_TRAIN_DS = '/diabetes75pc_100_times.csv'
DATASET_CSV_FILE = os.getcwd() + BIG_TRAIN_DS

EPOCH = 49

df_orig = pd.read_csv(DATASET_CSV_FILE)
feature_names = df_orig.columns[:8].tolist()
print(f'-- all features: {feature_names}')

X = df_orig[feature_names]
y = df_orig.Outcome
#print(X.shape)
#print(y.shape)

# Assumed best chosen features from the ref solution
selected_features = ['Pregnancies', 'Glucose', 'BMI', 'DiabetesPedigreeFunction']
print(f'-- selected features: {selected_features}')
# StandardScaler() scales each dimension to 0-mean and unit variance e.g. var == 1
X = StandardScaler().fit_transform(X[selected_features])
# Splitting  data into training and testing
X_train, X_test, y_train, y_test = train_test_split(
    X,
    y,
    test_size=0.25, # 25%
    stratify=y, # maintain balance between classes https://stackoverflow.com/questions/54600907/does-the-train-test-split-function-keep-the-balance-between-classes
    #random_state=1989
)
print(f'-- Train size = {X_train.shape[0]}')
print(f'-- Test size  = {X_test.shape[0]}')
train_0 = np.count_nonzero(y_train==0)
train_1 = np.count_nonzero(y_train==1)
test_0 = np.count_nonzero(y_test==0)
test_1 = np.count_nonzero(y_test==1)
print(f'-- in test: diabetic/non-diabetic  = {test_1/test_0}')
print(f'-- in train: diabetic/non-diabetic = {train_1/train_0}')
print(f'-- X_train shape: {X_train.shape}')
print(f'-- y_train shape: {y_train.shape}')

# start training
print('Training..')
nnet = NeuralNet_By_Numpy(X_train, y_train, batch_size=16, hidden_layers=[128])
for _ in range(EPOCH):
    nnet.iteration_train()
    y_pred = nnet.iteration_predict()
    y_pred = np.where(y_pred > 0.5, 1, 0)
    print('  -- Accuracy: {:.2f}%'.format(accuracy_score(y_train, y_pred) * 100))
    print('  -- F1      : {:.2f}%'.format(f1_score(y_train, y_pred) * 100))
# confusion matrix over training data
y_pred = nnet.predict(X_train)
y_pred = np.where(y_pred > 0.5, 1, 0)
do_confusion_matrix(y_train, y_pred, plt)

# test perf
print('Testing..')
y_pred = nnet.predict(X_test)
y_pred = np.where(y_pred > 0.5, 1, 0)
print('  -- Accuracy: {:.2f}%'.format(accuracy_score(y_test, y_pred) * 100))
print('  -- F1      : {:.2f}%'.format(f1_score(y_test, y_pred) * 100))
# confusion matrix over training data
do_confusion_matrix(y_test, y_pred, plt)

# check training perf
#train_pred = nnet.predict(X_train)
#train_pred = np.where(train_pred > 0.5, 1, 0)
# train: confusion matrix
#train_cm = confusion_matrix(y_train, train_pred)
#print(train_cm)

plt.show()


import typing

from sqlalchemy import and_
from sqlalchemy import Boolean
from sqlalchemy import cast
from sqlalchemy import column
from sqlalchemy import DateTime
from sqlalchemy import false
from sqlalchemy import Float
from sqlalchemy import func
from sqlalchemy import Integer
from sqlalchemy import or_
from sqlalchemy import select
from sqlalchemy import String
from sqlalchemy import true

# builtin.pyi stubs define object.__eq__() as returning bool,  which
# can't be overridden (it's final).  So for us to type `__eq__()` and
# `__ne__()`, we have to use type: ignore[override].  Test if does this mean
# the typing tools don't know the type, or if they just ignore the error.
# (it's fortunately the former)
expr1 = column("x", Integer) == 10

c1 = column("a", String)

c2 = column("a", Integer)

expr2 = c2.in_([1, 2, 3])

expr2_set = c2.in_({1, 2, 3})

expr2_gen = c2.in_((x for x in (1, 2, 3)))

nexpr2 = c2.not_in([1, 2, 3])

nexpr2_set = c2.not_in({1, 2, 3})

nexpr2_gen = c2.not_in((x for x in (1, 2, 3)))

short_cir1 = and_(True, c2 == 5)
short_cir2 = or_(False, c2 == 5)

short_cir3 = and_(true(), c2 == 5)
short_cir4 = or_(false(), c2 == 5)

# EXPECTED_MYPY: Missing positional argument "initial_clause" in call to "and_"
no_empty_1 = and_()

# EXPECTED_MYPY: Missing positional argument "initial_clause" in call to "or_"
no_empty_2 = or_()

expr3 = c2 / 5

expr4 = -c2

expr5 = ~(c2 == 5)

q = column("q", Boolean)
expr6 = ~q

expr7 = c1 + "x"

expr8 = c2 + 10

stmt = select(column("q")).where(lambda: column("g") > 5).where(c2 == 5)

expr9 = c1.bool_op("@@")(func.to_tsquery("some & query"))

def test_issue_9418() -> None:
    and_(c1.is_(q))
    and_(c1.is_not(q))
    and_(c1.isnot(q))
    and_(c1.not_in(["x"]))
    and_(c1.notin_(["x"]))
    and_(c1.not_like("x"))
    and_(c1.notlike("x"))
    and_(c1.not_ilike("x"))
    and_(c1.notilike("x"))

def test_issue_9451() -> None:
    # issue #9451
    c1.cast(Integer)
    c1.cast(Float)
    c1.op("foobar")("operand").cast(DateTime)
    cast(c1, Float)
    cast(c1.op("foobar")("operand"), DateTime)

def test_issue_9650_char() -> None:
    and_(c1.contains("x"))
    and_(c1.startswith("x"))
    and_(c1.endswith("x"))
    and_(c1.icontains("x"))
    and_(c1.istartswith("x"))
    and_(c1.iendswith("x"))

def test_issue_9650_bitwise() -> None:
    # EXPECTED_TYPE: BinaryExpression[Any]
    reveal_type(c2.bitwise_and(5))
    # EXPECTED_TYPE: BinaryExpression[Any]
    reveal_type(c2.bitwise_or(5))
    # EXPECTED_TYPE: BinaryExpression[Any]
    reveal_type(c2.bitwise_xor(5))
    # EXPECTED_TYPE: UnaryExpression[int]
    reveal_type(c2.bitwise_not())
    # EXPECTED_TYPE: BinaryExpression[Any]
    reveal_type(c2.bitwise_lshift(5))
    # EXPECTED_TYPE: BinaryExpression[Any]
    reveal_type(c2.bitwise_rshift(5))
    # EXPECTED_TYPE: ColumnElement[int]
    reveal_type(c2 << 5)
    # EXPECTED_TYPE: ColumnElement[int]
    reveal_type(c2 >> 5)

if typing.TYPE_CHECKING:
    # as far as if this is ColumnElement, BinaryElement, SQLCoreOperations,
    # that might change.  main thing is it's SomeSQLColThing[bool] and
    # not 'bool' or 'Any'.
    # EXPECTED_RE_TYPE: sqlalchemy..*ColumnElement\[builtins.bool\]
    reveal_type(expr1)

    # EXPECTED_RE_TYPE: sqlalchemy..*ColumnClause\[builtins.str.?\]
    reveal_type(c1)

    # EXPECTED_RE_TYPE: sqlalchemy..*ColumnClause\[builtins.int.?\]
    reveal_type(c2)

    # EXPECTED_RE_TYPE: sqlalchemy..*BinaryExpression\[builtins.bool\]
    reveal_type(expr2)

    # EXPECTED_RE_TYPE: sqlalchemy..*ColumnElement\[Union\[builtins.float, .*\.Decimal\]\]
    reveal_type(expr3)

    # EXPECTED_RE_TYPE: sqlalchemy..*UnaryExpression\[builtins.int.?\]
    reveal_type(expr4)

    # EXPECTED_RE_TYPE: sqlalchemy..*ColumnElement\[builtins.bool.?\]
    reveal_type(expr5)

    # EXPECTED_RE_TYPE: sqlalchemy..*ColumnElement\[builtins.bool.?\]
    reveal_type(expr6)

    # EXPECTED_RE_TYPE: sqlalchemy..*ColumnElement\[builtins.str\]
    reveal_type(expr7)

    # EXPECTED_RE_TYPE: sqlalchemy..*ColumnElement\[builtins.int.?\]
    reveal_type(expr8)

    # EXPECTED_TYPE: BinaryExpression[bool]
    reveal_type(expr9)


# -*- coding: utf-8 -*-

"""
 usage:

 $ baseline_taskAB.py gold_file system_file taskName

 - gold_file and system_file are tab-separated, UTF-8 encoded files
 - taskName is the name of the task (A|B)

"""

import argparse, sys
from sklearn.metrics import precision_recall_fscore_support as score
from sklearn.metrics import confusion_matrix

def preproc(infile, task):
    y = []

    reader = infile.readlines()
    for row in reader:
        if row.startswith("id"):
            continue
        if task == "A":
            label = row.split('\t')[2]
        elif task == "B":
            label = row.split('\t')[3].rstrip()
        y.append(label)

    return y

def eval(y_test, y_predicted):

    precision, recall, fscore, _ = score(y_test, y_predicted)
    print('\n     {0}   {1}'.format("0","1"))
    print('P: {}'.format(precision))
    print('R: {}'.format(recall))
    print('F: {}'.format(fscore))
    #"""
    _, _, fscore, _ = score(y_test, y_predicted, average='macro')
    print('Macro-F1: {}'.format(fscore))

    print('\n Confusion matrix:')
    print(confusion_matrix(y_test, y_predicted))

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="")
    parser.add_argument('test', help='haspeede test set')
    parser.add_argument('predicted', help='system output')
    parser.add_argument('task', type=str, help="the name of the task, i.e. A or B")
    args = parser.parse_args()

    with open(sys.argv[1], 'r',encoding="utf8") as f:
        y_test = preproc(f,sys.argv[3])

    with open(sys.argv[2], 'r', encoding="utf8") as f:
        y_predicted = preproc(f,sys.argv[3])

    eval(y_test, y_predicted)


import pygame
import itertools

screen_size = (1920, 1080)

# pastel pink
LOW_COLOR = (255,209,220)
# deep blue
MED_COLOR = (7, 42, 108)
# red
PARTY_COLOR = (255, 0, 0)

color_map = {
    'LOW': LOW_COLOR,
    'MED': MED_COLOR,
    'PARTY': PARTY_COLOR
}

class VibeLight:
    def __init__(self):
        self.is_on = False

    def start(self):
        pygame.init()
        self.is_on = True
        self.screen = pygame.display.set_mode(screen_size)

    def set_mood(self, mood):
        self.screen.fill(color_map[mood])
        pygame.display.flip()


from flask import Flask, render_template,request, redirect, session
app = Flask(__name__)
app.secret_key = 'what sup?'
# our index route will handle rendering our form
@app.route('/')
def index():
    return render_template("index.html")

@app.route('/process', methods=['POST'])
def submit_survey():
    print("Got Post Info")
    print(request.form)
    session['username'] = request.form['name']
    session['userlocation'] = request.form['location']
    session['userlanguage'] = request.form['language']
    session['usercomment'] = request.form['comment']
    return redirect('/show')

@app.route('/show')
def show_user():
    return render_template("show.html", name_on_template=session['username'], location_on_template=session['userlocation'],language_on_template=session['userlanguage'],comment_on_template=session['usercomment'])

if __name__ == "__main__":
    app.run(debug=True, port = 5001)


from model.misinfo_model import *

if __name__ == "__main__":
    for i in range(10):
        model = MisinfoPy(n_agents=1000)
        model(time_tracking=True, belief_update_fn=BeliefUpdate.SIT)
#              # all high:
#              # mlit_select=1.0, del_t=0.5, rank_t=0.5, rank_punish=-1.0, strikes_t=0.5)

#              # all medium:
#              # mlit_select=0.2, del_t=0.1, rank_t=0.2, rank_punish=-0.2, strikes_t=0.1)


"""
The PyRankine: the  hybrid steady-state simulator of Rankine Cycle

run:

python rankinesim_spec.py

Author: Cheng Maohua, Email:cmh@seu.edu.cn

"""
from platform import os
from rankine.utils import OutFiles, create_dictcycle_from_jsonfile
from rankine.simrankine import SimRankineCycle

curpath = os.path.abspath(os.path.dirname(__file__))
rankinefilename = curpath+'\\'+'./jsonmodel/N600.json'
dictcycle = create_dictcycle_from_jsonfile(rankinefilename)

cycle600 = SimRankineCycle(dictcycle)
# 1 1kg
# cycle600.simulator_sm()  # sequential-modular approach
cycle600.simulator_eo() # use sequential-modular approach
cycle600.simulator_performance()

# 2 Specified Net Output Power(MW)
Wcycledot = 600  # MW
cycle600.specified_simulator(set_power=Wcycledot)

# 3 Specified Mass Flow(kg/h)
mdot = cycle600.cycle.mdot  # 1709996  kg/h
cycle600.specified_simulator(set_mass=mdot)

# 4 Variable parameters: add the (p,t) of boiler output
# 4.1 init
curcycle = cycle600.cycle
obPort = curcycle.comps["BO"].oPort
bp = obPort.p
bt = obPort.t
beta = curcycle.efficiency_cycle
# 4.2 to change parameters
obPort.p = bp+1
obPort.t = bt+1
obPort.pt()
curcycle.simulator_eo()
print("\n--- Variable parameters: add the (p,t) of boiler output  ---\n")
print("p,    t,       eta")
print(bp, bt, beta)
print(obPort.p, obPort.t, curcycle.efficiency_cycle)


import datetime
from util.util import CGreeks, CSingleOptHolding
from util.COption import COption
import pandas as pd

class COptHolding(object):
    """期权持仓类"""

    def __init__(self, str_logfile_path=None):
        """
        初始化持仓数据及持仓汇总希腊字母
        class attributes:
        holdings: {'code': CSingleOptHolding}
        mode: 模式，逐步加仓、恒定gamma
        status: 状态，onposition=建仓，positioned=建仓完成，onliquidation=平仓
        cashinflow: 资金流入(卖出开仓、卖出平仓)
        cashoutflow: 资金流出(买入开仓、买入平仓)
        commission: 佣金
        pandl: 盈亏
        capital: 规模
        gammaexposure: gamma暴露
        greeks: CGreeks类
        logger: 日志类
        logfilepath: 日志文件路径
        """
        self.holdings = {}
        self.mode = ''
        self.status = ''
        self.cashinflow = 0.0
        self.cashoutflow = 0.0
        self.commission = 0.0
        self.pandl = 0.0
        self.capital = 0.0
        self.gammaexposure = 0.0
        self.greeks = CGreeks()
        # self.logger = None
        self.logfilepath = str_logfile_path

    def load_holdings(self, holding_filename):
        """加载持仓数据,同时加载期权的分钟行情数据"""
        # 先清空持仓数据
        self.holdings = {}
        with open(holding_filename, 'rt') as f:
            bIsPandLSection = False
            bIsHoldingSection = False
            while True:
                strline = f.readline().strip('\n')
                if not strline:
                    break
                if strline == '[begin of P&L]':
                    bIsPandLSection = True
                    strline = f.readline().strip('\n')
                if strline == '[end of P&L]':
                    bIsPandLSection = False
                    strline = f.readline().strip('\n')
                if strline == '[begin of holdings]':
                    bIsHoldingSection = True
                    strline = f.readline().strip('\n')
                if strline == '[end of holdings]':
                    bIsHoldingSection = False

                if bIsPandLSection:
                    ind_name, ind_value = strline.split(sep='=')
                    if ind_name == 'mode':
                        self.mode = ind_value
                    elif ind_name == 'status':
                        if self.status != 'onliquidation':
                            self.status = ind_value
                    elif ind_name == 'cashinflow':
                        self.cashinflow = float(ind_value)
                    elif ind_name == 'cashoutflow':
                        self.cashoutflow = float(ind_value)
                    elif ind_name == 'commission':
                        self.commission = float(ind_value)
                    elif ind_name == 'pandl':
                        self.pandl = float(ind_value)
                    elif ind_name == 'capital':
                        self.capital = float(ind_value)
                    elif ind_name == 'gammaexposure':
                        self.gammaexposure = float(ind_value)
                if bIsHoldingSection:
                    holding_data = strline.split(sep='|')
                    dict_holding = {}
                    for data in holding_data:
                        ind_name, ind_value = data.split(sep='=')
                        dict_holding[ind_name] = ind_value
                    if dict_holding['code'] not in self.holdings:
                        holding_side = int(dict_holding['holdingside'])
                        holding_vol = int(dict_holding['holdingvol'])
                        holding_opt = COption(dict_holding['code'])
                        self.holdings[dict_holding['code']] = CSingleOptHolding(holding_side, holding_vol, holding_opt)
                    else:
                        # strmsg = "持仓文件：" + holding_filename + "中期权" + dict_holding['code'] + "的持仓数据重复\n"
                        strmsg = "holding data of option:%s in holding file: %s was duplicated.\n" % (dict_holding['code'], holding_filename)
                        # self.logger.error(strmsg)
                        with open(self.logfilepath, 'at') as f:
                            f.write(strmsg)

    def save_holdings(self, holding_filename):
        """保存持仓数据"""
        # 如果当前的状态为onliquidation，那么把状态改为onposition
        if self.status == 'onliquidation':
            self.status = 'onposition'
        with open(holding_filename, 'wt') as f:
            f.write('[begin of P&L]\n')
            f.write('mode=%s\n' % self.mode)
            f.write('status=%s\n' % self.status)
            f.write('cashinflow=%0.2f\n' % self.cashinflow)
            f.write('cashoutflow=%0.2f\n' % self.cashoutflow)
            f.write('commission=%0.2f\n' % self.commission)
            f.write('pandl=%0.2f\n' % self.pandl)
            f.write('capital=%0.2f\n' % self.capital)
            f.write('gammaexposure=%0.2f\n' % self.gammaexposure)
            f.write('gamma_mv=%0.2f\n' % self.greeks.gamma_mv)
            f.write('delta_mv=%0.2f\n' % self.greeks.delta_mv)
            f.write('total_margin=%0.2f\n' % self.total_margin())
            f.write('[end of P&L]\n\n')

            f.write('[begin of holdings]\n')
            for opt_code, opt_holding in self.holdings.items():
                f.write('code=%s|holdingside=%d|holdingvol=%d\n' % (opt_code, opt_holding.holdingside, opt_holding.holdingvol))
            f.write('[end of holdings]\n')

    def calc_greeks(self, underlying_price, risk_free, dividend_rate, vol, calc_datetime=datetime.datetime.now()):
        """计算期权持仓的汇总希腊字母值"""
        self.greeks.delta = 0.0
        self.greeks.gamma = 0.0
        self.greeks.vega = 0.0
        self.greeks.delta_mv = 0.0
        self.greeks.gamma_mv = 0.0
        # 遍历持仓，先计算每个持仓期权的希腊字母，然后汇总希腊字母值
        for code, holding in self.holdings.items():
            holding.COption.calc_greeks(underlying_price, risk_free, dividend_rate, vol, calc_datetime)
            self.greeks.delta += (holding.COption.greeks.delta * holding.COption.multiplier *
                                  holding.holdingside * holding.holdingvol)
            self.greeks.gamma += (holding.COption.greeks.gamma * holding.COption.multiplier *
                                  holding.holdingside * holding.holdingvol)
            self.greeks.vega += (holding.COption.greeks.vega * holding.COption.multiplier *
                                 holding.holdingside * holding.holdingvol)
            self.greeks.delta_mv += (holding.COption.greeks.delta_mv * holding.holdingside * holding.holdingvol)
            self.greeks.gamma_mv += (holding.COption.greeks.gamma_mv * holding.holdingside * holding.holdingvol)

    def calc_margin(self, trading_day):
        """
        计算持仓期权的开仓保证金
        :param trading_day: 日期（类型=datetime.date）
        :return:
        """
        # 1.读取标的日K线时间序列
        underlying_quote = pd.read_csv('../data/underlying_daily_quote.csv', index_col=0, parse_dates=[0])
        underlying_pre_close = float(underlying_quote.ix[trading_day, 'pre_close'])
        # 2.读取样本期权的日行情
        strdate = trading_day.strftime('%Y-%m-%d')
        strfilepath = '../../opt_quote/' + strdate + '/50OptionDailyQuote.csv'
        opts_quote = pd.read_csv(strfilepath, usecols=range(1, 14), parse_dates=[0], encoding='gb18030', dtype={'option_code':str})
        opts_quote.set_index(keys='option_code', inplace=True)
        # 3.计算持仓期权的开仓保证金
        for optcode, holding in self.holdings.items():
            if optcode in opts_quote.index:
                opt_pre_settle = float(opts_quote.ix[optcode, 'pre_settle'])
                holding.COption.calc_margin(opt_pre_settle, underlying_pre_close)
            else:
                holding.COption.margin = 3000.0

    def total_margin(self):
        """返回持仓期权的合计保证金"""
        fmargin = 0.0
        for optcode, optholding in self.holdings.items():
            if optholding.holdingside == -1:
                fmargin += optholding.COption.margin * optholding.holdingvol
        return fmargin

    def margin_ratio(self):
        """返回保证金占资金规模的比例"""
        return self.total_margin() / self.capital

    def get_least_timevalue_opts(self, underlying_price, trading_datetime, exclusions=None):
        """
        取得持仓中时间价值最小的认购、认沽期权
        :param underlying_price: 标的最新价格
        :param trading_datetime: 交易时间(类型=datetime.datetime)
        :param exclusions: 需要排除的代码列表
        :return: tuple(认购COption,认沽COption)
        """
        if exclusions is None:
            exclusions = []
        opt_call = None
        opt_put = None
        for optcode, holding in self.holdings.items():
            if holding.holdingvol > 0 and optcode not in exclusions:
                if holding.COption.opt_type == "Call":
                    if opt_call is None:
                        opt_call = holding.COption
                    else:
                        if holding.COption.time_value(underlying_price, trading_datetime) < \
                                opt_call.time_value(underlying_price, trading_datetime):
                            opt_call = holding.COption
                else:
                    if opt_put is None:
                        opt_put = holding.COption
                    else:
                        if holding.COption.time_value(underlying_price, trading_datetime) < \
                                opt_put.time_value(underlying_price, trading_datetime):
                            opt_put = holding.COption
        return opt_call, opt_put

    def get_minmax_gamma_opts(self, exclusions=None):
        """
        取得持仓中gamma值最大的认购认沽期权，以及gamma值最小的认购认沽期权
        :param exclusions: 需要排除的代码列表
        :return: dict{'min'/'max':tuple(COption of call, COption of put)}
        """
        if exclusions is None:
            exclusions = []
        min_gamma_call = None
        min_gamma_put = None
        max_gamma_call = None
        max_gamma_put = None
        for optcode, holding in self.holdings.items():
            if holding.holdingvol > 0 and optcode not in exclusions:
                if holding.COption.opt_type == 'Call':
                    if min_gamma_call is None:
                        min_gamma_call = holding.COption
                    if max_gamma_call is None:
                        max_gamma_call = holding.COption
                    if holding.COption.greeks.gamma < min_gamma_call.greeks.gamma:
                        min_gamma_call = holding.COption
                    if holding.COption.greeks.gamma > max_gamma_call.greeks.gamma:
                        max_gamma_call = holding.COption
                else:
                    if min_gamma_put is None:
                        min_gamma_put = holding.COption
                    if max_gamma_put is None:
                        max_gamma_put = holding.COption
                    if holding.COption.greeks.gamma < min_gamma_put.greeks.gamma:
                        min_gamma_put = holding.COption
                    if holding.COption.greeks.gamma > max_gamma_put.greeks.gamma:
                        max_gamma_put = holding.COption
        return {'min': (min_gamma_call, min_gamma_put), 'max': (max_gamma_call, max_gamma_put)}

    def verify_update_tradedata(self, tradedata):
        """
        校验单条交易记录的有效性，若有效，那么同时更新交易数据至持仓数据
        :param tradedata: 单条期权交易记录
        :return: 校验通过并返回True，校验错误返回False
        """
        if tradedata.code in self.holdings:
            if tradedata.tradeside == 'buy' and tradedata.openclose == 'open':
                if self.holdings[tradedata.code].holdingside == 1:
                    self.holdings[tradedata.code].holdingvol += tradedata.tradevol
                    self.cashoutflow += tradedata.tradevalue
                    self.commission += tradedata.commission
                    # return True
                else:
                    return False
            elif tradedata.tradeside == 'buy' and tradedata.openclose == 'close':
                if self.holdings[tradedata.code].holdingside == -1:
                    self.holdings[tradedata.code].holdingvol -= tradedata.tradevol
                    self.cashoutflow += tradedata.tradevalue
                    self.commission += tradedata.commission
                    # return True
                else:
                    return False
            elif tradedata.tradeside == 'sell' and tradedata.openclose == 'open':
                if self.holdings[tradedata.code].holdingside == -1:
                    self.holdings[tradedata.code].holdingvol += tradedata.tradevol
                    self.cashinflow += tradedata.tradevalue
                    self.commission += tradedata.commission
                    # return True
                else:
                    return False
            elif tradedata.tradeside == 'sell' and tradedata.openclose == 'close':
                if self.holdings[tradedata.code].holdingside == 1:
                    self.holdings[tradedata.code].holdingvol -= tradedata.tradevol
                    self.cashinflow += tradedata.tradevalue
                    self.commission += tradedata.commission
                    # return True
                else:
                    return False
        else:
            if tradedata.openclose == 'close':
                return False
            elif tradedata.tradeside == 'buy' and tradedata.openclose == 'open':
                # holding = CSingleOptHolding(side=1, vol=tradedata.tradevol, opt=COption(tradedata.code))
                holding = CSingleOptHolding(side=1, vol=tradedata.tradevol, opt=tradedata.opt)
                self.cashoutflow += tradedata.tradevalue
                self.commission += tradedata.commission
                self.holdings[tradedata.code] = holding
                # return True
            elif tradedata.tradeside == 'sell' and tradedata.openclose == 'open':
                # holding = CSingleOptHolding(side=-1, vol=tradedata.tradevol, opt=COption(tradedata.code))
                holding = CSingleOptHolding(side=-1, vol=tradedata.tradevol, opt=tradedata.opt)
                self.cashinflow += tradedata.tradevalue
                self.commission += tradedata.commission
                self.holdings[tradedata.code] = holding
                # return True
            else:
                return
        # 如果该交易记录对应期权持仓量=0，那么删除该条期权持仓
        if self.holdings[tradedata.code].holdingvol == 0:
            del_holding = self.holdings.pop(tradedata.code)
            if del_holding is not None:
                # self.logger.info("期权%s的持仓量等于0，该持仓已删除。" % tradedata.code)
                # strmsg = "期权%s的持仓量等于0，该持仓已删除。\n" % tradedata.code
                strmsg = "holding vol of option: %s is equal to 0, the holding data was deleted.\n" % tradedata.code
            else:
                # self.logger.error("删除期权%s持仓数据失败！" % tradedata.code)
                # strmsg = "删除期权%s持仓数据失败！\n" % tradedata.code
                strmsg = "failed to delete holding data of option: %s\n" % tradedata.code
            with open(self.logfilepath, 'at') as f:
                f.write(strmsg)
        return True

    def update_holdings(self, tradedatas):
        """
        根据给定的期权交易数据列表，更新持仓数据
        :param tradedatas: 期权交易数据(COptTradeData)列表
        :return: 无
        """
        # 遍历期权交易数据列表，更新每条期权交易数据
        with open(self.logfilepath, 'at') as f:
            for tradedata in tradedatas:
                log_msg = "trade info: time=%s,code=%s,tradeside=%s,openclose=%s,price=%f,vol=%d,value=%f,commission=%f\n" % \
                          (tradedata.time.strftime('%H:%M:%S'), tradedata.code, tradedata.tradeside, tradedata.openclose,
                           tradedata.tradeprice, tradedata.tradevol, tradedata.tradevalue, tradedata.commission)
                # self.logger.info(log_msg)
                f.write(log_msg)
                if not self.verify_update_tradedata(tradedata):
                    # print('%s,期权%s的交易数据出错\n' % (tradedata.time.strftime('%H:%M:%S'), tradedata.code))
                    # self.logger.error('上一条交易数据错误')
                    f.write('the last trading data was wrong.\n')
        # 更新持仓期权的保证金
        if tradedatas:
            self.calc_margin(tradedatas[0].time.date())

    def holding_mv(self, trading_datetime):
        """
        计算持仓期权的总市值
        :param trading_datetime: 计算时间，类型=datetime.datetime
        :return:
        """
        opt_mv = 0.0
        for optcode, optholding in self.holdings.items():
            opt_price = optholding.COption.quote_1min.ix[trading_datetime, 'close']
            opt_mv += optholding.holdingside * optholding.holdingvol * opt_price * optholding.COption.multiplier
        return opt_mv

    def p_and_l(self, trading_datetime):
        """
        计算给定时间点时的盈亏
        :param trading_datetime: 计算时间，类型=datetime.datetime
        :return:
        """
        self.pandl = self.holding_mv(trading_datetime) + self.cashinflow - self.cashoutflow - self.commission
        return self.pandl


# -*- coding: utf-8 -*-
"""
Preppin' Data 2020: Week 9 - C&BS Co: Political Monitoring
https://preppindata.blogspot.com/2020/02/2020-week-9.html

- Input data
- Remove the Average Record for the polls
- Clean up your Dates
- Remove any Null Poll Results
- Form a Rank (modified competition) of the candidates per Poll based on their results
- Determine the spread for each poll from 1st Place to 2nd Place
- Rename Sample Types: RV = Registered Voter, LV = Likely Voter, null = Unknown
- Output the Data
- Optional: Build the Viz

Author: Kelly Gilbert
Created: 2022-02-02
Requirements:
  - input dataset:
      - PD 2020 Wk 9 Input - Sheet1.csv
  - output dataset (for results check only):
      - PD 2020 Wk 9 Output.csv
"""

from numpy import nan, where
import pandas as pd

#---------------------------------------------------------------------------------------------------
# input the data
#---------------------------------------------------------------------------------------------------

# read in the file, melt the candidates into rows, remove nulls, remove averages
df = pd.read_csv(r'.\inputs\PD 2020 Wk 9 Input - Sheet1.csv', na_values='--')\
       .melt(id_vars=['Poll', 'Date', 'Sample'], var_name='Candidate', value_name='Poll Results')\
       .dropna(subset=['Poll Results'])\
       .query("~Poll.str.contains('Average')", engine='python')

#---------------------------------------------------------------------------------------------------
# process the data
#---------------------------------------------------------------------------------------------------

# clean up end date
df['End Date'] = pd.to_datetime(df['Date'].str.extract('.*- (\d+/\d+)', expand=False) + '/2020')
df['End Date'] = where(df['End Date'].dt.month >= 7,
                       df['End Date'] + pd.DateOffset(years=-1),
                       df['End Date'])

# form a Rank (modified competition) of the candidates per Poll based on their results
df['Rank'] = df.groupby(['Poll', 'End Date', 'Sample'])['Poll Results'].rank(method='max', ascending=False)\
               .astype(int)

# difference in poll results between first and second
df['Spread from 1st to 2nd Place'] = \
    df.groupby(['Poll', 'End Date', 'Sample'], as_index=False)['Poll Results']\
      .transform(lambda x: x.max() - x.nlargest(2).min())

# rename sample types
sample_map = {'.*RV' : 'Registered Voter', '.*LV' : 'Likely Voter', nan : 'Unknown'}
df['Sample Type'] = df['Sample'].replace(sample_map, regex=True)

#---------------------------------------------------------------------------------------------------
# output the file
#---------------------------------------------------------------------------------------------------

out_cols = ['Candidate', 'Poll Results', 'Spread from 1st to 2nd Place', 'Rank', 'End Date',
            'Sample Type', 'Poll']
df.to_csv(r'.\outputs\output-2020-09.csv', index=False, columns=out_cols, date_format='%d/%m/%Y')

#---------------------------------------------------------------------------------------------------
# create chart
#---------------------------------------------------------------------------------------------------

from bokeh.io import output_file
from bokeh.layouts import row
from bokeh.models import CustomJS, Legend, DatetimeTickFormatter, Select, Title
from bokeh.plotting import figure, show

# color constants
color_selected = '#0066cc'
color_deselected = '#bab0ac'

# set the output file path
output_file('dimensions.html')

# subset of registered voters
df_rv = df.loc[df['Sample Type']=='Registered Voter']\
          .sort_values(by=['Candidate', 'End Date', 'Poll'])

# add a figure and format it
p = figure(width=900, height=500, x_axis_type='datetime')
p.add_layout(Title(text='Data from: realclearpolitics.com; Sample Type: Registered Voter',
                   text_font_size='9pt'), 'above')
p.add_layout(Title(text='2020 Democratic Presidential Nominations', text_font_size="24pt"), 'above')
p.xaxis.formatter=DatetimeTickFormatter(days=["%Y-%m-%d"])
p.y_range.flipped = True
p.add_layout(Legend(), 'right')

# add a line and circles for each candidate option
candidates = sorted(df_rv['Candidate'].unique())

line_dict = {}
circle_dict = {}
for i, c in enumerate(candidates):
    line_dict[c] = p.line(df_rv.loc[df['Candidate']==c]['End Date'],
                          df_rv.loc[df['Candidate']==c]['Rank'],
                          legend_label=c, line_width=2,
                          color=color_selected if i==0 else color_deselected)

    circle_dict[c] = p.circle(df_rv.loc[df['Candidate']==c]['End Date'],
                              df_rv.loc[df['Candidate']==c]['Rank'],
                              legend_label=c, size=7,
                              line_width=0,
                              fill_color=color_selected if i==0 else color_deselected)

# create a drop-down menu
menu = Select(options=candidates, value=candidates[0], title='Select an item:')

# link the plot and the button using a callback function

# cb_obj = the model that triggered the callback (e.g. button model, dropdown model)
# args = list of name=object values you want to have accessible inside the callback function
# can't assign cb_obj.value to a model property directly; you have to store it in a variable first

callback = CustomJS(args=dict(p=p, lines=line_dict, circles=circle_dict), code="""
const t = cb_obj.value;

// make the selected item's marks blue and the rest gray
for (let i in lines) {
  if (i == t) {
      lines[i].glyph.line_color = '""" + color_selected + """';
      circles[i].glyph.fill_color = '""" + color_selected + """';
  } else {
      lines[i].glyph.line_color = '""" + color_deselected + """';
      circles[i].glyph.fill_color = '""" + color_deselected + """';
  }
}
""")

menu.js_on_change('value', callback)

# display the layout
chart_layout = row(p, menu)
show(chart_layout)

#---------------------------------------------------------------------------------------------------
# check results
#---------------------------------------------------------------------------------------------------

solution_files = ['PD 2020 Wk 9 Output.csv']
my_files = ['output-2020-09.csv']
unique_cols = [['Candidate', 'Poll', 'End Date', 'Sample Type']]
col_order_matters = True
round_dec = 8

for i, solution_file in enumerate(solution_files):
    print('---------- Checking \'' + solution_file + '\' ----------\n')

    # read in the files
    df_sol = pd.read_csv('.\\outputs\\' + solution_file, encoding='utf-8')
    df_mine = pd.read_csv('.\\outputs\\' + my_files[i], encoding='utf-8')

    # are the columns the same?
    solution_cols = list(df_sol.columns)
    myCols = list(df_mine.columns)
    if not col_order_matters:
         solution_cols.sort()
         myCols.sort()

    col_match = False
    if solution_cols != myCols:
        print('*** Columns do not match ***')
        print('    Columns in solution: ' + str(list(df_sol.columns)))
        print('    Columns in mine    : ' + str(list(df_mine.columns)))
        print('\n\n')
    else:
        print('Columns match\n')
        col_match = True

    # are the values the same? (only check if the columns matched)
    if col_match:
        errors = 0
        df_compare = df_sol.merge(df_mine, how='outer', on=unique_cols[i],
                                  suffixes=['_sol', '_mine'], indicator=True)

        # extra/missing records
        if len(df_compare[df_compare['_merge'] != 'both']) > 0:
            print('*** Missing or extra records ***\n')
            print('In solution, not in mine:\n')
            print(df_compare[df_compare['_merge'] == 'left_only'][unique_cols[i]])
            print('\n\nIn mine, not in solution:\n')
            print(df_compare[df_compare['_merge'] == 'right_only'][unique_cols[i]])
            errors += 1

        # for the records that matched, check for mismatched values
        for c in [c for c in df_sol.columns if c not in unique_cols[i]]:
            if 'float' in df_compare[f'{c}_sol'].dtype.name:
                df_compare[f'{c}_sol'] = df_compare[f'{c}_sol'].round(round_dec)
                df_compare[f'{c}_mine'] = df_compare[f'{c}_mine'].round(round_dec)

            unmatched = df_compare[(df_compare['_merge']=='both')
                                   & (df_compare[f'{c}_sol'] != df_compare[f'{c}_mine'])]
            if len(unmatched) > 0:
                print(f'*** Values do not match: {c} ***\n')
                print(df_compare[(df_compare['_merge']=='both')
                                 & (df_compare[f'{c}_sol'] != df_compare[f'{c}_mine'])]\
                                [unique_cols[i] + [f'{c}_sol', f'{c}_mine']])
                print('\n')
                errors += 1

        if errors == 0:
            print('Values match')

    print('\n')


import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from pytorch3d.ops import knn_points, ball_query
from .logger import logger

def index_points(points, idx):
    """
    Input:
        points: input points data, [B, N, C]
        idx: sample index data, [B, S, [K]]
    Return:
        new_points:, indexed points data, [B, S, [K], C]
    """
    raw_size = idx.size()
    idx = idx.reshape(raw_size[0], -1)
    res = torch.gather(points, 1, idx[..., None].expand(-1, -1, points.size(-1)))
    return res.reshape(*raw_size, -1)

def sample_points_from_ball_query(pt_xyz, pt_feats, center_point, k, radius):
    _, ball_idx, xyz = ball_query(center_point, pt_xyz, K=k, radius=radius, return_nn=True)
    invalid = torch.sum(ball_idx == -1) > 0
    if invalid:
        logger.warning(f"ball query returns {torch.sum(ball_idx == -1)} / {torch.numel(ball_idx)} -1 in its index, "
                       f"which means you need to increase raidus or decrease K")

    points = index_points(pt_feats, ball_idx).squeeze(1)
    xyz = xyz.squeeze(1)
    return xyz, points

def sample_points_from_knn(pt_xyz, pt_feats, center_point, k):
    _, knn_idx, xyz = knn_points(center_point, pt_xyz, K=k, return_nn=True)
    points = index_points(pt_feats, knn_idx).squeeze(1)
    xyz = xyz.squeeze(1)
    return xyz, points


#### with lru caching method #########

from functools import lru_cache
@lru_cache(maxsize=16)
def fib(n):
    if n<=2 :
        return 1
    return fib(n-1) + fib(n-2)

print(fib(50))

######################### with memoization technique #########
def fib(n,memo):

    if (n in memo):
        return memo[n]
    if n<=2 :
        return 1
    memo[n]  = fib(n-1,memo) + fib(n-2,memo)

    return memo[n]
memo={}
print(fib(15,memo))


import torch
from .base_model import BaseModel
from .BigGAN_networks import *
from util.util import toggle_grad, loss_hinge_dis, loss_hinge_gen, ortho, default_ortho, toggle_grad, prepare_z_y, \
    make_one_hot, to_device, multiple_replace, random_word
import pandas as pd
from .OCR_network import *
from torch.nn import CTCLoss, MSELoss, L1Loss
from torch.nn.utils import clip_grad_norm_
import random
import unicodedata
import sys

activation_dict = {'inplace_relu': nn.ReLU(inplace=True),
                   'relu': nn.ReLU(inplace=False),
                   'ir': nn.ReLU(inplace=True),}

class ScrabbleGANBaseModel(BaseModel):

    @staticmethod
    def modify_commandline_options(parser, is_train=True):
        parser.set_defaults(G_shared=False)
        parser.set_defaults(first_layer=True)
        parser.set_defaults(one_hot=True)
        return parser

    def __init__(self, opt):
        BaseModel.__init__(self, opt)  # call the initialization method of BaseModel
        opt.G_activation = activation_dict[opt.G_nl]
        opt.D_activation = activation_dict[opt.D_nl]
        # load saved model to finetune:
        if self.isTrain and opt.saved_model!='':
            opt.G_init = os.path.join(opt.checkpoints_dir, opt.saved_model)
            opt.D_init = os.path.join(opt.checkpoints_dir, opt.saved_model)
            opt.OCR_init = os.path.join(opt.checkpoints_dir, opt.saved_model)
        # specify the training losses you want to print out. The program will call base_model.get_current_losses to plot the losses to the console and save them to the disk.
        self.loss_names = ['G', 'D', 'Dreal', 'Dfake', 'OCR_real', 'OCR_fake', 'grad_fake_OCR', 'grad_fake_adv']
        self.loss_G = torch.zeros(1)
        self.loss_D =torch.zeros(1)
        self.loss_Dreal =torch.zeros(1)
        self.loss_Dfake =torch.zeros(1)
        self.loss_OCR_real =torch.zeros(1)
        self.loss_OCR_fake =torch.zeros(1)
        self.loss_grad_fake_OCR =torch.zeros(1)
        self.loss_grad_fake_adv =torch.zeros(1)
        # specify the models you want to save to the disk. The program will call base_model.save_networks and base_model.load_networks to save and load networks.
        # you can use opt.isTrain to specify different behaviors for training and test. For example, some networks will not be used during test, and you don't need to load them.
        self.model_names = ['G', 'D', 'OCR']
        # define networks; you can use opt.isTrain to specify different behaviors for training and test.
        # Next, build the model
        opt.n_classes = len(opt.alphabet)
        self.netG = Generator(**vars(opt))
        self.Gradloss = torch.nn.L1Loss()

        self.netconverter = strLabelConverter(opt.alphabet)
        self.netOCR = CRNN(opt).to(self.device)
        if len(opt.gpu_ids) > 0:
            assert (torch.cuda.is_available())
            self.netOCR.to(opt.gpu_ids[0])
            self.netG.to(opt.gpu_ids[0])
            # net = torch.nn.DataParallel(net, gpu_ids)  # multi-GPUs
            if len(opt.gpu_ids) > 1:
                self.netOCR = torch.nn.DataParallel(self.netOCR, device_ids=opt.gpu_ids, dim=1, output_device=opt.gpu_ids[0]).cuda()
                self.netG = torch.nn.DataParallel(self.netG, device_ids=opt.gpu_ids, output_device=opt.gpu_ids[0]).cuda()

        self.OCR_criterion = CTCLoss(zero_infinity=True, reduction='none')
        print(self.netG)

        if self.isTrain:  # only defined during training time
            # define your loss functions. You can use losses provided by torch.nn such as torch.nn.L1Loss.
            # We also provide a GANLoss class "networks.GANLoss". self.criterionGAN = networks.GANLoss().to(self.device)
            # define and initialize optimizers. You can define one optimizer for each network.
            # If two networks are updated at the same time, you can use itertools.chain to group them. See cycle_gan_model.py for an example.
            self.optimizer_G = torch.optim.Adam(self.netG.parameters(),
                                                lr=opt.G_lr, betas=(opt.G_B1, opt.G_B2), weight_decay=0, eps=opt.adam_eps)
            self.optimizer_OCR = torch.optim.Adam(self.netOCR.parameters(),
                                                lr=opt.OCR_lr, betas=(opt.OCR_B1, opt.OCR_B2), weight_decay=0,
                                                eps=opt.adam_eps)
            self.optimizers = [self.optimizer_G, self.optimizer_OCR]

            self.optimizer_G.zero_grad()
            self.optimizer_OCR.zero_grad()

        exception_chars = ['ï', 'ü', '.', '_', 'ö', ',', 'ã', 'ñ']
        if opt.lex.endswith('.tsv'):
            self.lex = pd.read_csv(opt.lex, sep='\t')['lemme']
            self.lex = [word.split()[-1] for word in self.lex if
                        (pd.notnull(word) and all(char not in word for char in exception_chars))]
        elif opt.lex.endswith('.txt'):
            with open(opt.lex, 'rb') as f:
                self.lex = f.read().splitlines()
            lex=[]
            for word in self.lex:
                try:
                    word=word.decode("utf-8")
                except:
                    continue
                if len(word)<20:
                    lex.append(word)
            self.lex = lex
        else:
            raise ValueError('could not load lexicon ')
        self.fixed_noise_size = 2
        self.fixed_noise, self.fixed_fake_labels = prepare_z_y(self.fixed_noise_size, opt.dim_z,
                                       len(self.lex), device=self.device,
                                       fp16=opt.G_fp16, seed=opt.seed)
        self.fixed_noise.sample_()
        self.fixed_fake_labels.sample_()
        self.rep_dict = {"'":"", '"':'', ' ':'_', ';':'', '.':''}
        fixed_words_fake = [self.lex[int(i)].encode('utf-8') for i in self.fixed_fake_labels]
        self.fixed_text_encode_fake, self.fixed_text_len = self.netconverter.encode(fixed_words_fake)
        if self.opt.one_hot:
            self.one_hot_fixed = make_one_hot(self.fixed_text_encode_fake, self.fixed_text_len, self.opt.n_classes)
        # Todo change to display names of classes instead of numbers
        self.label_fix = [multiple_replace(word.decode("utf-8"), self.rep_dict) for word in fixed_words_fake]
        visual_names_fixed_noise = ['fake_fixed_' + 'label_' + label for label in self.label_fix]
        visual_names_grad_OCR = ['grad_OCR_fixed_' + 'label_' + label for label in self.label_fix]
        visual_names_grad_G = ['grad_G_fixed_' + 'label_' + label for label in self.label_fix]
        # specify the images you want to save and display. The program will call base_model.get_current_visuals to save and display these images.
        self.visual_names = ['real', 'fake']
        self.visual_names.extend(visual_names_fixed_noise)
        self.visual_names.extend(visual_names_grad_G)
        self.visual_names.extend(visual_names_grad_OCR)
        self.z, self.label_fake = prepare_z_y(opt.batch_size, opt.dim_z, len(self.lex),
                                   device=self.device, fp16=opt.G_fp16, z_dist=opt.z_dist, seed=opt.seed)
        if opt.single_writer:
            self.fixed_noise = self.z[0].repeat((self.fixed_noise_size, 1))
            self.z = self.z[0].repeat((opt.batch_size, 1)).to(self.device)
            self.z.requires_grad=True
            self.optimizer_z = torch.optim.SGD([self.z], lr=opt.G_lr)
            self.optimizer_z.zero_grad()
        self.l1_loss = L1Loss()
        self.mse_loss = MSELoss()
        self.OCRconverter = OCRLabelConverter(opt.alphabet)
        self.epsilon = 1e-7
        self.real_z = None
        self.real_z_mean = None

    def visualize_fixed_noise(self):
        if self.opt.single_writer:
            self.fixed_noise = self.z[0].repeat((self.fixed_noise_size, 1))
        if self.opt.one_hot:
            images = self.netG(self.fixed_noise, self.one_hot_fixed.to(self.device))
        else:
            images = self.netG(self.fixed_noise, self.fixed_text_encode_fake.to(self.device))

        loss_G = loss_hinge_gen(self.netD(**{'x': images, 'z': self.fixed_noise}), self.fixed_text_len.detach(), self.opt.mask_loss)
        # self.loss_G = loss_hinge_gen(self.netD(self.fake, self.rep_label_fake))
        # OCR loss on real data
        pred_fake_OCR = self.netOCR(images)
        preds_size = torch.IntTensor([pred_fake_OCR.size(0)] * len(self.fixed_text_len)).detach()
        # loss_OCR_fake = self.OCR_criterion(pred_fake_OCR.log_softmax(2), self.fixed_text_encode_fake.detach().to(self.device),
        #                                    preds_size, self.fixed_text_len.detach())
        loss_OCR_fake = self.OCR_criterion(pred_fake_OCR, self.fixed_text_encode_fake.detach().to(self.device),
                                           preds_size, self.fixed_text_len.detach())
        loss_OCR_fake = torch.mean(loss_OCR_fake[~torch.isnan(loss_OCR_fake)])

        grad_fixed_OCR = torch.autograd.grad(loss_OCR_fake, images)
        grad_fixed_adv = torch.autograd.grad(loss_G, images)
        _, preds = pred_fake_OCR.max(2)
        preds = preds.transpose(1, 0).contiguous().view(-1)
        sim_preds = self.OCRconverter.decode(preds.data, preds_size.data, raw=False)
        raw_preds = self.OCRconverter.decode(preds.data, preds_size.data, raw=True)
        print('######## fake images OCR prediction ########')
        for i in range(self.fixed_noise_size):
            print('%-20s => %-20s, gt: %-20s' % (raw_preds[i], sim_preds[i], self.lex[int(self.fixed_fake_labels[i])]))
            image = images[i].unsqueeze(0).detach()
            grad_OCR = torch.abs(grad_fixed_OCR[0][i]).unsqueeze(0).detach()
            grad_OCR = (grad_OCR / torch.max(grad_OCR)) * 2 - 1
            grad_adv = torch.abs(grad_fixed_adv[0][i]).unsqueeze(0).detach()
            grad_adv = (grad_adv / torch.max(grad_adv)) * 2 - 1
            label = self.label_fix[i]
            setattr(self, 'grad_OCR_fixed_' + 'label_' + label, grad_OCR)
            setattr(self, 'grad_G_fixed_' + 'label_' + label, grad_adv)
            setattr(self, 'fake_fixed_' + 'label_' + label, image)

    def get_current_visuals(self):

        self.visualize_fixed_noise()
        with torch.no_grad():
            preds_size = torch.IntTensor([self.pred_real_OCR.size(0)] * len(self.label)).detach()
            _, preds = self.pred_real_OCR.max(2)
            preds = preds.transpose(1, 0).contiguous().view(-1)
            sim_preds = self.OCRconverter.decode(preds.data, preds_size.data, raw=False)
            raw_preds = self.OCRconverter.decode(preds.data, preds_size.data, raw=True)
            print('######## real images OCR prediction ########')
            for i in range(min(3, len(self.label))):
                print('%-20s => %-20s, gt: %-20s' % (
                    raw_preds[i], sim_preds[i], self.label[i].decode('utf-8', 'strict')))
                self.netOCR.train()
            ones_img = torch.ones(eval('self.fake_fixed_' + 'label_' +
                                         self.label_fix[0]).shape, dtype=torch.float32)
            ones_img[:, :, :, 0:min(self.real.shape[3], ones_img.shape[3])] = self.real[0, :, :, 0:min(self.real.shape[3], ones_img.shape[3])]
            self.real = ones_img
            ones_img = torch.ones(eval('self.fake_fixed_' + 'label_' +
                                         self.label_fix[0]).shape, dtype=torch.float32)
            ones_img[:, :, :, 0:min(self.fake.shape[3], ones_img.shape[3])] = self.fake[0, :, :, 0:min(self.fake.shape[3], ones_img.shape[3])]
            self.fake = ones_img
            self.netG.train()
            return super(ScrabbleGANBaseModel, self).get_current_visuals()

    def set_input(self, input):
        """Unpack input data from the dataloader and perform necessary pre-processing steps.

        Parameters:
            input: a dictionary that contains the data itself and its metadata information.
        """
        # if hasattr(self, 'real'): del self.real, self.one_hot_real, self.text_encode, self.len_text
        self.real = input['img'].to(self.device)
        if 'label' in input.keys():
            self.label = input['label']
            self.text_encode, self.len_text = self.netconverter.encode(self.label)
            if self.opt.one_hot:
                self.one_hot_real = make_one_hot(self.text_encode, self.len_text, self.opt.n_classes).to(self.device).detach()
            self.text_encode = self.text_encode.to(self.device).detach()
            self.len_text = self.len_text.detach()
        self.img_path = input['img_path']  # get image paths
        self.idx_real = input['idx']  # get image paths

    def load_networks(self, epoch):
        BaseModel.load_networks(self, epoch)
        if self.opt.single_writer:
            load_filename = '%s_z.pkl' % (epoch)
            load_path = os.path.join(self.save_dir, load_filename)
            self.z = torch.load(load_path)

    def forward(self, words=None, z=None):
        """Run forward pass. This will be called by both functions <optimize_parameters> and <test>."""
        if hasattr(self, 'fake'): del self.fake, self.text_encode_fake, self.len_text_fake, self.one_hot_fake

        self.label_fake.sample_()
        if words is None:
            words = [self.lex[int(i)] for i in self.label_fake]
            if self.opt.capitalize:
                for i, word in enumerate(words):
                    if random.random()<0.5:
                        word = list(word)
                        word[0] = unicodedata.normalize('NFKD',word[0].upper()).encode('ascii', 'ignore').decode("utf-8")
                        word = ''.join(word)
                    words[i] = word
            words = [word.encode('utf-8') for word in words]
        if z is None:
            if not self.opt.single_writer:
                self.z.sample_()
        else:
            if z.shape[0]==1:
                self.z = z.repeat((len(words), 1))
                self.z = z.repeat((len(words), 1))
            else:
                self.z = z
        self.words = words
        self.text_encode_fake, self.len_text_fake = self.netconverter.encode(self.words)
        self.text_encode_fake = self.text_encode_fake.to(self.device)
        if self.opt.one_hot:
            self.one_hot_fake = make_one_hot(self.text_encode_fake, self.len_text_fake, self.opt.n_classes).to(self.device)
            try:
                self.fake = self.netG(self.z, self.one_hot_fake)
            except:
                print(words)
        else:
            self.fake = self.netG(self.z, self.text_encode_fake)  # generate output image given the input data_A

    def backward_D_OCR(self):
        # Real
        if self.real_z_mean is None:
            pred_real = self.netD(self.real.detach())
        else:
            pred_real = self.netD(**{'x': self.real.detach(), 'z': self.real_z_mean.detach()})
        # Fake
        try:
            pred_fake = self.netD(**{'x': self.fake.detach(), 'z': self.z.detach()})
        except:
            print('a')
        # Combined loss
        self.loss_Dreal, self.loss_Dfake = loss_hinge_dis(pred_fake, pred_real, self.len_text_fake.detach(), self.len_text.detach(), self.opt.mask_loss)
        self.loss_D = self.loss_Dreal + self.loss_Dfake
        # OCR loss on real data
        self.pred_real_OCR = self.netOCR(self.real.detach())
        preds_size = torch.IntTensor([self.pred_real_OCR.size(0)] * self.opt.batch_size).detach()
        loss_OCR_real = self.OCR_criterion(self.pred_real_OCR, self.text_encode.detach(), preds_size, self.len_text.detach())
        self.loss_OCR_real = torch.mean(loss_OCR_real[~torch.isnan(loss_OCR_real)])
        # total loss
        loss_total = self.loss_D + self.loss_OCR_real

        # backward
        loss_total.backward()
        for param in self.netOCR.parameters():
            param.grad[param.grad!=param.grad]=0
            param.grad[torch.isnan(param.grad)]=0
            param.grad[torch.isinf(param.grad)]=0
        if self.opt.clip_grad > 0:
             clip_grad_norm_(self.netD.parameters(), self.opt.clip_grad)

        return loss_total

    def backward_OCR(self):
        # OCR loss on real data
        self.pred_real_OCR = self.netOCR(self.real.detach())
        preds_size = torch.IntTensor([self.pred_real_OCR.size(0)] * self.opt.batch_size).detach()
        loss_OCR_real = self.OCR_criterion(self.pred_real_OCR, self.text_encode.detach(), preds_size, self.len_text.detach())
        self.loss_OCR_real = torch.mean(loss_OCR_real[~torch.isnan(loss_OCR_real)])

        # backward
        self.loss_OCR_real.backward()
        for param in self.netOCR.parameters():
            param.grad[param.grad!=param.grad]=0
            param.grad[torch.isnan(param.grad)]=0
            param.grad[torch.isinf(param.grad)]=0
        if self.opt.clip_grad > 0:
             clip_grad_norm_(self.netD.parameters(), self.opt.clip_grad)
        return self.loss_OCR_real

    def backward_D(self):
        # Real
        if self.real_z_mean is None:
            pred_real = self.netD(self.real.detach())
        else:
            pred_real = self.netD(**{'x': self.real.detach(), 'z': self.real_z_mean.detach()})
        pred_fake = self.netD(**{'x': self.fake.detach(), 'z': self.z.detach()})
        # Combined loss
        self.loss_Dreal, self.loss_Dfake = loss_hinge_dis(pred_fake, pred_real, self.len_text_fake.detach(), self.len_text.detach(), self.opt.mask_loss)
        self.loss_D = self.loss_Dreal + self.loss_Dfake
        # backward
        self.loss_D.backward()

        if self.opt.clip_grad > 0:
             clip_grad_norm_(self.netD.parameters(), self.opt.clip_grad)
        return self.loss_D

    def backward_G(self):
        self.loss_G = loss_hinge_gen(self.netD(**{'x': self.fake, 'z': self.z}), self.len_text_fake.detach(), self.opt.mask_loss)
        # OCR loss on real data

        pred_fake_OCR = self.netOCR(self.fake)
        preds_size = torch.IntTensor([pred_fake_OCR.size(0)] * self.opt.batch_size).detach()
        loss_OCR_fake = self.OCR_criterion(pred_fake_OCR, self.text_encode_fake.detach(), preds_size, self.len_text_fake.detach())
        self.loss_OCR_fake = torch.mean(loss_OCR_fake[~torch.isnan(loss_OCR_fake)])
        # total loss
        self.loss_T = self.loss_G + self.opt.gb_alpha*self.loss_OCR_fake
        grad_fake_OCR = torch.autograd.grad(self.loss_OCR_fake, self.fake, retain_graph=True)[0]
        self.loss_grad_fake_OCR = 10**6*torch.mean(grad_fake_OCR**2)
        grad_fake_adv = torch.autograd.grad(self.loss_G, self.fake, retain_graph=True)[0]
        self.loss_grad_fake_adv = 10**6*torch.mean(grad_fake_adv**2)
        if not self.opt.no_grad_balance:
            self.loss_T.backward(retain_graph=True)
            grad_fake_OCR = torch.autograd.grad(self.loss_OCR_fake, self.fake, create_graph=True, retain_graph=True)[0]
            grad_fake_adv = torch.autograd.grad(self.loss_G, self.fake, create_graph=True, retain_graph=True)[0]
            a = self.opt.gb_alpha * torch.div(torch.std(grad_fake_adv), self.epsilon+torch.std(grad_fake_OCR))
            if a is None:
                print(self.loss_OCR_fake, self.loss_G, torch.std(grad_fake_adv), torch.std(grad_fake_OCR))
            if a>1000 or a<0.0001:
                print(a)
            b = self.opt.gb_alpha * (torch.mean(grad_fake_adv) -
                                            torch.div(torch.std(grad_fake_adv), self.epsilon+torch.std(grad_fake_OCR))*
                                            torch.mean(grad_fake_OCR))
            # self.loss_OCR_fake = a.detach() * self.loss_OCR_fake + b.detach() * torch.sum(self.fake)
            self.loss_OCR_fake = a.detach() * self.loss_OCR_fake
            self.loss_T = (1-1*self.opt.onlyOCR)*self.loss_G + self.loss_OCR_fake
            self.loss_T.backward(retain_graph=True)
            grad_fake_OCR = torch.autograd.grad(self.loss_OCR_fake, self.fake, create_graph=False, retain_graph=True)[0]
            grad_fake_adv = torch.autograd.grad(self.loss_G, self.fake, create_graph=False, retain_graph=True)[0]
            self.loss_grad_fake_OCR = 10 ** 6 * torch.mean(grad_fake_OCR ** 2)
            self.loss_grad_fake_adv = 10 ** 6 * torch.mean(grad_fake_adv ** 2)
            with torch.no_grad():
                self.loss_T.backward()
        else:
            self.loss_T.backward()

        if self.opt.clip_grad > 0:
             clip_grad_norm_(self.netG.parameters(), self.opt.clip_grad)
        if any(torch.isnan(loss_OCR_fake)) or torch.isnan(self.loss_G):
            print('loss OCR fake: ', loss_OCR_fake, ' loss_G: ', self.loss_G, ' words: ', self.words)
            sys.exit()

    def optimize_D_OCR(self):
        self.forward()
        self.set_requires_grad([self.netD], True)
        self.set_requires_grad([self.netOCR], True)
        self.optimizer_D.zero_grad()
        if self.opt.OCR_init in ['glorot', 'xavier', 'ortho', 'N02']:
            self.optimizer_OCR.zero_grad()
        self.backward_D_OCR()

    def optimize_OCR(self):
        self.forward()
        self.set_requires_grad([self.netD], False)
        self.set_requires_grad([self.netOCR], True)
        if self.opt.OCR_init in ['glorot', 'xavier', 'ortho', 'N02']:
            self.optimizer_OCR.zero_grad()
        self.backward_OCR()

    def optimize_D(self):
        self.forward()
        self.set_requires_grad([self.netD], True)
        self.backward_D()

    def optimize_D_OCR_step(self):
        self.optimizer_D.step()
        if self.opt.OCR_init in ['glorot', 'xavier', 'ortho', 'N02']:
            self.optimizer_OCR.step()
        self.optimizer_D.zero_grad()
        self.optimizer_OCR.zero_grad()

    def optimize_D_step(self):
        self.optimizer_D.step()
        if any(torch.isnan(self.netD.infer_img.blocks[0][0].conv1.bias)):
            print('D is nan')
            sys.exit()
        self.optimizer_D.zero_grad()

    def optimize_G(self):
        self.forward()
        self.set_requires_grad([self.netD], False)
        self.set_requires_grad([self.netOCR], False)
        self.backward_G()

    def optimize_G_step(self):
        if self.opt.single_writer and self.opt.optimize_z:
            self.optimizer_z.step()
            self.optimizer_z.zero_grad()
        if not self.opt.not_optimize_G:
            self.optimizer_G.step()
            self.optimizer_G.zero_grad()

    def optimize_ocr(self):
        self.set_requires_grad([self.netOCR], True)
        # OCR loss on real data
        pred_real_OCR = self.netOCR(self.real)
        preds_size =torch.IntTensor([pred_real_OCR.size(0)] * self.opt.batch_size).detach()
        self.loss_OCR_real = self.OCR_criterion(pred_real_OCR, self.text_encode.detach(), preds_size, self.len_text.detach())
        self.loss_OCR_real.backward()
        self.optimizer_OCR.step()

    def optimize_z(self):
        self.set_requires_grad([self.z], True)

    def optimize_parameters(self):
        self.forward()
        self.set_requires_grad([self.netD], False)
        self.optimizer_G.zero_grad()
        self.backward_G()
        self.optimizer_G.step()

        self.set_requires_grad([self.netD], True)
        self.optimizer_D.zero_grad()
        self.backward_D()
        self.optimizer_D.step()

    def test(self):
        self.visual_names = ['fake']
        self.netG.eval()
        with torch.no_grad():
            self.forward()

    def train_GD(self):
        self.netG.train()
        self.netD.train()
        self.optimizer_G.zero_grad()
        self.optimizer_D.zero_grad()
        # How many chunks to split x and y into?
        x = torch.split(self.real, self.opt.batch_size)
        y = torch.split(self.label, self.opt.batch_size)
        counter = 0

        # Optionally toggle D and G's "require_grad"
        if self.opt.toggle_grads:
            toggle_grad(self.netD, True)
            toggle_grad(self.netG, False)

        for step_index in range(self.opt.num_critic_train):
            self.optimizer_D.zero_grad()
            with torch.set_grad_enabled(False):
                self.forward()
            D_input = torch.cat([self.fake, x[counter]], 0) if x is not None else self.fake
            D_class = torch.cat([self.label_fake, y[counter]], 0) if y[counter] is not None else y[counter]
            # Get Discriminator output
            D_out = self.netD(D_input, D_class)
            if x is not None:
                pred_fake, pred_real = torch.split(D_out, [self.fake.shape[0], x[counter].shape[0]])  # D_fake, D_real
            else:
                pred_fake = D_out
            # Combined loss
            self.loss_Dreal, self.loss_Dfake = loss_hinge_dis(pred_fake, pred_real, self.len_text_fake.detach(), self.len_text.detach(), self.opt.mask_loss)
            self.loss_D = self.loss_Dreal + self.loss_Dfake
            self.loss_D.backward()
            counter += 1
            self.optimizer_D.step()

        # Optionally toggle D and G's "require_grad"
        if self.opt.toggle_grads:
            toggle_grad(self.netD, False)
            toggle_grad(self.netG, True)
        # Zero G's gradients by default before training G, for safety
        self.optimizer_G.zero_grad()
        self.forward()
        self.loss_G = loss_hinge_gen(self.netD(self.fake, self.label_fake), self.len_text_fake.detach(), self.opt.mask_loss)
        self.loss_G.backward()
        self.optimizer_G.step()


# PYTHONCASEOK is use for case-sensitive match in windows
# urllib2, scrapy, pyquery, BeautifulSoap, etc. for web scrapping
# os, os.path, and shutil. fies related modules
# number, string, tuple, list, dictionary uses as data type in python.
# we use with statment with files open and close and execption hadling..
# Dictionaries are the best option when the data is labeled
# we use enumerate() to retrive index position after itrate through the sequence.
# 7 sequence types in python. str, list, tuple, unicode, bytearray, xrange, and buffer.

import subprocess
import sys

# install module dynamically
#def install(package):
#    subprocess.call([sys.executable, "-m", "pip", "install", package])

#install('mysql-connector')

# to assign multi line string
str = """ here is string
hear is string"""
#print(str)

# to check built in function of the crunt included packages.
#print(dir('builtins'))

list1 = [1, 2, 3, 4]
list2 = [2, 3, 4, 1]
#print(len(list1+list2))

#for i in [1, 2, 3, 4, 5][::-2] :
#    print (i)

x = 4
def f1():
 #global x
  x = 4 # here we are using auto global veriable x

def f2(a,b):
  return a+b+x

f1()
total = f2(1,2)
#print(total)

str = 'hello'
#print(str.count('aba'))
#print(type(type(int)))
#print(str[-1:])

a = {1:'a', 2:'b', 3:'c'}
for x,y in a:
  print(x,y)

a[1] = 1
a['1'] = 2

#print str.center
#for i in a.values():
#    print(i)


from msilib.schema import Error
import matplotlib.pyplot as plt
import numpy as np
import boto3
import os

def draw_chart(result, user_id, practice_id, gender):
    w_min_jitter = 1.599
    w_max_jitter = 2.310
    w_min_shimmer = 7.393
    w_max_shimmer = 12.221

    m_min_jitter = 2.159
    m_max_jitter = 3.023
    m_min_shimmer = 10.132
    m_max_shimmer = 13.526

    min_speed = 96
    max_speed = 124

    font_size = 12

    # 차트 그리기
    # 1) speed
    speed_x = ['slow', 'user', 'fast']
    speed_y = [min_speed, result['voice']['speed'], max_speed]
    speed_colors = ['lightgray', 'mediumpurple', 'silver']
    plt.xlabel('standard', size=font_size)
    plt.ylabel('Words Per Minute', size=font_size)
    bar_chart=plt.bar(speed_x, speed_y, color=speed_colors)

    for rect in bar_chart:
        height = rect.get_height()
        plt.text(rect.get_x()+rect.get_width()/2.0, height-font_size, height, ha='center', va='bottom', size=12)

    plt.savefig(f'./{user_id}_{practice_id}_speed.png')
    plt.clf()

    # 2) shimmer, jitter
    if gender == 'W':
        mins = [w_min_jitter, w_min_shimmer]
        maxs = [w_max_jitter, w_max_shimmer]
    else:
        mins = [m_min_jitter, m_min_shimmer]
        maxs = [m_max_jitter, m_max_shimmer]

    users = [result['voice']['jitter'], result['voice']['shimmer']]
    checks = ['jitter', 'shimmer']

    bar_width=0.25

    index = np.arange(2)

    b1 = plt.bar(index, mins, bar_width, color='lightgray', label='mins')
    b2 = plt.bar(index+bar_width, users, bar_width, color='mediumpurple', label='users')
    b3 = plt.bar(index+2*bar_width, maxs, bar_width, color='silver', label='maxs')

    plt.legend()
    plt.savefig(f'./{user_id}_{practice_id}_shimmer_jitter.png')
    plt.clf()

    # 3) closing remarks
    closing_remark_ratio = [result['voice']['closing_remarks'], 100-result['voice']['closing_remarks']]
    closing_remark_labels = ['correctly recognized', 'recognization failed']
    closing_remark_colors = ['thistle', 'mediumpurple']
    wedgeprops = {'width': 0.7, 'edgecolor': 'w', 'linewidth': 5}

    plt.pie(closing_remark_ratio, labels=closing_remark_labels, autopct='%.1f%%', startangle=180, colors = closing_remark_colors, wedgeprops=wedgeprops)
    plt.savefig(f'./{user_id}_{practice_id}_closing_remarks.png')
    plt.clf()

    # 4) movement
    move_duration = result['video']['pose']['inclined_duration']
    +result['video']['pose']['first_duration']
    +result['video']['pose']['second_duration']
    +result['video']['pose']['third_duration']
    +result['video']['eyes']['script_duration']
    + result['video']['eyes']['around_duration']
    + result['video']['eyes']['face_move_duration']
    move_ratio = [result['video']['pose']['total_duration'], move_duration]
    move_labels = ['normal', 'bad movement']
    move_colors = ['thistle', 'mediumpurple']

    plt.pie(move_ratio, labels=move_labels, autopct='%.1f%%', startangle=180, colors=move_colors, wedgeprops=wedgeprops)
    plt.savefig(f'./{user_id}_{practice_id}_movement.png')
    plt.clf()

    # 5) movement detail
    bad_x = ['around', 'inclined', 'first', 'second', 'third', 'script', 'face_move']
    bad_y = [
        result['video']['eyes']['around_duration'],
        result['video']['pose']['inclined_duration'],
        result['video']['pose']['first_duration'],
        result['video']['pose']['second_duration'],
        result['video']['pose']['third_duration'],
        result['video']['eyes']['script_duration'],
        result['video']['eyes']['face_move_duration']
    ]
    bad_colors = ['thistle', 'plum', 'mediumpurple', 'mediumslateblue', 'mediumorchid', 'darkorchid', 'rebeccapurple']

    plt.xlabel('sort', size=font_size)
    plt.ylabel('duration', size=font_size)
    bar_chart=plt.bar(bad_x, bad_y, color=bad_colors)
    plt.savefig(f'./{user_id}_{practice_id}_movement_detail.png')
    plt.clf()

def upload_chart_to_s3(user_id, practice_id):
    s3=boto3.client('s3')
    bucket_name = "sookpeech-wavfile"

    # chart 업로드하기
    files = {
        "closing_remarks" : f'./{user_id}_{practice_id}_closing_remarks.png',
        "movement" : f'./{user_id}_{practice_id}_movement.png',
        "movement_detail": f'./{user_id}_{practice_id}_movement_detail.png',
        "shimmer_jitter" : f'./{user_id}_{practice_id}_shimmer_jitter.png',
        "speed" : f'./{user_id}_{practice_id}_speed.png'
    }

    for key in files.keys():
        try:
            file = open(files[key], 'rb')
            s3.upload_fileobj(file, bucket_name, f'{user_id}/{practice_id}/{key}.png', ExtraArgs={'ContentType': 'image/png', 'ACL':'public-read'})
        except:
            print(f'failed to upload chart image = {key}')

def delete_image(user_id, practice_id):
    files = {
        "closing_remarks" : f'./{user_id}_{practice_id}_closing_remarks.png',
        "movement" : f'./{user_id}_{practice_id}_movement.png',
        "movement_detail": f'./{user_id}_{practice_id}_movement_detail.png',
        "shimmer_jitter" : f'./{user_id}_{practice_id}_shimmer_jitter.png',
        "speed" : f'./{user_id}_{practice_id}_speed.png'
    }

    for key in files.keys():
        if os.path.exists(files[key]):
            os.remove(files[key])


import pytest

import for_mocking

"""
1. Реализовать программу на Python. Программа может содержать любое количество методов и классов (>0), но обязательно должна иметь класс main.
2. Имитировать:
    a. Метод созданного класса (метод не должен являться генератором).
    b. Параметр внутри метода класса.
    c. Класс.
3. Спровоцировать имитированную ошибку.
4. Применить декоратор @patch на любом имитируемом объекте (классе, методе и т.д.).
Выбирается студентом самостоятельно.
5. Имитировать объект-генератор (см.пр. 26.6.3.3.[1]).
"""

def test_format_status(mocker):
    main_object = for_mocking.Main('Not OK')
    mocker.patch.object(main_object, 'format_status',
                        return_value=['Status: OK;'])
    assert main_object.format_status() == ['Status: OK;']
    assert main_object.status == 'Not OK'

def test_property_mocking(mocker):
    main_object = for_mocking.Main('Not OK')
    main_object = mocker.patch.object(main_object, 'format_status')
    main_object.format_status(mocker.ANY)
    main_object.format_status.assert_called_once_with(mocker.ANY)

def test_class_mocking(mocker):
    mocker.patch.object(for_mocking, 'Main')
    for_mocking.Main.format_status.return_value = 123
    mocked_instance = for_mocking.Main('wow')
    mocked_instance.format_status()
    mocked_instance.format_status.assert_called_once_with()  # pylint: disable=no-member

@pytest.mark.xfail(raises=Exception)
def test_generator(mocker):
    main_object = for_mocking.Main('Not OK')
    main_object = mocker.patch.object(main_object, 'format_status')
    main_object.format_status(123)

def test_letter_generator(mocker):
    main_object = for_mocking.Main('Not OK')
    mocker.patch.object(main_object, 'letter_generator',
                        return_value=iter('Status: OK;'))
    assert list(main_object.letter_generator()) == list('Status: OK;')


import asyncio

from supybot import callbacks, httpserver, log

from .helpers import github

# Import files that will hook themself up when imported
from .events import (  # noqa
    commit_comment,
    discussion,
    issue,
    pull_request,
    push,
    tag,
)
from .patches import gidgethub  # noqa
from .protocols import irc as irc_protocols  # noqa
from .protocols import discord as discord_protocols  # noqa

class GitHubCallback(httpserver.SupyHTTPServerCallback):
    name = "GitHub"
    defaultResponse = "404: page not found"

    def doPost(self, handler, path, form):
        if path != "/":
            self.send_response(404)
            self.send_header("Content-type", "text/plain")
            self.end_headers()
            self.wfile.write(b"404: page not found")
            return

        headers = {key.lower(): value for key, value in dict(self.headers).items()}

        try:
            asyncio.run(github.dispatch(headers, form))

            self.send_response(200)
            self.send_header("Content-Type", "text/plain")
            self.end_headers()
            self.wfile.write(b"200: OK")
        except Exception:
            log.exception("Failed to handle GitHub event")
            self.send_response(403)
            self.send_header("Content-type", "text/plain")
            self.end_headers()
            self.wfile.write(b"403: failed processing event")

class GitHub(callbacks.Plugin):
    """Translates GitHub events to IRC messages."""

    def __init__(self, irc):
        self.__parent = super(GitHub, self)
        callbacks.Plugin.__init__(self, irc)

        httpserver.hook("github", GitHubCallback())

    def die(self):
        self.__parent.die()
        httpserver.unhook("github")

Class = GitHub