import argparse
import os
import numpy as np
import math
import json
import torch
import torchvision.transforms as transforms
from torchvision.utils import save_image
from torch.utils.data import DataLoader
from torch.autograd import Variable
from dataset import CartoonDataset
from PIL import Image
from models_bce import ACGenerator, ACDiscriminator, ACGenerator_noconcat
from models_sn import ACDiscriminator_SN, ACDiscriminator_SNPJ

parser = argparse.ArgumentParser()
parser.add_argument("file_name",type=str)
parser.add_argument("--train_attr_file", type=str, default='./data/selected_cartoonset100k/cartoon_attr.txt')
parser.add_argument("--img_dir", type=str, default='./data/selected_cartoonset100k/images')
parser.add_argument("--n_epochs", type=int, default=500, help="number of epochs of training")
parser.add_argument("--batch_size", type=int, default=32, help="size of the batches")
parser.add_argument("--lr", type=float, default=0.001, help="adam: learning rate")
parser.add_argument("--b1", type=float, default=0.5, help="adam: decay of first order momentum of gradient")
parser.add_argument("--b2", type=float, default=0.9, help="adam: decay of first order momentum of gradient")

parser.add_argument("--latent_dim", type=int, default=128, help="dimensionality of the latent space")
parser.add_argument("--all_classes_dim",type=int, default=15)
parser.add_argument("--img_size", type=int, default=128, help="size of each image dimension")
parser.add_argument("--channels", type=int, default=3, help="number of image channels")
parser.add_argument("--n_layers", type=int, default=5)

parser.add_argument("--sample_interval", type=int, default=5, help="interval between image sampling")
parser.add_argument("--test_interval", type=int, default=10, help="interval between image testing")
parser.add_argument("--test_start", type=int, default=0, help="start test image")

parser.add_argument("--lambda_gp", type=int, default=1)
parser.add_argument("--lambda_aux", type=int, default=20)
parser.add_argument("--clip_value", type=float, default=0.1, help="lower and upper clip value for disc. weights")
parser.add_argument("--loss_type", type=str,  default='WGGP', help='MM, NS, WGCP, WGGP WGDIV')
parser.add_argument("--generator_type",type=str, default='concat', help='concat, noconcat')
parser.add_argument("--discriminator_type",type=str, default='SN', help='noSN, SN, SNPJ')
parser.add_argument("--seed",type=int,default=422)

opt = parser.parse_args()

cuda = True if torch.cuda.is_available() else False

os.makedirs("./saves/{}".format(opt.file_name),exist_ok=True)
os.makedirs("./saves/{}/train_sample".format(opt.file_name), exist_ok=True)
os.makedirs("./saves/{}/models".format(opt.file_name), exist_ok=True)

auxiliary_loss = torch.nn.BCELoss()

if opt.generator_type == 'concat':
    generator = ACGenerator(opt, n_filters = [512, 256, 128, 64, 32])
elif opt.generator_type == 'noconcat':
    generator = ACGenerator_noconcat(opt, n_filters = [512, 256, 128, 64, 32])

if opt.discriminator_type == 'noSN':
    discriminator = ACDiscriminator(opt, n_filters = [32, 64, 128, 256, 512])
elif opt.discriminator_type == 'SN':
    discriminator = ACDiscriminator_SN(opt, n_filters = [32, 64, 128, 256, 512])
elif opt.discriminator_type == 'SNPJ':
    discriminator = ACDiscriminator_SNPJ(opt, n_filters = [32, 64, 128, 256, 512])

if cuda:
    generator.cuda()
    discriminator.cuda()
    auxiliary_loss.cuda()

data_transform = transforms.Compose([
transforms.Resize(opt.img_size),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.5]*opt.channels, [0.5]*opt.channels)])

# Configure data loader
train_dataset = CartoonDataset(attr_file=opt.train_attr_file, img_dir=opt.img_dir, transform=data_transform)
train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=opt.batch_size,shuffle=True,num_workers=16,drop_last=True)

# Optimizers
optimizer_G = torch.optim.Adam(generator.parameters(), lr=opt.lr, betas=(opt.b1, opt.b2), weight_decay = 1e-4)
optimizer_D = torch.optim.Adam(discriminator.parameters(), lr=opt.lr, betas=(opt.b1, opt.b2), weight_decay = 1e-4)

FloatTensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor
LongTensor = torch.cuda.LongTensor if cuda else torch.LongTensor

np.random.seed(opt.seed)
fixed_z = FloatTensor(np.random.normal(0, 1, (12 ** 2, opt.latent_dim)))

def sample_image(n_row, epoch):
    hair_labels = np.eye(6)[np.array([num  for num in [0,0,1,1,2,2,3,3,4,4,5,5] for _ in range(n_row)])]
    eye_labels = np.eye(4)[np.array([num for _ in range(int(n_row/2)) for num in ([0]*6+[1]*6+[2]*6+[3]*6)])]
    face_labels = np.eye(3)[np.array([num for _ in range(n_row) for num in [0,0,1,1,2,2] * 2])]
    glasses_labels = np.eye(2)[np.array([num for _ in range(n_row) for num in [0,1] * 6])]
    condition = FloatTensor(np.concatenate((hair_labels,eye_labels,face_labels,glasses_labels),axis=1))
    gen_imgs = generator(fixed_z, condition)
    save_image(gen_imgs.data, "./saves/{}/train_sample/ep-{}_seed-{}.png".format(opt.file_name,epoch,opt.seed), nrow=n_row, normalize=True)

def compute_gradient_penalty_GP(D, real_samples, fake_samples, condition):
    alpha = torch.rand(real_samples.size(0), 1, 1, 1).cuda()
    alpha = alpha.expand_as(real_samples)
    interpolates = (alpha * real_samples + ((1 - alpha) * fake_samples))
    d_interpolates, _ = D(interpolates, condition)
    fake = torch.ones(d_interpolates.shape).cuda()
    # Get gradient w.r.t. interpolates
    gradients = torch.autograd.grad(
        outputs=d_interpolates,
        inputs=interpolates,
        grad_outputs=fake,
        create_graph=True,
        retain_graph=True,
        only_inputs=True,
    )[0]
    gradients = gradients.view(gradients.size(0), -1)
    gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean()
    return gradient_penalty *  opt.lambda_gp

def compute_gradient_penalty_DIV(real_pred, fake_pred, real_imgs, fake_imgs):
    p = 6
    real_grad_out = Variable(FloatTensor(real_imgs.size(0), 1).fill_(1.0), requires_grad=False)
    real_grad = torch.autograd.grad(real_pred, real_imgs, real_grad_out, create_graph=True, retain_graph=True, only_inputs=True)[0]
    real_grad_norm = real_grad.view(real_grad.size(0), -1).pow(2).sum(1) ** (p / 2)
    fake_grad_out = Variable(FloatTensor(fake_imgs.size(0), 1).fill_(1.0), requires_grad=False)
    fake_grad = torch.autograd.grad(fake_pred, fake_imgs, fake_grad_out, create_graph=True, retain_graph=True, only_inputs=True)[0]
    fake_grad_norm = fake_grad.view(fake_grad.size(0), -1).pow(2).sum(1) ** (p / 2)
    div_gp = torch.mean(real_grad_norm + fake_grad_norm) * opt.lambda_gp
    return div_gp
# ----------
#  Training
# ----------

plot = {
    'd_real_loss':[],
    'd_fake_loss':[],
    'g_adv_loss':[],
    'g_aux_loss':[],
    'd_acc':[],
}

for epoch in range(opt.n_epochs):

    batch_num = 0
    plot['d_real_loss'].append(0.0)
    plot['d_fake_loss'].append(0.0)
    plot['g_aux_loss'].append(0.0)
    plot['g_adv_loss'].append(0.0)
    plot['d_acc'].append(0.0)

    for i, sample in enumerate(train_dataloader):

        batch_size = sample['image'].shape[0]

        # Configure input
        real_imgs = Variable(sample['image'].type(FloatTensor), requires_grad=True)
        true_vec = sample['true_vec'].type(FloatTensor)
        # -----------------
        #  Train Generator
        # -----------------

        z = torch.randn(batch_size, opt.latent_dim).cuda()
        gen_imgs = generator(z, true_vec)
        # Loss measures generator's ability to fool the discriminator
        validity, pred_labels = discriminator(gen_imgs, true_vec)
        #g_adv_loss = adversarial_loss(validity, valid)
        if opt.loss_type == 'MM':
            g_adv_loss = torch.mean(torch.log(1 - torch.sigmoid(validity) + 1e-5))
            g_aux_loss = auxiliary_loss(pred_labels, true_vec)
        elif opt.loss_type == 'NS':
            g_adv_loss  = - torch.mean(torch.log(torch.sigmoid(validity) + 1e-5))
            g_aux_loss = auxiliary_loss(pred_labels, true_vec)
        elif opt.loss_type == 'WGCP' or opt.loss_type == 'WGGP' or opt.loss_type == 'WGDIV':
            g_adv_loss = - torch.mean(validity)
            g_aux_loss = auxiliary_loss(pred_labels, true_vec)  * opt.lambda_aux

        g_loss = (g_adv_loss + g_aux_loss)
        optimizer_G.zero_grad()
        g_loss.backward(retain_graph = True)
        optimizer_G.step()

        # ---------------------
        #  Train Discriminator
        # ---------------------
        z = torch.randn(batch_size, opt.latent_dim).cuda()
        gen_imgs = generator(z, true_vec)

        real_pred, real_labels = discriminator(real_imgs, true_vec)
        if opt.loss_type == 'WGDIV':
            fake_pred, fake_labels = discriminator(gen_imgs, true_vec)
        else:
            fake_pred, fake_labels = discriminator(gen_imgs.detach(), true_vec)

        if opt.loss_type == 'MM':
            d_real_adv_loss = torch.mean(torch.log(1 - torch.sigmoid(real_pred) + 1e-5))
            d_real_aux_loss = auxiliary_loss(real_labels, true_vec)
            d_fake_adv_loss = torch.mean(torch.log(1 - torch.sigmoid(fake_pred) + 1e-5))
            d_fake_aux_loss = auxiliary_loss(fake_labels, true_vec)
        elif opt.loss_type == 'NS':
            d_real_adv_loss  = - torch.mean(torch.log(torch.sigmoid(real_pred) + 1e-5))
            d_real_aux_loss = auxiliary_loss(real_labels, true_vec)
            d_fake_adv_loss  = - torch.mean(torch.log(torch.sigmoid(fake_pred) + 1e-5))
            d_fake_aux_loss = auxiliary_loss(fake_labels, true_vec)
        elif opt.loss_type == 'WGCP' or opt.loss_type == 'WGGP' or opt.loss_type=='WGDIV':
            d_real_adv_loss = - torch.mean(real_pred)
            d_real_aux_loss = auxiliary_loss(real_labels, true_vec) * opt.lambda_aux
            d_fake_adv_loss = torch.mean(fake_pred)
            d_fake_aux_loss = auxiliary_loss(fake_labels, true_vec) * opt.lambda_aux

        if opt.loss_type == 'WGGP':
            gradient_penalty = compute_gradient_penalty_GP(discriminator, real_imgs, gen_imgs, true_vec)

        if opt.loss_type == 'WGDIV':
            gradient_penalty = compute_gradient_penalty_DIV(real_pred, fake_pred, real_imgs, gen_imgs)

        # Total discriminator loss
        d_real_loss = d_real_adv_loss + d_real_aux_loss
        d_fake_loss = d_fake_adv_loss + d_fake_aux_loss
        optimizer_D.zero_grad()
        d_real_loss.backward(retain_graph=True)
        d_fake_loss.backward(retain_graph=True)

        if opt.loss_type == 'WGGP' or opt.loss_type=='WGDIV':
            gradient_penalty.backward(retain_graph=True)

        optimizer_D.step()

        if opt.loss_type == 'WGCP':
            for p in discriminator.parameters():
                p.data.clamp_(-opt.clip_value, opt.clip_value)

        # Calculate discriminator accuracy
        pred = np.concatenate([real_labels.data.cpu().numpy(), fake_labels.data.cpu().numpy()], axis=0).flatten()
        gt = np.concatenate([true_vec.data.cpu().numpy(), true_vec.data.cpu().numpy()], axis=0).flatten()
        pred[pred>=0.5] = 1.0
        pred[pred<0.5] = 0.0
        d_acc = np.mean(pred == gt)

        plot['d_real_loss'][epoch] += d_real_loss.item()
        plot['d_fake_loss'][epoch] += d_fake_loss.item()
        plot['g_adv_loss'][epoch] += g_adv_loss.item()
        plot['g_aux_loss'][epoch] += g_aux_loss.item()
        plot['d_acc'][epoch] += d_acc

        print("[Epoch %d/%d] [Batch %d/%d] [D real: %f, fake: %f, acc: %d%%] [G adv: %f, aux: %f]"
              % (epoch+1, opt.n_epochs, i, len(train_dataloader), plot['d_real_loss'][-1]/(i+1), plot['d_fake_loss'][-1]/(i+1),
                100 * plot['d_acc'][-1]/(i+1), plot['g_adv_loss'][-1]/(i+1), plot['g_aux_loss'][-1]/(i+1)), end='\r')

        batch_num += 1

    for k, v in plot.items():
        plot[k][epoch] = plot[k][epoch] / batch_num

    print("[Epoch %d/%d] [D real: %f, fake: %f, acc: %d%%] [G adv: %f, aux: %f]"
        % (epoch+1, opt.n_epochs, plot['d_real_loss'][-1], plot['d_fake_loss'][-1],
            100 * plot['d_acc'][-1],  plot['g_adv_loss'][-1], plot['g_aux_loss'][-1]))

    if ((epoch+1) >= opt.test_start) and ((epoch+1) % opt.sample_interval == 0):
        sample_image(n_row=12, epoch=epoch+1)

    with open('./saves/{}/plot.json'.format(opt.file_name), 'w') as f:
        json.dump(plot,f)

    if (epoch + 1) % 10 == 0:
        torch.save({
            'epoch': epoch + 1,
            'generator': generator.state_dict(),
            'discriminator': discriminator.state_dict(),
            'optimizer_G': optimizer_G.state_dict(),
            'optimizer_D': optimizer_D.state_dict()
            }, './saves/{}/models/epoch-{}.pth'.format(opt.file_name,epoch+1))


#!/usr/bin/env/python
#-*- coding:utf-8 -*-

__author__ = 'BlackYe.'

from thirdparty_libs.colorizer import colored

# Color names mapped to themselves.
m_colors = {
    None        : None,
    'blue'      : 'blue',
    'green'     : 'green',
    'cyan'      : 'cyan',
    'magenta'   : 'magenta',
    'grey'      : 'grey',
    'gray'      : 'grey',  # tomayto, tomahto...
    'red'       : 'red',
    'yellow'    : 'yellow',
    'white'     : 'white',

    # String log levels to color names.
    'informational' : 'blue',
    'low'           : 'cyan',
    'middle'        : None,
    'high'          : 'magenta',
    'critical'      : 'red',

    # Integer log levels to color names.
    0 : 'blue',
    1 : 'cyan',
    2 : None,
    3 : 'red',
    4 : 'yellow',
}

# Colors that need an increase in brightness.
m_make_brighter = ['blue', 'grey', 'red']

def get_terminal_size():
    import platform
    current_os = platform.system()
    tuple_xy=None
    if current_os == 'Windows':
        tuple_xy = _get_terminal_size_windows()
        if tuple_xy is None:
            tuple_xy = _get_terminal_size_tput()
            # needed for window's python in cygwin's xterm!
    if current_os == 'Linux' or current_os == 'Darwin' or  current_os.startswith('CYGWIN'):
        tuple_xy = _get_terminal_size_linux()
    if tuple_xy is None:
        tuple_xy = (80, 25)      # default value
    return tuple_xy

def _get_terminal_size_windows():
    res=None
    try:
        from ctypes import windll, create_string_buffer

        # stdin handle is -10
        # stdout handle is -11
        # stderr handle is -12

        h = windll.kernel32.GetStdHandle(-12)
        csbi = create_string_buffer(22)
        res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi)
    except:
        return None
    if res:
        import struct
        (bufx, bufy, curx, cury, wattr,
         left, top, right, bottom, maxx, maxy) = struct.unpack("hhhhHhhhhhh", csbi.raw)
        sizex = right - left + 1
        sizey = bottom - top + 1
        return sizex, sizey
    else:
        return None

def _get_terminal_size_tput():
    # get terminal width
    # src: http://stackoverflow.com/questions/263890/how-do-i-find-the-width-height-of-a-terminal-window
    try:
        import subprocess
        proc=subprocess.Popen(["tput", "cols"],stdin=subprocess.PIPE,stdout=subprocess.PIPE)
        output=proc.communicate(input=None)
        cols=int(output[0])
        proc=subprocess.Popen(["tput", "lines"],stdin=subprocess.PIPE,stdout=subprocess.PIPE)
        output=proc.communicate(input=None)
        rows=int(output[0])
        return (cols,rows)
    except:
        return None

def _get_terminal_size_linux():
    def ioctl_GWINSZ(fd):
        try:
            import fcntl, termios, struct
            cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ,'1234'))
        except:
            return None
        return cr
    cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
    if not cr:
        try:
            import os
            fd = os.open(os.ctermid(), os.O_RDONLY)
            cr = ioctl_GWINSZ(fd)
            os.close(fd)
        except:
            pass
    if not cr:
        try:
            import os
            env = os.environ
            cr = (env['LINES'], env['COLUMNS'])
        except:
            return None
    return int(cr[1]), int(cr[0])

#------------------------------------------------------------------------------
def colorize_substring(text, substring, level_or_color):
    """
    Colorize a substring within a text to be displayed on the console.

    :param text: Full text.
    :type text: str

    :param substring: Text to find and colorize.
    :type substring: str

    :param level_or_color:
        Color name or risk level name.
        See the documentation for colorize() for more details.
    :type level_or_color: str

    :returns: Colorized text.
    :rtype: str
    """

    #
    # XXX TODO:
    #
    # We also probably need to parse existing ANSI escape codes
    # to know what's the color of the surrounding text, otherwise
    # we'll only properly colorize substrings in non colored text.
    #
    # Maybe we can settle with this: indicate a color for the text
    # and a color for the substring. Should work in all situations
    # we _currently_ need to handle.
    #

    # Check for trivial cases.
    if text and substring and True:

        # Loop for each occurrence of the substring.
        m_pos = 0
        while 1:

            # Find the substring in the text.
            m_pos = text.find(substring, m_pos)

            # If not found, break out of the loop.
            if m_pos < 0:
                break

            # Split the text where the substring was found.
            m_prefix  = text[:m_pos]
            m_content = text[m_pos: m_pos + len(substring)]
            m_suffix  = text[m_pos + len(substring):]

            # Patch the text to colorize the substring.
            m_content = colorize(m_content, level_or_color)
            text = "%s%s%s" % (m_prefix, m_content, m_suffix)

            # Update the current position and keep searching.
            m_pos = len(m_prefix) + len(m_content)

    # Return the patched text.
    return text

#------------------------------------------------------------------------------
def colorize(text, level_or_color):
    """
    Colorize a text to be displayed on the console.

    The following color names may be used:

     - Blue
     - Cyan
     - Green
     - Grey (or gray)
     - Magenta
     - Red
     - Yellow
     - White

    The following risk levels may be used in lieu of colors:

     - Informational (0)
     - Low (1)
     - Middle (2)
     - High (3)
     - Critical (4)

    :param text: Text to colorize.
    :type text: str

    :param level_or_color: Color name or risk level name.
    :type level_or_color: str

    :returns: Colorized text.
    :rtype: str
    """

    # Check if colors are enabled.
    if True:

        # Parse the color name or level into
        # a color value that colored() expects.
        try:
            level_or_color = level_or_color.lower()
        except AttributeError:
            pass
        color = m_colors[level_or_color]

        # Colorize the text.
        if color:
            if color in m_make_brighter:
                text = colored(text, color, attrs=["bold"])
            else:
                text = colored(text, color)

    # Return the text.
    return text


"""
Tests for view - dashboard
"""
import pytest

from datetime import date, datetime, timedelta

from pytest_django.asserts import assertContains

from django.http import HttpResponse
from django.urls import reverse

from accessibility_monitoring_platform.apps.common.models import ChangeToPlatform

from ...cases.models import (
    Case,
    STATEMENT_COMPLIANCE_STATE_COMPLIANT,
    WEBSITE_COMPLIANCE_STATE_COMPLIANT,
    CASE_COMPLETED_NO_SEND,
    REPORT_APPROVED_STATUS_APPROVED,
    CASE_COMPLETED_SEND,
    ENFORCEMENT_BODY_PURSUING_YES_COMPLETED,
)
from ...cases.utils import create_case_and_compliance
from ...common.models import BOOLEAN_TRUE
from ...reminders.models import Reminder

def test_dashboard_loads_correctly_when_user_logged_in(admin_client):
    """Tests if a logged in user can access the Dashboard"""
    response: HttpResponse = admin_client.get(reverse("dashboard:home"))

    assert response.status_code == 200
    assertContains(response, "Home |")

def test_dashboard_redirects_to_login_when_user_not_logged_in(client):
    """Tests if a unauthenticated user returns a 302 response"""
    url: str = reverse("dashboard:home")
    response: HttpResponse = client.get(url)

    assert response.status_code == 302
    assert response.url == f"/account/login/?next={url}"  # type: ignore

@pytest.mark.parametrize(
    "dashboard_view",
    ["View+your+cases", "View+all+cases"],
)
def test_dashboard_shows_qa_auditors(dashboard_view, admin_client):
    """Tests if dashboard views are showing the expected QA auditors column"""
    response: HttpResponse = admin_client.get(
        f'{reverse("dashboard:home")}?view={dashboard_view}'
    )

    assert response.status_code == 200
    assertContains(response, "On call")

def test_dashboard_shows_oldest_unassigned_cases_first(admin_client):
    """Tests dashboard shows unassigned cases in order of oldest first"""
    case_1: Case = Case.objects.create(organisation_name="Organisation One")
    case_2: Case = Case.objects.create(organisation_name="Organisation Two")
    case_3: Case = Case.objects.create(organisation_name="Organisation Three")

    response: HttpResponse = admin_client.get(reverse("dashboard:home"))

    assert response.status_code == 200

    content: str = str(response.content)
    case_1_position: int = content.index(case_1.organisation_name)
    case_2_position: int = content.index(case_2.organisation_name)
    case_3_position: int = content.index(case_3.organisation_name)

    assert case_1_position < case_2_position
    assert case_2_position < case_3_position

def test_dashboard_shows_link_to_closed_and_sent_cases(admin_client, admin_user):
    """Check dashboard contains link to find closed and sent to equalities body cases"""
    create_case_and_compliance(
        home_page_url="https://www.website.com",
        organisation_name="org name",
        auditor=admin_user,
        website_compliance_state_initial=WEBSITE_COMPLIANCE_STATE_COMPLIANT,
        statement_compliance_state_initial=STATEMENT_COMPLIANCE_STATE_COMPLIANT,
        report_review_status=BOOLEAN_TRUE,
        report_approved_status=REPORT_APPROVED_STATUS_APPROVED,
        report_sent_date=datetime.now(),
        report_acknowledged_date=datetime.now(),
        twelve_week_update_requested_date=datetime.now(),
        twelve_week_correspondence_acknowledged_date=datetime.now(),
        case_completed=CASE_COMPLETED_SEND,
        sent_to_enforcement_body_sent_date=datetime.now(),
    )

    response: HttpResponse = admin_client.get(reverse("dashboard:home"))

    assert response.status_code == 200

    assertContains(
        response,
        f"""

                View all your cases with status "Case closed and sent to equalities body"

""",
        html=True,
    )

def test_dashboard_shows_link_to_completed_cases(admin_client, admin_user):
    """Check dashboard contains link to find completed cases"""
    create_case_and_compliance(
        home_page_url="https://www.website.com",
        organisation_name="org name",
        auditor=admin_user,
        website_compliance_state_initial=WEBSITE_COMPLIANCE_STATE_COMPLIANT,
        statement_compliance_state_initial=STATEMENT_COMPLIANCE_STATE_COMPLIANT,
        case_completed=CASE_COMPLETED_NO_SEND,
    )

    response: HttpResponse = admin_client.get(reverse("dashboard:home"))

    assert response.status_code == 200

    assertContains(
        response,
        f"""

                View all your cases with status "Complete"

""",
        html=True,
    )

def test_dashboard_shows_warning_of_recent_changes_to_platform(admin_client):
    """Check dashboard contains link to find completed cases"""
    ChangeToPlatform.objects.create(name="Recent change")

    response: HttpResponse = admin_client.get(reverse("dashboard:home"))

    assert response.status_code == 200

    assertContains(
        response,
        f"""

            !

                Warning
                An update has been made to the platform. View the update in

                    Settings > Platform version history

""",
        html=True,
    )

def test_dashboard_shows_correct_number_of_active_cases(admin_client, admin_user):
    """Check dashboard shows correct number of active cases"""
    # Creates unassigned case
    Case.objects.create()

    # Creates test in progress case
    Case.objects.create(
        home_page_url="https://www.website.com",
        organisation_name="org name",
        auditor=admin_user,
    )

    # Creates deactivated case
    Case.objects.create(
        home_page_url="https://www.website.com",
        organisation_name="org name",
        is_deactivated=True,
    )

    # Creates completed case
    create_case_and_compliance(
        home_page_url="https://www.website.com",
        organisation_name="org name",
        auditor=admin_user,
        statement_compliance_state_initial=STATEMENT_COMPLIANCE_STATE_COMPLIANT,
        website_compliance_state_initial=WEBSITE_COMPLIANCE_STATE_COMPLIANT,
        report_review_status=BOOLEAN_TRUE,
        report_approved_status=REPORT_APPROVED_STATUS_APPROVED,
        report_sent_date=datetime.now(),
        report_acknowledged_date=datetime.now(),
        twelve_week_update_requested_date=datetime.now(),
        twelve_week_correspondence_acknowledged_date=datetime.now(),
        case_completed=CASE_COMPLETED_SEND,
        sent_to_enforcement_body_sent_date=datetime.now(),
        enforcement_body_pursuing=ENFORCEMENT_BODY_PURSUING_YES_COMPLETED,
    )

    # Creates closed sent to equalities-body case
    create_case_and_compliance(
        home_page_url="https://www.website.com",
        organisation_name="org name",
        auditor=admin_user,
        statement_compliance_state_initial=STATEMENT_COMPLIANCE_STATE_COMPLIANT,
        website_compliance_state_initial=WEBSITE_COMPLIANCE_STATE_COMPLIANT,
        report_review_status=BOOLEAN_TRUE,
        report_approved_status=REPORT_APPROVED_STATUS_APPROVED,
        report_sent_date=datetime.now(),
        report_acknowledged_date=datetime.now(),
        twelve_week_update_requested_date=datetime.now(),
        twelve_week_correspondence_acknowledged_date=datetime.now(),
        case_completed=CASE_COMPLETED_SEND,
        sent_to_enforcement_body_sent_date=datetime.now(),
    )

    response: HttpResponse = admin_client.get(reverse("dashboard:home"))
    assert Case.objects.all().count() == 5
    assert response.status_code == 200
    expected_number_of_active_cases: str = """

 Total active cases

 2

    """
    assertContains(
        response,
        expected_number_of_active_cases,
        html=True,
    )

def test_dashboard_shows_correct_number_of_your_active_cases(admin_client, admin_user):
    """Check dashboard shows correct number of your active cases"""

    # Creates unassigned case
    Case.objects.create()
    Case.objects.create()

    # Case assigned to you
    Case.objects.create(auditor=admin_user)

    response: HttpResponse = admin_client.get(reverse("dashboard:home"))
    assert Case.objects.all().count() == 3
    assert response.status_code == 200
    expected_number_of_your_active_cases: str = """

 Your active cases

 1

    """
    assertContains(
        response,
        expected_number_of_your_active_cases,
        html=True,
    )

    expected_number_of_unnassigned_cases: str = """

 Unassigned cases

 2

    """
    assertContains(
        response,
        expected_number_of_unnassigned_cases,
        html=True,
    )

def test_dashboard_shows_link_to_reminder(admin_client, admin_user):
    """
    Tests dashboard shows link to reminder for cases of status
    reviewing changes
    """
    case: Case = Case.objects.create(
        home_page_url="https://www.website.com",
        organisation_name="org name",
        auditor=admin_user,
        report_review_status=BOOLEAN_TRUE,
        report_approved_status=REPORT_APPROVED_STATUS_APPROVED,
        report_sent_date=datetime.now(),
        report_acknowledged_date=datetime.now(),
        twelve_week_update_requested_date=datetime.now(),
        twelve_week_correspondence_acknowledged_date=datetime.now(),
    )
    case.compliance.statement_compliance_state_initial = (
        STATEMENT_COMPLIANCE_STATE_COMPLIANT
    )
    case.compliance.website_compliance_state_initial = (
        WEBSITE_COMPLIANCE_STATE_COMPLIANT
    )
    case.compliance.save()
    case.save()
    reminder: Reminder = Reminder.objects.create(
        case=case, due_date=date.today() + timedelta(days=10)
    )

    response: HttpResponse = admin_client.get(reverse("dashboard:home"))

    assert response.status_code == 200

    assertContains(
        response, reverse("reminders:edit-reminder", kwargs={"pk": reminder.id})
    )


from  tkinter import *
class Application(Frame):
    def __init__(self, root):
        super(Application, self,).__init__(root)
        self.grid()
        self.create_widgets()
    def create_widgets(self):
        Label(self, text='У кажите любимые жанры кино').grid(row=0, column=0, sticky=W)
        Label(self, text='Выберите все что вам по вкусу: ').grid(row=1, column=0, sticky=W)
        self.favorite=StringVar()
        self.favorite.set(None)
        Radiobutton(self, text='Комедия', variable=self.favorite, command=self.update_text, value='Комедия').grid(row=2, column=0, sticky=W)

        Radiobutton(self, text='Драма', variable=self.favorite, command=self.update_text, value='Драма').grid(row=3, column=0,sticky=W)

        Radiobutton(self, text='Романтика', variable=self.favorite, command=self.update_text, value='Романтика').grid(row=4, column=0,sticky=W)
        self.result_text=Text(self, width=40, height=5, wrap=WORD)
        self.result_text.grid(row=5, column=0, columnspan=3)

    def update_text(self):
        likes = "Ваш любимый киножанр- "
        likes+=self.favorite.get()

        self.result_text.delete(0.0, END)
        self.result_text.insert(0.0, likes)
root = Tk()
root.title("Киноман 2")
app = Application(root)
root.mainloop()

Киноман 2.py

Киноман 2.py


"""
Test the MarchMachineLearningManiaResults class.
"""
from ..results import MarchMachineLearningManiaResults
from os import path
import unittest

class MarchMachineLearningManiaResultsTest(unittest.TestCase):
    """
    """

    def test_object_creation(self):
        """
        Test that we can create a results object.
        """
        game_results = MarchMachineLearningManiaResults()

    def test_load_submission_compact(self):
        """
        Test that we can load the compact sample game results file from kaggle.
        """
        game_results = MarchMachineLearningManiaResults()
        game_results.load(path.join("..", "data-files", "tourney_compact_results_thru_2015.csv"))
        self.assertEqual(len(game_results.game_scores.keys()), 31)
        self.assertEqual(len(game_results.game_scores[2015].keys()), 12)

    def test_load_submission_detailed(self):
        """
        Test that we can load the detailed sample game results file from kaggle.
        """
        game_results = MarchMachineLearningManiaResults()
        game_results.load(path.join("..", "data-files", "tourney_detailed_results_thru_2015.csv"))
        self.assertEqual(len(game_results.game_scores.keys()), 13)
        self.assertEqual(len(game_results.game_scores[2015].keys()), 12)


from api.permissions import IsDeviceUser
from dashboard.models import DASHBOARD_SOURCES
from device.models import DeviceProperty
from django.conf import settings
from django.utils import timezone
from rest_framework import viewsets
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response

from dashboard.superset import SupersetDashboard

PERMISSIONS_ADMIN = settings.PERMISSIONS_ADMIN

DEFAULT_WIDGETS = [
    "total_devices",
    "active_devices",
    "total_energy_generated_today",
    "total_energy_consumed_today",
    "total_energy_exported_today",
    "total_profit_today"
]

class WidgetViewSet(viewsets.ViewSet):
    """
        API endpoint to provide user details.
    """
    permission_classes = (IsAuthenticated, IsDeviceUser)

    def get_widgets(self, request):

        user = request.user
        user_data = {
            "results": []
        }
        # user_widgets = user.userwidget_set.filter(
        #     active=True
        # )
        user_widgets = user.userwidget_set.filter()

        total_devices = 0
        active_devices = 0
        total_energy_generated_this_day = 0
        total_energy_generation_limit_this_day = 0
        total_energy_consumed_this_day = 0
        total_energy_consumption_limit_this_day = 0
        total_energy_imported_this_day = 0
        total_energy_import_limit_this_day = 0
        total_energy_exported_this_day = 0
        total_energy_export_limit_this_day = 0

        for device in user.device_list(return_objects=True):
            latest_data = device.get_latest_data()
            properties = {
                p.name: p.value for p in DeviceProperty.objects.filter(device=device)
            }

            total_devices += 1
            if latest_data is not None and latest_data.data_arrival_time.date() == timezone.now().date():
                active_devices += 1

            total_energy_generated_this_day += float(properties.get("energy_generated_this_day", 0.0))
            total_energy_generation_limit_this_day += float(properties.get("energy_generation_limit_this_day", 0.0))

            total_energy_consumed_this_day += float(properties.get("energy_consumed_this_day", 0.0))
            total_energy_consumption_limit_this_day += float(properties.get("energy_consumption_limit_this_day", 0.0))

            total_energy_imported_this_day += float(properties.get("energy_imported_this_day", 0.0))
            total_energy_import_limit_this_day += float(properties.get("energy_import_limit_this_day", 0.0))

            total_energy_exported_this_day += float(properties.get("energy_exported_this_day", 0.0))
            total_energy_export_limit_this_day += float(properties.get("energy_export_limit_this_day", 0.0))

        summary_daya = {
            "total_devices": total_devices,
            "active_devices": active_devices,
            "total_energy_generated_this_day": total_energy_generated_this_day,
            "total_energy_generation_limit_this_day": total_energy_generation_limit_this_day,
            "total_energy_consumed_this_day": total_energy_consumed_this_day,
            "total_energy_consumption_limit_this_day": total_energy_consumption_limit_this_day,
            "total_energy_imported_this_day": total_energy_imported_this_day,
            "total_energy_import_limit_this_day": total_energy_import_limit_this_day,
            "total_energy_exported_this_day": total_energy_exported_this_day,
            "total_energy_export_limit_this_day": total_energy_export_limit_this_day,
        }

        for widget in user_widgets:
            if not widget.active:
                continue
            dash = None
            if widget.widget.metadata != None and  widget.widget.metadata.get("source") == DASHBOARD_SOURCES.superset:
                dash = SupersetDashboard(widget.widget.metadata.get("name"))

            widget_data = {
                "widget": {
                    "name": widget.widget.name,
                    "type": widget.widget.type,
                    "description": widget.widget.description,
                    "metadata": widget.widget.metadata
                },
                "display_order": widget.display_order,
                "metadata": widget.metadata,
                "updated_at": widget.updated_at.strftime(settings.TIME_FORMAT_STRING)
            }

            if dash is not None:
                widget_data["dashboard_data"] = dash.get_details()

            user_data["results"].append(widget_data)

        user_data["results"].append({
            "widget": {
                "name": "summary",
                "type": "summary",
                "description": "summary",
                "metadata": summary_daya
            },
            "display_order": 0,
            "metadata": {},
            "updated_at": timezone.now().strftime(settings.TIME_FORMAT_STRING)
        })

        return Response(user_data)


import matplotlib.pyplot as plt
import numpy as np
import logging

from scipy.signal import find_peaks
from skimage import filters

def calculate_emittance(image, scale, slit_sep, drift):
    '''
    calculate emittance using processed beam image

    Arguments
    ---------
    image : ndarray, size (N,M)
        Post-processed screen image with screen removed and noise present

    scale : float
        Pixel scaling factor in meters per pixel

    slit_sep : float
        Slit seperation in meters

    drift : float
        Longitudinal drift distance between screen and slits

    threshold : float in range [0,1), optional
        Fractional threshold to set proj to zero as a fraction of global maximum,
        default 0.0 (no change)
    '''

    logger = logging.getLogger(__name__)
    logging.getLogger('matplotlib').setLevel(logging.WARNING)

    #get projection
    orig_proj = np.sum(image, axis = 1)
    orig_proj = orig_proj / np.max(orig_proj)

    #apply threshold
    #proj = np.where(proj > threshold * np.max(proj), proj, 0.0)
    try:
        triangle_threshold = filters.threshold_triangle(orig_proj)
    except ValueError:
        fig,ax = plt.subplots(1,2)
        ax[0].plot(orig_proj)
        ax[1].imshow(image)
        print(image)
    logger.debug(f'triangle_threshold: {triangle_threshold}')
    proj = np.where(orig_proj > triangle_threshold + 0.05, orig_proj, 0)
    #proj = orig_proj

    #plot proj if in debugging
    if 0:
        fig,ax = plt.subplots(1,3)
        ax[0].imshow(image)
        ax[1].plot(orig_proj)
        ax[2].plot(proj)

        plt.show()

    #we assume that the beam is divergent, as a result the peaks should be at least
    #2 mm apart
    peaks,_ = find_peaks(proj, distance = 0.5e-3 / scale)

    if len(peaks) < 5:
        logger.warning(f'detected only {len(peaks)} peaks '
                       '-- emittance might be underestimated')

    logger.debug(f'peak finding found {len(peaks)} peaks')
    logger.debug(f'found peaks at {peaks} px')
    logger.debug(f'mean seperation {np.mean(peaks[1:] - peaks[:-1])*scale:.2e}')
    logger.debug(f'rms seperation {np.std(peaks[1:] - peaks[:-1])*scale:.2e}')

    #calculate mid points and number of blobs
    mid_pts = (peaks[1:] + peaks[:-1])/2
    mid_pts = mid_pts.astype(np.int)
    mid_pts = np.array([0, *mid_pts, len(proj)])

    x_screen_px = np.arange(len(proj))
    n_blobs = len(mid_pts) - 1

    #calculate gaussian fit stats
    a = np.empty(n_blobs) # relative intensity
    b = np.empty(n_blobs) #'y' peak position
    c = np.empty(n_blobs) #'y' sigma

    for n in range(n_blobs):
        sub_proj = proj[mid_pts[n] : mid_pts[n+1]]
        sub_x = x_screen_px[mid_pts[n] : mid_pts[n+1]]

        a[n] = np.sum(sub_proj)
        b[n] = np.average(sub_x, weights = sub_proj)
        c[n] = np.sqrt(np.average((sub_x - b[n])**2, weights=sub_proj))

    #sort peaks by central position
    sorted_idx = np.argsort(b)
    a,b,c = a[sorted_idx],b[sorted_idx],c[sorted_idx]

    logger.debug(f'mean beamlet with_px: {np.mean(c)}')
    logger.debug(a)
    logger.debug(b)
    logger.debug(c)
    #convert pixel lengths to meters
    b = b * scale
    c = c * scale

    #define slit locations
    x_slit_m = np.linspace(-(n_blobs-1)/2*slit_sep, (n_blobs-1)/2*slit_sep, n_blobs)
    logger.debug(x_slit_m)
    #calculate beam centroid at slits
    ixi = np.sum(a * x_slit_m) / np.sum(a)

    #center b coords on beam center on screen
    b = b - np.average(b, weights = a)
    logger.debug(b)

    #calculate rms spread at slits
    ixxi = np.sum(a * (x_slit_m - ixi)**2) / np.sum(a)

    #calc mean divergence of each beamlet
    xp = (b - x_slit_m) / drift

    #calc mean divergence of beam at slits
    ixpi = np.sum(a * xp) / np.sum(a)

    #calc spread in divergence for each beamlet
    sp = c / drift

    #calc rms divergence at slits
    ixpxpi = np.sum(a * sp**2) / np.sum(a)
    #ixpxpi = np.sum(a * sp**2 + a * (xp - ixpi)**2) / np.sum(a)

    #calc correlation term at slits
    ixxpi = (np.sum(a * x_slit_m * xp) - np.sum(a) * ixi *ixpi) / np.sum(a)

    logger.debug('emittance calculation results')
    logger.debug(f'ixi:{ixi}')
    logger.debug(f'ixpi:{ixpi}')
    logger.debug(f'ixxi:{ixxi}')
    logger.debug(f'ixpxpi:{ixpxpi}')
    logger.debug(f'ixxpi:{ixxpi}')

    #calculate emittance
    #emittance = np.sqrt(ixxi * ixpxpi - ixxpi**2)
    emittance = np.sqrt(ixxi * ixpxpi)
    logger.info(f'calculated emittance: {emittance:.2e}, n_peaks:{len(peaks)}')

    return emittance, ixxi, ixpxpi, ixxpi

if __name__ == '__main__':
    logging.basicConfig(level = logging.DEBUG)
    #testing
    minscreen2mm = np.genfromtxt('test_images/onenc400200.txt',names=True)
    img, _, _ = np.histogram2d(minscreen2mm['x'], minscreen2mm['y'],
                               range = np.array(((-25.0,25.0),
                                                 (-25.0,25.0)))*1e-3,
                               bins=(700,700))

    scale = 50e-3 / 700
    drift = 1.5748
    slit_sep = 0.002

    calculate_emittance(img, scale, slit_sep, drift, threshold = 0.0)


def gcd(a,b):
    while b!=0:
        r = a%b
        a, b = b, r
    return a

import sys
a,b = map(int, sys.stdin.readline().split())
print(gcd(a,b))
print(a*b//gcd(a,b))

백준/Bronze/2609. 최대공약수와 최소공배수/최대공약수와 최소공배수.py

최대공약수와 최소공배수.py


from datetime import date
from datetime import timedelta
from datetime import datetime
from typing import Dict, Any
from pyspark.sql import DataFrame
from src.customer360.utilities.spark_util import get_spark_session
from pyspark.sql import functions as F
import logging

def create_l0_campaign_history_master_active(input_campaign_master: DataFrame, output: Dict[str, Any]) -> DataFrame:
    spark = get_spark_session()
    stmt = "Drop table if exists " + output['table']
    spark.sql(stmt)
    return input_campaign_master

def create_l5_campaign_distinct_contact_response(child_response_full: DataFrame,
                                                 child_response_params: Dict[str, Any]) -> DataFrame:
    child_response = child_response_full.fillna(0, subset=child_response_params["source_cols"])
    # source_cols: ['contact_trans_m0', 'contact_trans_m1', 'contact_trans_m2',
    # 'trans_resp_m0', 'trans_resp_m1', 'trans_resp_m2',
    # 'trans_nonres_m0', 'trans_nonres_m1', 'trans_nonres_m2', 'trans_inform_m0', 'trans_inform_m1', 'trans_inform_m2',
    # 'trans_null_m0', 'trans_null_m1', 'trans_null_m2']
    # #agg_cols: ['contact_trans_last_3mth', 'trans_response_last_3mth', 'trans_no_response_last_3mth', 'trans_inform_last_3mth','trans_null_last_3mth']
    child_response_agg = child_response
    iter_var = 0
    for agg_col in child_response_params["agg_cols"]:
        child_response_agg = child_response_agg.withColumn(agg_col, F.lit(
            F.col(child_response_params["source_cols"][(iter_var * 3) + 0]) +
            F.col(child_response_params["source_cols"][(iter_var * 3) + 1]) +
            F.col(child_response_params["source_cols"][(iter_var * 3) + 2])))
        iter_var = iter_var + 1
    # Example if each iterate:
    # child_response_agg = child_response_agg.withColumn('contact_last_3mth', F.lit(F.col('contact_trans_m0')+
    #                                                                               F.col('contact_trans_m1')+
    #                                                                               F.col('contact_trans_m2))
    # print("==========================SUCCESS=========================")
    return child_response_agg

def create_l5_response_percentage_report(distinct_child_response_agg: DataFrame,
                                         campaign_master: DataFrame,
                                         report_param: Dict[str, Any]) -> DataFrame:
    # excluding the offer_type != information because the new column includes the non tracking information campaigns
    campaign_mst_exclude_information = campaign_master.where('offer_type <> "information" and month_id = "2020-02-29"')
    # campaign_mst_exclude_information = campaign_master.where('month_id = "2020-02-29"')
    campaign_agg_no_info = distinct_child_response_agg.alias('agg') \
        .join(campaign_mst_exclude_information.alias('mst'),
              F.col('agg.' + report_param['join_cols']) == F.col('mst.' + report_param['join_cols']), 'left') \
        .select("agg.*")
    response_rate_df = campaign_agg_no_info \
        .withColumn(report_param['response_rate_cols'],
                    F.lit(F.col('trans_response_last_3mth') / F.col('contact_trans_last_3mth') * 100)) \
        .fillna(0, subset=report_param['response_rate_cols'])
    return response_rate_df

def filter_campaigns_by_category(base_campaigns: DataFrame,
                                 campaign_amount_per_category: int,
                                 sub_criteria: str) -> DataFrame:
    # Different category has different amount of contacts
    # The contacts_trans_last_3mth is the total times a campaign has contacted customers in the last 3 months
    # For churn-related and cross sell/upsell, the number of contacts are very high, so we filter only the campaigns that
    # have exceptionally high contacts, currently higher than 1M contacts for low response and 100K contacts for high response

    df_churn = base_campaigns.where('campaign_category = "Churn Prevention" '
                                    'or campaign_category = "Retention"') \
        .orderBy('contact_trans_last_3mth', ascending=False) \
        .limit(campaign_amount_per_category)
    if sub_criteria == 'low_response':
        df_churn = df_churn.where('contact_trans_last_3mth > 1000000')

    df_cross_upsell = base_campaigns.where('contact_trans_last_3mth > 1000000'
                                           ' and (campaign_category = "Cross/Up sell"'
                                           'or campaign_category = "Cross Sell")') \
        .orderBy('contact_trans_last_3mth', ascending=False) \
        .limit(campaign_amount_per_category)
    if sub_criteria == 'low_response':
        df_cross_upsell = df_cross_upsell.where('contact_trans_last_3mth > 1000000')

    df_handset = base_campaigns.where('campaign_category = "Handset"') \
        .orderBy('contact_trans_last_3mth', ascending=False) \
        .limit(campaign_amount_per_category)

    df_pre_to_post = base_campaigns.where('campaign_category = "Convert Pre to Post"') \
        .orderBy('contact_trans_last_3mth', ascending=False) \
        .limit(campaign_amount_per_category)

    df_top_up = base_campaigns.where('campaign_category = "Top up"') \
        .orderBy('contact_trans_last_3mth', ascending=False) \
        .limit(campaign_amount_per_category)

    df_vas = base_campaigns.where('campaign_category = "VAS"') \
        .orderBy('contact_trans_last_3mth', ascending=False) \
        .limit(campaign_amount_per_category)

    # Union all dataframes into one combined dataframe with all categories
    total_focus_campaigns = df_churn.union(df_cross_upsell) \
        .union(df_handset) \
        .union(df_pre_to_post) \
        .union(df_top_up) \
        .union(df_vas)
    return total_focus_campaigns

def create_focus_campaign_by_volume(response_percentage_report: DataFrame,
                                                 percentage_threshold_low: float,
                                                 percentage_threshold_high: float,
                                                 campaign_amount_per_category: int,
                                                 focus_campaign_param: Dict[str, Any]) -> DataFrame:
    # Exclude information campaigns and non-tracking campaigns
    df_not_inform_trackable_low_response = response_percentage_report.select(focus_campaign_param['focus_cols']) \
        .where("response_rate_percentage <= " + str(percentage_threshold_low) +
               " and trans_inform_last_3mth = 0 and trans_null_last_3mth = 0 ") \
        .orderBy('contact_trans_last_3mth', ascending=False)
    df_not_inform_trackable_high_response = response_percentage_report.select(focus_campaign_param['focus_cols']) \
        .where("response_rate_percentage >= " + str(percentage_threshold_high) +
               " and trans_inform_last_3mth = 0 and trans_null_last_3mth = 0 ") \
        .orderBy('contact_trans_last_3mth', ascending=False)

    # Select (amount) campaigns by categories and union into one dataframe
    # Add 2 criteria columns
    # main_criteria: High contact volume
    # sub_criteria: Reason to focus on the campaigns, whether low or high response
    total_focus_campaigns_low_response = filter_campaigns_by_category(df_not_inform_trackable_low_response,
                                                                      campaign_amount_per_category,
                                                                      'low_response')
    total_focus_campaigns_low_response = total_focus_campaigns_low_response.withColumn("main_criteria",
                                                                                F.lit("High_contact")) \
        .withColumn("sub_criteria", F.lit("Low_response"))

    total_focus_campaigns_high_response = filter_campaigns_by_category(df_not_inform_trackable_high_response,
                                                                       campaign_amount_per_category,
                                                                       'high_response')
    total_focus_campaigns_high_response = total_focus_campaigns_high_response.withColumn("main_criteria",
                                                                                F.lit("High_contact")) \
        .withColumn("sub_criteria", F.lit("High_response"))
    total_focus_campaign = total_focus_campaigns_high_response.union(total_focus_campaigns_low_response)
    return total_focus_campaign


import time
import random
import numpy

import pyadvantg
import Logger
import datetime
import SpacePartition1D
import SpacePartition3D
import matplotlib
import matplotlib.pyplot as pyplot

__author__ = 'etnc6d'

class EvolAlg(object):

    def __init__(self):
        self.max_evals = 1E5
        self.timeout = 30*60  # Seconds
        self.pop_size = 15
        self.children = 5

        today = datetime.datetime.now()
        Logger.Logger.initialize("logfile_" + str(today.year) + "_" + str(today.month) + "_" + str(today.day) + "_" + str(today.hour) + "_" + str(today.minute) + ".log")

    def finish(self):
        Logger.Logger.finish()

    def begin(self):
        eval_hist = []
        terminated = False
        start_time = time.time()
        best_solution = None
        generation = 0
        generation_hist = []
        best_solution_hist = []
        pop_size_hist = []
        gen_best_solution = []

        xpop = EvolAlg.initialize(self.pop_size, 10)
        # for i in range(len(xpop)):
        #    EvolAlg.fixify_x(xpop[i])
        ypop = EvolAlg.initialize(self.pop_size, 10)
        zpop = EvolAlg.initialize(self.pop_size, 10)

        xpop[0].relBins = [0, .3, .7, 1]
        xpop[1].relBins = [0, .6, .9, 1]

        # Comment

        print(xpop[0])
        print(xpop[1])
        print(str(xpop[0].diff(xpop[1])) + "   " + str(xpop[0].diff(xpop[1], True)))

        print(xpop[0])
        print(xpop[2])
        print(str(xpop[0].diff(xpop[2])) + "   " + str(xpop[0].diff(xpop[2], True)))

        print(xpop[0])
        print(xpop[3])
        print(str(xpop[0].diff(xpop[3])) + "   " + str(xpop[0].diff(xpop[3], True)))

        teams = EvolAlg.make_teams(xpop, ypop, zpop, 5)
        solutions = []
        for team in teams:
            sol = SpacePartition3D.SpacePartition3D()
            sol.x = team[0]
            sol.y = team[1]
            sol.z = team[2]
            sol.fit = EvolAlg.evaluate_team(sol.x, sol.y, sol.z)
            solutions.append(sol)
            Logger.Logger.write_eval(team, sol.fit)

        maxfit = -1
        for sol in solutions:
            if sol.fit.fomavg > maxfit:
                maxfit = sol.fit.fomavg
                best_solution = sol

        generation_hist.append(0)
        best_solution_hist.append(best_solution)
        gen_best_solution.append(best_solution)
        pop_size_hist.append(self.pop_size)
        eval_hist.append(len(solutions))

        while not terminated:
            generation += 1

            xparents, yparents, zparents = EvolAlg.select_parents(xpop, ypop, zpop)
            xoff, yoff, zoff = EvolAlg.xover_coev(xparents, yparents, zparents, self.children)

            xoff = [EvolAlg.mutate(x) for x in xoff]

            # for i in range(len(xoff)):
            #    EvolAlg.fixify_x(xoff[i])

            yoff = [EvolAlg.mutate(y) for y in yoff]
            zoff = [EvolAlg.mutate(z) for z in zoff]

            for xx in xoff:
                if xx.get_bin_count() < 4:
                    print("Problem!")

            for xx in yoff:
                if xx.get_bin_count() < 4:
                    print("Problem!")

            for xx in zoff:
                if xx.get_bin_count() < 4:
                    print("Problem!")

            xpop.extend(xoff)
            ypop.extend(yoff)
            zpop.extend(zoff)

            # xpop, ypop, zpop = EvolAlg.tournament(xpop, ypop, zpop, self.pop_size, solutions)
            teams = EvolAlg.make_teams(xpop, ypop, zpop, 5)
            # solutions = []
            for team in teams:
                sol = SpacePartition3D.SpacePartition3D()
                sol.x = team[0]
                sol.y = team[1]
                sol.z = team[2]
                sol.fit = EvolAlg.evaluate_team(sol.x, sol.y, sol.z)
                solutions.append(sol)
                Logger.Logger.write_eval(team, sol.fit)

            xpop, ypop, zpop = EvolAlg.tournament(xpop, ypop, zpop, self.pop_size, solutions)

            for sol in solutions:
                if sol.fit.fomavg > best_solution.fit.fomavg:
                    best_solution = sol

            generation_hist.append(generation)
            best_solution_hist.append(best_solution)
            pop_size_hist.append(len(xpop))
            eval_hist.append(len(solutions))

            if len(solutions) > self.max_evals:
                terminated = True
                print('Max evaluations reached')

            if time.time() - start_time > self.timeout:
                terminated = True
                print('Timeout')

        print("Generations: " + str(generation))
        print("Evaluations: " + str(len(solutions)))
        print(best_solution)

        '''
        matplotlib.rcParams.update({'font.size': 22})
        matplotlib.rcParams.update({'figure.autolayout': True})

        pyplot.figure()
        pyplot.plot(generation_hist, [s.fit.fomavg for s in best_solution_hist])
        pyplot.title('Best Fitness')
        pyplot.xlabel('Generation')
        pyplot.ylabel('Fitness')
        # pyplot.title('Best Fitness', fontname="Times New Roman")
        # pyplot.xlabel('Generation', fontname="Times New Roman")
        # pyplot.ylabel('Fitness', fontname="Times New Roman")

        pyplot.figure()
        pyplot.loglog(generation_hist, [s.fit.fomavg for s in best_solution_hist])
        pyplot.title('Best Fitness (Log Log)')
        pyplot.xlabel('Generation')
        pyplot.ylabel('Fitness')

        pyplot.figure()
        pyplot.plot([s.fit.fomavg for s in solutions])
        pyplot.title('Fitness with Evaluations')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('Fitness')

        pyplot.figure()
        pyplot.plot([s.x.get_bin_count() for s in solutions])
        pyplot.title('X Bin Size')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('X Bins')

        pyplot.figure()
        pyplot.plot([s.y.get_bin_count() for s in solutions])
        pyplot.title('Y Bin Size')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('Y Bins')

        pyplot.figure()
        pyplot.plot([s.z.get_bin_count() for s in solutions])
        pyplot.title('Z Bin Size')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('Z Bins')

        pyplot.figure()
        pyplot.plot([s.fit.mcnp_t for s in solutions])
        pyplot.title('MCNP Runtime')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('Runtime')

        pyplot.figure()
        pyplot.plot([s.fit.adv_t for s in solutions])
        pyplot.title('ADVANTG Runtime')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('Runtime')

        pyplot.figure()
        totaltime = [s.fit.adv_t + s.fit.mcnp_t for s in solutions]
        cumsumtime = numpy.cumsum(totaltime).tolist()
        globalfit = [s.fit.fomavg / ct for s, ct in zip(solutions, cumsumtime)]
        pyplot.plot(globalfit)
        pyplot.title('Global Fitness')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('Fitness')

        pyplot.figure()
        totaltime = [s.fit.adv_t + s.fit.mcnp_t for s in solutions]
        cumsumtime = numpy.cumsum(totaltime).tolist()
        globalfit = [s.fit.fomavg / ct for s, ct in zip(best_solution_hist, cumsumtime)]
        pyplot.plot(globalfit)
        pyplot.title('Global Best Fitness')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('Fitness')

        pyplot.figure()
        pyplot.plot([s.fit.avg_unc() for s in solutions])
        pyplot.title('Average Uncertainty')
        pyplot.xlabel('Evaluation')
        pyplot.ylabel('Uncertainty')

        fig = pyplot.figure()
        fig.set_alpha(0.5)

        voxel_count = [s.x.get_bin_count()*s.y.get_bin_count()*s.z.get_bin_count() for s in solutions]
        time_per_voxel = [t/v for t, v in zip(totaltime, voxel_count)]
        # pyplot.scatter(voxel_count, time_per_voxel, c='k', alpha=.2)
        pyplot.scatter(voxel_count, totaltime, c='k', alpha=.2)
        # pyplot.ylim(0, 1.01*max(time_per_voxel))
        pyplot.ylim(0, 1.01*max(totaltime))
        pyplot.title('Time per voxel')
        pyplot.xlabel('Voxel Count')
        pyplot.ylabel('Evaluation Time / Voxel [min]')

        pyplot.figure()
        pyplot.plot(generation_hist, pop_size_hist)
        pyplot.title('Population Size')
        pyplot.xlabel('Generation')
        pyplot.ylabel('# Of Individuals per dimension')

        pyplot.show()
        '''

    @staticmethod
    def evaluate_team(xindiv, yindiv, zindiv):
        """Takes an individual from each coev pop and evaluates the
        """
        return pyadvantg.eval_co_fitness(xindiv, yindiv, zindiv)  # random.random()

    @staticmethod
    def make_teams(xpop, ypop, zpop, pair_count):
        """Pairs members of the coev pops together
        xpop - A list of x SpacePartition1D individuals
        ypop - A list of y SpacePartition1D individuals
        zpop - A list of z SpacePartition1D individuals
        pair_count - The number of pairings that will be made
        """

        tups = []
        for i in range(pair_count):
            x = random.choice(xpop)
            y = random.choice(ypop)
            z = random.choice(zpop)
            tups.append((x, y, z))

        return tups

    @staticmethod
    def select_parents(xpop, ypop, zpop):
        return xpop, ypop, zpop

    @staticmethod
    def xover1D(parent1, parent2):
        children = []

        xpt = random.random()

        child1 = SpacePartition1D.SpacePartition1D()
        child2 = SpacePartition1D.SpacePartition1D()

        # Construct child 1 from parent1 and parent2
        for gene in parent1.relBins:
            if gene < xpt:
                child1.relBins.append(gene)
        for gene in parent2.relBins:
            if gene >= xpt:
                child1.relBins.append(gene)
        while child1.get_bin_count() < 4:
            EvolAlg.mutate_add(child1)

        # Construct child 2 form parent2 and parent1
        for gene in parent2.relBins:
            if gene < xpt:
                child2.relBins.append(gene)
        for gene in parent1.relBins:
            if gene >= xpt:
                child2.relBins.append(gene)
        while child2.get_bin_count() < 4:
            EvolAlg.mutate_add(child2)

        children.append(child1)
        children.append(child2)

        return children

    @staticmethod
    def xover3D(x1, y1, z1, x2, y2, z2):
        xchildren = EvolAlg.xover1D(x1, x2)
        ychildren = EvolAlg.xover1D(y1, y2)
        zchildren = EvolAlg.xover1D(z1, z2)
        return xchildren, ychildren, zchildren

    @staticmethod
    def xover_coev(xpop, ypop, zpop, xover_count):
        xchildren = []
        ychildren = []
        zchildren = []

        for i in range(xover_count):
            p1 = random.randint(0, len(xpop)-1)
            p2 = random.randint(0, len(xpop)-1)
            xc, yc, zc = EvolAlg.xover3D(xpop[p1], ypop[p1], zpop[p1], xpop[p2], ypop[p2], zpop[p2])

            #for i in range(len(xc)):
            #    EvolAlg.fixify_x(xc[i])
            xchildren.extend(xc)
            ychildren.extend(yc)
            zchildren.extend(zc)

        return xchildren, ychildren, zchildren

    @staticmethod
    def mutate(indiv):
        if random.random() < 0.5:
            EvolAlg.mutate_add(indiv)
        else:
            EvolAlg.mutate_delete(indiv)

        # EvolAlg.fixify(indiv)

        return indiv

    # @staticmethod
    # def fixify_x(indiv):
    #    if 107.5/112.5 not in indiv.relBins:
    #        indiv.relBins.append(107.5/112.5)
    #        indiv.relBins = sorted(indiv.relBins)

    @staticmethod
    def mutate_add(indiv):
        indiv.relBins.append(random.random())
        indiv.relBins = sorted(indiv.relBins)

    @staticmethod
    def mutate_densify(indiv):
        p = 0.3

        t1 = random.random()
        t2 = random.random()
        pt1 = min(t1, t2)
        pt2 = max(t1, t2)

        tlist = []
        for i in range(0, indiv.get_bin_count()):
            if random.random() < p and (pt1 < indiv.relBins[i] < pt2 or pt1 < indiv.relBins[i+1] < pt2):
                r = random.random()
                r = (indiv.relBins[i] - indiv.relBins[i+1])*r + indiv.relBins[i]
                tlist.append(r)

        indiv.relBins.extend(tlist)
        indiv.relBins = sorted(indiv.relBins)

    @staticmethod
    def mutate_sparcify(indiv):
        p = 0.3

        t1 = random.random()
        t2 = random.random()
        pt1 = min(t1, t2)
        pt2 = max(t1, t2)

        for i in range(indiv.get_bin_count(), -1, -1):
            if random.random() < p and pt1 < indiv.relBins[i] < pt2:
                del self.relBins[i]

    @staticmethod
    def mutate_delete(indiv):
        if indiv.get_bin_count() > 4:
            remove_gene = random.randint(0, len(indiv.relBins)-1)
            del indiv.relBins[remove_gene]

    @staticmethod
    def tournament(x, y, z, survivor_count, solutions):

        newx = EvolAlg.tournament_single(x, survivor_count, solutions)
        newy = EvolAlg.tournament_single(y, survivor_count, solutions)
        newz = EvolAlg.tournament_single(z, survivor_count, solutions)

        return newx, newy, newz

    @staticmethod
    def tournament_single(x, survivor_count, solutions):

        if survivor_count >= len(x):
            return x

        xfit = []
        for indiv in x:
            ifit = 0
            icnt = 0
            for sol in solutions:
                if sol.x == indiv:
                    ifit += sol.fit.fomavg
                    icnt += 1
            if icnt > 0:
                ifit /= icnt
            xfit.append(ifit)

        mostfit = max(xfit)
        pressure = 2.0

        xfit = [q+(mostfit/pressure) for q in xfit]

        xfitsum = sum(xfit)

        if xfitsum == 0:
            return numpy.random.choice(x, survivor_count, False).tolist()

        xfit = [q/xfitsum for q in xfit]

        return numpy.random.choice(x, survivor_count, False, xfit).tolist()

    @staticmethod
    def initialize(indivCount, max_genes):
        init_pop = []

        for i in range(indivCount):
            new_indiv = SpacePartition1D.SpacePartition1D()
            EvolAlg.randomize(new_indiv, random.randint(5, max_genes))
            init_pop.append(new_indiv)

        return init_pop

    @staticmethod
    def randomize(indiv, genes):
        for i in range(genes):
            indiv.relBins.append(random.random())
            indiv.relBins = sorted(indiv.relBins)
        return indiv


import numpy as np
from sklearn.neural_network import MLPClassifier

def good_log(x):
    x_log = np.log(x, where=(x != 0))
    x_log[np.where(x == 0)] = -(10.0**8)
    return x_log

def lse(x, axis=None, keepdims=False):
    x_max = np.max(x, axis=axis, keepdims=keepdims)
    x_diff = x - x_max
    sumexp = np.exp(x_diff).sum(axis=axis, keepdims=keepdims)
    return (x_max + np.log(sumexp))

def log_gaussian(o, mu, r):
    compute = (- 0.5 * good_log(r) - np.divide(
        np.square(o - mu), 2 * r) - 0.5 * np.log(2 * np.pi)).sum()
    return compute

def forward(pi, a, o, mu, r):
    # pi is initial probability over states, a is transition matrix
    T = o.shape[0]
    J = mu.shape[0]
    log_alpha = np.zeros((T, J))

    for j in range(J):
        log_alpha[0][j] = good_log(pi)[j] + log_gaussian(o[0], mu[j], r[j])

    for t in range(1, T):
        for j in range(J):
            log_alpha[t, j] = log_gaussian(
                o[t], mu[j], r[j]) + lse(good_log(a[:, j].T) + log_alpha[t - 1])

    return log_alpha

def context_expand(data):
    T = data.shape[0]
    data_1 = np.copy(data[0])
    data_T = np.copy(data[-1])
    for i in range(3):
        data = np.insert(data, 0, data_1, axis=0)
        data = np.insert(data, -1, data_T, axis=0)
    expand_data = np.zeros((T, 7 * data.shape[1]))
    for t in range(3, T + 3):
        np.concatenate((data[t - 3], data[t - 2], data[t - 1], data[t], data[t + 1], data[t + 2], data[t + 3]),
                       out=expand_data[t - 3])
    return expand_data

class SingleGaussian():
    def __init__(self):
        # Basic class variable initialized, feel free to add more (Use from project1)
        self.dim = None
        self.mu = None
        self.r = None

    def train(self, data):
        data = np.vstack(data)
        self.mu = np.mean(data, axis=0)
        self.r = np.mean(np.square(np.subtract(data, self.mu)), axis=0)

    # Computes the log-likelihood probability for the Gaussian model fitted to the data
    def loglike(self, data_mat):
        ll = 0
        for each_line in data_mat:
            ll += log_gaussian(each_line, self.mu, self.r)
        return ll

class HMM():
    def __init__(self, sg_model, nstate):
        # Basic class variable initialized, feel free to add more
        # nstate

        self.pi = np.zeros(nstate)
        self.pi[0] = 1
        self.mu = np.tile(sg_model.mu, (nstate, 1))
        self.r = np.tile(sg_model.r, (nstate, 1))
        self.nstate = nstate

    # Initialize the HMM for each voice, and equally divide and align the HMM state.
    def initStates(self, data):
        self.states = []
        for data_s in data:
            T = data_s.shape[0]
            state_seq = np.array(
                [self.nstate * t / T for t in range(T)], dtype=int)
            self.states.append(state_seq)

    # The HMM aligns the input sequence
    def get_state_seq(self, data):
        T = data.shape[0]
        J = self.nstate
        s_hat = np.zeros(T, dtype=int)
        log_a = good_log(self.a)
        log_delta = np.zeros((T, J))
        log_delta[0] = good_log(self.pi)
        psi = np.zeros((T, J))

        # initialize
        for j in range(J):
            log_delta[0, j] += log_gaussian(data[0], self.mu[j], self.r[j])

        # forward algorithm
        for t in range(1, T):
            for j in range(J):
                temp = np.zeros(J)
                for i in range(J):
                    temp[i] = log_delta[t - 1, i] + log_a[i, j] + \
                        log_gaussian(data[t], self.mu[j], self.r[j])
                log_delta[t, j] = np.max(temp)
                psi[t, j] = np.argmax(log_delta[t - 1] + log_a[:, j])

        s_hat[T - 1] = np.argmax(log_delta[T - 1])

        # backtracking state sequence
        for t in reversed(range(T - 1)):
            s_hat[t] = psi[t + 1, s_hat[t + 1]]

        return s_hat

    def viterbi(self, data):
        for u, data_u in enumerate(data):
            s_hat = self.get_state_seq(data_u)
            self.states[u] = s_hat

    # M_step update parameters
    def m_step(self, data):
        self.a = np.zeros((self.nstate, self.nstate))
        gamma_0 = np.zeros(self.nstate)

        gamma_1 = np.zeros((self.nstate, data[0].shape[1]))
        gamma_2 = np.zeros((self.nstate, data[0].shape[1]))

        for s in range(len(data)):
            T = data[s].shape[0]

            # state_seq is a list of states with length t
            state_seq = self.states[s]
            # gamma is emission_matrix
            gamma = np.zeros((T, self.nstate))

            # Statistical state transitions in the state sequence, gamma records the state path
            for t, j in enumerate(state_seq[:-1]):
                self.a[j, state_seq[t + 1]] += 1

            for t, j in enumerate(state_seq):
                gamma[t, j] = 1

            # Update the number of states gamma_0
            gamma_0 += np.sum(gamma, axis=0)

            # gGamma_1 is the sum of feature data in each hidden state, and gamma_2 is the sum of squares of feature data in each hidden state. For subsequent update of Gaussian parameters
            for t, j in enumerate(state_seq):
                gamma_1[j] += data[s][t]
                gamma_2[j] += np.square(data[s][t])
        # Update the transition matrix and the Gaussian mean variance of each hidden state
        for j in range(self.nstate):
            self.a[j] /= np.sum(self.a[j])
            self.mu[j] = gamma_1[j] / gamma_0[j]
            self.r[j] = (gamma_2[j] - np.multiply(gamma_0[j],
                         self.mu[j] ** 2)) / gamma_0[j]

    # Perform iter iterations of the EM algorithm

    def train(self, data, iter):
        if iter == 0:
            self.initStates(data)
        self.m_step(data)
        # renew states
        self.viterbi(data)

    # The forward algorithm calculates the likelihood probability of the HMM fitting data. The digit parameter is not used in this method in order to be unified with the loglike method of the DNNHMM model
    def loglike(self, data, digit=None):
        T = data.shape[0]
        log_alpha_t = forward(self.pi, self.a, data, self.mu, self.r)[-1]
        # log_sum所有状态概率
        ll = lse(log_alpha_t)
        return ll

# MLP+HMM

class HMMMLP():
    def __init__(self, mlp, hmm_model, S, uniq_state_dict):
        """
         mlp: multilayer perceptron as dnn
         hmm_model: trained hmm model
         S: The state sequence of all data is spliced on the x-axis
         uniq_state_dict: key: (digit, state) value: state_index
        """
        self.mlp = mlp
        self.hmm = hmm_model
        self.log_prior = self.com_log_prior(S)
        self.uniq_state_dict = uniq_state_dict

    # Calculate HMM hidden state prior probability based on alignment
    def com_log_prior(self, s):
        states, counts = np.unique(s, return_counts=True)
        p = np.zeros(len(states))
        for s, c in zip(states, counts):
            p[s] = c
        p_dis = p / np.sum(p)
        return good_log(p_dis)

    def forward_dnn(self, data, digit):
        T = data.shape[0]
        J = self.hmm.nstate

        # After expanding the left and right three frames of the voice, perform -3 to 3 7-frame context feature splicing
        o_expand = context_expand(data)

        # Neural Network Forward Propagation
        mlp_ll = self.mlp.predict_log_proba(o_expand)
        log_alpha = np.zeros((T, J))
        log_alpha[0] = good_log(self.hmm.pi)

        """
         HMM forward algorithm, note that it is different from the traditional Gaussian model to get P(observation|state),
         dnn gets P(state|observation), Bayesian conversion P(o|s) = P(s|o)*p(o)|p(s), where p(o) can be ignored.
        """
        for j in range(J):
            log_alpha[0] += np.array(mlp_ll[0][self.uniq_state_dict[(digit, j)]] - self.log_prior[
                self.uniq_state_dict[(digit, j)]])
        for t in range(1, T):
            for j in range(J):
                tmp = mlp_ll[t][self.uniq_state_dict[(
                    digit, j)]] - self.log_prior[self.uniq_state_dict[(digit, j)]]
                log_alpha[t, j] = tmp + \
                    lse(good_log(self.hmm.a[:, j].T) + log_alpha[t - 1])

        return log_alpha

    def loglike(self, data, digit):
        log_alpha_t = self.forward_dnn(data, digit)[-1]
        ll = lse(log_alpha_t)

        return ll

def sg_train(digits, train_data):
    model = {}
    for digit in digits:
        model[digit] = SingleGaussian()

    for digit in digits:
        data = train_data[digit]
        print("process %d data for digit %s" % (len(data), digit))
        model[digit].train(data)

    return model

def hmm_train(digits, train_data, sg_model, nstate, niter):
    print("hidden Markov model training, %d states, %d iterations" %
          (nstate, niter))

    hmm_model = {}

    # Initialize an HMM model for each digit
    for digit in digits:
        hmm_model[digit] = HMM(sg_model[digit], nstate=nstate)

    i = 0

    # training iterations
    while i < niter:
        print("iteration: %d" % i)
        total_log_like = 0.0
        for digit in digits:
            data = train_data[digit]
            print("process %d data for digit %s" % (len(data), digit))

            hmm_model[digit].train(data, i)

            for data_u in data:
                total_log_like += hmm_model[digit].loglike(data_u)

        print("log likelihood: %f" % (total_log_like))
        i += 1

    return hmm_model

def mlp_train(digits, train_data, hmm_model, uniq_state_dict, nunits=(128, 128), lr=0.01):
    """
     MLP-HMM model training, that is, using all data to train the neural network to classify the states of all HMMs,
     Then the Gaussian part in the original Gaussian HMM is replaced by a neural network to calculate the hidden state observation probability.
     train_data: all training data
     hmm_model: trained hmm model
     uniq_state_dict: hidden state dictionary
    """
    data_dict = {}
    seq_dict = {}
    for digit in digits:
        def uniq(t): return uniq_state_dict[(digit, t)]
        vfunc = np.vectorize(uniq)  # Vector function definition

        sequences = []
        data = train_data[digit]
        data_dict[digit] = data

        # Align the speech according to the incoming hmm model, and use the vector function for each alignment sequence to perform a state map to obtain a hidden state sequence
        for data_u in data:
            seq = hmm_model[digit].get_state_seq(data_u)
            sequences.append(vfunc(seq))

        seq_dict[digit] = sequences

    O = []
    S = []

    for digit in digits:
        data = data_dict[digit]
        sequences = seq_dict[digit]
        for data_u, seq in zip(data, sequences):
            data_u_expanded = context_expand(data_u)
            O.append(data_u_expanded)
            S.append(seq)

    # O is all data, S is the alignment status of all data and spliced by the x-axis
    O = np.vstack(O)
    S = np.concatenate(S, axis=0)

    # MLP network definition
    mlp = MLPClassifier(hidden_layer_sizes=nunits, random_state=1, early_stopping=True, verbose=True,
                        validation_fraction=0.1)
    # MLP network training, the number of output nodes is the number of hidden states of all HMMs nstate*num_digits
    mlp.fit(O, S)

    # Here mlp_model is the HMM_MLP model corresponding to each number
    mlp_model = {}
    for digit in digits:
        # variables to initialize HMMMLP are incomplete below, pass additional variables that are required
        mlp_model[digit] = HMMMLP(mlp, hmm_model[digit], S, uniq_state_dict)

    return mlp_model


from math import gcd
for _ in range(int(input())):
    n = int(input())
    a = list(map(int, input().split(" ")))
    if n == 1:
        print(a[0])
        continue
    lcm = lambda x, y : (x * y) // gcd(x, y)
    ans = lcm(a[0], a[1])
    for i in range(2, n):
        ans = lcm(ans, a[i])
    print(ans)


from typing import MutableSequence

def insertion_sort(a: MutableSequence) -> None:
    """단순 삽입 정렬"""
    n = len(a)
    for i in range(1,n):
        j = i
        tmp = a[i]
        while j > 0 and a[j-1]> tmp:
            a[j] = a[j-1]
            j -= 1
        a[j] = tmp

if __name__ == '__main__':
    print('단순 삽입 정렬을 수행합니다.')
    num = int(input('원소 수를 입력하세요:'))
    x = [None] * num

    for i in range(num):
        x[i] = int(input(f'x[{i}]:'))

    insertion_sort(x)

    print('오름차순으로 정렬')
    for i in range(num):
        print(f'x[{i}] = {x[i]}')

"""이 알고리즘은 서로 떨어져 있는 원소를 교환하지 않으므로 안정적이라고 할 수 있다.
    단순 정렬 알고리즘의 시간 복잡도는 모두 O(n^2)으로 프로그램의 효율이 좋지 않다."""

알고리즘/sort/단순 삽입 정렬.py


#!usr/bin/env python
# coding:utf-8

import zhconv
import codecs
import os
import re
import jieba

all_tang_song_verse = "../corpus/corpus-master/hans_全宋词_全唐五代词.txt"
tang_300_peom = "../corpus/corpus-master/hans_唐诗三百首.txt"
song_300_verse = "../corpus/corpus-master/hans_宋词三百首.txt"
shi_jing = "../corpus/corpus-master/hans_诗经.txt"
chu_ci = "../corpus/corpus-master/hans_楚辞.txt"
lun_yu = "../corpus/corpus-master/hans_论语.txt"
# 设置分句的标志符号；可以根据实际需要进行修改
cutlist = "。！？；".decode('utf-8')

def convert_file_to_hans(target_file):
    with codecs.open(target_file, 'r', encoding='utf-8') as f:
        content = f.read()
        dir_name, base_name = os.path.split(target_file)
        hans_base_name = "hans_" + zhconv.convert(base_name.decode('utf-8'), 'zh-hans').encode('utf-8')
        hans = zhconv.convert(content, 'zh-hans')
        with codecs.open(dir_name + "/" + hans_base_name, 'w', encoding='utf-8') as res:
            res.write(hans)

def convert_hant2hans():
    input_dir = raw_input("directory to convert: ")
    for f in os.listdir(input_dir):
        if f.endswith('.txt') and not f.startswith("hans"):
            convert_file_to_hans(input_dir + "/" + f)
    print("convert finished")

# 检查某字符是否分句标志符号的函数；如果是，返回True，否则返回False
def FindToken(cutlist, char):
    if char in cutlist:
        return True
    else:
        return False

# 进行分句的核心函数
def Cut(cutlist, lines):  # 参数1：引用分句标志符；参数2：被分句的文本，为一行中文字符
    l = []  # 句子列表，用于存储单个分句成功后的整句内容，为函数的返回值
    line = []  # 临时列表，用于存储捕获到分句标志符之前的每个字符，一旦发现分句符号后，就会将其内容全部赋给l，然后就会被清空

    for i in lines:  # 对函数参数2中的每一字符逐个进行检查 （本函数中，如果将if和else对换一下位置，会更好懂）
        if FindToken(cutlist, i):  # 如果当前字符是分句符号
            line.append(i)  # 将此字符放入临时列表中
            l.append(''.join(line))  # 并把当前临时列表的内容加入到句子列表中
            line = []  # 将符号列表清空，以便下次分句使用
        else:  # 如果当前字符不是分句符号，则将该字符直接放入临时列表中
            line.append(i)
    return l

def create_corpus(resource, peom_prefix="", impurities=[]):
    sentences_list = []
    with open(resource, 'r') as text_file:
        for text_line in text_file:
            sentences = Cut(list(cutlist), list(text_line.decode('utf-8')))
            for sentence in sentences:
                if sentence.strip() != "":
                    sentence = sentence.strip()
                    if len(peom_prefix.strip()) > 0:
                        sentence = sentence.replace(peom_prefix, u"\n")
                    for s in impurities:
                        sentence = re.sub(s, "", sentence)
                    sentences_list.append(sentence)
        print("Cut sentence finished")
    dir_name, base_name = os.path.split(resource)
    corpus = dir_name + '/corpus_' + base_name
    with open(corpus, 'w') as corpus:
        for sentence in sentences_list:
            s = sentence.encode('utf-8')
            corpus.write("%s\n" % s)
        print("corpus created")

# 全宋词_全唐五代词
def create_corpus_of_all_tang_song_verse():
    create_corpus(all_tang_song_verse, u"词文：", [u'【.*】?。?'])

# 300唐诗
def create_corpus_of_300_tang_peom():
    create_corpus(tang_300_peom, u'诗文:', [u"\(.*\)"])

# 300宋词
def create_corpus_of_300_song_verse():
    create_corpus(song_300_verse, u"词文:")

# 楚辞
def create_corpus_of_chu_ci():
    create_corpus(chu_ci)

# 诗经
def create_corpus_of_shi_jing():
    create_corpus(shi_jing)

# 论语
def create_corpus_of_lun_yu():
    create_corpus(lun_yu)

if __name__ == '__main__':

    # convert_hant2hans()

    # create_corpus_of_300_tang_peom()

    # create_corpus_of_300_song_verse()

    # create_corpus_of_chu_ci()

    # create_corpus_of_shi_jing()

    create_corpus_of_lun_yu()


from django.shortcuts import render
from django.utils import timezone

class Person():
    def __init__(self, name):
        self.name = name

    def say_hello(self):
        return 'hello'

def test(request):
    people = ['Amy', 'Josh', 'Tobey', 'Josh']
    person = Person('Amy')
    person_list = []
    now = timezone.now()
    past_dt = timezone.datetime(1971,8,22,0,0)
    criteria_dt = timezone.datetime(2001,3,19,0,0)
    future_dt = timezone.datetime(2037,1,1,0,0)

    value ='''
        Miracles happen to only those who believe in them.
        Think like a man of action and act like man of thought.
        Courage is very important. Like a muscle, it is strengthened by use.
        Life is the art of drawing sufficient conclusions from insufficient premises.
        By doubting we come at the truth.
        A man that has no virtue in himself, ever envies virtue in others.
        When money speaks, the truth keeps silent.
        Better the last smile than the first laughter.
    '''

    value1 = 'Joel is a slug'
    value2 = '
Joel is a slug
'
    value3 = "https://www.example.org/foo?a=b&c=d"
    value4 = "Check out www.djangoproject.com"
    value5 = "Send questions to foo@example.com"
    value6 = 'Joel \n is a slug'

    return render(request, 'mytest/test.html', {'people':people, 'person':person,
                                                'person_list':person_list,
                                                'datetime_obj':now, 'past_dt':past_dt,
                                                'criteria_dt':criteria_dt, 'future_dt':future_dt,
                                                'value':value, 'value1':value1,
                                                'value2':value2, 'value3':value3,
                                                'value4':value4, 'value5':value5, 'value6':value6})


import os
import pickle

import numpy as np
import matplotlib.pyplot as plt

from sklearn.preprocessing import OneHotEncoder

class ANN:
    def __init__(self, layers_size, model_dir: str=None, load: bool=False):
        self.layers_size = layers_size
        self.model_dir = model_dir
        self.load = load
        self.parameters = {}
        self.L = len(self.layers_size)
        self.n = 0
        self.train_costs = []
        self.val_costs = []
        self.train_accs = []
        self.val_accs = []

    @staticmethod
    def sigmoid(Z):
        return 1 / (1 + np.exp(-Z))

    @staticmethod
    def relu(Z):
        return np.where(Z >= 0, Z, 0)

    @staticmethod
    def softmax(Z):
        expZ = np.exp(Z - np.max(Z))
        return expZ / expZ.sum(axis=0, keepdims=True)

    def initialize_parameters(self):

        if not self.load:
            np.random.seed(1)

            for l in range(1, len(self.layers_size)):
                self.parameters["W_" + str(l)] = np.random.randn(self.layers_size[l - 1], self.layers_size[l]) / np.sqrt(
                    self.layers_size[l - 1])
                self.parameters["b_" + str(l)] = np.zeros((self.layers_size[l], 1))

        else:
            self.load_model()

    def forward(self, X):
        store = {}

        A = X.T
        for l in range(self.L - 1):
            Z = self.parameters["W_" + str(l + 1)].T.dot(A) + self.parameters["b_" + str(l + 1)]
            A = self.relu(Z)
            store["A_" + str(l + 1)] = A
            store["W_" + str(l + 1)] = self.parameters["W_" + str(l + 1)]
            store["Z_" + str(l + 1)] = Z

        Z = self.parameters["W_" + str(self.L)].T.dot(A) + self.parameters["b_" + str(self.L)]
        A = self.softmax(Z)
        store["A_" + str(self.L)] = A
        store["W_" + str(self.L)] = self.parameters["W_" + str(self.L)]
        store["Z_" + str(self.L)] = Z

        return A, store

    @staticmethod
    def sigmoid_derivative(Z):
        s = 1 / (1 + np.exp(-Z))
        return s * (1 - s)

    @staticmethod
    def relu_derivative(Z):
        return np.where(Z >= 0, 1, 0)

    def backward(self, X, Y, store):

        derivatives = {}

        store["A_0"] = X.T

        A = store["A_" + str(self.L)]
        dZ = A - Y.T

        dW = store["A_" + str(self.L - 1)].dot(dZ.T) / self.n
        db = np.sum(dZ, axis=1, keepdims=True) / self.n
        dAPrev = store["W_" + str(self.L)].dot(dZ)

        derivatives["dW_" + str(self.L)] = dW
        derivatives["db_" + str(self.L)] = db

        for l in range(self.L - 1, 0, -1):
            dZ = dAPrev * self.relu_derivative(store["Z_" + str(l)])
            dW = 1. / self.n * store["A_" + str(l - 1)].dot(dZ.T)
            db = 1. / self.n * np.sum(dZ, axis=1, keepdims=True)
            if l > 1:
                dAPrev = store["W_" + str(l)].dot(dZ)

            derivatives["dW_" + str(l)] = dW
            derivatives["db_" + str(l)] = db

        return derivatives

    def fit(self, train_x, train_y, val_x, val_y, learning_rate=0.01, batch_size=64, n_iterations=2500):
        np.random.seed(1)

        #self.n = X.shape[0]
        self.n = batch_size

        self.layers_size.insert(0, train_x.shape[1])
        indices = np.arange(train_x.shape[0])
        self.initialize_parameters()
        for loop in range(n_iterations):
            chosen_index = np.random.choice(indices, size=batch_size)
            x_batch = train_x[chosen_index]
            y_batch = train_y[chosen_index]
            A, store = self.forward(x_batch)
            #cost = -np.mean(y_batch * np.log(A.T+ 1e-8))
            derivatives = self.backward(x_batch, y_batch, store)

            for l in range(1, self.L + 1):
                self.parameters["W_" + str(l)] = self.parameters["W_" + str(l)] - learning_rate * derivatives[
                    "dW_" + str(l)]
                self.parameters["b_" + str(l)] = self.parameters["b_" + str(l)] - learning_rate * derivatives[
                    "db_" + str(l)]

            if loop % 100 == 0:

                train_A, _ = self.forward(train_x)
                train_cost = -np.mean(train_y * np.log(train_A.T + 1e-8) + (1 - train_y) * np.log(1 - train_A.T + 1e-8))
                train_accuracy = self.evaluate(train_x, train_y)
                val_A, _ = self.forward(val_x)
                val_cost = -np.mean(val_y * np.log(val_A.T + 1e-8) + (1 - val_y) * np.log(1 - val_A.T + 1e-8))
                val_accuracy = self.evaluate(val_x, val_y)
                print("Step: {}, Train cost: {}, Train accuracy: {}, Val cost: {}, Val accuracy: {}".format(loop, train_cost, train_accuracy, val_cost, val_accuracy))

            #if loop % 10 == 0:
                self.train_costs.append(train_cost)
                self.val_costs.append(val_cost)
                self.train_accs.append(train_accuracy)
                self.val_accs.append(val_accuracy)

        self.save_model()

    def evaluate(self, X, Y):
        A, cache = self.forward(X)
        y_hat = np.argmax(A, axis=0)
        Y = np.argmax(Y, axis=1)
        accuracy = (y_hat == Y).mean()
        return accuracy * 100

    def predict(self, X):
        A, cache = self.forward(X)
        return A

    def save_model(self):
        save_dict = {}
        save_dict["parameters"] = self.parameters
        save_dict["layers_size"] = self.layers_size
        save_dict["train_costs"] = self.train_costs
        save_dict["val_costs"] = self.val_costs
        save_dict["train_accs"] = self.train_accs
        save_dict["val_accs"] = self.val_accs
        if not os.path.exists(self.model_dir):
            os.makedirs(self.model_dir, exist_ok=True)
        with open(os.path.join(self.model_dir, "model.pkl"), "wb") as f:
            pickle.dump(save_dict, f)
            f.close()

        print("Model saved")

    def load_model(self):
        if self.model_dir is None:
            raise ValueError("Model dir must not be None")
        with open(os.path.join(self.model_dir, "model.pkl"), "rb") as f:
            save_dict = pickle.load(f)
            self.parameters = save_dict["parameters"]
            self.layers_size = save_dict["layers_size"]
            self.L = len(self.layers_size) - 1
            f.close()

        print("Model loaded")

    def plot_cost(self):
        #plt.figure()
        plt.subplots(2, 1, sharex=True)
        plt.subplot(2, 1, 1)
        plt.plot(np.arange(len(self.train_costs)), self.train_costs, label="train cost")
        plt.plot(np.arange(len(self.val_costs)), self.val_costs, label="val cost")
        plt.xlabel("steps")
        plt.ylabel("cost")
        plt.legend()
        plt.grid()
        plt.subplot(2, 1, 2)
        plt.plot(np.arange(len(self.train_accs)), self.train_accs, label="train accuracy")
        plt.plot(np.arange(len(self.val_accs)), self.val_accs, label="val accuracy")
        plt.xlabel("steps")
        plt.ylabel("accuracy")
        plt.legend()
        plt.grid()
        plt.show()

def pre_process_data(train_x, train_y, test_x, test_y):
    # Normalize
    train_x = train_x / 255.
    test_x = test_x / 255.

    train_x = np.reshape(train_x, newshape=(train_x.shape[0], -1))
    test_x = np.reshape(test_x, newshape=(test_x.shape[0], -1))

    enc = OneHotEncoder(sparse=False, categories='auto')
    train_y = enc.fit_transform(train_y.reshape(len(train_y), -1))

    test_y = enc.transform(test_y.reshape(len(test_y), -1))

    return train_x, train_y, test_x, test_y


import pytest ; pytest

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

# External imports
from flaky import flaky
from selenium.webdriver.common.keys import Keys

# Bokeh imports
from bokeh._testing.util.selenium import RECORD
from bokeh.layouts import row
from bokeh.models import (
    Circle,
    ColumnDataSource,
    CustomAction,
    CustomJS,
    MultiChoice,
    Plot,
    Range1d,
)

#-----------------------------------------------------------------------------
# Tests
#-----------------------------------------------------------------------------

pytest_plugins = (
    "bokeh._testing.plugins.project",
)

def modify_doc(doc):
    source = ColumnDataSource(dict(x=[1, 2], y=[1, 1], val=["a", "b"]))
    plot = Plot(plot_height=400, plot_width=400, x_range=Range1d(0, 1), y_range=Range1d(0, 1), min_border=0)
    plot.add_glyph(source, Circle(x='x', y='y', size=20))
    plot.add_tools(CustomAction(callback=CustomJS(args=dict(s=source), code=RECORD("data", "s.data"))))
    input_box = MultiChoice(css_classes=["foo"])
    input_box.title = "title"
    input_box.options = ["100001", "12344556", "12344557", "3194567289", "209374209374"]
    input_box.value = ["12344556", "12344557"]
    def cb(attr, old, new):
        source.data['val'] = [old, new]
    input_box.on_change('value', cb)
    doc.add_root(row(input_box, plot))

@pytest.mark.selenium
class Test_MultiChoice(object):

    def test_displays_multi_choice(self, bokeh_model_page) -> None:
        text_input = MultiChoice(css_classes=["foo"], options = ["100001", "12344556", "12344557", "3194567289", "209374209374"])

        page = bokeh_model_page(text_input)

        el = page.driver.find_element_by_css_selector('.foo input')
        assert el.get_attribute('type') == "text"

        assert page.has_no_console_errors()

    def test_displays_title(self, bokeh_model_page) -> None:
        text_input = MultiChoice(title="title", css_classes=["foo"], options = ["100001", "12344556", "12344557", "3194567289", "209374209374"])

        page = bokeh_model_page(text_input)

        el = page.driver.find_element_by_css_selector('.foo label')
        assert el.text == "title"
        el = page.driver.find_element_by_css_selector('.foo input')
        assert el.get_attribute('placeholder') == ""
        assert el.get_attribute('type') == "text"

        assert page.has_no_console_errors()

    @flaky(max_runs=10)
    def test_displays_menu(self, bokeh_model_page) -> None:
        text_input = MultiChoice(title="title", css_classes=["foo"], options = ["100001", "12344556", "12344557", "3194567289", "209374209374"])

        page = bokeh_model_page(text_input)

        el = page.driver.find_element_by_css_selector('.foo .choices__list--dropdown')
        assert 'is-active' not in el.get_attribute('class')

        # double click to highlight and overwrite old text
        inp = page.driver.find_element_by_css_selector('.foo input')
        inp.click()
        assert 'is-active' in el.get_attribute('class')

        inp.send_keys(Keys.ENTER)

        selected = page.driver.find_element_by_css_selector('.foo .choices__list--multiple')
        items = selected.find_elements_by_css_selector("div")
        assert len(items) == 1

        item = page.driver.find_element_by_css_selector('.foo .choices__list--multiple div.choices__item')
        assert '100001' == item.get_attribute('data-value')

        delete_button = page.driver.find_element_by_css_selector('.foo .choices__item button')
        assert "Remove item: '100001'" == delete_button.get_attribute('aria-label')

        delete_button.click()

        selected = page.driver.find_element_by_css_selector('.foo .choices__list--multiple')
        items = selected.find_elements_by_css_selector("div")
        assert len(items) == 0

        assert page.has_no_console_errors()

    @flaky(max_runs=10)
    def test_server_on_change_round_trip_on_enter(self, bokeh_server_page) -> None:
        page = bokeh_server_page(modify_doc)

        inp = page.driver.find_element_by_css_selector('.foo input')
        inp.click()

        inp.send_keys(Keys.ENTER)

        button = page.driver.find_element_by_class_name("bk-toolbar-button-custom-action")
        button.click()

        results = page.results
        assert results['data']['val'] == [['12344556', '12344557'], ['12344556', '12344557', '100001']]


__all__ = ["set_LBFGS_options"]

from ..backend import backend_name

LBFGS_options = {}

def set_LBFGS_options(
    maxcor=100,
    ftol=0,
    gtol=1e-8,
    maxiter=15000,
    maxfun=None,
    maxls=50,
):
    """Sets the hyperparameters of L-BFGS.

    The L-BFGS optimizer used in each backend:

    - TensorFlow 1.x: `scipy.optimize.minimize <https://docs.scipy.org/doc/scipy/reference/optimize.minimize-lbfgsb.html#optimize-minimize-lbfgsb>`_
    - TensorFlow 2.x: `tfp.optimizer.lbfgs_minimize <https://www.tensorflow.org/probability/api_docs/python/tfp/optimizer/lbfgs_minimize>`_
    - PyTorch: `torch.optim.LBFGS <https://pytorch.org/docs/stable/generated/torch.optim.LBFGS.html>`_
    - Paddle: `paddle.incubate.optimizers.LBFGS <https://www.paddlepaddle.org.cn/documentation/docs/en/develop/api/paddle/incubate/optimizer/LBFGS_en.html>`_

    I find empirically that torch.optim.LBFGS and scipy.optimize.minimize are better than
    tfp.optimizer.lbfgs_minimize in terms of the final loss value.

    Args:
        maxcor (int): `maxcor` (scipy), `num_correction_pairs` (tfp), `history_size` (torch), `history_size` (paddle).
            The maximum number of variable metric corrections used to define the limited
            memory matrix. (The limited memory BFGS method does not store the full
            hessian but uses this many terms in an approximation to it.)
        ftol (float): `ftol` (scipy), `f_relative_tolerance` (tfp), `tolerance_change` (torch), `tolerance_change` (paddle).
            The iteration stops when `(f^k - f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= ftol`.
        gtol (float): `gtol` (scipy), `tolerance` (tfp), `tolerance_grad` (torch), `tolerance_grad` (paddle).
            The iteration will stop when `max{|proj g_i | i = 1, ..., n} <= gtol` where
            `pg_i` is the i-th component of the projected gradient.
        maxiter (int): `maxiter` (scipy), `max_iterations` (tfp), `max_iter` (torch), `max_iter` (paddle).
            Maximum number of iterations.
        maxfun (int): `maxfun` (scipy), `max_eval` (torch), `max_eval` (paddle).
            Maximum number of function evaluations. If ``None``, `maxiter` * 1.25.
        maxls (int): `maxls` (scipy), `max_line_search_iterations` (tfp).
            Maximum number of line search steps (per iteration).

    Warning:
        If L-BFGS stops earlier than expected, set the default float type to 'float64':

        .. code-block:: python

            dde.config.set_default_float("float64")
    """
    global LBFGS_options
    LBFGS_options["maxcor"] = maxcor
    LBFGS_options["ftol"] = ftol
    LBFGS_options["gtol"] = gtol
    LBFGS_options["maxiter"] = maxiter
    LBFGS_options["maxfun"] = maxfun if maxfun is not None else int(maxiter * 1.25)
    LBFGS_options["maxls"] = maxls

set_LBFGS_options()

# Backend-dependent options
if backend_name in ["pytorch", "paddle"]:
    # number of iterations per optimization call
    LBFGS_options["iter_per_step"] = min(1000, LBFGS_options["maxiter"])
    LBFGS_options["fun_per_step"] = (
        LBFGS_options["maxfun"]
        * LBFGS_options["iter_per_step"]
        // LBFGS_options["maxiter"]
    )


import sys

N = int(input())

li = []

for i in range(N):
    name, score = sys.stdin.readline().rstrip().split()
    score = int(score)
    li.append((name, score))

li.sort(key = lambda x : x[1])

for name, score in li:
    print(name)