"""
Converts MethodSchema's (data configurations) into Deform schemas.  There is also a helper function
(get_method_schema_preview) for turning the created schema into HTML for preview purposes on the methods page.
"""

from collections import namedtuple
from datetime import date
from beaker.cache import cache_region
from deform.form import Form
import random
import string
import colander
import deform
from jcudc24provisioning.models import DBSession
from jcudc24provisioning.models.ca_model import CAModel
from jcudc24provisioning.models.project import MethodSchema, field_types, Project
from jcudc24provisioning.models.file_upload import upload_widget

__author__ = 'xjc01266'

# Indexes of custom_field's, field_types.
INTEGER_INDEX = 0
DECIMAL_INDEX = 1
TEXT_INPUT_INDEX = 2
TEXT_AREA_INDEX = 3
CHECKBOX_INDEX = 4
SELECT_INDEX = 5
RADIO_INDEX = 6
FILE_INDEX = 7
WEBSITE_INDEX = 8
EMAIL_INDEX = 9
PHONE_INDEX = 10
DATE_INDEX = 11
HIDDEN_INDEX = 12

class DummySession(object):
    """
    Pretend/dummy session that allows file upload widgets to work in the HTML output from get_method_schema_preview.
    """
    def setdefault(self, arg1, arg2):
        pass

    def changed(self):
        pass

#@cache_region('long_term')
def get_method_schema_preview(method_schema_id):
    """
    Create and render the method schema identified by method_schema_id as HTML.

    :param method_schema_id: ID of the MethodSchema to preview.
    :return: Deform rendered HTML form for the identified MethodSchema (Note: The <form> element is removed).
    """
    method_schema = DBSession.query(MethodSchema).filter_by(id=method_schema_id).first()
    if method_schema is None:
        return "
Please create your data mapping using the standardised and custom fields.
"

    model_schema = DataTypeSchema(method_schema)

    # Create a dummy request object to make file upload widgets display properly for preview purposes.
    settings = {'workflows.files': "./"}
    Registry = namedtuple('registry', 'settings')
    Request = namedtuple('registry', ['registry', 'session'])
    dummy_request = Request(registry=Registry(settings=settings), session=DummySession())

    model_schema._bind({'request': dummy_request})   # Use _bind instead of bind so the schema isn't cloned
    form = Form(model_schema, action="")
    display = form.render({})
    display = display[display.index(">")+1:].replace("</form>", "").strip()
    return display

class DataTypeModel(CAModel):
    def __init__(self, schema=None, appstruct=None):
        self.id = None

        if schema is not None:
            attrs = {child.name: None for child in schema.children}
            self.__dict__.update(attrs)

            model_class = type(schema.name, (DataTypeModel,), attrs)
            test2 = model_class()

            self._model_class = model_class

        super(DataTypeModel, self).__init__(schema=schema, appstruct=appstruct)

class DataTypeSchema(colander.SchemaNode):
    """
    Base Deform schema that dynamically adds all elements of a MethodSchema (including parent schema elements).
    """
    def __init__(self, method_schema):
        params = {}
        self.__dict__['params'] = params
        super(DataTypeSchema, self).__init__(colander.Mapping('ignore'), **params)

        if isinstance(method_schema, MethodSchema):
            fields = get_schema_fields(method_schema)

            for field in fields:
                self.add(field)

#def method_schema_to_model(method_schema):
#    """
#    This is another way of generating the schema from MethodSchema models.
#    """
#    fields = get_schema_fields(method_schema)
#    model_schema = colander._SchemaMeta(str(method_schema.name), (colander._SchemaNode,), fields)
#    return model_schema

#@cache_region('long_term')
def get_schema_fields(method_schema):
    """
    Create all fields/elements of the MethodSchema, this includes:
    - Hierarchically add all elements of parent schemas.
    - Add Deform element display attributes such as description and placeholder.
    - Dynamically create the correct widget with associated settings such as select values and mask regex.

    :param method_schema: MethodSchema to generate a Deform schema from.
    :return: Deform schema (that can be rendered to HTML as a form).
    """
    fields = []

    for parent in method_schema.parents:
        fields.extend(get_schema_fields(parent))

    for field in method_schema.custom_fields:
        field_type = field.type == field_types[CHECKBOX_INDEX][0] and colander.Boolean() or \
               field.type == field_types[DATE_INDEX][0] and colander.DateTime() or \
               colander.String()

        python_type = field.type == field_types[INTEGER_INDEX][0] and int or\
                      field.type == field_types[DECIMAL_INDEX][0] and float or\
                      field.type == field_types[FILE_INDEX][0] and file or\
                      field.type == field_types[DATE_INDEX][0] and date or\
                      str

        if field.values is not None:
            value_items = field.values.split(",")
            values = ()
            for value in value_items:
                values = values + ((value.strip(", ").lower().replace(" ", "_"), value),)

        # Website regex is basic but should validate blatant mistakes such as user misinterpreting the field for email
        widget = field.type == field_types[INTEGER_INDEX][0] and deform.widget.TextInputWidget(regex_mask="^\\\\d*$", strip=False) or\
                 field.type == field_types[DECIMAL_INDEX][0] and deform.widget.TextInputWidget(regex_mask="^(((\\\\.\\\\d*)?)|(\\\\d+(\\\\.\\\\d*)?))$", strip=False) or\
                 field.type == field_types[TEXT_AREA_INDEX][0] and deform.widget.TextAreaWidget() or\
                 field.type == field_types[CHECKBOX_INDEX][0] and deform.widget.CheckboxWidget() or\
                 field.type == field_types[SELECT_INDEX][0] and deform.widget.SelectWidget(values=values) or\
                 field.type == field_types[RADIO_INDEX][0] and deform.widget.RadioChoiceWidget(values=values) or\
                 field.type == field_types[FILE_INDEX][0] and upload_widget or\
                 field.type == field_types[WEBSITE_INDEX][0] and deform.widget.TextInputWidget(
                     regex_mask="(http://)?(www\.)?([^@. ]+)(\.[^.@ ]+)(\.[^@. ]+)?(\.[^@. ]+)?(\.[^@. ]+)?", strip=False) or \
                 field.type == field_types[EMAIL_INDEX][0] and deform.widget.TextInputWidget(
                     regex_mask="[^@ ]+@[^@ ]+\.[^@ ]+", strip=False) or\
                 field.type == field_types[PHONE_INDEX][0] and deform.widget.TextInputWidget(
                     regex_mask="(\d{3}[-\.\s]??\d{3}[-\.\s]??\d{4}|\(\d{3}\)\s*\d{3}[-\.\s]??\d{4}|\d{3}[-\.\s]??\d{4})", strip=False) or\
                 field.type == field_types[DATE_INDEX][0] and deform.widget.DateInputWidget() or\
                 field.type == field_types[HIDDEN_INDEX][0] and deform.widget.HiddenWidget() or\
                 deform.widget.TextInputWidget()

        children = []
        params = {
            'name': field.internal_name,
            'title': "%s%s" % (field.name, field.units and " (%s)" % field.units or ""),
            'widget': widget,
            'description': field.description,
            'placeholder': field.placeholder,
            'default': field.default,
            'python_type': python_type,
        }

        fields.append(colander.SchemaNode(field_type, *children, **params))

    return fields


import os
import uuid
import json
import s3_handler
import processing_requests as pr

import pika
import aio_pika
import asyncio
import logging

from aiogram import Bot, types
from aiogram.dispatcher import Dispatcher
from aiogram.utils import executor
from aiogram.contrib.fsm_storage.memory import MemoryStorage
from aiogram.contrib.middlewares.logging import LoggingMiddleware

from config import TOKEN, inline_choose_filters, MESSAGES, input_path
from state import BotStates

bot = Bot(token=TOKEN)
dp = Dispatcher(bot, storage=MemoryStorage())
dp.middleware.setup(LoggingMiddleware())
s3handler = s3_handler.s3Handler()
amqp_url = os.environ["AMQP_URL"]

conn_params = pika.URLParameters(amqp_url)

async def consume():

    connection2 = None
    while connection2 == None:
        try:
            connection2 = await aio_pika.connect_robust(amqp_url)
            logging.info('Connected')
        except:
            logging.info('Waiting for connection')
            await asyncio.sleep(50)

    queue_name = "second"
    async with connection2:
        channel2 = await connection2.channel()
        await channel2.set_qos(prefetch_count=10)

        queue = await channel2.declare_queue(queue_name)

        async with queue.iterator() as it:
            async for message in it:
                async with message.process():
                    input = pr.requests_queue.json_deserializer(message.body)
                    print(input)
                    outp_path = input['outp_path']
                    s3handler.download(input['outp_path'])
                    photo = open(outp_path, 'rb')
                    await bot.send_photo(input["chat_id"], photo)
                    await dp.current_state(user=input["chat_id"]).finish()

@dp.message_handler(state='*', commands=['start'])
async def process_start_command(message: types.Message):
    await dp.current_state(user=message.from_user.id).set_state('bot_wait_a_photo')
    await message.reply(MESSAGES['start'])

@dp.message_handler(state=BotStates.BOT_WAIT_A_PHOTO, content_types=['photo'])
async def process_get_image(message: types.Message):
    await dp.current_state(user=message.from_user.id).set_state('bot_wait_a_filter')
    await message.reply(text=MESSAGES['get_image'], reply_markup=inline_choose_filters)
    file_url = message.photo[-1]
    file_info = await bot.get_file(file_url.file_id)
    file_name = str(uuid.uuid4())
    file_path = input_path + file_name + '.' + file_info.file_path.split('photos/')[1].split('.')[-1]
    await file_url.download(file_path)
    s3handler.upload(file_path)
    pr.requests_queue.new_request(message.from_user.id, file_name)
    pr.requests_queue.add_name(message.from_user.id, file_name)
    os.remove(file_path)

@dp.callback_query_handler(state=BotStates.BOT_WAIT_A_FILTER)
async def process_callback_filter(callback_query: types.CallbackQuery):
    await dp.current_state(user=callback_query.from_user.id).set_state('bot_wait_a_process')
    await bot.answer_callback_query(callback_query.id)
    pr.requests_queue.add_filter(callback_query.from_user.id, str(callback_query.data))
    await bot.send_message(callback_query.from_user.id, MESSAGES['filter'])
    logging.info(pr.requests_queue.get_request(callback_query.from_user.id))
    req = pr.requests_queue.json_serializer(pr.requests_queue.get_request(callback_query.from_user.id))
    connection = pika.BlockingConnection(conn_params)
    channel = connection.channel()
    channel.queue_declare(queue='first')
    channel.basic_publish(exchange='', routing_key='first', body=req)
    connection.close()

@dp.callback_query_handler(state=BotStates.BOT_WAIT_A_PROCESS)
async def process_callback_wait(callback_query: types.CallbackQuery):
    await bot.answer_callback_query(callback_query.id)
    await bot.send_message(callback_query.from_user.id, MESSAGES['filter_wait'])

if __name__ == '__main__':
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    loop.create_task(consume())
    executor.start_polling(dp)


from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render
from managment.models import CoreSettings

from .forms import ParticipantsForm, TeamsForm

def index(request):
    settings = CoreSettings.objects.all()
    return render(request, "base.html", {"settings": settings})

def register(request):
    if request.method == "POST":
        form = ParticipantsForm(request.POST)
        if form.is_valid():
            instance = form.save()
            instance.save()
            return HttpResponse(
                "
Ваша заявка успешно сохранена!
\r\n
Вернуться на главную страницу
"
            )
    else:
        return render(
            request,
            "registration.html",
            {
                "form": ParticipantsForm,
                "title": "Вступить в существующую команду",
            },
        )

def register_team(request):
    if request.method == "POST":
        form = TeamsForm(request.POST)

        if form.is_valid():
            instance = form.save()
            instance.save()
            return HttpResponseRedirect("/registration")
    else:
        return render(
            request,
            "registration.html",
            {
                "form": TeamsForm,
                "title": "Регистрация команды",
            },
        )


# All rights reserved by forest fairy.
# You cannot modify or share anything without sacrifice.
# If you don't agree, keep calm and don't look at code bellow!

__author__ = "VirtualV <https://github.com/virtualvfix>"
__date__ = "$Apr 13, 2014 8:47:25 PM$"

import re
from config import CONFIG
from tests.exceptions import ResultsNotFoundError
from tests.benchmarks.tools.base import OnlineBenchmark

class Vellamo3(OnlineBenchmark):
    def __init__(self, attributes, serial):
        OnlineBenchmark.__init__(self, attributes, serial)
        self._pull = attributes['pull']
        self.failed_fields = attributes['failed_fields']

    def start(self):
        try:
            self.sh('rm -r ' + self._pull + '*.html', errors=False, empty=True)
        except:
            pass
        super(Vellamo3, self).start()

    def __pull_logs(self):
        raw_res = []
        lslist = [x.replace('\r', '').strip() for x in self.sh('ls {} | grep .html'.format(self._pull)).split('\n')
                  if x.strip() != '\n' and x.strip() != '']
        if len(lslist) == 0: raise ResultsNotFoundError('Vellamo HTML results not found.')
        postfix = '_{}_{}.html'.format(CONFIG.CURRENT_CYCLE, CONFIG.LOCAL_CURRENT_CYCLE)
        for x in lslist:
            self.logger.info('Pulling {} log...'.format(x))
            self.pull(self._pull + x, CONFIG.LOG_PATH + x[:-5]+postfix)
            with open(CONFIG.LOG_PATH + x[:-5]+postfix, 'rb') as file:
                raw_res.append(file.read())
        return zip(lslist, raw_res)

    def collect_results(self, res_doc):
        raw_res_list = self.__pull_logs()
        for log, raw_res in raw_res_list:
            match = re.search('h2\sstyle.*?>([\w\s/.-]+)<.*?Score:.*?>([\d]+)', raw_res, re.DOTALL|re.I)
            res_doc.add_name(str(match.group(1).split(' ')[2].strip())+' ['+log+']')
            res_doc.add_result(match.group(2))
            rows = re.findall('
(.*?)
', raw_res, re.DOTALL|re.I)
            for row in rows:
                match = re.search('([\w\s/\(\).-]+)<.*?>([\w.\s]+)', row, re.DOTALL|re.I)
                res_doc.add_name(match.group(1))
                # failed fields
                if match.group(1) in self.failed_fields and match.group(2).strip() in ['0', '']:
                    res_doc.add_result('Failed')
                    for x in self.failed_fields[match.group(1)]:
                        res_doc.add_name(x)
                        res_doc.add_result('')
                else:
                    res_doc.add_result(match.group(2))
                    values = re.findall('<li\sstyle.*?([\w\s/.-]+):\s([\w.\s]+)
', row, re.DOTALL|re.I)
                    for value in values:
                        # skip Invalid CPU mode error in result
                        if 'Invalid CPU' in value[1]:
                            continue
                        # skip failed field
                        if value[0].lower() == 'failed':
                            continue
                        res_doc.add_name(value[0])
                        res_doc.add_result(value[1])
        self.sh('rm -r ' + self._pull + '*.html', errors=False)


import allure
from allure_commons.types import AttachmentType
from locators.manager_locators import ManagerLocators
import pytest
from enums.errors_enums import ErrorMassages
from allure import severity_level

@allure.story('Scope of tests "manager page"')
class TestManagerPageElements:

    buttons_locator_list = [
        (ManagerLocators.HOME_BUTTON, "HOME_BUTTON"),
        (ManagerLocators.ADD_CONSUMER_BUTTON, "ADD_CONSUMER_BUTTON"),
        (ManagerLocators.OPEN_ACCOUNT_BUTTON, "OPEN_ACCOUNT_BUTTON"),
        (ManagerLocators.CONSUMERS_BUTTON, "CONSUMERS_BUTTON"),
    ]

    @allure.severity(severity_level.NORMAL)
    @allure.description('Checking that all elements are visible')
    @pytest.mark.parametrize("button_locator, test_name", buttons_locator_list)
    def test_elements_are_visible(self, driver, manager_page, button_locator, test_name):

        assert manager_page.element_is_visible(button_locator), ErrorMassages.ELEMENT_IS_NOT_VISIBLE.value.format(button_locator)
        allure.attach(driver.get_screenshot_as_png(), name="screenshot", attachment_type=AttachmentType.PNG)

    @allure.severity(severity_level.NORMAL)
    @allure.description('Checking that all elements are clickable')
    @pytest.mark.parametrize("button_locator, test_name", buttons_locator_list)
    def test_elements_are_clickable(self, driver, manager_page, button_locator, test_name):

        assert manager_page.element_is_clickable(button_locator), ErrorMassages.ELEMENT_IS_NOT_CLICKABLE.value.format(button_locator)


"""

strategy to solve the problem:
    #daily temperature problem: find the number of day from the ith day will have warmer temperature

    #using stack strategy:
        res (list): to hold result list
        stack (list): to hold the index, temperature of daly
        #for each iteration we will know the index (date) and it's temperature
            #if
"""

#############################################reference solution
from typing import List
class Solution:
    def dailyTemperatures(self, temperatures: List[int]) -> List[int]:
        res = [0] * len(temperatures)
        stack = []  # pair: [temp, index]

        for i, t in enumerate(temperatures):
            while stack and t > stack[-1][0]: #find result of all date in stack
                stackT, stackInd = stack.pop()
                res[stackInd] = i - stackInd
            stack.append((t, i)) #build stack
        return res


import sys
sys.path.append('./')
import time
import torch
import json
import logging

from tqdm import tqdm
from common.const import *
from common.utils import *
from torch.utils.data.distributed import DistributedSampler
from torch import nn
from torch.optim import lr_scheduler
from module.common.common_executor import *
from collections import Counter

class JointModelExecutor(CommonExecutor):
    def __init__(self, model, train_dataset, dev_dataset, test_dataset, conf):
        super(JointModelExecutor, self).__init__(model, train_dataset, dev_dataset, test_dataset, conf)
        self.max_seq_len = conf.get('max_seq_len', 256)
        self.pretrained_model_name = conf.get('pretrained_model_name', 'bert-base-chinese')
        # self.tokenizer = BertTokenizer.from_pretrained(self.pretrained_model_name)
        self.max_ent_len = conf.get("max_ent_len", 10)
        self.max_evt_len = conf.get("max_evt_len", 3)
        self.role_list = conf.get("role_list", [])
        self.possible_role_list = conf.get("possible_role_list", [])
        self.eval_output_path = conf.get('eval_output_path', 'output/joint_eval.json')
        self.dev_path = conf.get("dev_path", 'data/tmp_dev.json')
        self.entity_list = conf.get("entity_list", [])
        self.event_list = conf.get("event_list", [])
        self.use_crf = conf.get('use_crf', 0)
        self.use_lstm = conf.get('use_lstm', 0)
        self.retrain = conf.get("retrain", 0)

    def get_loss(self, logits, labels):
        evt_logit, ent_logit, role_logit = logits
        evt_label, ent_label, role_label = labels

        evt_label = evt_label.view(-1)
        ent_label = ent_label.view(-1)
        evt_logit = evt_logit.view(evt_label.size()[0], -1)
        ent_logit = ent_logit.view(ent_label.size()[0], -1)

        loss = 0.33 * self.loss_function(evt_logit, evt_label)
        loss += 0.33 * self.loss_function(ent_logit, ent_label)

        role_label = role_label.view(-1)
        role_logit = role_logit.view(role_label.size()[0], -1)

        loss += 0.34 * self.loss_function(role_logit, role_label)
        return loss

    def optimize(self):
        # nn.utils.clip_grad_norm_(model.parameters(), max_norm=1, norm_type=2)
        self.optimizer.step()
        self.optimizer.zero_grad()

    def get_metric(self, y_pred, y_true, average='micro'):
        evt_pred, ent_pred, role_pred = y_pred
        evt_true, ent_true, role_true = y_true
        evt_p, evt_r, evt_f1 = calc_metrics(evt_true, evt_pred, [i for i in range(1, 2 * len(self.event_list) + 1)], average=average)
        ent_p, ent_r, ent_f1 = calc_metrics(ent_true, ent_pred, [1, 2], average=average)
        role_p, role_r, role_f1 = calc_metrics(role_true, role_pred, [i for i in range(1, len(self.role_list))], average=average)

        precision = (evt_p, ent_p, role_p)
        recall = (evt_r, ent_r, role_r)
        f1 = (evt_f1, ent_f1, role_f1)

        metric = (evt_f1 + ent_f1 + role_f1) / 3
        return precision, recall, f1, metric

    def get_result(self, logits, labels, input_data=None, ignore_index=-100):
        final_preds = tuple()
        final_labels = tuple()
        # evt_logit, ent_logit, role_logit = logits
        # evt_label, ent_label, role_label = labels
        for pred, label in zip(logits, labels):
            label = label.view(-1)
            pred = pred.view(label.size()[0], -1)
            proba = torch.log_softmax(pred, dim=1)
            pred_cls = torch.argmax(proba, dim=1)
            pred_cls = pred_cls.cpu().view(-1).numpy()
            label = label.cpu().view(-1).numpy()

            tmp_preds, tmp_labels = [], []
            for idx, (p, l) in enumerate(zip(pred_cls, label)):
                if l == ignore_index:
                    continue
                tmp_preds.append(p)
                tmp_labels.append(l)
            final_preds += (tmp_preds, )
            final_labels += (tmp_labels, )

        return final_preds, final_labels

    def train(self, checkpoint=None):
        logging.info('  开始训练')
        train_data_loader = torch.utils.data.DataLoader(dataset=self.train_dataset, batch_size=self.batch_size,
                                                        shuffle=True, num_workers=0)
#         dev_data_loader = torch.utils.data.DataLoader(dataset=self.dev_dataset, batch_size=self.batch_size,
#                                                       shuffle=False, num_workers=0)
        # init
        if self.use_lstm:
            logging.info('  锁定bert参数')
            bert_params = self.model.module.bert.parameters() if self.use_gpu else self.model.bert.parameters()
            for i in bert_params:
                i.requires_grad = False
        if self.retrain:
            checkpoint = self.best_model_save_path
            self.load_model(checkpoint)
            logging.info('  释放bert参数')
            bert_params = self.model.module.bert.parameters() if self.use_gpu else self.model.bert.parameters()
            for i in bert_params:
                i.requires_grad = True
        self.setup()

        # # load checkpoint when needed
        # if checkpoint != '' and checkpoint is not None:
        #     self.load_checkpoint(checkpoint)

        # train
        while self.epoch < self.epochs:
            logging.info('  epoch: ' + str(self.epoch))

            self.model.train()
            # output lr info
            for lrs in self.optimizer.state_dict()['param_groups']:
                logging.info('  学习率: ' + str(lrs['lr']))
            # training loop
            bar = tqdm(list(enumerate(train_data_loader)))
            for step, input_data in bar:
                inputs = input_data[:-3]
                labels = input_data[-3:]
                # print(input_data)
                if self.use_gpu:
                    inputs = tuple(x.cuda() for x in inputs)
                    labels = tuple(x.cuda() for x in labels)

                logits = self.model(inputs)
                loss = self.get_loss(logits, labels)
                loss.backward()

                # params optimization
                if ((step + 1) % self.accumulate_step) == self.accumulate_step - 1 or step == len(
                        train_data_loader) - 1:
                    self.optimize()

                # calc metric info
                y_pred, y_true = self.get_result(logits, labels)
                precision, recall, f1, metric = self.get_metric(y_pred, y_true)
                bar.set_description(f"step:{step}: pre:{round(precision[0], 3)}, {round(precision[1], 3)}, {round(precision[2], 3)} loss:{loss}")

            # adjust learning rate
            self.scheduler.step()
            # evaluate on dev dataset
#             metric, _ = self.eval(self.model, dev_data_loader)
            metric = self.eval(self.model)
            # model save when achieve best metric
            if metric > self.best_metric:
                self.save_checkpoint(self.model, metric)
                self.best_metric = metric
                self.early_stop = 0
            else:
                self.early_stop += 1
            # early stop
            if self.early_stop > self.early_stop_threshold:
                break
            self.epoch += 1

        logging.info('  训练结束!\n')
        return True

    def calc_standard_metrics(self, pred_path, ground_truth_path):
        pred_items = [json.loads(line) for line in open(pred_path)]
        ground_truth_items = json.load(open(ground_truth_path))
        pred_num = 0
        true_num = 0
        gt_num = 0
        for p, gt in zip(pred_items, ground_truth_items):
            assert p['text'] == gt['text']
            pe = set()
            pa = set()
            for e in p.get("event_list", []):
                e_key = tuple(e['trigger'])
                pe.add(e_key)
                for a in e['argument']:
                    pa.add(tuple(a) + tuple(e_key))
            ge = set()
            ga = set()
            for e in gt.get("event_list", []):
                e_key = (e['event_type'], int(e['beg']), e['event_content'])
                ge.add(e_key)
                for a in e['argument_list']:
                    ga.add((a['argument_role'], int(a['beg']), a['argument_content']) + e_key)
            pred_num += len(pe) + len(pa)
            true_num += len(pe & ge) + len(pa & ga)
            gt_num += len(ge) + len(ga)
        p = float(true_num / pred_num) if pred_num != 0 else 0
        r = float(true_num / gt_num) if gt_num != 0 else 0
        f1 = 2 * p * r / (p + r) if p + r != 0 else 0
        logging.info(f"dev:{self.epoch}: {round(f1, 3)}")
        return f1

    def eval(self, model, mode='dev'):
        self.test(input_path=self.dev_path, output_path=self.eval_output_path, dataset=self.dev_dataset, model=model)
        metrics = self.calc_standard_metrics(self.eval_output_path, self.dev_path)
        return metrics

    def test(self, input_path=None, output_path=None, dataset=None, checkpoint=None, model=None):
        if not output_path:
            output_path = self.test_output_path
        logging.info('  测试开始')
        if not dataset:
            dataset = self.test_dataset
        if not model:
            # load checkpoint
            if not checkpoint:
                checkpoint = self.best_model_save_path
            self.load_model(checkpoint)
        else:
            self.model = model
        # load test dataset
        data_loader = torch.utils.data.DataLoader(dataset=dataset, batch_size=self.batch_size,
                                                       shuffle=False, num_workers=0)
        test_result = []
        if not input_path:
            test_contents = [json.loads(line) for line in open(self.test_path)]
        else:
            test_contents = json.load(open(input_path))

        offset = 0
        # test
        with torch.no_grad():
            self.model.eval()
            bar = tqdm(list(enumerate(data_loader)))
            for step, input_data in bar:
                # inputs = tuple(x.to(self.device) for x in input_data[:-1])
                inputs = input_data[:2]
                if self.use_gpu:
                    inputs = tuple(x.cuda() for x in inputs)
                    result = self.model.module.predict(inputs)
                else:
                    result = self.model.predict(inputs, use_gpu=False)

                for i in range(len(result)):
                    test_info = test_contents[offset+i]
                    pred_info = self.handle_result(result[i], test_info['text'])

                    pred_info['doc_id'] = test_info.get('doc_id', 0)
                    test_result.append(pred_info)

                offset += len(result)
        logging.info("存储test结果：" + output_path)

        import jsonlines
        with jsonlines.open(output_path, mode='w') as writer:
            for i in test_result:
#                 if 'text' in i:
#                     del i['text']
                writer.write(i)
#         json.dump(test_result, open(output_path, 'w'), indent=2, ensure_ascii=False)
        return True

    def handle_result(self, result, text):
        event_list = []
        if not result:
            return {"text": text, "event_list": []}

        args, ents, evts = result

        evt_arg_map = {}
        # for beg, end, t in evts:
        #     ent[beg + '@#@' + end] = t

        for k in args:
            for j in k:
                evt_ofs, ent_ofs, role_info = j[0], j[1], j[2]
                if role_info == 0 or evt_ofs[-1] == 0:
                    continue
                role = self.role_list[role_info]
                evt_type = self.event_list[evt_ofs[-1]]
                trigger = text[evt_ofs[0]: evt_ofs[1]]

                if not trigger:
                    continue
                argument = text[ent_ofs[0]: ent_ofs[1]]

                if not argument:
                    continue
                evt_type_key = (evt_type, evt_ofs[0], trigger)
                if evt_type_key not in evt_arg_map:
                    evt_arg_map[evt_type_key] = set()
                # check if evt + role possible
                if evt_type + '@#@' + role not in self.possible_role_list:
                    continue
                evt_arg_map[evt_type_key].add((role, ent_ofs[0], argument))

        for et, arg_set in evt_arg_map.items():
            arguments = []
            for arg_info in arg_set:
                arguments.append(list(arg_info))
            event = {
                'trigger': list(et),
                'argument': arguments
            }
            event_list.append(event)

        return {"text": text, "event_list": event_list}


import chainer
from chainer import functions
from chainer import links

from chainer_pointnet.models.conv_block import ConvBlock
from chainer_pointnet.models.kdcontextnet.kdcontextconv_block import \
    KDContextConvBlock

class KDContextDeconvBlock(chainer.Chain):

    """KD Context Deconvolution Block

    It performs each level feature learning & aggregation

    Args:
        in_channels (int or None): channel size for input `x`
        m (int): Number to group to be aggregated.
        out_deconv_channels (int or None): output channels size for deconv part
        in_channels_skip (int or None): channel size for input `x_skip`
        feature_learning_mlp (list): list of int, specifies MLP size for
            feature learning stage.
        feature_aggregation_mlp (list): list of int, specifies MLP size for
            feature aggregation stage.
        ksize (int or tuple): kernel size
        stride (int or tuple): stride size
        pad (int or tuple): padding size
        nobias (bool): use bias `b` or not.
        initialW: initiallizer of `W`
        initial_bias: initializer of `b`
        use_bn (bool): use batch normalization or not
        activation (callable): activation function
        dropout_ratio (float): dropout ratio, set negative value to skip
            dropout
        normalize (bool): apply normalization to calculate global context cues
        residual (bool): use residual connection or not
    """

    def __init__(self, in_channels, m, in_channels_skip=None,
                 out_deconv_channels=None,
                 feature_learning_mlp=None,
                 feature_aggregation_mlp=None,
                 ksize=1, stride=1, pad=0,
                 nobias=False, initialW=None, initial_bias=None, use_bn=True,
                 activation=functions.relu, dropout_ratio=-1, normalize=False,
                 residual=False):
        super(KDContextDeconvBlock, self).__init__()
        if in_channels_skip is None:
            in_channels_context = None
        else:
            in_channels_context = in_channels_skip + out_deconv_channels
        if out_deconv_channels is None:
            raise ValueError('currently out_deconv_channels must be set '
                             'specifically')
        with self.init_scope():
            # deconvolution part
            self.conv = links.Convolution2D(
                in_channels, out_deconv_channels * m, ksize=ksize,
                stride=stride, pad=pad, nobias=nobias, initialW=initialW,
                initial_bias=initial_bias)
            # set `aggregation=False` to keep num_point of output.
            self.kdcontextconv = KDContextConvBlock(
                in_channels_context, m,
                feature_learning_mlp=feature_learning_mlp,
                feature_aggregation_mlp=feature_aggregation_mlp,
                ksize=ksize, stride=stride, pad=pad,
                nobias=nobias, initialW=initialW, initial_bias=initial_bias,
                use_bn=use_bn, activation=activation,
                dropout_ratio=dropout_ratio, aggregation=False,
                normalize=normalize, residual=residual)
        self.m = m
        self.out_deconv_channels = out_deconv_channels

    def __call__(self, x, x_skip):
        """Main forward computation

        Args:
            x (array or Variable): Input from parent node.
                4-dim array (batchsize, in_channels, num_point, 1)
            x_skip (array or Variable): Input from skip connection.
                4-dim array (batchsize, in_channels_skip, num_point*m, 1)

        Returns (array or Variable):
            4-dim array (batchsize, out_channels, num_point*m, 1)
            Number of output points are increased to `num_point*m` after
            deconvolution.
        """
        assert x.ndim == 4  # (bs, ch, N, 1)
        assert x_skip.ndim == 4  # (bs, ch, N, 1)
        bs, ch, num_point, w = x.shape
        bs, ch_skip, num_pointm, w_skip = x_skip.shape
        assert w == 1
        assert w_skip == 1
        assert num_point * self.m == num_pointm
        # --- deconvolution from parent node to children part ---
        # (bs, ch, n, 1) -> (bs, out_ch*m, n, 1) -> (bs, out_ch, n*m, 1)
        h = self.conv(x)
        h = functions.reshape(h, (bs, self.out_deconv_channels,
                                  self.m * num_point, 1))
        # --- skip connection part ---
        # Do nothing, just concat.
        # Other way is to put one convolution operation,
        # and we can reduce channels for `h_skip` here.
        # h_skip = self.conv_skip(x_skip)
        h_skip = x_skip  # (bs, ch_skip, n*m, 1)

        h = functions.concat([h, h_skip], axis=1)
        h = self.kdcontextconv(h)
        return h

if __name__ == '__main__':
    import numpy
    from chainer_pointnet.utils.kdtree import construct_kdtree_data

    batchsize = 1
    num_point = 135  # try 100, 128, 135
    max_level = 7  # 2^7 -> 128. Final num_point will be 128
    dim = 3
    point_set = numpy.random.rand(num_point, dim).astype(numpy.float32)
    point_set2 = numpy.random.rand(num_point, dim).astype(numpy.float32)
    print('point_set', point_set.shape)
    points, split_dims, inds, kdtree, split_positions = construct_kdtree_data(
        point_set, max_level=7, calc_split_positions=True)
    points2, split_dims2, inds2, kdtree2, split_positions2 = construct_kdtree_data(
        point_set2, max_level=7, calc_split_positions=True)
    print('points', points.shape)  # 128 point here!
    kdconvblock = KDContextConvBlock(
        3, m=2**3, feature_learning_mlp=[16], feature_aggregation_mlp=[24])
    kddeconvblock = KDContextDeconvBlock(
        24, m=2**3, out_deconv_channels=12,
        feature_learning_mlp=[28], feature_aggregation_mlp=[30])
    # split_dim = numpy.array([split_dims[-1], split_dims2[-1]])
    # print('split_dim', split_dim.shape)
    pts = numpy.array([points, points2])
    pts = numpy.transpose(pts, (0, 2, 1))[:, :, :, None]
    print('pts', pts.shape)  # (2, 3, 128, 1)
    pts2 = kdconvblock(pts)
    print('pts2', pts2.shape)  # (2, 24, 16, 1)
    out = kddeconvblock(pts2, pts)
    print('out', out.shape)  # (2, 30, 128, 1)


__author__ = 'yutao'

from pymongo import MongoClient

col = MongoClient(host="166.111.7.105")["douban"]["photo"]
f_in = open("ids.txt")
ids = []
cnt = 0
for line in f_in:
    print(cnt)
    cnt += 1
    x = line.strip()
    if not col.find_one({"movie_id": line.strip()}):
        ids.append(x)

f_out = open("ids.txt", "w")
for i in ids:
    f_out.write(i + "\n")

f_out.close()


import matplotlib
import numpy as np
import pygame

from abm.projects.visual_flocking.vf_agent import vf_supcalc
from abm.agent.agent import Agent
from abm.projects.visual_flocking.vf_contrib import vf_params
from abm.contrib import colors
import importlib

class VFAgent(Agent):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        importlib.reload(vf_supcalc)
        importlib.reload(vf_params)

        # creating agent status
        # flocking or emergency
        self.agent_state = "flocking"

        # introducing agent-specific movement params
        # if they are None, they will be read from vf_contrib.vf_params
        self.ALP0 = None
        self.BET0 = None
        self.V0 = None

        # lines to follow
        self.lines = []
        self.sensor_distance = 20  # sensor distance to follow lines
        self.sensor_size = 9
        self.line_map = np.zeros((self.WIDTH + self.window_pad, self.HEIGHT + self.window_pad))

        # getting additional information from supplementary calculation functions
        self.verbose_supcalc = False
        # boundary conditions
        # infinite or walls
        self.boundary_cond = vf_params.BOUNDARY
        self.limit_movement = vf_params.LIMIT_MOVEMENT
        self.max_vel = vf_params.MAX_VEL
        self.max_th = vf_params.MAX_TH

        # preparing phi values for algorithm according to FOV
        self.PHI = np.arange(-np.pi, np.pi, (2*np.pi)/self.v_field_res)

        # social information: visual field projections
        self.soc_v_field_proj = np.zeros(self.v_field_res)

        # saving collision times
        self.coll_times = []

    def update(self, agents):
        """
        main update method of the agent. This method is called in every timestep
        to calculate the new state/position
        of the agent and visualize it in the environment
        :param agents: a list of all obstacle/agents coordinates as (X, Y) in
        the environment. These are not necessarily socially relevant, i.e. all
        agents.
        """
        # update agent information
        self.update_social_info(agents)

        # update agent's state
        self.update_state()

        # perform agent's action i.e. exploration, taxis, relocation or flocking
        self.perform_action()

        # boundary conditions if applicable
        if self.boundary_cond == "walls":
            self.reflect_from_walls()
        elif self.boundary_cond == "infinite":
            self.teleport_infinite_arena()

        # updating agent visualization
        self.draw_update()

    def reflect_from_walls(self):
        """reflecting agent from environment boundaries according to a desired x, y coordinate. If this is over any
        boundaries of the environment, the agents position and orientation will be changed such that the agent is
         reflected from these boundaries."""
        if len(self.lines) == 0:
            super().reflect_from_walls()
        else:
            # Boundary conditions according to center of agent (simple)
            x = self.position[0] + self.radius
            y = self.position[1] + self.radius

            # Reflection from left wall
            if x < self.boundaries_x[0]:
                self.position[0] = self.boundaries_x[0] - self.radius

                if np.pi / 2 <= self.orientation < np.pi:
                    self.orientation -= np.pi
                elif np.pi <= self.orientation <= 3 * np.pi / 2:
                    self.orientation += np.pi
                self.prove_orientation()  # bounding orientation into 0 and 2pi

            # Reflection from right wall
            if x > self.boundaries_x[1]:

                self.position[0] = self.boundaries_x[1] - self.radius - 1

                if 3 * np.pi / 2 <= self.orientation < 2 * np.pi:
                    self.orientation -= np.pi
                elif 0 <= self.orientation <= np.pi / 2:
                    self.orientation += np.pi
                self.prove_orientation()  # bounding orientation into 0 and 2pi

            # Reflection from upper wall
            if y < self.boundaries_y[0]:
                self.position[1] = self.boundaries_y[0] - self.radius

                if np.pi / 2 <= self.orientation <= np.pi:
                    self.orientation += np.pi
                elif 0 <= self.orientation < np.pi / 2:
                    self.orientation -= np.pi
                self.prove_orientation()  # bounding orientation into 0 and 2pi

            # Reflection from lower wall
            if y > self.boundaries_y[1]:
                self.position[1] = self.boundaries_y[1] - self.radius - 1
                if 3 * np.pi / 2 <= self.orientation <= 2 * np.pi:
                    self.orientation += np.pi
                elif np.pi <= self.orientation < 3 * np.pi / 2:
                    self.orientation -= np.pi
                self.prove_orientation()  # bounding orientation into 0 and 2pi

    def draw_update(self):
        """
        updating the outlook of the agent according to position and orientation
        """
        # update position
        self.rect.x = self.position[0]
        self.rect.y = self.position[1]

        # change agent color according to mode
        self.change_color()

        # update surface according to new orientation
        # creating visualization surface for agent as a filled circle
        self.image = pygame.Surface([self.radius * 2, self.radius * 2])
        self.image.fill(colors.BACKGROUND)
        self.image.set_colorkey(colors.BACKGROUND)
        if self.is_moved_with_cursor:
            pygame.gfxdraw.filled_circle(
                self.image,
                self.radius,
                self.radius,
                self.radius,
                self.selected_color
            )
            pygame.gfxdraw.aacircle(self.image,
                                    self.radius,
                                    self.radius,
                                    self.radius,
                                    colors.BACKGROUND)
        else:
            pygame.gfxdraw.filled_circle(
                self.image,
                self.radius,
                self.radius,
                self.radius-1,
                self.color[0:-1]
            )
            pygame.gfxdraw.aacircle(self.image,
                                    self.radius,
                                    self.radius,
                                    self.radius-1,
                                    colors.BLACK)

            # pygame.draw.circle(
            #     self.image, self.color, (self.radius, self.radius), self.radius
            # )

        # showing agent orientation with a line towards agent orientation
        new_white = (255, 255, 254)
        pygame.draw.line(self.image, new_white, (self.radius, self.radius),
                         ((1 + np.cos(self.orientation)) * self.radius, (1 - np.sin(self.orientation)) * self.radius),
                         3)
        self.image = pygame.transform.smoothscale(self.image, [self.radius * 2, self.radius * 2])
        self.mask = pygame.mask.from_surface(self.image)

    def teleport_infinite_arena(self):
        """In case the boundary conditions are infinite (as in now reflection from walls is requested) the
        agents are teleported on a torus when reaching walls."""

        # Boundary conditions according to center of agent (simple)
        x = self.position[0] + self.radius
        y = self.position[1] + self.radius

        if x < self.boundaries_x[0]:
            self.position[0] = self.boundaries_x[1] - self.radius
        elif x > self.boundaries_x[1]:
            self.position[0] = self.boundaries_x[0] + self.radius

        if y < self.boundaries_y[0]:
            self.position[1] = self.boundaries_y[1] - self.radius
        elif y > self.boundaries_y[1]:
            self.position[1] = self.boundaries_y[0] + self.radius

    def update_social_info(self, agents):
        # calculate socially relevant projection field (e.g. according to
        # signalling agents)
        self.soc_v_field = self.calc_soc_v_proj(agents)
        try:
            dphi2 = vf_supcalc.follow_lines_local(self.position, self.radius, self.orientation, self.line_map, self.velocity,
                                                  sensor_radius=self.sensor_size, sensor_distance=self.sensor_distance)
        except:
            print("Error during line following!")

    def calc_soc_v_proj(self, agents):
        """
        :param agents: all agents
        """
        agents_of_interest = [ag for ag in agents if ag.id != self.id]
        visual_field = vf_supcalc.projection_field(
            fov=self.FOV,
            v_field_resolution=self.v_field_res,
            position=np.array(self.position),
            radius=self.radius,
            orientation=self.orientation,
            object_positions=[np.array(ag.position) for ag in agents_of_interest],
            object_sizes=[ag.radius for ag in agents_of_interest],
            boundary_cond=self.boundary_cond,
            arena_width=self.WIDTH,
            arena_height=self.HEIGHT,
            ag_id=self.id)
        # sum of all agents projections at each point in visual field
        svfield = visual_field.sum(axis=0)
        # binarize v field
        svfield[svfield > 0] = 1
        return svfield

    def update_state(self):
        # update agent state based on the decision-making process
        self.agent_state = "flocking"

    def update_linemap(self):
        """Updating background line map to follow"""
        subsurface = pygame.Surface((self.WIDTH + self.window_pad, self.HEIGHT + self.window_pad))
        subsurface.fill(colors.BACKGROUND)
        subsurface.set_colorkey(colors.WHITE)
        subsurface.set_alpha(255)
        for line in self.lines:
            for pi in range(1, len(line)):
                point1 = line[pi-1]
                point2 = line[pi]
                color = colors.BLACK
                pygame.draw.line(subsurface, color, point1, point2, 1)
        from scipy.ndimage.filters import gaussian_filter
        line_map = pygame.surfarray.array3d(subsurface)
        line_map = gaussian_filter(line_map, sigma=1)
        line_map[line_map<253] = 0
        line_map = gaussian_filter(line_map, sigma=1)
        self.line_map = 1 - line_map.swapaxes(0, 1)[:, :, 0]/255

    def perform_action(self):
        # we freeze agents when we move them
        if not self.is_moved_with_cursor:
            # perform the agent's action according to the current state
            if self.agent_state == "flocking":
                if len(self.PHI) == len(self.soc_v_field):
                    if not self.verbose_supcalc:
                        dv, dphi = vf_supcalc.VSWRM_flocking_state_variables(self.velocity, self.PHI, np.flip(self.soc_v_field),
                                                                             vf_params, verbose=self.verbose_supcalc,
                                                                             ALP0=self.ALP0, BET0=self.BET0, V0=self.V0)
                        if len(self.lines) != 0:
                            dphi2 = vf_supcalc.follow_lines_local(self.position, self.radius, self.orientation, self.line_map, self.velocity,
                                                            sensor_radius=self.sensor_size, sensor_distance=self.sensor_distance)
                            dphi = dphi2
                    else:
                        self.dv, self.dphi, self.ablob, self.aedge, self.bblob, self.bedge = vf_supcalc.VSWRM_flocking_state_variables(self.velocity, self.PHI, np.flip(self.soc_v_field),
                                                                             vf_params, verbose=self.verbose_supcalc,
                                                                             ALP0=self.ALP0, BET0=self.BET0, V0=self.V0)
                        dv, dphi = self.dv, self.dphi
                else:
                    dv, dphi = 0, 0
                    print("Warning: temporary skip calculation due to mismatch in phi and vfield resolutions!")
                self.update_agent_position(dphi, dv)
            elif self.agent_state == "emergency":
                pass

    def update_agent_position(self, theta, vel):
        # updating agent's state variables according to calculated vel and
        # theta
        # maximum turning angle per timestep
        if self.limit_movement:
            theta = self.prove_turning(theta, theta_lim=self.max_th)
        self.orientation += theta
        self.prove_orientation()  # bounding orientation into 0 and 2pi

        self.velocity += vel
        if self.limit_movement:
            self.prove_velocity(velocity_limit=self.max_vel)  # possibly bounding velocity of agent

        # new agent's position
        new_pos = (
            self.position[0] + self.velocity * np.cos(self.orientation),
            self.position[1] - self.velocity * np.sin(self.orientation)
        )
        # update the agent's position with constraints (reflection from the
        # walls) or with the new position
        self.position = list(new_pos)

    def prove_turning(self, theta, theta_lim=0.2):
        t_sign = np.sign(theta)
        if t_sign == 0:
            t_sign = +1
        if np.abs(theta) > theta_lim:
            theta = theta_lim * t_sign
        return theta

    def prove_velocity(self, velocity_limit=1):
        """Restricting the absolute velocity of the agent"""
        vel_sign = np.sign(self.velocity)
        if vel_sign == 0:
            vel_sign = +1
        if np.abs(self.velocity) > velocity_limit:
            self.velocity = velocity_limit * vel_sign

    def change_color(self):
        """
        Changing color of agent according to the behavioral mode the agent is
        currently in.
        """
        cmap = matplotlib.cm.get_cmap('jet')
        rgba = np.array(cmap(self.orientation / (2 * np.pi)))
        # rescaling color for pygame
        rgba[0:3] *= 255
        self.color = rgba


import nltk
import re
import unicodedata
import string
from bs4 import BeautifulSoup
from nltk.corpus import stopwords
import warnings

warnings.filterwarnings("ignore")

def strip_html_tags(text: str):
    try:
        soup = BeautifulSoup(text, "html.parser")
        stripped_text = soup.get_text()
    except Exception as e:
        return text
    return stripped_text

def remove_accented_chars(text: str):
    try:
        text = (
            unicodedata.normalize("NFKD", text)
            .encode("ascii", "ignore")
            .decode("utf-8", "ignore")
        )
    except Exception as e:
        return text
    return text

def remove_url(statement: str):
    patterns = [r"http\S+", r"www\S+"]
    out = statement
    for pattern in patterns:
        out = re.sub(pattern, "", out)
    return out

def tokenize_statement(statement: str):
    tokens = nltk.word_tokenize(statement)
    return " ".join(token for token in tokens)

def strip_punctuation(statement: str):
    regex = re.compile("[%s]" % re.escape(string.punctuation))
    statement = regex.sub(" ", statement)
    return statement

def lower_case_statement(statement: str):
    return statement.lower()

def remove_stop_words(statement: str):
    stop_words = set(stopwords.words("english"))
    word_tokens = statement.split()
    filtered_statement = " ".join(w for w in word_tokens if w not in stop_words)
    return filtered_statement

def remove_special_characters(text: str, remove_digits=False):
    pattern = r"[^a-zA-z0-9\s]" if not remove_digits else r"[^a-zA-z\s]"
    text = re.sub(pattern, "", text)
    return text

def process(statement):
    try:
        statement = strip_html_tags(statement)
        statement = remove_accented_chars(statement)
        statement = strip_punctuation(statement)
        statement = remove_url(statement)
        statement = tokenize_statement(statement)
        statement = lower_case_statement(statement)
    except Exception as e:
        pass
    return statement


import os.path
import csv
from csv import Dialect
import pika
import json
import unicodecsv

from flatten_json import flatten
from rabbitmq import RabbitMQProducer
from rabbitmq.RabbitMQConsumer import RabbitMQConsumer
from mdm_logging import *
from RabbitMQConsumer import *
from RabbitMQProducer import *
import sys

setup_logging()
logger = logging.getLogger(__name__)

# https://stackoverflow.com/questions/50813108/get-transferred-file-name-in-rabbitmq-using-python-pika
# for transferring csv files

class TrafficJsonToCsv:

    def JSONtoCsvArea(self, **data):
        logger = logging.getLogger(__name__)
        # filename = 'mdm_data_parkingArea.csv'
        global filenameArea
        filenameArea = 'mdm_data_parkingArea.csv'
        with open(filenameArea, "a", encoding='utf-8') as csvfile:
        #with open(filenameArea, 'rwb') as csvfile:
            fileEmpty = os.stat(filenameArea).st_size == 0
            headers = list(data)
            writer = csv.DictWriter(csvfile, delimiter=',', lineterminator='\n', fieldnames=headers)
            if fileEmpty:
                # print("Header is written now")
                logger.info('Header is written now')
                writer.writeheader()  # file doesn't exist yet, write a header
            #cannot deal with the letter "ö" in Wilhelmshöhe
            writer.writerow(data)
            # print('Received one set of data')
            logger.info('Received one set of data')
        csvfile.close()

    def JSONtoCsvFacility(self, **data):
        logger = logging.getLogger(__name__)
        # filename = 'mdm_data_parkingFacility.csv'
        global filenameFacility
        filenameFacility = 'mdm_data_parkingFacility.csv'
        #with open(filenameFacility, "ab") as csvfile:
        with open(filenameFacility, 'a', encoding='utf-8') as csvfile:
            fileEmpty = os.stat(filenameFacility).st_size == 0
            headers = list(data)
            writer = csv.DictWriter(csvfile, delimiter=',', lineterminator='\n', fieldnames=headers)
            if fileEmpty:
                # print("Header is written now")
                logger.info('Header is written now')
                writer.writeheader()
            writer.writerow(data)
            # print('Received one set of data')
            logger.info('Received one set of data')
        csvfile.close()

    def getCsv(self, filename):
        with open(filename, 'rb') as csv_file:
            return filename + csv_file.read().decode()

consumer_config = json.dumps({
    "exchangeName": "topic_datas",
    "host": "localhost",
    "routingKey": "bc",
    "exchangeType": "direct",
    "queueName": "bc",
    "exchangeOptions": {
        "passive": False,
        "durable": False,
        "autoDelete": False,
        "internal": False
    },
    "queueOptions": {
        "passive": False,
        "durable": False,
        "exclusive": False,
        "autoDelete": False
    }
})

producer_config = json.dumps({
    "exchangeName": "topic_datas",
    "exchangeType": "direct",
    "host": "localhost",
    "routingKey": "cd"

})

producer = RabbitMQProducer(json.loads(producer_config))

filenameArea = None
filenameFacility = None
k = 0

def main():
    # consume from Queue
    with RabbitMQConsumer(json.loads(consumer_config)) as consumer:
        logger.info('Consume')
        consumer.consume(resolve_message)

def resolve_message(data):

    print(" [x] Receiving message %r" % data)

    task = TrafficJsonToCsv()
    global k

    if k < 11:
        json_data = json.loads(data)
        #a = task.byteify(json_data)
        x = flatten(json_data)
        firstkey = list(x.keys())[0]
        # taskOne = Task_JsonCsv()
        if firstkey == 'parkingAreaOccupancy':
            print(x)
            task.JSONtoCsvArea(**x)
            logger.info('Stored data in csvArea')
        else:
            task.JSONtoCsvFacility(**x)
            logger.info('Stored data in csvFacility')
        k = k + 1
        print(k)
    else:
        json_data = json.loads(data)
        x = flatten(json_data)
        firstkey = list(x.keys())[0]
        # taskOne = Task_JsonCsv()
        if firstkey == 'parkingAreaOccupancy':
            task.JSONtoCsvArea(**x)
            logger.info('Stored data in csvArea')
        else:
            task.JSONtoCsvFacility(**x)
            logger.info('Stored data in csvFacility')

        i = task.getCsv(filenameArea)
        j = task.getCsv(filenameFacility)
        producer.publish(i)
        logger.info('Csv area files have been pushed to the queue')
        producer.publish(j)
        logger.info('Csv facility files have been pushed to the queue')
        k = k + 1
        print(k)

main()


import itertools
from functools import wraps, partial

from app import db
from app.history_meta import create_history

def transactional(func):
    @wraps(func)
    def commit_or_rollback(*args, **kwargs):
        from flask import current_app
        try:
            res = func(*args, **kwargs)
            db.session.commit()
            return res
        except Exception as e:
            current_app.logger.error(e)
            db.session.rollback()
            raise
    return commit_or_rollback

def version_class(model_class, history_cls=None):
    create_hist = partial(create_history, history_cls=history_cls)

    def versioned(func):
        @wraps(func)
        def record_version(*args, **kwargs):
            func(*args, **kwargs)
            history_objects = [create_hist(obj) for obj in
                               itertools.chain(db.session.new, db.session.dirty)
                               if isinstance(obj, model_class)]
            for h_obj in history_objects:
                db.session.add(h_obj)
        return record_version
    return versioned

def dao_rollback():
    db.session.rollback()

def is_duplicate_key_integrity_error(e):
    if hasattr(e, 'orig') and hasattr(e.orig, 'pgerror') and e.orig.pgerror \
            and ('duplicate key value violates unique constraint' in e.orig.pgerror):
        return True
    return False


"""
@author  : MG
@Time    : 2021/1/27 11:15
@File    : reversion_rights_md.py
@contact : mmmaaaggg@163.com
@desc    : 用于产生复权行情数据
"""

import logging
import typing
from datetime import timedelta
from enum import Enum

import pandas as pd
from ibats_utils.db import with_db_session, bunch_insert_on_duplicate_update
from ibats_utils.mess import str_2_date
from sqlalchemy.dialects.mysql import DOUBLE
from sqlalchemy.types import String, Date

from tasks import config
from tasks.backend import engine_md

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)

class Method(Enum):
    """
    方法枚举，value为对应的默认 adj_factor 因子值
    """
    division = 1
    diff = 0

def generate_md_with_adj_factor(instrument_type: str,
                                method: typing.Optional[Method] = Method.division):
    """
    将指定期货品种生产复权后价格。
    主力合约名称 f"{instrument_type.upper()}9999"
    次主力合约名称 f"{instrument_type.upper()}8888"
    仅针对 OHLC 四个价格进行复权处理
    """
    table_name = 'ifind_future_adj_factor'
    sql_str = f"""SELECT trade_date, instrument_id_main, adj_factor_main,
        instrument_id_secondary, adj_factor_secondary
        FROM {table_name}
        where instrument_type=%s and method=%s"""
    adj_factor_df = pd.read_sql(sql_str, engine_md, params=[instrument_type, method.name])
    adj_factor_df['trade_date_larger_than'] = adj_factor_df['trade_date'].shift(1)
    adj_factor_df.set_index(['trade_date_larger_than', 'trade_date'], inplace=True)

    instrument_id_set = set(adj_factor_df['instrument_id_main']) | set(adj_factor_df['instrument_id_secondary'])
    in_clause_str = "'" + "', '".join(instrument_id_set) + "'"
    # daily_table_name = 'ifind_future_daily'
    sql_str = f"""SELECT ths_code Contract, `time` trade_date, `Open`, `High`, `Low`, `Close`, Volume, openInterest OI
    FROM ifind_future_daily
    where ths_code in ({in_clause_str})
    """
    daily_df = pd.read_sql(sql_str, engine_md)
    n_final = adj_factor_df.shape[0]
    data_count = 0
    for n, ((trade_date_larger_than, trade_date), adj_factor_s) in enumerate(adj_factor_df.iterrows(), start=1):
        # 截止当日下午3点收盘，保守计算，延迟1个小时，到16点
        if pd.isna(trade_date_larger_than):
            start = None
        else:
            start = trade_date_larger_than + timedelta(days=1)

        if n == n_final:
            if start is None:
                start = str_2_date('1990-01-01')
                end = str_2_date('2090-12-31')
            else:
                end = start + timedelta(days=365)
        else:
            end = trade_date
            if start is None:
                start = end - timedelta(days=365)

        # 对 主力、次主力合约复权
        is_match = (
                           daily_df['code'] >= pd.to_datetime(start)
                   ) & (
                           daily_df['code'] < pd.to_datetime(end)
                   )
        instrument_id_main = adj_factor_s['instrument_id_main']
        adj_factor_main = adj_factor_s['adj_factor_main']
        main_df = daily_df[(daily_df['code'] == instrument_id_main) & is_match].copy()
        instrument_id_secondary = adj_factor_s['instrument_id_secondary']
        adj_factor_secondary = adj_factor_s['adj_factor_secondary']
        sec_df = daily_df[(daily_df['code'] == instrument_id_secondary) & is_match].copy()
        rename_dic = {_: f"{_}Next" for _ in sec_df.columns if _ != 'trade_date'}
        sec_df.rename(columns=rename_dic, inplace=True)
        main_sec_df = pd.merge(main_df, sec_df, on='trade_date')
        main_sec_df['instrument_type'] = instrument_type
        main_sec_df['adj_factor_main'] = adj_factor_main
        main_sec_df['adj_factor_secondary'] = adj_factor_secondary
        dtype = {
            'trade_date': Date,
            'Contract': String(20),
            'ContractNext': String(20),
            'instrument_type': String(20),
            'Close': DOUBLE,
            'CloseNext': DOUBLE,
            'Volume': DOUBLE,
            'VolumeNext': DOUBLE,
            'OI': DOUBLE,
            'OINext': DOUBLE,
            'Open': DOUBLE,
            'OpenNext': DOUBLE,
            'High': DOUBLE,
            'HighNext': DOUBLE,
            'Low': DOUBLE,
            'LowNext': DOUBLE,
            'adj_factor_main': DOUBLE,
            'adj_factor_secondary': DOUBLE
        }
        table_name = 'wind_future_continuous_no_adj'
        bunch_insert_on_duplicate_update(
            main_sec_df, table_name, engine_md, dtype=dtype,
            primary_keys=['instrument_type', 'trade_date'], schema=config.DB_SCHEMA_MD
        )

        main_sec_df['Open'] *= adj_factor_main
        main_sec_df['High'] *= adj_factor_main
        main_sec_df['Low'] *= adj_factor_main
        main_sec_df['Close'] *= adj_factor_main
        main_sec_df['Volume'] *= adj_factor_main
        main_sec_df['OI'] *= adj_factor_main
        main_sec_df['OpenNext'] *= adj_factor_secondary
        main_sec_df['HighNext'] *= adj_factor_secondary
        main_sec_df['LowNext'] *= adj_factor_secondary
        main_sec_df['CloseNext'] *= adj_factor_secondary
        main_sec_df['VolumeNext'] *= adj_factor_secondary
        main_sec_df['OINext'] *= adj_factor_secondary
        table_name = 'wind_future_continuous_adj'
        bunch_insert_on_duplicate_update(
            main_sec_df, table_name, engine_md, dtype=dtype,
            primary_keys=['instrument_type', 'trade_date'], schema=config.DB_SCHEMA_MD
        )
        data_count += main_sec_df.shape[0]
        logger.info("%s [%s ~ %s] 包含 %d 条数据，复权保存完成",
                    instrument_type, start, end, main_sec_df.shape[0])

    logger.info(f'{instrument_type.upper()} {data_count} 条记录 复权保存完成')

def _test_generate_md_with_adj_factor():
    instrument_type = 'rb'
    generate_md_with_adj_factor(instrument_type, method=Method.division)

def generate_adj_md(instrument_types: typing.Optional[list], method: Method, until_instrument_type=None):
    """
    对指定类型进行复权数据生成
    :param instrument_types 类型列表，如果为None则为全部生成
    :param method 方法筛选
    :param until_instrument_type 指定当前品种才开始运行（仅同调试使用），有时候运行中途出错，再次运行时，从报错的品种开始重新运行，而不是全部重新运行
    """
    if instrument_types is None:
        sql_str = """SELECT distinct instrument_type
        FROM ifind_future_adj_factor where method = :method"""
        with with_db_session(engine_md) as session:
            instrument_types = [
                row[0] for row in session.execute(sql_str, params={'method': method.name}).fetchall()]

    for instrument_type in instrument_types:
        generate_md_with_adj_factor(instrument_type, method=method)

def _test_generate_adj_md():
    instrument_types = None
    # instrument_types = [
    #     'rb', 'hc', 'i', 'j', 'jm', 'ru', 'bu', 'ma', 'cu', 'fg', 'jd',
    #     'ap', 'cj', 'a', 'b', 'm', 'oi', 'y', 'rm', 'p', 'rs', 'sr', 'cf'
    # ]
    generate_adj_md(
        instrument_types,
        method=Method.division,
        # until_instrument_type='V',
    )

if __name__ == "__main__":
    # _test_generate_md_with_adj_factor()
    _test_generate_adj_md()


import datetime
import gzip
import json
import urllib.request

import jsonschema

def get_indy_ada_daily_closing_prices() -> dict[datetime.date, float]:
    """Return all daily closing prices of INDY, denominated in ADA.

    Closing price for e.g. 2023-03-25 is the price at 2023-03-26 00:00.
    """
    ada_usd = get_ada_usd_daily_closing_prices()
    indy_usd = get_indy_usd_daily_closing_prices()
    indy_ada = {}
    for date in indy_usd.keys():
        if date in ada_usd.keys():
            indy_ada[date] = indy_usd[date] / ada_usd[date]
        else:
            raise Exception(f"Found INDY USD price for {date}, but no ADA USD price.")
    return indy_ada

def get_ada_usd_daily_closing_prices() -> dict[datetime.date, float]:
    return _get_daily_usd_prices(975)

def get_indy_usd_daily_closing_prices() -> dict[datetime.date, float]:
    return _get_daily_usd_prices(28303)

def _get_daily_usd_prices(asset_id: int) -> dict[datetime.date, float]:
    raw = _get_chart_data(asset_id)
    usd_prices: dict[datetime.date, float] = {}
    for x in raw["stats"]:
        timestamp = x[0] / 1000.0
        dt = datetime.datetime.utcfromtimestamp(timestamp)
        dt_zero_time = dt.replace(hour=0, minute=0, second=0, microsecond=0)
        minus_one_date = dt.date() - datetime.timedelta(days=1)
        if dt == dt_zero_time:
            # We're technically interested in closing prices of dates, but Coingecko
            # prices are opening prices for the next day. Which is the same price,
            # with an offset date.
            usd_prices[minus_one_date] = x[1]
        else:
            # There's a window after UTC 00:00, e.g. 00:26, when the price for 00:00
            # isn't separately included yet. In that case, fill it out with the current
            # price.
            if minus_one_date not in usd_prices:
                usd_prices[minus_one_date] = x[1]

            # Add the latest price. This is a bit of a hack. The only price with a
            # time other than 00:00 is the current price. The current price is needed
            # for some commands in case they're executed for the current day, after
            # 21:46 (after the Indigo snapshot) but before 00:00 UTC (daily closing
            # price availability). Appending the current price here avoids an extra
            # network request.
            usd_prices[dt.date()] = x[1]

    return usd_prices

def _get_chart_data(asset_id: int) -> dict:
    """Return daily opening prices and volumes for an asset.

    Uses an undocumented Coingecko API, which seems more efficient and easy than the
    official API. Gets years' worth of daily prices (and volumes) in a single HTTP
    request.
    """
    req = urllib.request.Request(
        f"https://www.coingecko.com/price_charts/{asset_id}/usd/max.json",
        headers={
            "accept": (
                "text/html,application/xhtml+xml,application/xml;q=0.9,"
                "image/avif,image/webp,image/apng,*/*;q=0.8"
            ),
            "accept-encoding": "gzip",
            "accept-language": "en-US,en;q=0.9",
            "cache-control": "max-age=0",
            "User-Agent": (
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
                " (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
            ),
        },
    )

    with urllib.request.urlopen(req) as response:
        if response.info().get("Content-Encoding") == "gzip":
            decompressed_content = gzip.decompress(response.read())
        else:
            decompressed_content = response.read()
        decoded_content = decompressed_content.decode("utf-8")
        json_response = json.loads(decoded_content)

    schema = {
        "$schema": "http://json-schema.org/draft-07/schema#",
        "type": "object",
        "properties": {
            "stats": {
                "type": "array",
                "items": {
                    "type": "array",
                    "items": [{"type": "integer"}, {"type": "number"}],
                },
            },
            "total_volumes": {
                "type": "array",
                "items": {
                    "type": "array",
                    "items": [{"type": "integer"}, {"type": "number"}],
                },
            },
        },
        "required": ["stats", "total_volumes"],
    }

    jsonschema.validate(json_response, schema)

    return json_response


from datetime import datetime
import env_var
from ApplicationService import request_info_service, reply_service
from repositories import (
    user_match_repository,
    user_repository,
    match_repository,
    hanchan_repository,
)

class ReplyHistoryUseCase:

    def execute(self) -> None:
        req_line_id = request_info_service.req_line_user_id

        users = user_repository.find(
            query={'line_user_id': req_line_id},
        )
        if len(users) == 0:
            reply_service.add_message(
                'ユーザーが登録されていません。友達追加してください。'
            )
            reply_service.add_message(
                '既に友達の場合は一度ブロックして、ブロック解除を行ってください。'
            )
            return

        umList = user_match_repository.find(
            query={'user_id': users[0]._id},
        )
        matches = match_repository.find(
            query={'$and': [
                {'_id': {'$in': [um.match_id for um in umList]}},
                {'status': 2},
            ]},
        )

        if len(matches) == 0:
            reply_service.add_message(
                '対局履歴がありません。'
            )
            return
        all_hanchans = hanchan_repository.find(
            query={'$and': [
                {'match_id': {'$in': [match._id for match in matches]}},
                {'status': 2},
            ]},
        )

        if len(all_hanchans) == 0:
            raise ValueError('半荘情報を取得できませんでした。')

        message = ''
        total = 0

        # グラフ描画用プロットデータ
        history = {}

        for match in matches:
            score = 0
            for hanchan in [hanchan for hanchan in all_hanchans if hanchan.match_id == match._id]:
                if req_line_id in hanchan.converted_scores:
                    score += hanchan.converted_scores[req_line_id]
            total += score
            strScore = ('+' + str(score)) if score > 0 else str(score)
            message += match.created_at.strftime(
                "%Y/%m/%d") + ': ' + strScore + '\n'

            history[match.created_at] = total

        # 今までの全半荘一覧
        reply_service.add_message(
            message
        )

        # 合計
        strTotal = ('+' + str(total)) if total > 0 else str(total)
        reply_service.add_message(
            '累計: ' + strTotal
        )

        # グラフ描画
        # 初回値に0を追加
        from datetime import timedelta
        start_date: datetime = min(history.keys())
        end_date: datetime = max(history.keys())
        history[start_date - timedelta(minutes=2)] = 0
        history[start_date - timedelta(minutes=1)] = 0
        history = dict(sorted(history.items()))

        x =[]
        y =[]
        for k, v in history.items():
            x.append(k)
            y.append(v)

        import matplotlib
        import matplotlib.pyplot as plt
        import matplotlib.dates as mdates
        matplotlib.use('agg')

        fig, ax = plt.subplots()
        plt.step(history.keys(), history.values(), where='mid')

        plt.grid(which='major', axis='y')
        plt.xlim([start_date - timedelta(minutes=2), end_date])
        plt.xticks(rotation=30)
        # locator = mdates.DayLocator()
        # ax.xaxis.set_major_locator(locator)
        ax.xaxis.set_major_formatter(mdates.DateFormatter("%m/%d %H時"))

        try:
            fig.savefig(f"src/uploads/personal_history/{req_line_id}.png")
        except FileNotFoundError:
            reply_service.reset()
            reply_service.add_message(text='システムエラーが発生しました。')
            messages = [
                '対戦履歴の画像アップロードに失敗しました',
                '送信者: ' + req_line_id,
            ]
            reply_service.push_a_message(
                to=env_var.SERVER_ADMIN_LINE_USER_ID,
                message='\n'.join(messages),
            )
            return
        plt.clf()
        plt.close()

        path = f'uploads/personal_history/{req_line_id}.png'
        image_url = f'{env_var.SERVER_URL}{path}'
        reply_service.add_image(image_url)


from django.shortcuts import render
from django.http import HttpResponse, JsonResponse
from django.views.generic import View
import json
from . import Clean
from .forms import SymptomForm
from .store import getsymptoms

def home(request):
    context = {'b':'Helloworld'}
    return render(request,'main.html',context)
class AjaxHandler(View):
    def get(self, request):
        text = request.GET.get('values')
        is_ajax = request.META.get('HTTP_X_REQUESTED_WITH') == 'XMLHttpRequest'

        if is_ajax:
            sympts = text.split(',')

            sympts = [int(e) for e in sympts[:-1]]
            mainList = getsymptoms()
            mainList.sort()

            symptoms = [mainList[i] for i in sympts]
            diseases = Clean.runApp(symptoms)
            dis = ', '.join(diseases)

            return JsonResponse({'ans':dis, 'dis1': diseases[0], 'dis2':diseases[1], 'dis3':diseases[2]}, status=200)

        return render(request, 'trail.html')


import json, os

processed_files_path = "project/additional_files/processed_articles"
main_content_path = "project/additional_files/main_content" #abstract and data/methods
other_content_path = "project/additional_files/other_content"

if not os.path.exists(main_content_path):
    os.makedirs(main_content_path)

if not os.path.exists(other_content_path):
    os.makedirs(other_content_path)

for filename in os.listdir(processed_files_path):
    with open(os.path.join(processed_files_path, filename)) as f:
        print(filename)
        data = json.load(f)
        main_text = []
        other_text = []
        for key in data:
            if "abstract" == key:
                main_text.append(data["abstract"])
            elif "methodology" == key:
                main_text.append(data["methodology"])
            elif "subject" == key:
                continue
            elif "keywords" == key:
                other_text.append(" ".join(data["keywords"]))
            else:
                other_text.append(data[key])

        with open(os.path.join(main_content_path, filename), "w") as f:
            f.write("\n".join(main_text))

        with open(os.path.join(other_content_path, filename), "w") as f:
            f.write("\n".join(other_text))

print("done.")


import random
import string

def guess():
    guess = input('Enter one alphabet:')
    return guess

def str_to_list(word):
    mylist = []
    for i in word:
        mylist.append(i)
    return mylist

def printlist(mylist):
    word = ''
    for i in mylist:
        word += i
    return word

def update_list(mystring , random_word, myguess):
    for i in range (0, len(random_word)):
        if random_word[i] == myguess and mystring[i] == '_':
            mystring[i] = random_word[i]
    return mystring

def is_complete(mystring):
    for i in mystring:
        if i == '_':
            return False
    return True

def display_remainings(guessed_letter):
    a_to_z = string.ascii_lowercase
    remainings = ''
    for i in a_to_z:
        if i not in guessed_letter:
            remainings += i
    return remainings

def play_game():
    fruits = ["apple", "banana", "cherry", "date", "elderberry", "fig", "grape", "honeydew", "kiwi", "lemon", "mango", "orange", "papaya", "quince", "raspberry", "strawberry", "tangerine", "watermelon"]

    random_word = str_to_list(random.choice(fruits))
    word_length = len(random_word)
    chances = 10
    mystring = str_to_list(word_length *'_')
    guessed_letters = []

    print('Guess a fruit name')

    while chances > 0:
        if is_complete(mystring):
            print(f"You guess '{printlist(mystring)}' is correct")
            break
        else:
            print(f'\nalready guessed: {printlist(guessed_letters)} \nremaining guess letters: {display_remainings(guessed_letters)}')
            print(printlist(mystring))
            myguess = guess()
            if len(myguess) == 1:
                if myguess in guessed_letters:
                    print('You already guessed that letter')
                elif myguess in random_word:
                    guessed_letters.append(myguess)
                    mystring = update_list(mystring, random_word, myguess)
                else:
                    guessed_letters.append(myguess)
                    chances -= 1
                    print(f'Wrong guess you have {chances} chances left')
            else:
                print('Enter 1 alphabet only')
                guess()

    else:
        print(f"Your lost the game \nThe answer was '{printlist(random_word)}'")

play_again = '1'
while play_again != '2':
    play_game()
    play_again = input('Enter 1 to play again and 2 to exit')
    if play_again == '2':
        break
    else:
        print('Enter correc input')


from flask.helpers import flash
from server import result
from mysqlconnection import connectToMySQL
import re

class Dojo():

    def __init__(self,data):
        self.id = data['id']
        self.name = data['name']
        self.location = data['location']
        self.language = data['language']
        self.comment = data['comment']
        self.created_at = data['created_at']
        self.updated_at = data['updated_at']

    @classmethod
    def new_ninja(cls, data):
        query = 'INSERT INTO dojos(name, location, language, comment) VALUES( %(name)s, %(location)s, %(language)s, %(comment)s);'

        new_ninja = connectToMySQL('dojo_survey_schema').query_db(query,data)

        return new_ninja

    @classmethod
    def get_ninja_by_id(cls,data):
        query = 'SELECT * FROM dojos WHERE id = %(id)s;'

        result = connectToMySQL('dojo_survey_schema').query_db(query,data)

        single_ninja = cls(result[0])

        return single_ninja

    @staticmethod
    def validate_name(user):
        name_regex = re.compile(r"^[a-zA-Z '-]{1,45}$")
        is_valid = True
        if not name_regex.match(user['name']):
            flash('Your name must be between 1 and 45 characters')
            is_valid = False
        return is_valid


# Game Settings
WIDTH = 1080
HEIGHT = 720
TITLE = 'Flappy Chicken'
FPS = 60

# Colors
BLACK = (0, 0, 0)
WHITE = (255, 255, 255)
YELLOW = (255, 255, 0)
GREEN = (0, 255, 0)
RED = (255, 0, 0)
BLUE = (0, 0, 255)
GREY = (100, 100, 100)
DARKGREEN = (34, 139, 34)

# Adjustments for the game mechanics
WALLDISTANCE = 200     # Maintains distance between the walls
OBJECTSPEED = -6      # Regulates wall speed
BACKGROUNDSPEED = -1   # Regulates cloud speed
WALLCOUNTER = 1


def in_range(nums, lowest, highest):
    """Print numbers inside range.

    - nums: list of numbers
    - lowest: lowest number to print
    - highest: highest number to print

    For example:

      in_range([10, 20, 30, 40], 15, 30)

    should print:

      20 fits
      30 fits
    """

    # YOUR CODE HERE
    for i in range(lowest, highest):
        if nums.count(i):
            print(i, "fits")

in_range([10, 20, 30, 40, 50], 15, 30)


# 13-5. Sideways Shooter Part 2: We’ve come a long way since Exercise 12-6,
# Sideways Shooter. For this exercise, try to develop Sideways Shooter to the
# same point we’ve brought Alien Invasion to. Add a fleet of aliens, and make
# them move sideways toward the ship. Or, write code that places aliens at
# random positions along the right side of the screen and then sends them
# toward the ship. Also, write code that makes the aliens disappear when
# they’re hit.

import sys
import pygame
from pygame.sprite import Sprite

class Screen:
    def __init__(self):
        pygame.init()
        self.screen_width = 1200
        self.screen_height = 800
        self.screen_color = (230, 230, 230)
        self.screen = pygame.display.set_mode(
            (self.screen_width, self.screen_height))
        self.screen_rect = self.screen.get_rect()

        self.ship_image = pygame.image.load("exercise_13_5_ship.bmp")
        self.ship_rect = self.ship_image.get_rect()
        self.ship_rect.midleft = self.screen_rect.midleft
        self.ship_speed = 1
        self.ship_move_up = False
        self.ship_move_down = False

        self.bullets = pygame.sprite.Group()

        self.alien_move_vertical_speed = 1
        self.alien_move_horizontal_speed = 5
        self.alien_move_direction = 1
        self.aliens = pygame.sprite.Group()
        self._create_alien_fleet()

    def run_game(self):
        while True:
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    sys.exit()
                elif event.type == pygame.KEYDOWN:
                    if event.key == pygame.K_UP:
                        self.ship_move_up = True
                    elif event.key == pygame.K_DOWN:
                        self.ship_move_down = True
                    elif event.key == pygame.K_SPACE:
                        new_bullet = Bullet(self)
                        self.bullets.add(new_bullet)
                elif event.type == pygame.KEYUP:
                    if event.key == pygame.K_UP:
                        self.ship_move_up = False
                    elif event.key == pygame.K_DOWN:
                        self.ship_move_down = False

            self.ship_update()
            self.bullets.update()
            for bullet in self.bullets.copy():
                if bullet.rect.left > self.screen_rect.right:
                    self.bullets.remove(bullet)
            self.screen.fill(self.screen_color)
            self.screen.blit(self.ship_image, self.ship_rect)
            for bullet in self.bullets.sprites():
                bullet.draw_bullet()
            self.aliens.update()
            self._check_bullet_alien_collision()
            self._check_fleet_edges()
            self.aliens.draw(self.screen)
            pygame.display.flip()

    def ship_update(self):
        if self.ship_move_up and self.ship_rect.y > 0:
            self.ship_rect.y -= self.ship_speed
        if self.ship_move_down \
                and self.ship_rect.bottom < self.screen_rect.bottom:
            self.ship_rect.y += self.ship_speed

    def _create_alien_fleet(self):
        alien = Alien(self)
        number_x = 4
        number_y = (self.screen_rect.height - 2 *
                    alien.rect.height) // (2 * alien.rect.height)

        for number_column in range(number_x):
            for number_row in range(number_y):
                alien = Alien(self)
                alien.rect.y = alien.rect.height * (number_row + 1) + \
                    alien.rect.height * number_row
                alien.rect.x = self.screen_rect.width - 2 * \
                    (number_column + 1) * alien.rect.width
                self.aliens.add(alien)

    def _check_fleet_edges(self):
        for alien in self.aliens:
            if alien.rect.top <= 0 \
                    or alien.rect.bottom >= self.screen_rect.bottom:
                self.alien_move_direction *= -1
                self._move_fleet_left()
                break

    def _move_fleet_left(self):
        for alien in self.aliens:
            alien.rect.x -= self.alien_move_horizontal_speed

    def _check_bullet_alien_collision(self):
        collision = pygame.sprite.groupcollide(
            self.aliens, self.bullets, True, True)

class Bullet(Sprite):
    def __init__(self, screen):
        super().__init__()
        self.screen = screen
        self.bullet_speed = 1.0
        self.bullet_width = 15
        self.bullet_height = 3
        self.bullet_color = (60, 60, 60)
        self.rect = pygame.Rect(0, 0, self.bullet_width, self.bullet_height)
        self.rect.midright = self.screen.ship_rect.midright

    def update(self):
        self.rect.x += int(self.bullet_speed)

    def draw_bullet(self):
        pygame.draw.rect(self.screen.screen, self.bullet_color, self.rect)

class Alien(Sprite):
    def __init__(self, screen):
        self.screen = screen
        super().__init__()
        self.image = pygame.image.load("exercise_13_5_alien.bmp")
        self.rect = self.image.get_rect()

    def update(self):
        self.rect.y += self.screen.alien_move_direction * \
            self.screen.alien_move_vertical_speed

if __name__ == "__main__":
    screen = Screen()
    screen.run_game()


import numpy as np
from Submit import tools
from Submit import rdh
from Submit import extract
import sys
from PIL import Image

def embed_extract(origin_img, to_embed_info, pe_1, theta_1, pe_2):

    # to embed message

    psnr = 0

    image_x, image_y = origin_img.shape

    ST = 0
    pe_optimal = pe_1
    theta = theta_1

    lmap = tools.build_location_map(origin_img, 0)

    to_embed_binary_message = tools.encode(to_embed_info)
    to_embed_binary_message = list(to_embed_binary_message)

    cur_embed_index = [0]

    # embed
    extract_img, enough = rdh.embedding(origin_img, image_x, image_y,
                                        ST, pe_optimal, theta, lmap,
                                        to_embed_binary_message, cur_embed_index, 0)
    print(len(to_embed_binary_message))

    if enough:
        # just embed once
        # extract
        print('only use embedding one time')
        to_extract_img = extract_img.copy()
        psnr = tools.compute_psnr(origin_img, extract_img)

        return True, to_extract_img, psnr, 1

    else:
        # embed twice
        print('one stage not enough')
        # stage 1 capacity
        print('stage 1 capacity: %d' % cur_embed_index[0])
        stage_capacity_1 = cur_embed_index[0]
        aux_len_1 = tools.compute_aux_size(lmap)

        to_embed_info = tools.encode(to_embed_info)

        cur_embed_index_1 = [0]
        to_embed_binary_message_1 = list(to_embed_info[:stage_capacity_1-aux_len_1])

        cur_embed_index_2 = [0]
        to_embed_binary_message_2 = list(to_embed_info[stage_capacity_1 - aux_len_1:])

        ST2 = 1
        pe_optimal_2 = pe_2
        theta_2 = theta_1

        embed_once_img, enough = rdh.embedding(origin_img, image_x, image_y, ST, pe_optimal, theta, lmap,
                                            to_embed_binary_message_1, cur_embed_index_1,
                                               1)

        embed_once_img_ = embed_once_img.copy()

        lmap_2 = tools.build_location_map(embed_once_img_, 1)

        embed_twice_img, enough2 = rdh.embedding(embed_once_img_, image_x, image_y, ST2, pe_optimal_2, theta_2, lmap_2,
                                               to_embed_binary_message_2, cur_embed_index_2,
                                                 1)

        print('stage 1 capacity: %d' % cur_embed_index_1[0])
        print('stage 2 capacity: %d' % cur_embed_index_2[0])
        print('all %d' % (cur_embed_index_1[0] + cur_embed_index_2[0]))

        if enough2:
            embed_twice_img_ = embed_twice_img.copy()
            psnr = tools.compute_psnr(origin_img, embed_twice_img)

            return True, embed_twice_img_, psnr, 2

        else:
            return False, None, None, 0

def embed_main(origin_img, p1, p2, theta, embed_info):
    # to_embed_info = 2 * 'hello world  hhhhhhhhhhhhhhhhhhhhhhhh yzx yzx 2020 2020 01234567890123456789asdfghjkl01234567890123456789asdfghjkl01234567890123456789asdfghjkl'

    embed_success, extract_img, psnr, embed_stages = embed_extract(origin_img, embed_info, p1, theta, p2)

    return embed_success, extract_img, psnr, embed_stages

def extract_main(extract_img):
    to_extract_img = extract_img.copy()

    img_first = to_extract_img[0][0]
    bin_lsb = bin(img_first)
    if bin_lsb[-1] == '0':
        info = extract.extracting(to_extract_img, 0)
    else:
        info2 = extract.extracting(to_extract_img, 1)
        info1 = extract.extracting(to_extract_img, 0)
        info = info1+info2

    return info, to_extract_img

def test():
    img_path = '../test_img/'
    for i in range(1,7):
        origin_img = tools.read_image(img_path + str(i) + '.bmp')
        #print(origin_img.shape)
        height,width = origin_img.shape
        cnt = 0
        for j in range(height):
            for k in range(width):
                if origin_img[j][k] == 255 or origin_img[j][k] == 0 or origin_img[j][k] == 254 or origin_img[j][k] == 1:
                    cnt += 1
                    # print(origin_img[j][k])
                    # print(j,k)

        print(i)
        print(cnt)
        print("")


# -*- coding: utf-8 -*-
from selenium import webdriver
from.htmltestrunner import HTMLTestRunner

class Common(object):

    @classmethod
    def get_chrome_driver(cls):
        return webdriver.Chrome()

    @classmethod
    def create_report(cls, fileName, revT, revD, revTest):
        with open('../TestReport/' + fileName + '.html', 'wb') as HtmlFile:
            HTMLTestRunner(
                stream=HtmlFile,  # 文本流
                verbosity=2,  # 报告详情级别,一般都用2
                title=revT,  # 标题
                description=revD  # 描述
            ).run(revTest)


"""
before doing this, need to do the following:
1. clone https://github.com/mcordts/cityscapesScripts.git
2. run cityscapesScripts/cityscapesscripts/preparation/createTrainIdLabelImgs.py
```
CITYSCAPES_DATASET=xxxxxx python cityscapesscripts/preparation/createTrainIdLabelImgs.py
```
"""

from pathlib import Path
from shutil import copy
from tqdm import tqdm
import json

src_dir = Path('./data/cityscapes')
dst_dir = Path('./data/cityscapes_ss')

for phase in ['train', 'val']:
    images = {}
    labels = {}
    for src_p in tqdm((src_dir/f'leftImg8bit/{phase}').glob('**/*.png')):
        images[src_p.name[:-16]] = src_p.relative_to(src_dir).as_posix()
        dst_p = dst_dir/src_p.relative_to(src_dir)
        dst_p.parent.mkdir(exist_ok=True, parents=True)
        copy(src_p, dst_p)
    for src_p in tqdm((src_dir/f'gtFine/{phase}').glob('**/*_labelTrainIds.png')):
        labels[src_p.name[:-25]] = src_p.relative_to(src_dir).as_posix()
        dst_p = dst_dir/src_p.relative_to(src_dir)
        dst_p.parent.mkdir(exist_ok=True, parents=True)
        copy(src_p, dst_p)
    assert images.keys() == labels.keys()
    path_list = [f'{images[key]} {labels[key]}\n' for key in sorted(images.keys())]
    with open(dst_dir/f'{phase}.txt', 'w') as f:
        f.writelines(path_list)
with open(dst_dir/'test.txt', 'w') as f:
    f.writelines(path_list)

with open(dst_dir/'labelmap.json', 'w') as f:
    json.dump({
        '0': 'road',
        '1': 'sidewalk',
        '2': 'building',
        '3': 'wall',
        '4': 'fence',
        '5': 'pole',
        '6': 'trafficlight',
        '7': 'trafficsign',
        '8': 'vegetation',
        '9': 'terrain',
        '10': 'sky',
        '11': 'person',
        '12': 'rider',
        '13': 'car',
        '14': 'truck',
        '15': 'bus',
        '16': 'train',
        '17': 'motorcycle',
        '18': 'bicycle'
    }, f, indent=4)


import ctypes
from ctypes import *
import torch
import numpy as np
import os
import time

class PyScheduler:

    def __init__(self, sched_lib, num_clients):

        torch.cuda.set_device(0)
        self._scheduler = sched_lib.sched_init()
        self._sched_lib = sched_lib
        self._num_clients = num_clients

    def run_scheduler(
        self,
        barriers,
        tids,
        model_names,
        kernel_files,
        additional_kernel_files,
        num_kernels,
        additional_num_kernels,
        num_iters,
        profile,
        run_eval,
        reef,
        sequential,
        reef_depth,
        hp_limit,
        update_start,
        train
    ):

        print(f"REEF IS {reef}, SEQUENTIAL IS {sequential}")

        model_names_ctypes = [x.encode('utf-8') for x in model_names]
        lib_names = [x.encode('utf-8') for x in kernel_files]

        # convert
        IntAr = c_int * self._num_clients
        tids_ar = IntAr(*tids)
        num_kernels_ar = IntAr(*num_kernels)
        num_iters_ar = IntAr(*num_iters)

        CharAr = c_char_p * self._num_clients
        model_names_ctypes_ar = CharAr(*model_names_ctypes)
        lib_names_ar = CharAr(*lib_names)

        BoolAr = c_bool * self._num_clients
        train_ar = BoolAr(*train)

        print(train)
        self._sched_lib.argtypes = [c_void_p, c_int, POINTER(c_int), POINTER(c_char_p), POINTER(c_char_p), POINTER(c_int), POINTER(c_bool)]

        print(model_names, lib_names, tids)

        self._sched_lib.setup(self._scheduler, self._num_clients, tids_ar, model_names_ctypes_ar, lib_names_ar, num_kernels_ar, num_iters_ar, train_ar, reef)

        num_clients = len(tids)
        print(f"Num clients is {num_clients}")

        print(f"before starting, profile is {profile}")
        timings=[]

        if run_eval:
            if profile:
                barriers[0].wait()
                # run once to warm-up and setup
                self._sched_lib.schedule(self._scheduler, num_clients, True, 0, True, 1, reef, sequential, reef_depth, hp_limit, update_start)
                torch.cuda.synchronize()

                for j in range(num_clients):
                    if (additional_kernel_files[j] is not None):
                        new_kernel_file = additional_kernel_files[j].encode('utf-8')
                        self._sched_lib.setup_change(self._scheduler, j, new_kernel_file, additional_num_kernels[j])

                print("wait here")
                barriers[0].wait() #FIXME
                print("done!")

                # warmup
                self._sched_lib.schedule(self._scheduler, num_clients, True, 0, True, 10, reef, sequential, reef_depth, hp_limit, update_start)
                torch.cuda.synchronize()
                barriers[0].wait()

                start = time.time()
                print("call schedule")
                self._sched_lib.schedule(self._scheduler, num_clients, True, 0, False, 0, reef, sequential, reef_depth, hp_limit, update_start)
                barriers[0].wait()
                torch.cuda.synchronize()
                print(f"Total time is {time.time()-start}")

        else:
            for i in range(num_iters[0]):

                print(f"Start {i} iteration")
                if profile:
                    barriers[0].wait()
                    # needed for backward
                    if (i==1):
                        for j in range(num_clients):
                            if (additional_kernel_files[j] is not None):
                                new_kernel_file = additional_kernel_files[j].encode('utf-8')
                                self._sched_lib.setup_change(self._scheduler, j, new_kernel_file, additional_num_kernels[j])
                        barriers[0].wait() #FIXME

                    start = time.time()
                    print("call schedule")
                    self._sched_lib.schedule(self._scheduler, num_clients, True, i)
                    torch.cuda.synchronize()

                # or this
                else:
                    start = time.time()
                    for j in range(num_clients):
                        barriers[j].wait()
                        self._sched_lib.schedule_one(self._scheduler, j)
                        torch.cuda.synchronize()

                total_time = time.time()-start
                print(f"Iteration {i} took {total_time} sec")
                timings.append(total_time)
            timings = timings[3:]
            print(f"Avg is {np.median(np.asarray(timings))}, Min is {min(timings)} sec")


from imageai.Prediction import ImagePrediction
from os import getcwd,path
execution_path=getcwd()
#test

prediction = ImagePrediction()
prediction.setModelTypeAsDenseNet()
prediction.setModelPath(path.join(execution_path, "DenseNet-BC-121-32.h5"))
prediction.loadModel()

predictions, probabilities = prediction.predictImage(path.join(execution_path, "giraffe.jpg"), result_count=5 )
for eachPrediction, eachProbability in zip(predictions, probabilities):
    print(eachPrediction , " : " , eachProbability)