import os
from flask import Flask, request, abort, jsonify
from flask_sqlalchemy import SQLAlchemy
from flask_cors import CORS
import random

from models import setup_db, Question, Category

#-----------------------------------------------------------------------------#
# function_to_return_paginated_questions
#-----------------------------------------------------------------------------#

QUESTIONS_PER_PAGE = 10

def paginate_questions(request, selection):
    page = request.args.get('page', 1, type=int)
    start = (page-1) * QUESTIONS_PER_PAGE
    end = start * QUESTIONS_PER_PAGE

    questions = [Question.format() for question in selection]
    current_questions = questions[start:end]

    return current_questions

def create_app(test_config=None):
    # create and configure the app
    app = Flask(__name__)
    setup_db(app)

# -----------------------------------------------------------------------#
# cors-set-up. Allow '*' for origins
# -----------------------------------------------------------------------#

    cors = CORS(app, resources={r"/*": {"origins": "*"}})

# -----------------------------------------------------------------------#
# After_request decorator to set Access-Control-Allow
# -----------------------------------------------------------------------#

    @app.after_request
    def after_request(response):
        response.headers.add('Access-Control-Allow-Headers',
                             'Content-Type, Authorization')
        response.headers.add('Access-Control_allow-Methods',
                             'GET, POST, PATCH, DELETE, OPTIONS')
        return response

# -----------------------------------------------------------------------#
# Endpoint to handle GET requests
# for all available categories.
# -----------------------------------------------------------------------#

    @app.route('/categories')
    def retrieve_categories():
        selection = Category.query.order_by(Category.id).all()

        if len(selection) == 0:
            abort(404)

        categories = [Category.format() for category in selection]

        return jsonify({
            'success': True,
            'categories': categories
        })

# -----------------------------------------------------------------------#
# endpoint to handle GET requests for questions,
# including pagination (every 10 questions).
# This endpoint returns a list of questions,
# number of total questions, current category, categories.
# -----------------------------------------------------------------------#

    @app.route("/questions")
    def retrieve_questions():
        selection = Question.query.order_by(Question.id).all()
        current_questions = paginate_questions(request, selection)
        categories = list(map(Category.format, Category.query.all()))

        if len(current_questions) == 0:
            abort(404)

        return jsonify({
            'success': True,
            'questions': current_questions,
            'total_questions': len(Question.query.all()),
            'current_category': None,
            'categories': categories
        })

# -----------------------------------------------------------------------#
# Endpoint to DELETE question using a question ID.
# -----------------------------------------------------------------------#

    @app.route('/questions/<int:q_id>', methods=['DELETE'])
    def delete_question(q_id):
        try:
            question = Question.query.filter(Question.id == q_id).one_or_none()

            if question is None:
                abort(404)

            question.delete()
            selection = Question.query.order_by(Question.id).all()
            current_questions = paginate_questions(request, selection)

            return jsonify({
                'success': True,
                'deleted': q_id
            })
        finally:
          abort(422)

# -----------------------------------------------------------------------#
# Endpoint to POST a new question,
# which will require the question and answer text,
# category, and difficulty score.
# -----------------------------------------------------------------------#

    @app.route('/questions', methods=['POST'])
    def post_question():
        body = request.get_json()

        new_question = body.get('question', None)
        new_answer = body.get('answer', None)
        new_category = body.get('category', None)
        new_difficulty = body.get('difficulty', None)

        try:
            question = Question(question=new_question, answer=new_answer,
                                category=new_category, difficulty=new_difficulty)
            question.insert()

            selection = Question.query.order_by(Question.id).all()
            current_questions = paginate_questions(request, selection)

            return jsonify({
                'success': True,
                'created': question.id,
            })

        finally:
            abort(422)

# -----------------------------------------------------------------------#
# POST endpoint to get questions based on a search term.
# It returns any questions for whom the search term
# is a substring of the question.
# -----------------------------------------------------------------------#

    @app.route('/questions/search', methods=['POST'])
    def search_question():
        try:
            body = request.get_json()
            new_searchTerm = body.get('searchTerm', None)

            if new_searchTerm is not None:
                selection = Question.query.filter(Question.question
                            .ilike('%' + new_searchTerm + '%'))
                searched_questions = paginate_questions(request, selection)

                if len(searched_questions) > 0:
                    return jsonify({
                        'success': True,
                        'questions': searched_questions,
                        'total_questions': len(Question.query.all())
                    })

                else:
                    abort(404)

            else:
                abort(404)

        finally:
            abort(422)

# -----------------------------------------------------------------------#
# GET endpoint to get questions based on category.
#-----------------------------------------------------------------------#

    @app.route('/categories/<int:category_id>/questions')
    def question(category_id):
        try:
            questions = Question.query.filter(Question.category ==
                                              str(category_id)).one_or_none()

            return jsonify({
                'success': True,
                'questions': [questions.format() for question in questions],
                'total_questions': len(questions),
                'current_category': category_id
            })

        finally:
            abort(404)

# -----------------------------------------------------------------------#
# POST endpoint to get questions to play the quiz.
# This endpoint should take category and previous question parameters
# and return a random questions within the given category,
# if provided, and that is not one of the previous questions.
# -----------------------------------------------------------------------#

    @app.route('/quizzes', methods=['POST'])
    def retrieve_quiz_questions():
        body = request.get_json()
        if (('quiz_category' in body
             and 'id' in body['quiz_category'])
             and 'previous_questions' in body):
            questions_query = Question.query.filter_by(
            category=body['quiz_category']['id']
            ).filter(
            Question.id.notin_(body["previous_questions"])
            ).all()
            length_of_available_question = len(questions_query)
            if length_of_available_question > 0:
                return jsonify({
                    "success": True,
                    "question": Question.format(
                                questions_query[random.randrange(
                                0,
                                length_of_available_question
                                )]
                                )
                 })
            else:
                return jsonify({
                    "success": False,
                    "question": None
                })

        abort(422)

# -----------------------------------------------------------------------#
# Error handlers
# -----------------------------------------------------------------------#

    @app.errorhandler(400)
    def bad_request(error):
        return jsonify({
            'success': False,
            'message': "Bad Request",
            'error': 400
        }), 400

    @app.errorhandler(422)
    def unprocessable(error):
        return jsonify({
            'success': False,
            'message': 'Unprocessable Request',
            'error': 422
        }), 422

    @app.errorhandler(404)
    def not_found(error):
        return jsonify({
            'success': False,
            'message': 'File Not Found',
            'error': 404
        }), 404

    return app


# -*- coding:utf-8 -*-
from . import module
from flask import jsonify
import psutil
import platform
from datetime import datetime
import socket
from unified import MODULES
from lib import logger

def get_system_info():

	datetime_format = '%d-%B-%Y %H:%M:%S (UTC)'

	raw_boot_time = psutil.boot_time()

	boot_time = datetime.utcfromtimestamp(raw_boot_time)
	boot_time = boot_time.strftime(datetime_format)
	hostname = platform.node()

	os_name = '{} {}'.format(platform.system(), platform.release()).strip()
	if 'Linux' in os_name:
	    import distro
	    os_name = ' '.join(distro.linux_distribution()).strip()

	architecture = platform.machine()

	python_version = '{} ver. {}'.format(
	    platform.python_implementation(), platform.python_version()
	)
	data = {}
	data['SERVER_START_TIME'] = boot_time
	data['BUILD_TIME'] = '{BUILD_TIME}'
	data['BUILD_NUMBER'] = '{BUILD_NUMBER}'
	data['HOSTNAME'] = hostname
	data['OS_NAME'] = os_name
	data['ARCHITECTURE'] = architecture
	data['PYTHON_VERSION'] = python_version
	data['PRIVATE_IP'] = socket.gethostbyname(hostname)
	data['MODULES'] = MODULES
	data['MODULES'] = MODULES
	return data

@module.route("/_version", methods=['GET'])
def ping():
    return jsonify(get_system_info())


import os
import pickle
import random

import numpy as np
import pandas as pd
import sklearn.model_selection as ms

class Classific:

    def __init__(self):
        curr_dir = os.path.dirname(os.path.realpath(__file__))
        self.model_rf = pickle.load(open(os.path.join(curr_dir, 'rf_model.pkl'), 'rb'))
        #self.model_dt = pickle.load(open(os.path.join(curr_dir, 'dt_model.pkl'), 'rb'))
        #self.model_svc = pickle.load(open(os.path.join(curr_dir, 'svc_model.pkl'), 'rb'))

    def classify_rf(self, data: pd.DataFrame):
        return self.model_rf.predict(data)

    #def classify_dt(self, data: pd.DataFrame):
    #    return self.model_dt.predict(data)

    #def classify_rf(self, data: pd.DataFrame):
    #    return self.model_svc.predict(data)

    def classify(self, **kwargs):
        data = {
            "gender":      [kwargs['gender']],
            "age":         [kwargs['age']],
            "height":      [kwargs['height']],
            "weight":      [kwargs['weight']],
            "bmi":         [kwargs['bmi']],
            "employ_info": [kwargs['employ_info']],
            "insur_info":  [kwargs['insur_info']],
            "fam_hist":    [kwargs['fam_hist']],
            "med_hist":    [kwargs['med_hist']],
            "med_key":     [kwargs['med_key']]
        }
        data = pd.DataFrame(data)

        result_rf = self.classify_rf(data)
        #result_dt = self.classify_dt(data)
        #result_svc = self.classify_svc(data)

        return result_rf#, result_dt, result_svc


import cv2 as cv
import numpy as np

img = cv.imread('j.png',0)
kernel = np.ones((5,5),np.uint8)
cv.imshow('Original', img)

# erosion
erosion = cv.erode(img, kernel, iterations = 1)
cv.imshow('Erosion', erosion)

# dilation
dilation = cv.dilate(img, kernel, iterations = 1)
cv.imshow('Dilation', dilation)

# opening (erosion followed by dilation, good for removing noise)
opening = cv.morphologyEx(img, cv.MORPH_OPEN, kernel)
cv.imshow('Opening', opening)

# closing (dilation followed by erosion, good for closing holes inside foreground objects)
closing = cv.morphologyEx(img, cv.MORPH_CLOSE, kernel)
cv.imshow('Closing', closing)

# gradient (difference between dilation and erosion of an image)
gradient = cv.morphologyEx(img, cv.MORPH_GRADIENT, kernel)
cv.imshow('Gradient', gradient)

# top hat (difference between input image and opening of the image)
tophat = cv.morphologyEx(img, cv.MORPH_TOPHAT, kernel)
cv.imshow('Top Hat', tophat)

cv.waitKey(0)
cv.destroyAllWindows()


import sys
from string import strip

numero=sys.argv[1]

f=open("solotext2cleaned-div-"+numero+".xml", "r")
f2=open("P_uso-"+numero+".xml", "w")

lineas=0
d=dict()
for line in f.readlines():
	lineas=lineas+1
	palabras=line.split(" ")
	for pal in palabras:
		p=strip(pal)
		if p in d.keys():
			d[p]=d[p]+1
		else:
			d[p]=1

lineas=lineas-1
f.close()
f2.write("Cantidad de lineas: "+repr(lineas)+"\n")
for k in d.keys():
	if k!='':

		f2.write(repr(k)+ "->"+repr(d[k])+"\n")
f2.close()


s1 = "Alice likes Bob"

def reverseWords(str: str) -> str:
        # write your code here

    s = list(str)

    s.reverse()

    def reverseRange(s, l, r):

        while l < r:
            s[l], s[r] = s[r], s[l]
            l, r = l + 1, r - 1

    start = 0

    while True:
        try:
            end = s.index(' ', start)
            reverseRange(s, start, end - 1)
            start = end + 1
        except:
            break

        # Reverse final word
    reverseRange(s, start, len(s) - 1)
    return "".join(s)

print(reverseWords(s1))

# def reverseWordsInSentence(s):

#     finalRes = ""
#     curRes = ""
#     for i in range(len(s) - 1, -1, -1):

#         if s[i] != " ":
#             curRes += s[i]
#         else:
#             finalRes += (curRes[::-1] + s[i])
#             curRes = ""

#     finalRes += curRes[::-1]

#     return finalRes


# -*- coding: utf-8 -*-
"""Folder urls are in a seperate file because they are included with account url."""
from django.conf.urls import url

from documents.views import (RootFolderDetailView, FolderDetailView, SharedFolderView, DocumentAddView,
                             FolderAddView, FolderEditView, FolderDeleteView, FolderLookupView, FolderMoveView,
                             FolderOrderingAjaxView, PermissionDetailView, PermissionAddView, PermissionDeleteView)

app_name = 'folders'

urlpatterns = [
    url(r'^$', RootFolderDetailView.as_view(), name='rootfolder_detail'),
    url(r'^lookup-folder/$', FolderLookupView.as_view(), name='folder_lookup'),
    url(r'^shared/$', SharedFolderView.as_view(), name='shared_folder'),
    url(r'^(?P<slug>[-\w]+)/$', FolderDetailView.as_view(), name='folder_detail'),
    url(r'^(?P<slug>[-\w]+)/add/$', DocumentAddView.as_view(), name='document_add'),
    url(r'^(?P<slug>[-\w]+)/add-folder/$', FolderAddView.as_view(), name='folder_add'),
    url(r'^(?P<slug>[-\w]+)/edit/$', FolderEditView.as_view(), name='folder_edit'),
    url(r'^(?P<slug>[-\w]+)/delete/$', FolderDeleteView.as_view(), name='folder_delete'),
    url(r'^(?P<slug>[-\w]+)/move/$', FolderMoveView.as_view(), name='folder_move'),
    url(r'^(?P<slug>[-\w]+)/share/$', PermissionDetailView.as_view(), name='share_detail'),
    url(r'^(?P<slug>[-\w]+)/share/add/$', PermissionAddView.as_view(), name='share_add'),
    url(r'^(?P<slug>[-\w]+)/share/delete/$', PermissionDeleteView.as_view(), name='share_delete'),
    url(r'^(?P<slug>[-\w]+)/ordering/$', FolderOrderingAjaxView.as_view(), name='folder_ordering'),
]


import math
def mean(data):
    return sum(data)/len(data)
def median(data):
    a=sorted(data)
    b=int(len(data)/2)
    return a[b]
def mode(data):
    k=0
    max_num_occurances=0
    mode=data[0]
    while k<len(data):
        num_occurances=data.count(data[k])
        if num_occurances>max_num_occurances:
            max_num_occurances=num_occurances
            mode=data[k]
        k+=1
    return mode
def variance(data):
    #avg=mean(data)
    #return sum([(a-avg)*(a-avg) for a in data])/len(data)
    return sum([a*a for a in data])/len(data)-sum(data)*sum(data)/(len(data)*len(data))
def standard_deviation(data):
    return math.sqrt(variance(data))
def new_mean(oldmean,n,x):
    return (n*oldmean+x)/(n+1)

house_prices_1=[170000,190000,165000,180000,165000]
house_prices_2=[2400000,125000,148000,160000,110000,325000,180000]
age=[4,3,32,33,4,32,3,38,4,35,36]
nums=[ 5, 9, 100, 9, 97, 6, 9, 98, 9]
nums_2=[3, 9, 3, 8, 2, 9, 1, 9, 2, 4]
nums_3=[3,4,5,6,7]
nums_4=[8,9,10,11,12]
nums_4_1=[8,9,10,11,12,8]
nums_5=[15,20,25,30,35]
nums_6=[3,3,3,3,3]
friends=[17,19,18,17,19]
family=[7,38,4,23,18]
data1=[1,2,5,10,-20,5,5]
data2=[13.04, 1.32, 22.65, 17.44, 29.54, 23.22, 17.65, 10.12, 26.73, 16.43]
list_of_interest=nums_4_1
print("mean:",mean(list_of_interest))
print("new mean",new_mean(mean(list_of_interest),len(list_of_interest),8))
print("median:",median(list_of_interest))
print("mode",mode(list_of_interest))
print("variance",variance(list_of_interest))
print("standard deviation",standard_deviation(list_of_interest))


#!/usr/bin/env python
# -*- coding:utf-8 -*-
"""
@File    :   002_add_two_numbers.py
@Time    :   2020/08/22 16:55
@Author  :   TrenywanG
@Version :   1.0
@Contact :   brtxbrc0521@126.com
"""

# here put the import lib
class ListNode():
    def __init__(self, val):
        if isinstance(val, int):
            self.val = val
            self.next = None

        elif isinstance(val, list):
            self.val = val[0]
            self.next = None
            cur = self
            for i in val[1:]:
                cur.next = ListNode(i)
                cur = cur.next

    # def gatherAttrs(self):
    #     return ", ".join("{}: {}".format(k, getattr(self, k)) for k in self.__dict__.keys())
    #
    # def __str__(self):
    #     return self.__class__.__name__ + " {" + "{}".format(self.gatherAttrs()) + "}"

def printBackward(listnode):
    if listnode == None:return
    printBackward(listnode.next)
    print(listnode.val) #调换以上两句的顺序即可完成顺序打印

class Solution:
    def addTwoNumbers(self, l1, l2, ):
        def list_str(list):
            result = []
            while list:
                result.append(str(list.val))
                list = list.next
            return result

        def str_list(res):
            ResList = ListNode(int(res[0]))
            dummy = ResList
            for i in range(1, len(res)):
                ResList.next = ListNode(int(res[i]))
                ResList = ResList.next
            return dummy

        res1 = ''.join(list_str(l1))[::-1]
        res2 = ''.join(list_str(l2))[::-1]
        res = str(int(res1) + int(res2))[::-1]
        return str_list(res)

if __name__ == "__main__":
    solution1 = Solution()
    l1 = ListNode([2, 4, 3])
    l2 = ListNode([5, 6, 4])
    printBackward(solution1.addTwoNumbers(l1, l2))
    # print(solution1.addTwoNumbers(l1,l2))


user_id1 = ["frodo", "fradi", "crodo", "abc123", "frodoc"]
banned_id1 = ["fr*d*", "abc1**"]
#2
user_id2 = ["frodo", "fradi", "crodo", "abc123", "frodoc"]
banned_id2 = ["*rodo", "*rodo", "******"]
# candidate = [crodo frodo][crodo frodo][abc123 frodoc]

#2
user_id3 = ["frodo", "fradi", "crodo", "abc123", "frodoc"]
banned_id3 =["fr*d*", "*rodo", "******", "******"]
#3

from itertools import product
from collections import deque
import copy

def solution(user_id, banned_id):
    banned_list = []
    for idx in banned_id:
        banned_list.append([idx[i] for i in range(len(idx))])
    cur_stat = True
    candidate_list=[]
    for ban in banned_list:
        temp_list = []
        for user in user_id:
            if len(ban) == len(user):
                for i in range(len(ban)):
                    if ban[i] != user[i]:
                        if ban[i] =='*':
                            continue
                        else:
                            cur_stat=False
                            break
                if cur_stat:
                    temp_list.append(user)
                cur_stat=True
        candidate_list.append(temp_list)
    answer = list(product(*candidate_list))
    finished = False
    temp_ans = copy.deepcopy(answer)
    for i in answer:
        if len(set(i)) != len(i):
            temp_ans.remove(i)

    print(temp_ans)

    temp_temp_ans = []
    # for i in range(len(temp_ans)):
    #     print(temp_ans[i])

    #     temp_temp_ans.append(set(temp_ans[i]))

    for i in range(len(temp_ans)):
        temp_ans[i] = tuple(sorted(temp_ans[i]))
    print(len(set(tuple(temp_ans))))
    return answer

solution(user_id2,banned_id2)


import random

import pygame
from pygame.sprite import AbstractGroup

from core.animated_artefacts import AnimatedArtefact
from core.rl_movement import MovementRight2Left

class Ptera(AnimatedArtefact):

    def __init__(self, *groups: AbstractGroup):
        x, y = random.randrange(750, 1000), random.choice([175, 156, 110])
        image = "sprites/ptera{}.png"
        frames = (1, 2)
        super().__init__(image, x, y, frames, *groups)
        self.images = AnimatedArtefact.adjust_images(self.images, 50, 40)
        self.movement = MovementRight2Left(self, 8, -100)
        self.tick_limit = 3

    def update(self):
        super().update()
        self.movement.move()


import requests
from flask import Blueprint, request, jsonify
from dotenv import load_dotenv
from umami.lib.cms import initialize_firebase
import os

# You might use some configuration function or environment variables to get your Cloudflare API credentials
# from config import get_cloudflare_api_key, get_cloudflare_email

# Create a Flask Blueprint. This allows us to divide our routes into separate files
cdn = Blueprint('cdn', __name__)

load_dotenv()
db = initialize_firebase()

CLOUDFLARE_API_URL = "https://api.cloudflare.com/client/v4/"

def get_headers():
    """
    This function returns the required headers to make API requests to Cloudflare.
    """
    api_key = os.environ["CLOUD_FLARE_STREAM_AND_IMAGES_API_TOKEN"]
    email = os.environ["CLOUDFLARE_EMAIL"]

    headers = {
        "X-Auth-Email": email,
        "X-Auth-Key": api_key,
        "Content-Type": "application/json"
    }

    return headers

@cdn.route('/upload', methods=['POST'])
def upload():
    """
    Upload an image to Cloudflare.
    """
    image = request.files.get('image')

    if not image:
        return jsonify({"error": "No image provided"}), 400

    # Make your API call to Cloudflare to upload the image

    # Return some response
    return jsonify({"message": "Image uploaded successfully"})

@cdn.route('/delete/<image_id>', methods=['DELETE'])
def delete(image_id):
    """
    Delete an image from Cloudflare.
    """
    # Make your API call to Cloudflare to delete the image

    # Return some response
    return jsonify({"message": "Image deleted successfully"})

# More routes and helper functions can be added as needed.
# https://developers.cloudflare.com/api/operations/cloudflare-images-list-images
@cdn.route('/list_images', methods=['GET'])
def list_images():
    url = f"https://api.cloudflare.com/client/v4/accounts/{os.environ['CLOUDFLARE_IMAGES_ACCOUNT_ID']}/images/v1"
    headers = {
        'Authorization': os.environ["CLOUD_FLARE_STREAM_AND_IMAGES_API_TOKEN"],
        'Content-Type': 'application/json'
    }

    response = requests.get(url, headers=headers)

    if response.status_code == 200:
        return jsonify(response.json())
    else:
        return jsonify({"error": "Unable to fetch images"}), response.status_code

@cdn.route('/verify', methods=['GET'])
def verify_token():
    url = "https://api.cloudflare.com/client/v4/user/tokens/verify"
    headers = {
        "Authorization": os.environ["CLOUD_FLARE_STREAM_AND_IMAGES_API_TOKEN"],
        "Content-Type": "application/json",
    }
    response = requests.get(url, headers=headers)
    if response.status_code != 200:
        # Handle error response
        return jsonify({"error": "Failed to verify token"}), response.status_code

    return jsonify(response.json())

# list videos
@cdn.route('/list_videos', methods=['GET'])
def get_videos():
    url = f"https://api.cloudflare.com/client/v4/accounts/{os.environ['CLOUDFLARE_IMAGES_ACCOUNT_ID']}/stream"
    headers = {'Authorization': os.environ["CLOUD_FLARE_STREAM_AND_IMAGES_API_TOKEN"]}
    response = requests.get(url, headers=headers)
    print(response.text)  # Add this line to print the response to the console
    if response.status_code == 200:
        return jsonify(response.json()), 200
    else:
        return jsonify(response.json()), response.status_code


import logging
import requests
from twilio.rest import Client

from .configs.config import TWILLO_FROM, TO_NUMBER, ACCOUNT_SID, AUTH_TOKEN

logger = logging.getLogger(__name__)

def send_twillo_sms_notification(content:str):
    """Sends a sms to default number with a provided message
    * Twillo account required - will use Twillo credit.
    * See [Twillo SMS](https://www.twilio.com/docs/sms) for more information.
    """
    logger.info('Sending text message')
    client = Client(ACCOUNT_SID, AUTH_TOKEN)
    message = client.messages \
                    .create(
                        body = content,
                        from_ = TWILLO_FROM,
                        to = TO_NUMBER
                    )
    logger.info(f"Message Status: {message.status}")

def send_textbelt_sms_notification(content:str):
    """Sends a sms to Env variable number with a provided message
    * Textbelt api offers 1 free text a day with key = 'textbelt'
    * See [Textbelt](https://textbelt.com/) for more details.
    """
    logger.info('Sending text message')
    resp = requests.post('https://textbelt.com/text',
        {
            'phone': TO_NUMBER,
            'message': content,
            'key': 'textbelt',
        }
    )
    logger.info(resp.json())


import torch
# import logging

def get_reward(new_state, game, enemy_alive, pokemon_alive):
    # NEW STATE HAS SIZE 840, FIRST 420 is agent pokemon, next 420 is enemy pokemon
    # SINCE STATE HAS VALUES OF RANGE [0,1], max is 420, min is 0
    # MAX REWARD IS 420, MIN REWARD IS -420, normalize using these values
    agent_weight, enemy_weight = 1,1
    tie_bonus = (new_state.size()[0]/2)/4
    win_bonus = (new_state.size()[0]/2)/2
    lose_bonus = -win_bonus
    # logging.debug('WIN BONUS:' + str(win_bonus))
    # logging.debug('LOSE BONUS:' + str(lose_bonus))
    # logging.debug('TIE BONUS:' + str(tie_bonus))

    agent_state, enemy_state = torch.split(new_state, int(new_state.size()[0]/2))
    # logging.debug('AGENT STATE SIZE: '+ str(agent_state.size()))
    # logging.debug('ENEMY STATE SIZE: ' +  str(enemy_state.size()))
    new_enemy_alive = [1 if enemy_state[i]>0 else 0 for i in range(0,enemy_state.size(0), 62)]
    new_pokemon_alive = [1 if agent_state[i]>0 else 0 for i in range(0, agent_state.size(0), 62)]
    # print(f'New enemy alive: {new_enemy_alive}')

    reward = sum([a-b for a,b in zip(enemy_alive, new_enemy_alive)]) * 0.5
    reward += sum([a-b for a,b in zip(pokemon_alive, new_pokemon_alive)]) * -0.3

    # agent_sum = agent_state.sum()
    # enemy_sum = enemy_state.sum()

    # reward = ((agent_weight*agent_sum) - (enemy_weight*enemy_sum)).item()
    if game.tie: # ADD BONUS
        # reward += tie_bonus
        reward = 0.5
        # reward = torch.clamp(torch.tensor(reward), 0.0, 1.0)
        # return reward
        # logging.debug('RECEIVE TIE BONUS')
    elif game.win == 'p1': # AGENT WON
        # logging.debug('RECEIVE WIN BONUS')
        # reward += win_bonus
        reward = 1
        # reward = torch.clamp(torch.tensor(reward), 0.0, 1.0)
        # return reward
    elif game.win == 'p2': # AGENT LOST
        # logging.debug('RECEIVE LOSE BONUS')
        # reward += lose_bonus
        reward = -1
        # reward = torch.clamp(torch.tensor(reward), 0.0, 1.0)
        # return reward

    # minimum = -int(new_state.size()[0]/2)
    # maximum = int(new_state.size()[0]/2)

    # reward = torch.clamp(torch.tensor(reward), minimum, maximum)
    # reward = (reward - minimum)/(maximum - minimum)

    # logging.debug('REWARD:' + str(reward))
    # print(f'reward: {reward}')
    return torch.tensor(reward, dtype = torch.float), new_enemy_alive, new_pokemon_alive


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('configurations', '0006_auto_20151118_2116'),
        ('profiles', '0027_profilemore'),
    ]

    operations = [
        migrations.AddField(
            model_name='profile',
            name='languages',
            field=models.ManyToManyField(help_text='languages of service help text', verbose_name='languages of service', related_name='services', to='configurations.ServiceLanguage'),
        ),
        migrations.AddField(
            model_name='profile',
            name='tags',
            field=models.ManyToManyField(help_text='tags of service help text', verbose_name='tags of service', related_name='services', to='configurations.ServiceTag'),
        ),
    ]


import pathlib
import unittest

import onnx
from onnx import TensorProto, helper, shape_inference

from ..mobile_helpers.usability_checker import check_shapes
from ..onnx_model_utils import (
    fix_output_shapes,
    get_producer_consumer_maps,
    is_fixed_size_tensor,
    make_dim_param_fixed,
    make_input_shape_fixed,
)

script_dir = pathlib.Path(__file__).parent
ort_root = script_dir.parents[3]

# example usage from <ort root>/tools/python
# python -m unittest util/test/test_onnx_model_utils.py
# NOTE: at least on Windows you must use that as the working directory for all the imports to be happy

class TestGetProducerConsumerMaps(unittest.TestCase):
    @staticmethod
    def _create_model():
        # create a model with subgraphs and various types of shadowing
        body = helper.make_graph(
            [
                # shadow a1 in main graph.
                # LoopAdd_SubgraphOutput should be linked to this and a1 should not be an implicit input
                helper.make_node("Add", ["loop_state_in", "iter"], ["a1"], "LoopAdd_Shadows"),
                # main_graph_initializer should be handled (implicit input but no producer node)
                # graph input 'x' from main graph should also be handled
                helper.make_node("Add", ["main_graph_initializer", "x"], ["a2"], "LoopAdd_OuterScopeInitializer"),
                # implicit input should be handled - 'z' can be accessed from outside scope
                # Add2 in main graph should be implicit input of the Loop node
                helper.make_node("Add", ["z", "a1"], ["a3"], "LoopAdd_ImplicitInput"),
                # create subgraph output
                helper.make_node("Add", ["a2", "a3"], ["loop_state_out"], "LoopAdd_SubgraphOutput"),
            ],
            "Loop_body",
            [
                helper.make_tensor_value_info("iter", TensorProto.INT64, [1]),
                helper.make_tensor_value_info("cond", TensorProto.BOOL, [1]),
                helper.make_tensor_value_info("loop_state_in", TensorProto.FLOAT, [1]),
            ],
            [
                helper.make_tensor_value_info("cond", TensorProto.BOOL, [1]),
                helper.make_tensor_value_info("loop_state_out", TensorProto.FLOAT, [1]),
            ],
            [],
        )

        # Create the main graph
        graph_proto = helper.make_graph(
            [
                # create 'a1' which is shadowed in the subgraph. node should not be joined to Loop1
                helper.make_node("Add", ["x", "y"], ["a1"], "Add1"),
                # create 'z' which is an implicit input to subgraph. node should be joined to Loop1
                helper.make_node("Add", ["a1", "main_graph_initializer"], ["z"], "Add2"),
                # rename 'z' to use as explicit input to Loop
                helper.make_node("Identity", ["z"], ["state_var_in"], "RenameZ"),
                helper.make_node(
                    "Loop", ["max_trip_count", "keep_going", "state_var_in"], ["state_var_out"], "Loop1", body=body
                ),
                helper.make_node("Sub", ["a1", "state_var_out"], ["graph_output"], "sub_1"),
            ],
            "Main_graph",
            [
                helper.make_tensor_value_info("x", TensorProto.FLOAT, [1]),
                helper.make_tensor_value_info("y", TensorProto.FLOAT, [1]),
            ],
            [
                helper.make_tensor_value_info("graph_output", TensorProto.FLOAT, [1]),
            ],
            [
                helper.make_tensor("max_trip_count", TensorProto.INT64, [1], [2]),
                helper.make_tensor("main_graph_initializer", TensorProto.FLOAT, [1], [1.0]),
                helper.make_tensor("keep_going", TensorProto.BOOL, [1], [True]),
            ],
        )

        return helper.make_model(graph_proto)

    def test_model_with_subgraph(self):
        """
        Test a manually created model that has a subgraph and implicit inputs of all possible types.
        """

        model = self._create_model()
        node_to_producers, node_to_consumers = get_producer_consumer_maps(model.graph)

        main_graph_add_create_a1 = model.graph.node[0]
        main_graph_add_create_z = model.graph.node[1]
        main_graph_rename_z = model.graph.node[2]
        main_graph_loop = model.graph.node[3]
        main_graph_sub = model.graph.node[4]

        subgraph = main_graph_loop.attribute[0].g
        loop_add_shadow = subgraph.node[0]
        loop_add_outer_scope_init = subgraph.node[1]
        loop_add_implicit_input = subgraph.node[2]
        loop_add_subgraph_output = subgraph.node[3]

        def node_name(node):
            return f"{node.name}:{node.op_type}"

        def check_linked(producer, consumer):
            self.assertTrue(
                producer in node_to_producers[consumer],
                f"{node_name(producer)} not in producers for {node_name(consumer)}",
            )
            self.assertTrue(
                consumer in node_to_consumers[producer],
                f"{node_name(consumer)} not in consumers for {node_name(producer)}",
            )

        def check_not_linked(producer, consumer):
            self.assertFalse(
                producer in node_to_producers[consumer], f"{node_name(producer)} in producers for {node_name(consumer)}"
            )
            self.assertFalse(
                consumer in node_to_consumers[producer],
                f"{node_name(consumer)} not in consumers for {node_name(producer)}",
            )

        check_linked(main_graph_add_create_a1, main_graph_add_create_z)
        # a1 in main graph shouldn't be implicit input to loop as it is shadowed
        check_not_linked(main_graph_add_create_a1, main_graph_loop)
        # z is implicit input
        check_linked(main_graph_add_create_z, main_graph_loop)
        check_linked(main_graph_rename_z, main_graph_loop)
        check_linked(main_graph_loop, main_graph_sub)

        # check subgraph
        check_linked(loop_add_shadow, loop_add_implicit_input)
        check_linked(loop_add_outer_scope_init, loop_add_subgraph_output)
        check_linked(loop_add_implicit_input, loop_add_subgraph_output)

class TestDynamicDimReplacement(unittest.TestCase):
    def test_replace_symbolic_dim(self):
        """
        Update a model with a single symbolic input dimension. After replacement run shape inferencing to verify that
        all shapes in the model have fixed sizes.
        """
        model_path = (
            ort_root / "onnxruntime" / "test" / "testdata" / "CNTK" / "test_LSTM.tanh.bidirectional" / "model.onnx"
        )

        model = onnx.load_model(str(model_path))

        # validate the expected input after inferring shape info
        m2 = shape_inference.infer_shapes(model, True)
        dynamic_inputs, num_dynamic_values = check_shapes(m2.graph)
        self.assertEqual(len(dynamic_inputs), 1)
        self.assertEqual(dynamic_inputs[0].name, "Input3")
        self.assertGreater(num_dynamic_values, 0)

        # update original model
        make_dim_param_fixed(model.graph, "None", 4)

        # and validate the model no longer has dynamic values
        model = shape_inference.infer_shapes(model, True)
        dynamic_inputs, num_dynamic_values = check_shapes(model.graph)
        self.assertFalse(dynamic_inputs)
        self.assertEqual(num_dynamic_values, 0)

    def test_replace_input_shape(self):
        """
        Replace the entire shape for an input. This can be used when the model has inputs with unknown dimensions
        i.e. the dimension has no value and no symbolic name so it's harder to replace.
        """
        model_path = ort_root / "onnxruntime" / "test" / "testdata" / "gh_issue_9671.onnx"

        model = onnx.load_model(str(model_path))

        # validate the expected input after inferring shape info
        m2 = shape_inference.infer_shapes(model, True)
        dynamic_inputs, num_dynamic_values = check_shapes(m2.graph)
        self.assertEqual(len(dynamic_inputs), 3)
        self.assertEqual(dynamic_inputs[0].name, "X1")
        self.assertEqual(dynamic_inputs[1].name, "X2")
        self.assertEqual(dynamic_inputs[2].name, "X3")
        self.assertGreater(num_dynamic_values, 0)

        # update original model
        make_input_shape_fixed(model.graph, "X1", [2, 2, 4])
        make_input_shape_fixed(model.graph, "X2", [2, 4])
        make_input_shape_fixed(model.graph, "X3", [2, 2, 4])

        # and validate the model no longer has dynamic values
        model = shape_inference.infer_shapes(model, True)
        dynamic_inputs, num_dynamic_values = check_shapes(model.graph)
        self.assertFalse(dynamic_inputs)

    def test_replace_input_shape_with_dim_params(self):
        # replace the input shape where the existing shape also has dim_param entries.
        # in this case we should also iterate the rest of the model and replace other instances
        # of the dim_param with the new value.
        model_path = ort_root / "onnxruntime" / "test" / "testdata" / "fuse_mul_1.onnx"
        model = onnx.load_model(str(model_path))

        m2 = shape_inference.infer_shapes(model, True)
        dynamic_inputs, num_dynamic_values = check_shapes(m2.graph)
        self.assertEqual(len(dynamic_inputs), 1)
        self.assertEqual(dynamic_inputs[0].name, "X1")
        # input as well as other values in model have shape ['D'] so check > 1
        self.assertGreater(num_dynamic_values, 1)

        # replace X1's shape of ['D'] -> [4]
        make_input_shape_fixed(model.graph, "X1", [4])

        # validate the model no longer has dynamic values
        # we don't run shape_inference here as 'D' is the only dimension in the whole model, and we should have
        # replaced every instance of it if _make_input_shape_fixed worked as expected.
        model = shape_inference.infer_shapes(model, True)
        dynamic_inputs, num_dynamic_values = check_shapes(model.graph)
        self.assertFalse(dynamic_inputs)
        self.assertEqual(num_dynamic_values, 0)

    def test_fix_output_shape(self):
        """
        Replace an input shape in a model where that won't update the output shape automatically.
        Manually fix the output so the usage of the model is clearer.
        """
        model_path = ort_root / "onnxruntime" / "test" / "testdata" / "transform" / "fusion" / "bias_gelu_fusion.onnx"
        model = onnx.load_model(str(model_path))

        make_input_shape_fixed(model.graph, "A", [2, 2, 3072])

        # symbolic dim names in graph inputs don't match graph outputs so they won't have been updated yet
        self.assertFalse(is_fixed_size_tensor(model.graph.output[0]))
        fix_output_shapes(model)
        self.assertTrue(is_fixed_size_tensor(model.graph.output[0]))

    def test_invalid_replace_input_shape(self):
        model_path = ort_root / "onnxruntime" / "test" / "testdata" / "sklearn_bin_voting_classifier_soft.onnx"
        model = onnx.load_model(str(model_path))
        # test some invalid usages
        self.assertRaisesRegex(
            ValueError,
            "Rank mismatch. Existing:2 Replacement:3",
            make_input_shape_fixed,
            model.graph,
            "input",
            [1, 2, 3],
        )

        self.assertRaisesRegex(
            ValueError,
            "Can't replace existing fixed size of 2 with 3 for dimension 2",
            make_input_shape_fixed,
            model.graph,
            "input",
            [4, 3],
        )

        self.assertRaisesRegex(
            ValueError, "Input X1 was not found in graph inputs.", make_input_shape_fixed, model.graph, "X1", [2, 3]
        )


# -*- coding: utf-8 -*-

from datetime import datetime

import base64
from io import BytesIO

from odoo import models, fields, api, _
from odoo.exceptions import ValidationError

import xlsxwriter

class CvReportExcelWizard(models.TransientModel):
    _name = 'hr.employee.cv.report.wizard'

    state = fields.Selection([
        ("0", "For One"),
        ("1", "For All")]
        , required=True, default="0", string='Employee Filter')
    employee_ref = fields.Many2one('hr.employee', invisible=1, copy=False, string="Employee", required=True,
                                   states={'1': [('required', False)]})
    def check_cv(self):
        cv_lines = self.env['hr.employee.cv'].search([('employee_ref.id', '=', self.employee_ref.id)])
        if len(cv_lines) == 0:
            raise ValidationError(_("Employee's  CV was not found"))

    def get_excel(self):
        file_name = _('cv report.xlsx')
        fp = BytesIO()

        workbook = xlsxwriter.Workbook(fp)

        self.check_cv()
        self.get_data(workbook)

        workbook.close()
        file_download = base64.b64encode(fp.getvalue())
        fp.close()
        # Two Alternative
        #excel_context = self.with_context(default_name=file_name, default_file_download=file_download)
        # 'context': excel_context._context,
        dict_context = {'default_name': file_name, 'default_file_download': file_download}

        return {
            'name': 'cv report Download',
            'view_type': 'form',
            'view_mode': 'form',
            'res_model': 'hr.employee.cv.report.excel',
            'type': 'ir.actions.act_window',
            'target': 'new',
            'context': dict_context,
        }

    def get_data(self, workbook):
        worksheet = workbook.add_worksheet('CV Report')
        # Template format
        heading_format = workbook.add_format({'align': 'center',
                                              'valign': 'vcenter',
                                              'bold': True, 'size': 14})
        cell_text_format_1 = workbook.add_format({'align': 'center',
                                                  'bold': True, 'size': 9,
                                                  })
        cell_text_format_2 = workbook.add_format({'align': 'center',
                                                  'bold': False, 'size': 9,
                                                  })
        worksheet.set_column('A:A', 20)
        worksheet.set_column('B:B', 20)
        worksheet.set_column('C:C', 20)
        worksheet.set_column('D:D', 20)
        worksheet.set_column('E:E', 20)
        worksheet.set_column('F:F', 20)
        worksheet.set_column('G:G', 20)
        worksheet.set_column('H:H', 20)
        worksheet.set_column('I:I', 20)
        worksheet.set_column('J:J', 20)
        worksheet.set_column('K:K', 20)
        worksheet.set_column('L:L', 20)
        worksheet.set_column('M:M', 20)
        worksheet.set_column('N:N', 20)

        # Write

        today = datetime.today()
        worksheet.merge_range('A1:F2', 'Curriculum Vitae', heading_format)
        worksheet.write('F3', "%s %s %s" % (today.day, today.strftime("%B"), today.year), cell_text_format_1)

        if self.state == "0":
            cv_lines = self.env['hr.employee.cv'].search([('employee_ref.id', '=', self.employee_ref.id)])
        else:
            cv_lines = self.env['hr.employee.cv'].search([])

        cols = 0
        rows = 0
        worksheet.write(6, 0, "Employee", cell_text_format_1)
        worksheet.write(6, 1, "Nationality", cell_text_format_1)
        worksheet.write(6, 2, "Document Type", cell_text_format_1)
        worksheet.write(6, 3, "No.", cell_text_format_1)
        worksheet.write(6, 4, "Gender", cell_text_format_1)
        worksheet.write(6, 5, "Job Position", cell_text_format_1)
        worksheet.write(6, 6, "Email", cell_text_format_1)
        worksheet.write(6, 7, "Mobile Phone", cell_text_format_1)
        worksheet.write(6, 8, "Field of Study", cell_text_format_1)
        worksheet.write(6, 9, "School", cell_text_format_1)
        worksheet.write(6, 10, "State", cell_text_format_1)

        for line in cv_lines:
            worksheet.write(7 + rows, 0, "%s" % line.name if line.name!=False else '', cell_text_format_2)
            worksheet.write(7 + rows, 1, "%s" % line.country_id.name if line.country_id!=False else '', cell_text_format_2)
            worksheet.write(7 + rows, 2, "%s" % line.document_type if line.document_type!=False else '', cell_text_format_2)
            worksheet.write(7 + rows, 3, "%s" % line.identification_id if line.identification_id!=False else '', cell_text_format_2)
            worksheet.write(7 + rows, 4, "%s" % line.gender if line.gender!=False else '', cell_text_format_2)
            worksheet.write(7 + rows, 5, "%s" % line.job_id.name if line.job_id.name!=False else '', cell_text_format_2)
            worksheet.write(7 + rows, 6, "%s" % line.email if line.email!=False else '', cell_text_format_2)
            worksheet.write(7 + rows, 7, "%s" % line.mobile_phone if line.mobile_phone!=False else '', cell_text_format_2)
            if len(line.academic_line_ids) != 0:
                study_field, study_school, state = str(), str(), str()
                for academic_line in line.academic_line_ids:
                    study_field += "%s \n" % academic_line.study_field
                    study_school += "%s \n" % academic_line.study_school
                    state += "%s \n" % academic_line.state
                worksheet.write(7 + rows, 8, "%s" % study_field if study_field != None else "", cell_text_format_2)
                worksheet.write(7 + rows, 9, "%s" % study_school if study_school != None else "", cell_text_format_2)
                worksheet.write(7 + rows, 10, "%s" % state if state != None else "", cell_text_format_2)
                rows += 1
            else:
                rows += 1

class CvReportExcel(models.TransientModel):
    _name = 'hr.employee.cv.report.excel'

    name = fields.Char('File Name', size=256, readonly=True)
    file_download = fields.Binary('Download payroll', readonly=True)


#coding: utf-8
# В backends.py находится другой класс Fix для особо долгоиграющих задач
# Imports {{{1
import webapp2
from time import time as Time
import logging

#}}}
#class ClearCommonProps(webapp2.RequestHandler) {{{1
class ClearCommonProps(webapp2.RequestHandler):
    def get(self):
        # U = User.all()
        # for u in U:
            # u.common_props = []
            # u.put()
        # self.response.write("SUCCESS")
        pass

#class RenameProperty(webapp2.RequestHandler) {{{1
class RenameProperty(webapp2.RequestHandler):
    """Переименовать свойство в какой-нибудь модели. Нужная вещь. Информация подчерпнута здесь:
    http://stackoverflow.com/questions/2906746/updating-model-schema-in-google-app-engine"""

    def get(self):
        T1 = Time()
        U = User.get_by_key_name('VPKulesh@gmail.com')
        UTs = UserTrend.all().ancestor(U)

        for UT in UTs:
            props = UT.dynamic_properties()
            for prop in props:
                if prop.startswith('item'):
                    value = getattr(UT, prop)
                    newProp = 'prop%s' % prop[4:]
                    setattr(UT, newProp, value)
                    delattr(UT, prop)
                    logging.warning("%s=%s ==> %s=%s" % (prop, value, newProp, value))
            UT.put()

        T2 = Time()
        self.response.write("It has been taken %.2f sec." % (T2-T1))

class FixSunAndMoonData(webapp2.RequestHandler): #{{{1
    """Начиная с 23.10.2014 данные по фазе и освещённости Луны собирались неправильно.
    Берём данные из resources/sun_and_moon_data.yaml, который генерируется скриптом
    download_sun_and_moon_data.py
    """
    def get(self): #{{{2
        import yaml
        from datetime import date, time
        from models import CommonCityData

        T1 = Time()

        with open('resources/sun_and_moon_data.yaml') as F:
            DATA = yaml.load(F)
        # Ключи первого уровня в DATA могут соответствовать нескольких объектам Residence
        Rmap = {
            'bangkok': ({'R_keyname':'id_170509', 'dateFrom':date(2014,10,28), 'dateTo':date.today()},),
            'nashville': ({'R_keyname':'id_257387', 'dateFrom':date(2014,10,28), 'dateTo':date.today()},),
            'saint-petersburg': (
                {'R_keyname':'id_7285', 'dateFrom':date(2014,8,12), 'dateTo':date.today()},
                {'R_keyname':'id_1536', 'dateFrom':date(2014,8,12), 'dateTo':date.today()},
                {'R_keyname':'id_3328', 'dateFrom':date(2014,8,12), 'dateTo':date.today()})
        }

        for city, dates in DATA.iteritems():
            if city in Rmap:
                for Rdata in Rmap[city]:
                    R_keyname, dateFrom, dateTo = Rdata['R_keyname'], Rdata['dateFrom'], Rdata['dateTo']
                    for Date, values in dates.iteritems():
                        if dateFrom <= Date <= dateTo:
                            CCD_keyname = '%s-%s' % (R_keyname, Date.strftime('%Y%m%d'))
                            CCD = CommonCityData.get_by_key_name(CCD_keyname)
                            CCD.sunrise = values['sunrise']
                            CCD.sunset = values['sunset']
                            # округление day_duration до минут
                            day_duration = values['day_duration']
                            h, m, s = day_duration.hour, day_duration.minute, day_duration.second
                            if s > 30 or (s==30 and m%2==1):
                                m += 1
                            if m == 60:
                                h += 1
                                m = 0
                            CCD.day_duration = time(h, m)
                            CCD.moonrise = values['moonrise']
                            CCD.moonset = values['moonset']
                            CCD.moon_phase = values['moon_phase']
                            CCD.moon_illuminated = int(round(values['moon_illuminated']))
                            CCD.save()

        T2 = Time()
        self.response.write("It has taken %.2f sec.
\n" % (T2-T1))
        #}}}
    #}}}
#class Kulesh(webapp2.RequestHandler) {{{1
class Kulesh(webapp2.RequestHandler):
    def get(self):
        """Создать пользователя для Кулеша В.П. Создать тестовые параметры и заполнить данными с
        различными распределениями.
        """
        T0 = Time()

        from google.appengine.api import users
        user = users.User("VPKulesh@gmail.com", "gmail.com")
        # Добавление статичных данных пользователя (prof.Profile.post.submit_static)
        city_id = "7285" # Санкт-Петербург
        keyname = "id_" + city_id
        R = Residence.get_by_key_name(keyname)
        # такое место жительства уже есть
        U = User(
            key_name = user.email(),
            user_obj = user,
            nickname = user.nickname(),
            residence = R,
            name = u'Валерий',
            surname = u'Кулеш',
            patronymic = u'Петрович',
            gender = 'M'
        )
        # Отмечаем общие параметры
        U.common_props = [u"solar_radio_flux", u"mean_planetary_A_index", u"mean_planetary_Kp_index", u"temperature", u"cloud_cover", u"precipitation", u"pressure", u"humidity", u"wind_velocity", u"wind_direction", u"day_duration", u"moon_phase", u"moon_illuminated"]
        U.put()
        # Добавляем параметры
        # 4 простых шкальных параметра. Потом будут добавляться id
        props = [
            {"name": u"Равномерное распределение"},
            {"name": u"Нормальное распределение μ=5 σ=1.5"},
            {"name": u"Бета распределение α=0.5 β=0.5"},
            {"name": u"Показательное распределение λ=1"}
        ]
        for prop_dict in props:
            prop = Prop(
                user = U,
                prop_name = prop_dict['name'],
                prop_kind = 'simple'
            )
            prop.put()
            propitem = PropItem(
                user = U,
                prop = prop,
                item_name = prop_dict['name'],
                item_type = 'scale',
                scale_attrs = [0, 10, 1]
            )
            propitem.put()
            # добавляем к словарю id. Нужно на фазе заполнения данными
            prop_dict['id'] = propitem.key().id()
        # 1 сложный параметр с 4 элементами. Потом будут добавляться id
        prop = Prop(
            user = U,
            prop_name = u"Реальные данные",
            prop_kind = "complex"
        )
        prop.put()
        propitems = [
            {"name":u"Средняя температура в Торонто", "type":"float", "measure":u"°C"},
            {"name":u"Курс евро к рублю", "type":"float", "measure":u"руб."},
            {"name":u"Количество активных доменов в зоне UZ", "type":"int", "measure":u"шт."},
            {"name":u"Цена на золото", "type":"float", "measure":u"руб./грамм"}
        ]
        for propitem_dict in propitems:
            propitem = PropItem(
                user = U,
                prop = prop,
                item_name = propitem_dict['name'],
                item_type = propitem_dict['type'],
                item_measure = propitem_dict['measure']
            )
            propitem.put()
            # добавляем к словарю id. Нужно на фазе заполнения данными
            propitem_dict['id'] = propitem.key().id()

        ################################
        # Заполнение тестовыми данными #
        ################################
        import random
        import csv
        # даты и данные по реальным данным хранятся во внешнем файле
        F = open('resources/real_test_data.csv')
        data = {}
        first_row_passed = False
        for row in csv.reader(F, delimiter=','):
            # первую строку пропускаем (заголовки)
            if not first_row_passed:
                first_row_passed = True
                continue
            Date = datetime.strptime(row[0], "%d.%m.%Y")
            chunk = []
            # равномерное распределение
            chunk.append(random.randint(0,10))
            # нормальное распределение
            x = random.normalvariate(mu=5, sigma=1.5)
            x = int(round(x))
            if x < 0: x = 0   # P(x<) = scipy.stats.norm.cdf(-0.5, 5, 1.5) = 1.23e-4
            if x > 10: x = 10 # P(x>10) = 1-scipy.stats.norm.cdf(10.5, 5, 1.5) = 1.23e-4
            chunk.append(x)
            # бета распределение
            x = random.betavariate(alpha=0.5, beta=0.5) * 10
            x = int(round(x))
            chunk.append(x)
            # показательное распределение
            x = random.expovariate(lambd=1)
            x = int(round(x))
            if x > 10: x = 10 # P(x>10) = 1-scipy.stats.expon.cdf(10.5, 1) = 7.5e-5
            chunk.append(x)
            # температура в Торонто
            x = float(row[1])
            chunk.append(x)
            # курс евро к рублю
            if row[2] != '':
                x = float(row[2])
            else:
                x = None
            chunk.append(x)
            # количество доменов
            x = int(row[3])
            chunk.append(x)
            # цена на золото
            if row[4] != '':
                x = float(row[4])
            else:
                x = None
            chunk.append(x)
            # присваиваем кусок данных дате
            data[Date] = chunk
        F.close()

        # Заносим данные из data в хранилище
        for Date, chunk in data.iteritems():
            keyname = U.nickname+"_"+Date.strftime("%Y%m%d")
            UT = UserTrend(key_name=keyname, date=Date.date(), user=U)
            # элементы простых параметров
            for i,prop_dict in enumerate(props):
                item_id_name = "item%d" % prop_dict['id']
                setattr(UT, item_id_name, chunk[i])
            # элементы сложных параметров
            for i,propitem_dict in enumerate(propitems):
                item_id_name = "item%d" % propitem_dict['id']
                if chunk[4+i] is not None:
                    setattr(UT, item_id_name, chunk[4+i])
            # сохранение
            UT.put()

        T1 = Time()
        self.response.write("It has taken %.2f sec.
\n" % (T1-T0))

#class AddTestData(webapp2.RequestHandler) {{{1
class AddTestData(webapp2.RequestHandler):
    def get(self):
        """Добавить тестовые данные.
        * Данные добавляются пользователю TestData. Если его нет, то он создаётся.
        * Если у пользователя TestData есть параметры и тренд, затираем.
        * Город для пользователя выбирается из тех, за которые есть данные на заполняемые даты.
        * Тестовые данные должны содержать значения всех типов, а также пропуски (в тестовых целях).
        """
        from google.appengine.api import users
        import random

        T0 = Time()
        # создаём пользователя, если нет
        U = User.get_by_key_name('TestData@gmail.com')
        if U is None:
            U = User(
                key_name = "TestData@gmail.com",
                user_obj = users.User("TestData@gmail.com"),
                nickname = "TestData",
                residence = Residence.get_by_key_name('id_7285'), # Санкт-Петербург
                common_props = ['solar_radio_flux','mean_planetary_A_index','mean_planetary_Kp_index',
                    'temperature','cloud_cover','precipitation','pressure','humidity','wind_velocity',
                    'wind_direction','day_duration','moon_phase','moon_illuminated'] # все
            )
            U.put()

        # затираем данные пользователя
        db.delete(db.query_descendants(U))

        # создаём тестовые параметры всех типов
        @db.transactional
        def create_simple_prop(name, comment, type_, scale_attrs=[]):
            P = Prop(parent=U, prop_name=name, prop_kind='simple', prop_comment=comment)
            P.put()
            PI = PropItem(parent=U, prop=P, item_name=name, item_comment=comment, item_type=type_)
            PI.item_measure = ''
            if type_ == 'scale':
                PI.scale_attrs = scale_attrs
            PI.put()

        create_simple_prop('Scale', u'Шкаловые значения', 'scale', scale_attrs=[2,5,1])
        create_simple_prop('String', u'Строки', 'str')
        create_simple_prop('Integer', u'Целые числа', 'int')
        create_simple_prop('Float', u'Числа с плавающей точкой', 'float')
        create_simple_prop('Boolean', u'Булевы значения', 'bool')
        create_simple_prop('Time', u'Временные отметки', 'time')

        P = Prop(parent=U, prop_name='Complex', prop_kind='complex', prop_comment=u'Сложный параметр')
        P.put()

        @db.transactional
        def create_element(name, comment, type_, scale_attrs=[]):
            PI = PropItem(parent=U, prop=P, item_name=name, item_comment=comment, item_type=type_)
            PI.item_measure = ''
            if type_ == 'scale':
                PI.scale_attrs = scale_attrs
            PI.put()

        create_element('Scale', u'Шкаловые значения', 'scale', scale_attrs=[0,12,1])
        create_element('String', u'Строки', 'str')
        create_element('Integer', u'Целые числа', 'int')
        create_element('Float', u'Числа с плавающей точкой', 'float')
        create_element('Boolean', u'Булевы значения', 'bool')
        create_element('Time', u'Временные отметки', 'time')

        # заполнение тестовыми данными
        def get_random_value(type_, scale_attrs=[]):
            # 1/6, что значение будет None
            if random.randint(0,5) == 0:
                return None
            if type_ == 'scale':
                choices = range(scale_attrs[0], scale_attrs[1]+scale_attrs[2], scale_attrs[2])
                return random.choice(choices)
            elif type_ == 'str':
                chars = "abcdefghijklmnopqrstuvwxyz"
                string = ""
                for i in range(random.randint(3,6)):
                    string += random.choice(chars)
                return string
            elif type_ == 'int':
                return random.randint(0,100)
            elif type_ == 'float':
                return random.random() * 100
            elif type_ == 'bool':
                return random.choice((True, False))
            elif type_ == 'time':
                return datetime.strptime('%d:%d'%(random.randint(0,23), random.randint(0,59)), '%H:%M')

        Dates = (datetime.strptime('%d.2012'%day, '%j.%Y').date() for day in xrange(1,367)) # список дат
        PIs = PropItem.all().ancestor(U)
        for Date in Dates:
            keyname = "%s_%s" % (U.nickname, Date.strftime("%Y%m%d"))
            ccd_key = "%s-%s" % (U.residence.key().name(), Date.strftime("%Y%m%d"))
            UT = UserTrend(key_name=keyname, parent=U, date=Date, ccd_key=ccd_key)
            for PI in PIs:
                dyn_prop = 'item%d' % PI.key().id()
                value = get_random_value(PI.item_type, PI.scale_attrs)
                setattr(UT, dyn_prop, value)
            UT.put()

        T1 = Time()
        self.response.write("It has been taken %.2f sec
\n" % (T1-T0))

#class FixFalseFromArchive(webapp2.RequestHandler) {{{1
class FixFalseFromArchive(webapp2.RequestHandler):
    def get(self):
        """Скачать архив погоды во всем местам жительства за все даты, заканчивая вчерашней,
        если 'from_archive' == False.
        В данном случае для простоты мы скачиваем архив погоды для всех мест жительства
        с 23.05.2012 по вчерашнюю дату.
        """
        T0 = Time()
        from tasks import RP5Archive
        begin_date = date(2012, 5, 23)
        end_date = date.today() - timedelta(days=1)
        Date = begin_date
        while Date <= end_date:
            logging.warning("Processing date %s" % Date.strftime("%d.%m.%Y"))
            rp5_archive = RP5Archive()
            rp5_archive.get(day=Date)
            Date += timedelta(days=1)

        T1 = Time()
        self.response.write("It has taken %.2f sec." % (T1-T0))

#class MoveFalseToCCDM(webapp2.RequestHandler) {{{1
class MoveFalseToCCDM(webapp2.RequestHandler):
    def get(self):
        """Заполнить модель CommonCityDataMissed используя те объекты модели CommonCityData,
        у которых свойство from_archive==False и дата ранее сегодняшней."""
        today = date.today()
        CCDs = CommonCityData.all().filter('from_archive', False)
        for CCD in CCDs:
            if CCD.date < today:
                keyname = CCD.key().name()
                CCDM = CommonCityDataMissed(key_name=keyname, date=CCD.date, residence=CCD.residence)
                CCDM.put()

        self.response.write("DONE!
\n")

#class AddAstroData(webapp2.RequestHandler) {{{1
class AddAstroData(webapp2.RequestHandler):
    def get(self):
        import csv
        from datetime import datetime

        T0 = Time()
        F = open('resources/planets.csv')
        reader = csv.reader(F, delimiter=';')
        reader.next() # пропуск первой строки
        for row in reader:
            if row[0].startswith('2015'):
                Date = datetime.strptime(row[0], '%Y-%m-%d').date()
                sun_asc = float(row[1])
                sun_dec = float(row[2])
                sun_dis = float(row[3])
                moon_asc = float(row[4])
                moon_dec = float(row[5])
                moon_dis = float(row[6])
                mercury_asc = float(row[7])
                mercury_dec = float(row[8])
                mercury_dis = float(row[9])
                venus_asc = float(row[10])
                venus_dec = float(row[11])
                venus_dis = float(row[12])
                mars_asc = float(row[13])
                mars_dec = float(row[14])
                mars_dis = float(row[15])
                jupiter_asc = float(row[16])
                jupiter_dec = float(row[17])
                jupiter_dis = float(row[18])
                saturn_asc = float(row[19])
                saturn_dec = float(row[20])
                saturn_dis = float(row[21])
                uranus_asc = float(row[22])
                uranus_dec = float(row[23])
                uranus_dis = float(row[24])
                neptune_asc = float(row[25])
                neptune_dec = float(row[26])
                neptune_dis = float(row[27])
                pluto_asc = float(row[28])
                pluto_dec = float(row[29])
                pluto_dis = float(row[30])

                # Заносим в хранилище
                AD = AstroData(key_name=row[0], date=Date)
                AD.sun_asc = sun_asc
                AD.sun_dec = sun_dec
                AD.sun_dis = sun_dis
                AD.moon_asc = moon_asc
                AD.moon_dec = moon_dec
                AD.moon_dis = moon_dis
                AD.mercury_asc = mercury_asc
                AD.mercury_dec = mercury_dec
                AD.mercury_dis = mercury_dis
                AD.venus_asc = venus_asc
                AD.venus_dec = venus_dec
                AD.venus_dis = venus_dis
                AD.mars_asc = mars_asc
                AD.mars_dec = mars_dec
                AD.mars_dis = mars_dis
                AD.jupiter_asc = jupiter_asc
                AD.jupiter_dec = jupiter_dec
                AD.jupiter_dis = jupiter_dis
                AD.saturn_asc = saturn_asc
                AD.saturn_dec = saturn_dec
                AD.saturn_dis = saturn_dis
                AD.uranus_asc = uranus_asc
                AD.uranus_dec = uranus_dec
                AD.uranus_dis = uranus_dis
                AD.neptune_asc = neptune_asc
                AD.neptune_dec = neptune_dec
                AD.neptune_dis = neptune_dis
                AD.pluto_asc = pluto_asc
                AD.pluto_dec = pluto_dec
                AD.pluto_dis = pluto_dis
                AD.put()
        F.close()
        T1 = Time()
        self.response.write("It has been taken %.2f sec." % (T1-T0))

#class PropOrder(webapp2.RequestHandler) {{{1
class PropOrder(webapp2.RequestHandler):
    def get(self):
        """Присваиваем порядок пользовательским параметрам."""
        Us = User.all()
        for U in Us:
            Ps = Prop.all().ancestor(U).order('name')
            order = 1
            for P in Ps:
                P.order = order
                P.put()
                order += 1

        self.response.write("DONE")

#class AddEcData(webapp2.RequestHandler) {{{1
class AddEcData(webapp2.RequestHandler):
    #def get(self) {{{2
    def get(self):
        """Парсим сайт investfunds.ru и вытаскиваем данные с dateStart по dateEnd."""
        #def fetch(url) {{{3
        def fetch(url, **kwargs):
            trials = 3
            try:
                P = urlfetch.fetch(url, **kwargs)
            except DownloadError:
                logging.error("DownloadError with url %s" % url)
                raise
            except DeadlineExceededError:
                logging.error("DeadlineExceededError with url %s" % url)
                raise
            else:
                if P.status_code != 200:
                    logging.error("Status Code %d with url %s" % (P.status_code, url))
                    raise DownloadError
                else:
                    return P
        #}}}

        log = logging.info

        # Imports
        import re
        import lxml.html as lxhtml
        import urllib
        from google.appengine.api import urlfetch
        from google.appengine.api.urlfetch_errors import DownloadError, DeadlineExceededError

        # Переменные
        today = date.today()
        dateStart = date(2014, 9, 30)
        date0 = date(2012, 1, 1)
        dateEnd = today + timedelta(days=1)

        dateRE = re.compile(r'^\d{2}\.\d{2}\.\d{4}$')
        timeRE = re.compile(r'^\d{2}:\d{2}$')

        T0 = Time()

        props = {
            # индексы
            'micex': {'model':'Indice', 'url':'http://stocks.investfunds.ru/indicators/view/216/', 'dv':{}}, #dv=date->value
            'rts': {'model':'Indice', 'url':'http://stocks.investfunds.ru/indicators/view/218/', 'dv':{}},
            'dow': {'model':'Indice', 'url':'http://stocks.investfunds.ru/indicators/view/221/', 'dv':{}},
            'sap500': {'model':'Indice', 'url':'http://stocks.investfunds.ru/indicators/view/222/', 'dv':{}},
            'ftse100': {'model':'Indice', 'url':'http://stocks.investfunds.ru/indicators/view/223/', 'dv':{}},
            'sse': {'model':'Indice', 'url':'http://stocks.investfunds.ru/indicators/view/263/', 'dv':{}},
            # товары
            'oil': {'model':'Commodity', 'url':'http://stocks.investfunds.ru/indicators/view/624/', 'dv':{}},
            'gold': {'model':'Commodity', 'url':'http://stocks.investfunds.ru/indicators/view/224/', 'dv':{}},
            'silver': {'model':'Commodity', 'url':'http://stocks.investfunds.ru/indicators/view/225/', 'dv':{}},
            'aluminium': {'model':'Commodity', 'url':'http://stocks.investfunds.ru/indicators/view/1565/', 'dv':{}},
            # валютные пары
            'usd_rub': {'model':'Currency', 'url':'http://stocks.investfunds.ru/indicators/view/39/', 'dv':{}},
            'eur_rub': {'model':'Currency', 'url':'http://stocks.investfunds.ru/indicators/view/132/', 'dv':{}},
            'eur_usd': {'model':'Currency', 'url':'http://stocks.investfunds.ru/indicators/view/521/', 'dv':{}},
            'gbp_usd': {'model':'Currency', 'url':'http://valuta.investfunds.ru/indicators/view/529/', 'dv':{}},
            'eur_gbp': {'model':'Currency', 'url':'http://valuta.investfunds.ru/indicators/view/522/', 'dv':{}},
            'eur_jpy': {'model':'Currency', 'url':'http://valuta.investfunds.ru/indicators/view/524/', 'dv':{}},
            'usd_jpy': {'model':'Currency', 'url':'http://valuta.investfunds.ru/indicators/view/525/', 'dv':{}}
        }

        # Заполняем словари dv всеми датами, начиная с dateStart
        dates = []
        d = dateStart
        while d <= dateEnd:
            dates.append(d)
            d += timedelta(days=1)

        for prop in props.itervalues():
            for d in dates:
                prop['dv'][d] = None

        for propname, prop in props.iteritems():
            url = prop['url']
            log("Fetching %s data, url=%s" % (propname, url))
            baseUrl = url.split('.ru/')[0] + '.ru'

            # скачиваем базовую страницу индикатора
            P = fetch(url, deadline=30)
            root = lxhtml.fromstring(P.content)
            # Меняем дату "с"
            Ds = root.get_element_by_id('date_start')
            Ds.value = dateStart.strftime('%d.%m.%Y')
            F = root.xpath('//form[@name="sform"]')[0]

            # Отправка формы, перезагрузка страницы
            formValues = F.form_values()
            formData = urllib.urlencode(formValues)
            url = baseUrl + F.action
            P = fetch(url, payload=formData, method=urlfetch.POST, deadline=30,
                headers={'Content-Type':'application/x-www-form-urlencoded'})
            root = lxhtml.fromstring(P.content)

            try:
                pageCount = root.xpath('//select[@id="startlist"]/following-sibling::b')[0].text
            except IndexError: # когда одна страница, полей выбора страниц нет
                pageCount = 1

            # Переключаемся по страницам пока они не закончатся
            while True:
                if pageCount > 1:
                    pageSelected = root.xpath('//select[@id="startlist"]/option[@selected]')[0].text
                else:
                    pageSelected = 1
                log("Page %s of %s" % (pageSelected, pageCount))
                # Парсим таблицу с данными
                trs = root.xpath('(//table[@class="table-data"])[3]/tr')[1:]
                for tr in trs:
                    tds = tr.getchildren()
                    if len(tds) < 2:
                        log("len(tds) < 2")
                        continue
                    td0Text = tds[0].text_content().strip()

                    if dateRE.match(td0Text):
                        Date = datetime.strptime(td0Text, '%d.%m.%Y').date()
                    elif timeRE.match(td0Text):
                        Date = today
                    else:
                        logging.error("td0Text is not date nor time")
                        raise ValueError

                    value = float(tds[1].text.replace(' ', ''))

                    # Добавляем значение в словарь dv
                    prop['dv'][Date] = value

                # Переходим на следующую страницу, если она есть
                if pageCount > 1:
                    tdNext = root.xpath('//table[@summary="Pagination table"]//td[4]')[0]
                    a = tdNext.find('a')
                    if a is None: # страницы закончились
                        break

                    url = baseUrl + a.get('href')
                    P = fetch(url, deadline=30)
                    root = lxhtml.fromstring(P.content)
                    continue
                else:
                    break

        # Заполняем пустые значения в словарях dv
        log("Filling empty values")
        for prop in props.itervalues():
            dv = prop['dv']
            lastValue = None
            for d in dates:
                if dv[d] is None:
                    dv[d] = lastValue
                else:
                    lastValue = dv[d]

        # Заносим данные в хранилище
        log("Saving data into the datastore")
        for d in dates:
            # Данные раньше date0 не заносим
            if d < date0:
                continue
            if d.day == 1:
                log("Saving date %s" % d.isoformat())

            I = Indice.get_or_insert(key_name=d.isoformat(), date=d)
            Com = Commodity.get_or_insert(key_name=d.isoformat(), date=d)
            Cur = Currency.get_or_insert(key_name=d.isoformat(), date=d)

            for propname, prop in props.iteritems():
                value = prop['dv'][d]
                model = prop['model']
                if model == 'Indice':
                    setattr(I, propname, value)
                elif model == 'Commodity':
                    setattr(Com, propname, value)
                elif model == 'Currency':
                    setattr(Cur, propname, value)

            I.save()
            Com.save()
            Cur.save()

        T1 = Time()

        self.response.write("
It has been taken %.2f sec.
\n" % (T1-T0))
        log("AddEcData complete!")

#class AddEnResidencies(webapp2.RequestHandler) {{{1
class AddEnResidencies(webapp2.RequestHandler):
    #def get(self) {{{2
    def get(self):
        """Просто добавляем парочку населённых пунктов с url, ссылающимся на англискую версию.
        Нужны для контроля правильности парсинга погоды."""
        R = Residence(key_name='id_257387', id=257387, name='Nashville', url='http://rp5.ru/257387/en')
        R.put()
        R = Residence(key_name='id_170509', id=170509, name='Bangkok', url='http://rp5.ru/170509/en')
        R.put()
        self.response.write("Done!")

#class AddPropsLimit(webapp2.RequestHandler) {{{1
class AddPropsLimit(webapp2.RequestHandler):
    def get(self):
        Us = User.all()
        for U in Us:
            cPs = Prop.all().ancestor(U).count()
            if cPs > 6:
                propsLimit = cPs
            else:
                propsLimit = 6
            U.propsLimit = propsLimit
            U.put()

        self.response.write("Done!")

class FixMissedData(webapp2.RequestHandler): #{{{1
    def get(self): #{{{2
        import lxml.html as LH
        import re

        phase_map = {
            'New moon': u'Новолуние',
            'Waxing crescent moon': u'Растущий серп',
            'First quarter moon': u'Первая четверть',
            'Waxing gibbous moon': u'Растущая луна',
            'Full moon': u'Полнолуние',
            'Waning gibbous moon': u'Убывающая луна',
            'Last quarter moon': u'Последняя четверть',
            'Waning crescent moon': u'Убывающий серп'
        }
        data = {}

        CCDPs = CommonCityDataPages.all()
        for CCDP in CCDPs:
            keyname = CCDP.key().name()
            id = keyname.split('_')[1].split('-')[0]
            if id not in data:
                data[id] = {}
            location = data[id]
            html = CCDP.html
            P = LH.fromstring(html)
            lang = P.get('lang')
            forecastTable = P.find('.//table[@id="forecastTable"]')
            trs = forecastTable.getchildren()
            Ds = []

            def td_groups(tr): #{{{3
                groups = []

                tds = tr.getchildren()[1:]
                col = 0
                for td in tds:
                    colspan = int(td.get('colspan', '1'))
                    if col == 0 or (col-firstColspan)%usualColspan == 0:
                        groups.append([])
                    groups[-1].append(td)
                    col += colspan

                return groups
            #}}}

            # Строка с датами
            tr = trs[0]
            tds = tr.getchildren()[1:]
            # для td_groups
            firstColspan = int(tds[0].get('colspan'))
            usualColspan = int(tds[1].get('colspan'))

            for td in tds:
                text = td.find('.//span[@class="monthDay"]').text
                if lang == 'ru':
                    day = int(text.split()[0]) # 28 октября
                elif lang == 'en':
                    day = int(text.split()[1].replace('.', '')) # October 28
                if day > 20:
                    month = 1
                else:
                    month = 2
                D = date(2015, month, day)
                if D not in location:
                    location[D] = {}
                Ds.append(D)

            # Строка с восходом и заходом Солнца
            tdGroups = td_groups(trs[10])
            for i,D in enumerate(Ds):
                sunrise = None
                sunset = None
                tdGroup = tdGroups[i]
                for td in tdGroup:
                    text = td.text
                    if not text or text.find(':') == -1: # время не указано
                        pass
                    else:
                        T = datetime.strptime(text, '%H:%M').time()
                        if 'litegrey' in td.get('class').split():   # заход
                            sunset = T
                        else:
                            sunrise = T
                if 'sunrise' not in location[D] or location[D]['sunrise'] is None:
                    location[D]['sunrise'] = sunrise
                if 'sunset' not in location[D] or location[D]['sunset'] is None:
                    location[D]['sunset'] = sunset

            # Строка с восходом и заходом Луны
            tdGroups = td_groups(trs[11])
            for i,D in enumerate(Ds):
                moonrise = None
                moonset = None
                tdGroup = tdGroups[i]
                for td in tdGroup:
                    text = td.text
                    if not text or text.find(':') == -1: # время не указано
                        pass
                    else:
                        T = datetime.strptime(text, '%H:%M').time()
                        if 'litegrey' in td.get('class').split():   # заход
                            moonset = T
                        else:
                            moonrise = T
                if 'moonrise' not in location[D] or location[D]['moonrise'] is None:
                    location[D]['moonrise'] = moonrise
                if 'moonset' not in location[D] or location[D]['moonset'] is None:
                    location[D]['moonset'] = moonset

            # Строка с фазой и освещённостью Луны
            tr = trs[12]
            tds = tr.getchildren()[1:] # количество ячеек равно таковому с датой
            for i in range(len(Ds)):
                td = tds[i]
                div = td.find('div')
                onmouseover = div.get('onmouseover')
                phrase = onmouseover.split("'")[1]
                moon_phase = phrase.split(',')[0]
                if lang == 'en':
                    moon_phase = phase_map[moon_phase]
                moon_illuminated = int(re.search(r'(\d+)%', phrase, re.U).group(1))
                D = Ds[i]
                location[D]['moon_phase'] = moon_phase
                location[D]['moon_illuminated'] = moon_illuminated

        self.response.write(data)

        # day_duration
        for id, location in data.iteritems():
            for D, dayData in location.iteritems():
                if dayData['sunrise'] and dayData['sunset']:
                    sunrise = datetime.strptime(dayData['sunrise'].isoformat(), "%H:%M:%S")
                    sunset = datetime.strptime(dayData['sunset'].isoformat(), "%H:%M:%S")
                    delta = sunset - sunrise
                    dayData['day_duration'] = datetime.strptime(str(delta), '%H:%M:%S').time()
                else:
                    dayData['day_duration'] = None
                keyname = "id_%s-%s" % (id, D.strftime('%Y%m%d'))
                CCD = CommonCityData.get_by_key_name(keyname)
                if CCD:
                    if CCD.sunrise is None and dayData['sunrise'] is not None:
                        CCD.sunrise = dayData['sunrise']
                    if CCD.sunset is None and dayData['sunset'] is not None:
                        CCD.sunset = dayData['sunset']
                    if CCD.moonrise is None and dayData['moonrise'] is not None:
                        CCD.moonrise = dayData['moonrise']
                    if CCD.moonset is None and dayData['moonset'] is not None:
                        CCD.moonset = dayData['moonset']
                    if CCD.day_duration is None and dayData['day_duration'] is not None:
                        CCD.day_duration = dayData['day_duration']
                    if CCD.moon_phase is None and dayData['moon_phase'] is not None:
                        CCD.moon_phase = dayData['moon_phase']
                    if CCD.moon_illuminated is None and dayData['moon_illuminated'] is not None:
                        CCD.moon_illuminated = dayData['moon_illuminated']

                    CCD.put()

        self.response.write(data)

# app = webapp2.WSGIApplication(...) {{{1
app = webapp2.WSGIApplication([
    (r'/quickfix/clear_common_props', ClearCommonProps),
    (r'/quickfix/rename_property', RenameProperty),
    (r'/quickfix/fix_sun_and_moon_data', FixSunAndMoonData),
    (r'/quickfix/kulesh', Kulesh),
    (r'/quickfix/add_test_data', AddTestData),
    (r'/quickfix/fix_false_from_archive', FixFalseFromArchive),
    (r'/quickfix/move_false_to_ccdm', MoveFalseToCCDM),
    (r'/quickfix/add_astro_data', AddAstroData),
    (r'/quickfix/prop_order', PropOrder),
    (r'/quickfix/add_ec_data', AddEcData),
    (r'/quickfix/add_en_residencies', AddEnResidencies),
    (r'/quickfix/add_props_limit', AddPropsLimit),
    (r'/quickfix/fix_missed_data', FixMissedData),
], debug=True)


"""
https://en.wikipedia.org/wiki/Strongly_connected_component
A directed graph is said to be strongly connected if every vertex is reachable from every other vertex.
A strongly connected component (SCC) of a directed graph is a maximal strongly connected subgraph.

O(V + E) to find all the strongly connected components
https://en.wikipedia.org/wiki/Kosaraju%27s_algorithm
https://www.geeksforgeeks.org/strongly-connected-components/
https://cp-algorithms.com/graph/strongly-connected-components.html

SPOJ CAPCITY
"""

from typing import List

def Kosaraju(g: List[List[int]]) -> List[int]:
    """
    Kosaraju's algorithm O(V + E)
    g: directed graph, graph[u] = [v where u -> v]
    return the number of scc and the partition of vertices
    """

    n = len(g)
    order, visited = [], [False] * n
    for u in range(n):
        if visited[u]: continue
        visited[u] = True
        stack = [(u, 0)]
        while stack:  # use stack instead of dfs to avoid stack overflow from deep recursion
            cur, iter = stack.pop()
            if iter < len(g[cur]):
                stack.append((cur, iter + 1))
                v = g[cur][iter]
                if not visited[v]:
                    visited[v] = True
                    stack.append((v, 0))
            else:
                order.append(cur)
    gt = [[] for _ in range(n)]  # transpose of graph
    for u in range(n):
        for v in g[u]:
            gt[v].append(u)
    scc, cnt = [-1] * n, 0
    for u in reversed(order):
        if scc[u] >= 0: continue
        scc[u] = cnt
        q = [u]
        while q:
            for v in gt[q.pop()]:
                if scc[v] < 0:
                    scc[v] = cnt
                    q.append(v)
        cnt += 1
    return cnt, scc

'''
https://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm
'''

# To implement