import os
import os.path as osp
import numpy as np
import math
import cv2
import argparse

from paddlex import transforms as T
import paddlex as pdx

# 读数后处理中有把圆形表盘转成矩形的操作，矩形的宽即为圆形的外周长
# 因此要求表盘图像大小为固定大小，这里设置为[512, 512]
METER_SHAPE = [512, 512]  # 高x宽
# 圆形表盘的中心点
CIRCLE_CENTER = [256, 256]  # 高x宽
# 圆形表盘的半径
CIRCLE_RADIUS = 250
# 圆周率
PI = 3.1415926536
# 在把圆形表盘转成矩形后矩形的高
# 当前设置值约为半径的一半，原因是：圆形表盘的中心区域除了指针根部就是背景了
# 我们只需要把外围的刻度、指针的尖部保存下来就可以定位出指针指向的刻度
RECTANGLE_HEIGHT = 120
# 矩形表盘的宽，即圆形表盘的外周长
RECTANGLE_WIDTH = 1570
# 当前案例中只使用了两种类型的表盘，第一种表盘的刻度根数为50
# 第二种表盘的刻度根数为32。因此，我们通过预测的刻度根数来判断表盘类型
# 刻度根数超过阈值的即为第一种，否则是第二种
TYPE_THRESHOLD = 40
# 两种表盘的配置信息，包含每根刻度的值，量程，单位
METER_CONFIG = [{
    'scale_interval_value': 25.0 / 50.0,
    'range': 25.0,
    'unit': "(MPa)"
}, {
    'scale_interval_value': 1.6 / 32.0,
    'range': 1.6,
    'unit': "(MPa)"
}]
# 分割模型预测类别id与类别名的对应关系
SEG_CNAME2CLSID = {'background': 0, 'pointer': 1, 'scale': 2}

def parse_args():
    parser = argparse.ArgumentParser(description='Meter Reader Infering')
    parser.add_argument(
        '--det_model_dir',
        dest='det_model_dir',
        help='The directory of the detection model',
        type=str)
    parser.add_argument(
        '--seg_model_dir',
        dest='seg_model_dir',
        help='The directory of the segmentation model',
        type=str)
    parser.add_argument(
        '--image_dir',
        dest='image_dir',
        help='The directory of images to be inferred',
        type=str,
        default=None)
    parser.add_argument(
        '--image',
        dest='image',
        help='The image to be inferred',
        type=str,
        default=None)
    parser.add_argument(
        '--use_erode',
        dest='use_erode',
        help='Whether erode the lable map predicted from a segmentation model',
        action='store_true')
    parser.add_argument(
        '--erode_kernel',
        dest='erode_kernel',
        help='Erode kernel size',
        type=int,
        default=4)
    parser.add_argument(
        '--save_dir',
        dest='save_dir',
        help='The directory for saving the predicted results',
        type=str,
        default='./output/result')
    parser.add_argument(
        '--score_threshold',
        dest='score_threshold',
        help="Predicted bounding boxes whose scores are lower than this threshlod are filtered",
        type=float,
        default=0.5)
    parser.add_argument(
        '--seg_batch_size',
        dest='seg_batch_size',
        help="The number of images fed into the segmentation model during one forward propagation",
        type=int,
        default=2)

    return parser.parse_args()

def is_pic(img_name):
    """判断是否是图片

    参数：
        img_name (str): 图片路径

    返回：
        flag (bool): 判断值。
    """
    valid_suffix = ['JPEG', 'jpeg', 'JPG', 'jpg', 'BMP', 'bmp', 'PNG', 'png']
    suffix = img_name.split('.')[-1]
    flag = True
    if suffix not in valid_suffix:
        flag = False
    return flag

class MeterReader:
    """检测表盘的位置，分割各表盘内刻度和指针的位置，并根据分割结果计算出各表盘的读数

    参数：
        det_model_dir (str): 用于定位表盘的检测模型所在路径。
        seg_model_dir (str): 用于分割刻度和指针的分割模型所在路径。

    """

    def __init__(self, det_model_dir, seg_model_dir):
        if not osp.exists(det_model_dir):
            raise Exception("Model path {} does not exist".format(
                det_model_dir))
        if not osp.exists(seg_model_dir):
            raise Exception("Model path {} does not exist".format(
                seg_model_dir))
        self.detector = pdx.load_model(det_model_dir)
        self.segmenter = pdx.load_model(seg_model_dir)

    def decode(self, img_file):
        """图像解码

        参数：
            img_file (str|np.array): 图像路径，或者是已解码的BGR图像数组。

        返回：
            img (np.array): BGR图像数组。
        """

        if isinstance(img_file, str):
            img = cv2.imread(img_file).astype('float32')
        else:
            img = img_file.copy()
        return img

    def filter_bboxes(self, det_results, score_threshold):
        """过滤置信度低于阈值的检测框

        参数：
            det_results (list[dict]): 检测模型预测接口的返回值。
            score_threshold (float)：置信度阈值。

        返回：
            filtered_results (list[dict]): 过滤后的检测狂。

        """
        filtered_results = list()
        for res in det_results:
            if res['score'] > score_threshold:
                filtered_results.append(res)
        return filtered_results

    def roi_crop(self, img, det_results):
        """抠取图像上各检测框的图像区域

        参数：
            img (np.array)：BRG图像数组。
            det_results (list[dict]): 检测模型预测接口的返回值。

        返回：
            sub_imgs (list[np.array]): 各检测框的图像区域。

        """
        sub_imgs = []
        for res in det_results:
            # Crop the bbox area
            xmin, ymin, w, h = res['bbox']
            xmin = max(0, int(xmin))
            ymin = max(0, int(ymin))
            xmax = min(img.shape[1], int(xmin + w - 1))
            ymax = min(img.shape[0], int(ymin + h - 1))
            sub_img = img[ymin:(ymax + 1), xmin:(xmax + 1), :]
            sub_imgs.append(sub_img)
        return sub_imgs

    def resize(self, imgs, target_size, interp=cv2.INTER_LINEAR):
        """图像缩放至固定大小

        参数：
            imgs (list[np.array])：批量BGR图像数组。
            target_size (list|tuple)：缩放后的图像大小，格式为[高, 宽]。
            interp (int)：图像差值方法。默认值为cv2.INTER_LINEAR。

        返回：
            resized_imgs (list[np.array])：缩放后的批量BGR图像数组。

        """

        resized_imgs = list()
        for img in imgs:
            img_shape = img.shape
            scale_x = float(target_size[1]) / float(img_shape[1])
            scale_y = float(target_size[0]) / float(img_shape[0])
            resize_img = cv2.resize(
                img, None, None, fx=scale_x, fy=scale_y, interpolation=interp)
            resized_imgs.append(resize_img)
        return resized_imgs

    def seg_predict(self, segmenter, imgs, batch_size):
        """分割模型完成预测

        参数：
            segmenter (pdx.seg.model)：加载后的分割模型。
            imgs (list[np.array])：待预测的输入BGR图像数组。
            batch_size (int): 分割模型前向预测一次时输入图像的批量大小。

        返回：
            seg_results (list[dict]): 输入图像的预测结果。

        """
        seg_results = list()
        num_imgs = len(imgs)
        for i in range(0, num_imgs, batch_size):
            batch = imgs[i:min(num_imgs, i + batch_size)]
            result = segmenter.predict(batch)
            seg_results.extend(result)
        return seg_results

    def erode(self, seg_results, erode_kernel):
        """对分割模型预测结果中label_map做图像腐蚀操作

        参数：
            seg_results (list[dict])：分割模型的预测结果。
            erode_kernel (int): 图像腐蚀的卷积核的大小。

        返回：
            eroded_results (list[dict])：对label_map进行腐蚀后的分割模型预测结果。

        """
        kernel = np.ones((erode_kernel, erode_kernel), np.uint8)
        eroded_results = seg_results
        for i in range(len(seg_results)):
            eroded_results[i]['label_map'] = cv2.erode(
                seg_results[i]['label_map'].astype(np.uint8), kernel)
        return eroded_results

    def circle_to_rectangle(self, seg_results):
        """将圆形表盘的预测结果label_map转换成矩形

        圆形到矩形的计算方法：
            因本案例中两种表盘的刻度起始值都在左下方，故以圆形的中心点为坐标原点，
            从-y轴开始逆时针计算极坐标到x-y坐标的对应关系：
              x = r + r * cos(theta)
              y = r - r * sin(theta)
            注意：
                1. 因为是从-y轴开始逆时针计算，所以r * sin(theta)前有负号。
                2. 还是因为从-y轴开始逆时针计算，所以矩形从上往下对应圆形从外到内，
                   可以想象把圆形从-y轴切开再往左右拉平时，圆形的外围是上面，内围在下面。

        参数：
            seg_results (list[dict])：分割模型的预测结果。

        返回值：
            rectangle_meters (list[np.array])：矩形表盘的预测结果label_map。

        """
        rectangle_meters = list()
        for i, seg_result in enumerate(seg_results):
            label_map = seg_result['label_map']
            # rectangle_meter的大小已经由预先设置的全局变量RECTANGLE_HEIGHT, RECTANGLE_WIDTH决定
            rectangle_meter = np.zeros(
                (RECTANGLE_HEIGHT, RECTANGLE_WIDTH), dtype=np.uint8)
            for row in range(RECTANGLE_HEIGHT):
                for col in range(RECTANGLE_WIDTH):
                    theta = PI * 2 * (col + 1) / RECTANGLE_WIDTH
                    # 矩形从上往下对应圆形从外到内
                    rho = CIRCLE_RADIUS - row - 1
                    y = int(CIRCLE_CENTER[0] + rho * math.cos(theta) + 0.5)
                    x = int(CIRCLE_CENTER[1] - rho * math.sin(theta) + 0.5)
                    rectangle_meter[row, col] = label_map[y, x]
            rectangle_meters.append(rectangle_meter)
        return rectangle_meters

    def rectangle_to_line(self, rectangle_meters):
        """从矩形表盘的预测结果中提取指针和刻度预测结果并沿高度方向压缩成线状格式。

        参数：
            rectangle_meters (list[np.array])：矩形表盘的预测结果label_map。

        返回：
            line_scales (list[np.array])：刻度的线状预测结果。
            line_pointers (list[np.array])：指针的线状预测结果。

        """
        line_scales = list()
        line_pointers = list()

        for rectangle_meter in rectangle_meters:
            height, width = rectangle_meter.shape[0:2]
            line_scale = np.zeros((width), dtype=np.uint8)
            line_pointer = np.zeros((width), dtype=np.uint8)
            for col in range(width):
                for row in range(height):
                    if rectangle_meter[row, col] == SEG_CNAME2CLSID['pointer']:
                        line_pointer[col] += 1
                    elif rectangle_meter[row, col] == SEG_CNAME2CLSID['scale']:
                        line_scale[col] += 1
            line_scales.append(line_scale)
            line_pointers.append(line_pointer)
        return line_scales, line_pointers

    def mean_binarization(self, data_list):
        """对图像进行均值二值化操作

        参数：
            data_list (list[np.array])：待二值化的批量数组。

        返回：
            binaried_data_list (list[np.array])：二值化后的批量数组。

        """
        batch_size = len(data_list)
        binaried_data_list = data_list
        for i in range(batch_size):
            mean_data = np.mean(data_list[i])
            width = data_list[i].shape[0]
            for col in range(width):
                if data_list[i][col] < mean_data:
                    binaried_data_list[i][col] = 0
                else:
                    binaried_data_list[i][col] = 1
        return binaried_data_list

    def locate_scale(self, line_scales):
        """在线状预测结果中找到每根刻度的中心位置

        参数：
            line_scales (list[np.array])：批量的二值化后的刻度线状预测结果。

        返回：
            scale_locations (list[list])：各图像中每根刻度的中心位置。

        """
        batch_size = len(line_scales)
        scale_locations = list()
        for i in range(batch_size):
            line_scale = line_scales[i]
            width = line_scale.shape[0]
            find_start = False
            one_scale_start = 0
            one_scale_end = 0
            locations = list()
            for j in range(width - 1):
                if line_scale[j] > 0 and line_scale[j + 1] > 0:
                    if find_start == False:
                        one_scale_start = j
                        find_start = True
                if find_start:
                    if line_scale[j] == 0 and line_scale[j + 1] == 0:
                        one_scale_end = j - 1
                        one_scale_location = (
                            one_scale_start + one_scale_end) / 2
                        locations.append(one_scale_location)
                        one_scale_start = 0
                        one_scale_end = 0
                        find_start = False
            scale_locations.append(locations)
        return scale_locations

    def locate_pointer(self, line_pointers):
        """在线状预测结果中找到指针的中心位置

        参数：
            line_scales (list[np.array])：批量的指针线状预测结果。

        返回：
            scale_locations (list[list])：各图像中指针的中心位置。

        """
        batch_size = len(line_pointers)
        pointer_locations = list()
        for i in range(batch_size):
            line_pointer = line_pointers[i]
            find_start = False
            pointer_start = 0
            pointer_end = 0
            location = 0
            width = line_pointer.shape[0]
            for j in range(width - 1):
                if line_pointer[j] > 0 and line_pointer[j + 1] > 0:
                    if find_start == False:
                        pointer_start = j
                        find_start = True
                if find_start:
                    if line_pointer[j] == 0 and line_pointer[j + 1] == 0:
                        pointer_end = j - 1
                        location = (pointer_start + pointer_end) / 2
                        find_start = False
                        break
            pointer_locations.append(location)
        return pointer_locations

    def get_relative_location(self, scale_locations, pointer_locations):
        """找到指针指向了第几根刻度

        参数：
            scale_locations (list[list])：批量的每根刻度的中心点位置。
            pointer_locations (list[list])：批量的指针的中心点位置。

        返回：
            pointed_scales (list[dict])：每个表的结果组成的list。每个表的结果由字典表示，
                字典有两个关键词：'num_scales'、'pointed_scale'，分别表示预测的刻度根数、
                预测的指针指向了第几根刻度。

        """

        pointed_scales = list()
        for scale_location, pointer_location in zip(scale_locations,
                                                    pointer_locations):
            num_scales = len(scale_location)
            pointed_scale = -1
            if num_scales > 0:
                for i in range(num_scales - 1):
                    if scale_location[
                            i] <= pointer_location and pointer_location < scale_location[
                                i + 1]:
                        pointed_scale = i + (
                            pointer_location - scale_location[i]
                        ) / (scale_location[i + 1] - scale_location[i] + 1e-05
                             ) + 1
            result = {'num_scales': num_scales, 'pointed_scale': pointed_scale}
            pointed_scales.append(result)
        return pointed_scales

    def calculate_reading(self, pointed_scales):
        """根据刻度的间隔值和指针指向的刻度根数计算表盘的读数
        """
        readings = list()
        batch_size = len(pointed_scales)
        for i in range(batch_size):
            pointed_scale = pointed_scales[i]
            # 刻度根数大于阈值的为第一种表盘
            if pointed_scale['num_scales'] > TYPE_THRESHOLD:
                reading = pointed_scale['pointed_scale'] * METER_CONFIG[0][
                    'scale_interval_value']
            else:
                reading = pointed_scale['pointed_scale'] * METER_CONFIG[1][
                    'scale_interval_value']
            readings.append(reading)

        return readings

    def get_meter_reading(self, seg_results):
        """对分割结果进行读数后处理得到各表盘的读数

        参数：
            seg_results (list[dict]): 分割模型的预测结果。

        返回：
            meter_readings (list[dcit]): 各表盘的读数。

        """

        rectangle_meters = self.circle_to_rectangle(seg_results)
        line_scales, line_pointers = self.rectangle_to_line(rectangle_meters)
        binaried_scales = self.mean_binarization(line_scales)
        binaried_pointers = self.mean_binarization(line_pointers)
        scale_locations = self.locate_scale(binaried_scales)
        pointer_locations = self.locate_pointer(binaried_pointers)
        pointed_scales = self.get_relative_location(scale_locations,
                                                    pointer_locations)
        meter_readings = self.calculate_reading(pointed_scales)
        return meter_readings

    def print_meter_readings(self, meter_readings):
        """打印各表盘的读数

        参数：
            meter_readings (list[dict])：各表盘的读数
        """
        for i in range(len(meter_readings)):
            print("Meter {}: {}".format(i + 1, meter_readings[i]))

    def visualize(self, img, det_results, meter_readings, save_dir="./"):
        """可视化图像中各表盘的位置和读数

        参数：
            img (str|np.array): 图像路径，或者是已解码的BGR图像数组。
            det_results (dict): 检测模型的预测结果。
            meter_readings (list): 各表盘的读数。
            save_dir (str)：可视化后的图片保存路径。

        """
        vis_results = list()
        for i, res in enumerate(det_results):
            # 将检测结果中的关键词`score`替换成读数，就可以调用pdx.det.visualize画图了
            res['score'] = meter_readings[i]
            vis_results.append(res)
        # 检测结果可视化时会滤除score低于threshold的框，这里读数都是>=-1的，所以设置thresh=-1
        pdx.det.visualize(img, vis_results, threshold=-1, save_dir=save_dir)

    def predict(self,
                img_file,
                save_dir='./',
                use_erode=True,
                erode_kernel=4,
                score_threshold=0.5,
                seg_batch_size=2):
        """检测图像中的表盘，而后分割出各表盘中的指针和刻度，对分割结果进行读数后处理后得到各表盘的读数。

        参数：
            img_file (str)：待预测的图片路径。
            save_dir (str): 可视化结果的保存路径。
            use_erode (bool, optional): 是否对分割预测结果做图像腐蚀。默认值：True。
            erode_kernel (int, optional): 图像腐蚀的卷积核大小。默认值: 4。
            score_threshold (float, optional): 用于滤除检测框的置信度阈值。默认值：0.5。
            seg_batch_size (int, optional)：分割模型前向推理一次时输入表盘图像的批量大小。默认值为：2。
        """

        img = self.decode(img_file)
        det_results = self.detector.predict(img)
        filtered_results = self.filter_bboxes(det_results, score_threshold)
        sub_imgs = self.roi_crop(img, filtered_results)
        sub_imgs = self.resize(sub_imgs, METER_SHAPE)
        seg_results = self.seg_predict(self.segmenter, sub_imgs,
                                       seg_batch_size)
        seg_results = self.erode(seg_results, erode_kernel)
        meter_readings = self.get_meter_reading(seg_results)
        self.print_meter_readings(meter_readings)
        self.visualize(img, filtered_results, meter_readings, save_dir)

def infer(args):
    image_lists = list()
    if args.image is not None:
        if not osp.exists(args.image):
            raise Exception("Image {} does not exist.".format(args.image))
        if not is_pic(args.image):
            raise Exception("{} is not a picture.".format(args.image))
        image_lists.append(args.image)
    elif args.image_dir is not None:
        if not osp.exists(args.image_dir):
            raise Exception("Directory {} does not exist.".format(
                args.image_dir))
        for im_file in os.listdir(args.image_dir):
            if not is_pic(im_file):
                continue
            im_file = osp.join(args.image_dir, im_file)
            image_lists.append(im_file)

    meter_reader = MeterReader(args.det_model_dir, args.seg_model_dir)
    if len(image_lists) > 0:
        for image in image_lists:
            meter_reader.predict(image, args.save_dir, args.use_erode,
                                 args.erode_kernel, args.score_threshold,
                                 args.seg_batch_size)

if __name__ == '__main__':
    args = parse_args()
    infer(args)


import logging
from bdeep.context import getJobArgs, getConfig

log = logging.getLogger("BDEEP")

args = getJobArgs()
config = getConfig()
environment = config["environment"]

log.debug("Hello, %s. We're running in %s", args["name"], environment)


import numpy as np
import cv2
import math

# read input
img = cv2.imread("images/image1.jpeg")
img2 = cv2.imread("images/image.jpeg")
print(img)
print(img.shape[:2])
hh, ww = [x * 2 for x in img.shape[:2]]
print(hh)
print(ww)

# specify input coordinates for corners of red quadrilateral in order TL, TR, BR, BL as x,
input = np.float32([[512, 634], [1490, 644], [1914,878], [108,832]])

# get top and left dimensions and set to output dimensions of red rectangle
width = round(math.hypot(input[0,0]-input[1,0], input[0,1]-input[1,1]))
height = round(math.hypot(input[0,0]-input[3,0], input[0,1]-input[3,1]))
print("width:",width, "height:",height)

# set upper left coordinates for output rectangle
x = input[0,0]
y = input[0,1]

# specify output coordinates for corners of red quadrilateral in order TL, TR, BR, BL as x,
output = np.float32([[x,y], [x+width-1,y], [x+width-1,y+height-1], [x,y+height-1]])

# compute perspective matrix
matrix = cv2.getPerspectiveTransform(input,output)
print(matrix)

# do perspective transformation setting area outside input to black
# Note that output size is the same as the input image size
imgOutput = cv2.warpPerspective(img, matrix, (ww,hh), cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=(0,0,0))

# save the warped output
cv2.imwrite("output_images/output_"+"image"+".jpeg", imgOutput)

imgOutput2 = cv2.warpPerspective(img2, matrix, (ww,hh), cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=(0,0,0))

# save the warped output
cv2.imwrite("output_images/output_"+"image2"+".jpeg", imgOutput2)

# show the result
cv2.imshow("result", imgOutput2)
cv2.waitKey(0)
cv2.destroyAllWindows()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/9/9/009 23:27
# @Author  : Woe
# @Site    :
# @File    : Consumer.py
# @Software: PyCharm

import asyncio

class response:
    def __init__(self,body,urls):
        self.body = body
        self.urls = urls

    def body(self):
        return self.body

    def urls(self):
        return self.urls

class BaseTask(object):
    task_config = {
        'bitch_size': 50,
        'timehout': 5,
    }

    def __init__(self, urls):
        self.urls = urls

    @classmethod
    def consumer_html(self, queue: object, requests_queue: object, fields: object) -> object:
        """
        :param queue:
        :return:
        """
        asyncio.set_event_loop(asyncio.new_event_loop())
        loop = asyncio.get_event_loop()
        task = []
        while 1:
            try:
                res = queue.get(block=True, timeout=self.task_config['timehout'])
                item = response(body=res[1],urls=requests_queue)
                task.append(asyncio.ensure_future(fields[res[0]].parse(item)))
                if len(task) > self.task_config['bitch_size']:
                    loop.run_until_complete(asyncio.wait(task))
                    task = []
            except Exception as e:
                if task != []:
                    loop.run_until_complete(asyncio.wait(task))

    @classmethod
    async def parse(self, res):
        """
        :param res: 返回的html
        :param requests_queue: 需要再进行解析的queue　参数为(url,topic)
        :return:
        """
        from MpsiSpider.utils.logger import get_logger
        logger = get_logger('Task')
        logger.error('please set parse (self,res)')


#!/usr/bin/env python3
""" binomial distribution file"""

class Binomial:
    """ the binomial distribution class """

    def __init__(self, data=None, n=1., p=0.5):
        """constructor for binomial distribution
           n trials, p prob for success
        """
        if data is None:
            self.n = int(n)
            self.p = float(p)
            if n <= 0:
                raise ValueError("n must be a positive value")
            self.n = int(n)
            if p <= 0 or p >= 1:
                raise ValueError("p must be greater than 0 and less than 1")
        else:
            if not isinstance(data, list):
                raise TypeError("data must be a list")
            if len(data) < 2:
                raise ValueError("data must contain multiple values")
            mean = sum(data) / len(data)
            variance = 0
            for number in data:
                variance += (number - mean) ** 2
            variance /= (len(data))
            stddev = variance ** .5
            q = variance / mean
            p = 1 - q
            n = round(mean / p)
            self.p = mean / n
            self.n = n

    def pmf(self, k):
        """calculates pmf for k successes"""
        if not isinstance(k, int):
            k = int(k)
        if k < 0 or k > self.n:
            return 0
        n = self.n
        p = self.p
        nfact = 1
        for i in range(k, n + 1):
            nfact *= i
        kfact = 1
        for i in range(1, n + 1):
            kfact *= i * p
        nkfact = 1
        for i in range(1, n - k + 1):
            nkfact *= i * (1 - p)
        # the above mthod causes int overflow and gives wrong answer
        # print(nfact, kfact, nkfact, sep="\n")
        # print("nfact is", nfact)
        # print("nfact should be", Binomial.fact(self.n))
        # print("the combin is:", nfact / (kfact * nkfact))
        # print(p ** k, .6 ** 30)
        nfact = Binomial.fact(self.n)
        kfact = Binomial.fact(k)
        nkfact = Binomial.fact(self.n - k)
        return nfact / kfact / nkfact * self.p ** k * (1 - p) ** (n - k)
        retval = 1
        for i in range(1, n + 1):
            retval *= i
            if i <= k:
                retval /= i
                retval *= self.p
            if i <= n - k:
                retval /= i
                retval *= (1 - self.p)
        return retval

    def cdf(self, k):
        """the cdf for the binomial distribution for k variable """
        if not isinstance(k, int):
            k = int(k)
        if k < 0 or k > self.n:
            return 0
        retval = 0
        for i in range(0, k + 1):
            retval += self.pmf(i)
        return retval

    @staticmethod
    def fact(x):
        """ factorial of x """
        if x <= 1:
            return 1
        return x * Binomial.fact(x - 1)


from sklearn.decomposition import PCA
import matplotlib.pyplot as plt

def reduce_to_k_dim(M, k=2):
    pca = PCA(n_components=k)
    principalComponents = pca.fit_transform(M)
    return principalComponents

def plot_embeddings(M_reduced, word2Ind, words,epoch):
    """ Plot in a scatterplot the embeddings of the words specified in the list "words".
        NOTE: do not plot all the words listed in M_reduced / word2Ind.
        Include a label next to each point.

        Params:
            M_reduced (numpy matrix of shape (number of unique words in the corpus , k)): matrix of k-dimensioal word embeddings
            word2Ind (dict): dictionary that maps word to indices for matrix M
            words (list of strings): words whose embeddings we want to visualize
    """

    word_indices = [word2Ind[word] for word in words]

    for count, i in enumerate(word_indices):
        x = M_reduced[i][0]
        y = M_reduced[i][1]
        plt.scatter(x,y, marker='x', color='red')
        plt.text(x+0.0001, y+0.0001, words[count], fontsize=9)

    plt.savefig('plots/vector_space_epoch_{}.png'.format(epoch))
    print('saved vector_space_epoch_{}.png'.format(epoch))
    plt.show()


# coding: utf-8

"""
    ETSI ISG CIM / NGSI-LD API

    This OAS file describes the NGSI-LD API defined by the ETSI ISG CIM group. This Cross-domain Context Information Management API allows to provide, consume and subscribe to context information in multiple scenarios and involving multiple stakeholders  # noqa: E501

    The version of the OpenAPI document: latest
    Contact: NGSI-LD@etsi.org
    Generated by: https://openapi-generator.tech
"""

from setuptools import setup, find_packages  # noqa: H301

NAME = "ngsi-ld-client"
VERSION = "1.0.0"
# To install the library, run the following
#
# python setup.py install
#
# prerequisite: setuptools
# http://pypi.python.org/pypi/setuptools

REQUIRES = [
  "urllib3 >= 1.15",
  "certifi",
  "python-dateutil",
  "frozendict >= 2.0.3",
]

setup(
    name=NAME,
    version=VERSION,
    description="ETSI ISG CIM / NGSI-LD API",
    author="OpenAPI Generator community",
    author_email="NGSI-LD@etsi.org",
    url="",
    keywords=["OpenAPI", "OpenAPI-Generator", "ETSI ISG CIM / NGSI-LD API"],
    python_requires=">=3.9",
    install_requires=REQUIRES,
    packages=find_packages(exclude=["test", "tests"]),
    include_package_data=True,
    long_description="""\
    This OAS file describes the NGSI-LD API defined by the ETSI ISG CIM group. This Cross-domain Context Information Management API allows to provide, consume and subscribe to context information in multiple scenarios and involving multiple stakeholders  # noqa: E501
    """
)


# -------------------------------------------------------------------------------------------------------------------------------------
# Following code curated for Bio-Diffusion (https://github.com/BioinfoMachineLearning/bio-diffusion):
# -------------------------------------------------------------------------------------------------------------------------------------

import hydra
import signal
import time
import warnings

from importlib.util import find_spec
from pathlib import Path
from typing import Any, Callable, Dict, List, Union

from contextlib import contextmanager
from omegaconf import DictConfig
from pytorch_lightning import Callback
from pytorch_lightning.loggers import LightningLoggerBase
from pytorch_lightning.utilities import rank_zero_only

from src.utils import pylogger, rich_utils

from torchtyping import patch_typeguard
from typeguard import typechecked

patch_typeguard()  # use before @typechecked

log = pylogger.get_pylogger(__name__)

def task_wrapper(task_func: Callable) -> Callable:
    """Optional decorator that wraps the task function in extra utilities.

    Makes multirun more resistant to failure.

    Utilities:
    - Calling the `utils.extras()` before the task is started
    - Calling the `utils.close_loggers()` after the task is finished
    - Logging the exception if occurs
    - Logging the task total execution time
    - Logging the output dir
    """

    def wrap(cfg: DictConfig):

        # apply extra utilities
        extras(cfg)

        # execute the task
        try:
            start_time = time.time()
            metric_dict, object_dict = task_func(cfg=cfg)
        except Exception as ex:
            log.exception("")  # save exception to `.log` file
            raise ex
        finally:
            path = Path(cfg.paths.output_dir, "exec_time.log")
            content = f"'{cfg.task_name}' execution time: {time.time() - start_time} (s)"
            save_file(path, content)  # save task execution time (even if exception occurs)
            close_loggers()  # close loggers (even if exception occurs so multirun won't fail)

        log.info(f"Output dir: {cfg.paths.output_dir}")

        return metric_dict, object_dict

    return wrap

def extras(cfg: DictConfig) -> None:
    """Applies optional utilities before the task is started.

    Utilities:
    - Ignoring python warnings
    - Setting tags from command line
    - Rich config printing
    """

    # return if no `extras` config
    if not cfg.get("extras"):
        log.warning("Extras config not found! <cfg.extras=null>")
        return

    # disable python warnings
    if cfg.extras.get("ignore_warnings"):
        log.info("Disabling python warnings! <cfg.extras.ignore_warnings=True>")
        warnings.filterwarnings("ignore")

    # prompt user to input tags from command line if none are provided in the config
    if cfg.extras.get("enforce_tags"):
        log.info("Enforcing tags! <cfg.extras.enforce_tags=True>")
        rich_utils.enforce_tags(cfg, save_to_file=True)

    # pretty print config tree using Rich library
    if cfg.extras.get("print_config"):
        log.info("Printing config tree with Rich! <cfg.extras.print_config=True>")
        rich_utils.print_config_tree(cfg, resolve=True, save_to_file=True)

@rank_zero_only
def save_file(path: str, content: str) -> None:
    """Save file in rank zero mode (only on one process in multi-GPU setup)."""
    with open(path, "w+") as file:
        file.write(content)

def instantiate_callbacks(callbacks_cfg: DictConfig) -> List[Callback]:
    """Instantiates callbacks from config."""
    callbacks: List[Callback] = []

    if not callbacks_cfg:
        log.warning("Callbacks config is empty.")
        return callbacks

    if not isinstance(callbacks_cfg, DictConfig):
        raise TypeError("Callbacks config must be a DictConfig!")

    for _, cb_conf in callbacks_cfg.items():
        if isinstance(cb_conf, DictConfig) and "_target_" in cb_conf:
            log.info(f"Instantiating callback <{cb_conf._target_}>")
            callbacks.append(hydra.utils.instantiate(cb_conf))

    return callbacks

def instantiate_loggers(logger_cfg: DictConfig) -> List[LightningLoggerBase]:
    """Instantiates loggers from config."""
    logger: List[LightningLoggerBase] = []

    if not logger_cfg:
        log.warning("Logger config is empty.")
        return logger

    if not isinstance(logger_cfg, DictConfig):
        raise TypeError("Logger config must be a DictConfig!")

    for _, lg_conf in logger_cfg.items():
        if isinstance(lg_conf, DictConfig) and "_target_" in lg_conf:
            log.info(f"Instantiating logger <{lg_conf._target_}>")
            logger.append(hydra.utils.instantiate(lg_conf))

    return logger

@rank_zero_only
def log_hyperparameters(object_dict: dict) -> None:
    """Controls which config parts are saved by lightning loggers.

    Additionally saves:
    - Number of model parameters
    """

    hparams = {}

    cfg = object_dict["cfg"]
    model = object_dict["model"]
    trainer = object_dict["trainer"]

    if not trainer.logger:
        log.warning("Logger not found! Skipping hyperparameter logging...")
        return

    hparams["model"] = cfg["model"]

    # save number of model parameters
    hparams["model/params/total"] = sum(p.numel() for p in model.parameters())
    hparams["model/params/trainable"] = sum(
        p.numel() for p in model.parameters() if p.requires_grad
    )
    hparams["model/params/non_trainable"] = sum(
        p.numel() for p in model.parameters() if not p.requires_grad
    )

    hparams["datamodule"] = cfg["datamodule"]
    hparams["trainer"] = cfg["trainer"]

    hparams["callbacks"] = cfg.get("callbacks")
    hparams["extras"] = cfg.get("extras")

    hparams["task_name"] = cfg.get("task_name")
    hparams["tags"] = cfg.get("tags")
    hparams["ckpt_path"] = cfg.get("ckpt_path")
    hparams["seed"] = cfg.get("seed")

    # send hparams to all loggers
    trainer.logger.log_hyperparams(hparams)

def get_metric_value(metric_dict: dict, metric_name: str) -> float:
    """Safely retrieves value of the metric logged in LightningModule."""

    if not metric_name:
        log.info("Metric name is None! Skipping metric value retrieval...")
        return None

    if metric_name not in metric_dict:
        raise Exception(
            f"Metric value not found! <metric_name={metric_name}>\n"
            "Make sure metric name logged in LightningModule is correct!\n"
            "Make sure `optimized_metric` name in `hparams_search` config is correct!"
        )

    metric_value = metric_dict[metric_name].item()
    log.info(f"Retrieved metric value! <{metric_name}={metric_value}>")

    return metric_value

def close_loggers() -> None:
    """Makes sure all loggers closed properly (prevents logging failure during multirun)."""

    log.info("Closing loggers...")

    if find_spec("wandb"):  # if wandb is installed
        import wandb

        if wandb.run:
            log.info("Closing wandb!")
            wandb.finish()

@typechecked
def replace_dict_str_values_unconditionally(
    obj: Any,
    unconditional_key_value_replacements: Dict[str, str]
) -> Any:
    """Using a replacement mapping, replace all string values found in `obj` recursively."""
    if isinstance(obj, dict):
        return {key: replace_dict_str_values_unconditionally(val, unconditional_key_value_replacements) for key, val in obj.items()}
    elif isinstance(obj, list):
        if len(obj) > 0 and isinstance(obj[0], str):
            new_obj_list = []
            for o in obj:
                filled_obj = o
                for u, r in unconditional_key_value_replacements.items():
                    filled_obj = filled_obj.replace(u, r)
                new_obj_list.append(filled_obj)
            return new_obj_list
        elif len(obj) > 0 and isinstance(obj[0], dict):
            return [
                {
                    key: replace_dict_str_values_unconditionally(val, unconditional_key_value_replacements)
                    for key, val in o.items()
                }
                for o in obj
            ]
        else:
            return obj
    elif isinstance(obj, str):
        filled_obj = obj
        for u, r in unconditional_key_value_replacements.items():
            filled_obj = filled_obj.replace(u, r)
        return filled_obj
    else:
        return obj

def replace_dict_str_values_conditionally_for_dicts(
    obj: dict,
    conditional_key_value_replacements: Dict[str, Union[str, List[str]]]
):
    """Using a replacement mapping for dictionaries, recursively replace all string values found in `obj` that correspond to keys in the mapping."""
    for k in obj.keys():
        if k in conditional_key_value_replacements:
            obj[k] = conditional_key_value_replacements[k]
    return {key: replace_dict_str_values_conditionally(val, conditional_key_value_replacements) for key, val in obj.items()}

@typechecked
def replace_dict_str_values_conditionally(
    obj: Any,
    conditional_key_value_replacements: Dict[str, Union[str, List[str]]]
) -> Any:
    """Using a replacement mapping, recursively replace all string values found in `obj` that correspond to keys in the mapping."""
    if isinstance(obj, dict):
        return replace_dict_str_values_conditionally_for_dicts(
            obj, conditional_key_value_replacements=conditional_key_value_replacements
        )
    elif isinstance(obj, list):
        new_obj_list = []
        for o in obj:
            if isinstance(o, dict):
                new_obj_list.append(
                    replace_dict_str_values_conditionally_for_dicts(
                        o, conditional_key_value_replacements=conditional_key_value_replacements
                    )
                )
            else:
                new_obj_list.append(o)
        return new_obj_list
    else:
        return obj

class TimeoutException(Exception):
    pass

@contextmanager
@typechecked
def time_limit(seconds: int):
    def signal_handler(signum, frame):
        raise TimeoutException("Timed out!")
    signal.signal(signal.SIGALRM, signal_handler)
    signal.alarm(seconds)
    try:
        yield
    finally:
        signal.alarm(0)


import sys
import os

sys.path.append(os.path.abspath("."))
sys.dont_write_bytecode = True

__author__ = "bigfatnoob"

import numpy as np

from utils import logger
from utils.lib import O
from misconceptions.common import mongo_driver, props, differences

LOGGER = logger.get_logger(os.path.basename(__file__.split(".")[0]))

class SemanticSummary(O):
  def __init__(self, **kwargs):
    self.sim_score = None
    self.n_mismatched = 0
    self.size_diff = None
    self.row_diff = None
    self.col_diff = None
    self.n_val1_empty = 0
    self.n_val2_empty = 0
    self.n_both_empty = 0
    O.__init__(self, **kwargs)

  def update_summary(self, diff):
    if diff.message == "Mismatched types":
      self.n_mismatched += 1
      return
    elif isinstance(diff, differences.DataFrameDiffMeta) or isinstance(diff, differences.MatrixDiffMeta):
      if diff.row_diff is not None:
        if not self.row_diff: self.row_diff = []
        self.row_diff.append(diff.row_diff)
      if diff.col_diff is not None:
        if not self.col_diff: self.col_diff = []
        self.col_diff.append(diff.col_diff)
    elif isinstance(diff, differences.ArrayDiffMeta):
      if diff.size_diff is not None:
        if not self.size_diff: self.size_diff = []
        self.size_diff.append(diff.size_diff)
    self.n_both_empty += 1 if diff.is_val1_empty and diff.is_val2_empty else 0
    self.n_val1_empty += 1 if diff.is_val1_empty else 0
    self.n_val2_empty += 1 if diff.is_val2_empty else 0
    if not self.sim_score:
      self.sim_score = []
    self.sim_score.append(diff.sim_score)

  def summarize(self):
    report = {}
    if self.sim_score:
      report["semantic_score"] = np.mean(self.sim_score)
    report["n_mismatched"] = self.n_mismatched
    if self.row_diff:
      report["row_diff"] = np.mean(self.row_diff)
    if self.col_diff:
      report["col_diff"] = np.mean(self.col_diff)
    if self.size_diff:
      report["size_diff"] = np.mean(self.size_diff)
    report["n_both_empty"] = self.n_both_empty
    report["n_val1_empty"] = self.n_val1_empty
    report["n_val2_empty"] = self.n_val2_empty
    return report

def update_semantic_scores(start=0, end=None, log_interval=100, limit=0):
  store = mongo_driver.MongoStore(props.DATASET)
  diff_records = store.load_differences(additional_queries={"n_mismatched": {"$exists": False}}, limit=limit)
  n_records = diff_records.count()
  LOGGER.info("Retrieved %d records ..." % n_records)
  semantic_columns = ["semantic_score", "n_mismatched", "row_diff", "col_diff", "size_diff",
                      "n_both_empty", "n_val1_empty", "n_val2_empty"]
  store.create_semantic_indices(semantic_columns)
  for i, diff_record in enumerate(diff_records):
    if i < start or (end and i >= end):
      continue
    if (i + 1) % log_interval == 0:
      LOGGER.info("Processing semantic difference for %d / %d" % (i+1, n_records))
    summary = SemanticSummary()
    for d in diff_record["diff"]:
      diff = differences.DiffMeta.from_dict(d)
      summary.update_summary(diff)
    query = {"_id": diff_record["_id"]}
    updates = summary.summarize()
    store.update_difference(query, updates)

if __name__ == "__main__":
  update_semantic_scores()


#!/usr/bin/env python
# coding: utf-8

# # MNIST with Recurrent Neural Networks
# This program runs in about 46.033 seconds (Windows 11, i7, 16 GB)

# ## Prerequisites
#
# Install the following packages

try:
    from cloudmesh.common.StopWatch import StopWatch
except:  # noqa: E722
    get_ipython().system(' pip install cloudmesh-common')
    from cloudmesh.common.StopWatch import StopWatch

# ## Exporting Output Graphs

def save(file):
    if os.path.exists("images"):
        pass
    else:
        Shell.mkdir("images")
    plot_model(file, to_file=f'images/rnn-mnist.png', show_shapes=True)
    plot_model(file, to_file=f'images/rnn-mnist.pdf', show_shapes=True)

# ## Import Libraries

StopWatch.start("total")
StopWatch.start("import")
StopWatch.progress(0)

import numpy as np    # noqa: E402
from keras.models import Sequential    # noqa: E402
from keras.layers import Dense, Activation, SimpleRNN    # noqa: E402
from keras.utils import to_categorical, plot_model    # noqa: E402
from keras.datasets import mnist    # noqa: E402
from cloudmesh.common.StopWatch import StopWatch    # noqa: E402
from cloudmesh.common.Shell import Shell    # noqa: E402
import os    # noqa: E402

StopWatch.stop("import")
StopWatch.progress(7)

# ## Download Data and Pre-Process

StopWatch.start("data-load")

(x_train, y_train), (x_test, y_test) = mnist.load_data()

StopWatch.stop("data-load")
StopWatch.progress(8)

StopWatch.start("data-pre-process")

num_labels = len(np.unique(y_train))

y_train = to_categorical(y_train)
y_test = to_categorical(y_test)

image_size = x_train.shape[1]
x_train = np.reshape(x_train,[-1, image_size, image_size])
x_test = np.reshape(x_test,[-1, image_size, image_size])
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255

StopWatch.stop("data-pre-process")
StopWatch.progress(9)

# ## Define Model

StopWatch.start("compile")

input_shape = (image_size, image_size)
batch_size = 128
units = 256
dropout = 0.2

model = Sequential()
model.add(SimpleRNN(units=units,
                    dropout=dropout,
                    input_shape=input_shape, return_sequences=True))
model.add(SimpleRNN(units=units,
                    dropout=dropout,
                    return_sequences=True))
model.add(SimpleRNN(units=units,
                    dropout=dropout,
                    return_sequences=False))
model.add(Dense(num_labels))
model.add(Activation('softmax'))
model.summary()
save(model)

model.compile(loss='categorical_crossentropy',
              optimizer='sgd',
              metrics=['accuracy'])

StopWatch.stop("compile")
StopWatch.progress(10)

# ## Train

StopWatch.start("train")

model.fit(x_train, y_train, epochs=1, batch_size=batch_size)

StopWatch.stop("train")
StopWatch.progress(95)

# ## Test

StopWatch.start("evaluate")

loss, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
print("\nTest accuracy: %.1f%%" % (100.0 * acc))

StopWatch.stop("evaluate")
StopWatch.stop("total")
StopWatch.benchmark()
StopWatch.progress(100)


import pygame as pg
from enum import Enum
from math import sqrt
import neat
import pickle
from game_code.Bird import Bird
from game_code.Spikes import Spikes

class C(Enum):
    WHITE = (255, 255, 255)
    PINK = (220, 0, 63)
    L_BLUE = (179, 255, 255)

class Game_Board(object):
    def __init__(self, win_w, win_h, change_width=0):
        self.scale = 9/10

        self.mid = (win_w/2+change_width/2, win_h/2)

        self.board_h = self.scale*win_h
        self.board_w = self.scale*win_w

        self.left = (1-self.scale)*win_w/2+change_width/2
        self.right = self.left+self.board_w
        self.top = (1-self.scale)*win_h/2
        self.bottom = self.top + self.board_h

    def draw(self, screen):
        screen.fill(C.L_BLUE.value, (self.left, self.top,
                                     self.board_w, self.board_h))

gen = 0

class Game(object):
    def __init__(self):
        pg.font.init()
        self.RUNNING = True
        self.HEIGHT = 825
        self.WIDTH = 525
        self.size_change = 0
        self.g_board = self.spikes = self.bird = None
        self.clock = pg.time.Clock()
        self.screen = None
        icon = pg.image.load('data/Bird.png')
        pg.display.set_icon(icon)

    def run(self):
        '''Main game loop'''
        self.screen = pg.display.set_mode(
            [self.WIDTH, self.HEIGHT], pg.RESIZABLE)
        self.init_objects(self.WIDTH, self.HEIGHT)
        while self.RUNNING:
            self.clock.tick(60)
            self.draw(self.bird.score, self.g_board.mid)
            self.events_handler()

            if self.bird.alive:
                self.bird.move()
                self.bird.collision_check(self.spikes)
            elif not self.bird.alive:
                self.spikes.spikes = self.spikes.initial_spikes()

            pg.display.update()

    def watch_NEAT(self, NNet, config):
        '''Main game loop'''
        NNets = [NNet]
        self.screen = pg.display.set_mode([self.WIDTH, self.HEIGHT])
        self.init_objects(self.WIDTH, self.HEIGHT, True)
        self.bird = Bird(self.g_board, True)

        self.bird.alive = True
        fps = 60
        while self.RUNNING:
            self.clock.tick(fps)
            self.draw(self.bird.score, self.g_board.mid)

            events = pg.event.get()
            for e in events:
                if e.type == pg.QUIT:
                    self.RUNNING = False
                    pg.quit()
                    quit()
                if e.type == pg.KEYDOWN:
                    if e.key == pg.K_UP and fps != 120:
                        fps = fps*2
                    if e.key == pg.K_DOWN and fps != 60:
                        fps = fps/2
            if self.bird.alive:
                self.bird.move()

                prev_score = self.bird.score
                self.bird.collision_check(self.spikes)
                if self.bird.score > prev_score:
                    self.bird.find_best_gap(self.spikes)

                if self.bird.vel < 0:
                    site = self.g_board.left+self.bird.image.get_width()/2
                else:
                    site = self.g_board.right-self.bird.image.get_width()/2

                by = self.bird.y+self.bird.image.get_height()/2
                output = NNets[0].activate(
                    (self.bird.x, by, abs(self.bird.x-site), abs(by-self.bird.best_gap[1])))

                if output[0] > 0.5:
                    self.bird.jump()
            else:
                self.RUNNING = False

            pg.display.update()

    def train_NEAT(self, genomes, config):
        '''Main game loop for AI'''
        self.screen = pg.display.set_mode([self.WIDTH, self.HEIGHT])

        self.init_objects(self.WIDTH, self.HEIGHT, True)
        self.spikes.spikes = self.spikes.initial_spikes()
        self.RUNNING = True
        NNets, ge, self.bird = [], [], []

        for _, g in genomes:
            net = neat.nn.FeedForwardNetwork.create(g, config)
            NNets.append(net)
            self.bird.append(Bird(self.g_board, True))
            g.fitness = 0
            ge.append(g)

        prev_score, site = 0, 0
        fps = 240
        while self.RUNNING:
            self.clock.tick(fps)

            self.draw(self.bird[0].score, self.g_board.mid, True)

            events = pg.event.get()
            for e in events:
                if e.type == pg.QUIT:
                    self.RUNNING = False
                    pg.quit()
                    quit()
                if e.type == pg.KEYDOWN:
                    if e.key == pg.K_UP and fps != 240:
                        fps = fps*2
                    if e.key == pg.K_DOWN and fps != 60:
                        fps = fps/2

            if self.bird[0].score >= 500:
                pickle.dump(NNets[0], open("best.pickle", "wb"))

            image_height = self.bird[0].image.get_height()
            for x, brd in enumerate(self.bird):
                if brd.alive:
                    brd.move()
                    if brd.vel < 0:
                        site = self.g_board.left+brd.image.get_width()/2
                    else:
                        site = self.g_board.right-brd.image.get_width()/2

                    by = brd.y + brd.image.get_height()/2
                    output = NNets[self.bird.index(brd)].activate(
                        (brd.x, by, abs(brd.x-site), abs(by-brd.best_gap[1])))

                    pg.draw.line(self.screen, (0, 255, 0),
                                 (brd.x, by), (site, brd.best_gap[1]))

                    if output[0] > 0.5:
                        brd.jump()

                    prev_score = brd.score
                    if not brd.collision_check(self.spikes, True):
                        self.bird.pop(x)
                        ge[x].fitness -= 7
                        NNets.pop(x)
                        ge.pop(x)
                    else:
                        if brd.score > prev_score:
                            ge[x].fitness += 0.1
                            if by <= brd.best_gap[1]+25 and by >= brd.best_gap[1]-25 and brd.score > 0:
                                ge[x].fitness += 50
                                if brd.score > 20:
                                    ge[x].fitness += brd.score // 20 * 10
                            for brd in self.bird:
                                brd.find_best_gap(self.spikes)

            if len(self.bird) == 0:
                self.RUNNING = False

            pg.display.update()

        global gen
        gen += 1
        print(f"generation: {gen}\t score: {prev_score}")

    def init_objects(self, width, height, AI=False):
        self.g_board = Game_Board(width, height, self.size_change)
        if AI:
            self.spikes = Spikes(self.g_board, True)
        else:
            self.spikes = Spikes(self.g_board)
            self.bird = Bird(self.g_board)

    def events_handler(self):
        events = pg.event.get()
        for e in events:
            if e.type == pg.QUIT:
                self.RUNNING = False
                pg.quit()
                quit()
            if e.type == pg.VIDEORESIZE:
                self.screen = pg.display.set_mode([e.w, e.h], pg.RESIZABLE)
                self.WIDTH, self.HEIGHT = e.w, e.h
                self.size_change = e.w-e.h*7/11  # 7/11 is proportion of the game window
                self.init_objects(self.HEIGHT*7/11, self.HEIGHT)
            if e.type == pg.KEYDOWN:
                if e.key == pg.K_SPACE:
                    self.bird.jump()
                elif e.key == pg.K_ESCAPE:
                    self.RUNNING = False

    def draw(self, score, mid_pt, AI=False):
        pg.display.set_caption(
            f'Spikes Touch NO!     FPS:{int(self.clock.get_fps())}')
        self.screen.fill(C.PINK.value)

        self.g_board.draw(self.screen)
        self.spikes.draw(self.screen)

        font = pg.font.SysFont('techno', int(self.HEIGHT/5))
        score_text = font.render(str(score), True, C.L_BLUE.value)

        pg.draw.circle(self.screen, C.WHITE.value, (int(
            mid_pt[0]), int(mid_pt[1]*0.7)), int(self.HEIGHT/9))
        self.screen.blit(score_text, score_text.get_rect(
            center=(int(mid_pt[0]), int(mid_pt[1] * 0.7))))

        if AI:
            global gen
            font = pg.font.SysFont('techno', int(self.HEIGHT/16))
            gen_text = font.render(f"GEN: {gen}", True, C.WHITE.value)
            self.screen.blit(gen_text, gen_text.get_rect(
                center=(int(self.g_board.mid[0]), int(self.g_board.mid[1] * 0.3))))
            for brd in self.bird:
                brd.draw(self.screen)
        else:
            self.bird.draw(self.screen)


from __future__ import (
    division, print_function, absolute_import, unicode_literals)

import numpy as np
from scipy.ndimage import binary_dilation, label
from scipy.special import erf
from astropy import units as u

def find_signal_in_cube(
        cube, noisecube, mask=None,
        nchan_hi=3, snr_hi=3.5, nchan_lo=2, snr_lo=2,
        prune_by_npix=None, prune_by_fracbeam=1.,
        expand_by_npix=None, expand_by_fracbeam=0.,
        expand_by_nchan=2, verbose=False):
    """
    Identify (positive) signal in a cube based on S/N ratio.

    This is a revised version of the signal identification scheme
    described in Sun et al. (2018, ApJ, 860, 172).

    Here is a description of the scheme:
    1. Generate a 'core mask' by identifying detections with
       S/N >= `snr_hi` over at least `nchan_hi` consecutive channels.
       (if a user-specified mask exists, do this step inside the mask)
    2. Generate a 'wing mask' by identifying detections with
       S/N >= `snr_lo` over at least `nchan_lo` consecutive channels.
       (if a user-specified mask exists, do this step inside the mask)
    3. Dilate the 'core mask' inside the 'wing mask' to get
       a 'signal mask' that defines 'detections'.
    4. Label 'detections' by connectivity, and prune 'detections'
       in the 'signal mask' if projected area on sky smaller than
       a given number of pixels or a fraction of the beam area.
       (fraction specified by `prune_by_npix` or `prune_by_fracbeam`)
    5. Expand the 'signal mask' along the spatial dimensions by
       a given number of pixels or a fraction of the beam FWHM.
       (fraction specified by `expand_by_npix` or `expand_by_fracbeam`)
    6. Expand the 'signal mask' along the spectral dimension by
       a given number of channels.
       (# of channels specified by `expand_by_nchan`)

    Parameters
    ----------
    cube : SpectralCube object
        Input spectral cube
    noisecube : SpectralCube object
        Estimated rms noise cube
    mask : `np.ndarray` object, optional
        User-specified mask, within which all steps are performed
    nchan_hi : int, optional
        # of consecutive channels specified for the 'core mask'
        (Default: 3)
    snr_hi : float, optional
        S/N threshold specified for the 'core mask'
        (Default: 3.5)
    nchan_lo : int, optional
        # of consecutive channels specified for the 'wing mask'
        (Default: 2)
    snr_lo : float, optional
        S/N threshold specified for the 'wing mask'
        (Default: 2.0)
    prune_by_npix : int, optional
        Threshold for pruning. All detections with projected area
        smaller than this number of pixels will be pruned.
        If not None, `prune_by_fracbeam` will be ignored.
        (Default: None)
    prune_by_fracbeam : float, optional
        Threshold for pruning. All detections with projected area
        smaller than this threshold times the beam area will be pruned.
        (Default: 1.0)
    expand_by_npix : int, optional
        Expand the signal mask along the spatial dimensions by this
        number of pixels. If not None, `expand_by_fracbeam` will be
        ignored. (Default: None)
    expand_by_fracbeam : float, optional
        Expand the signal mask along the spatial dimensions by this
        fraction times the beam HWHM.
        (Default: 0.0)
    expand_by_nchan : int, optional
        Expand the signal mask along the spectral dimensions by this
        number of channels.
        (Default: 2)
    verbose : bool, optional
        Whether to print the detailed processing information
        Default is to not print.

    Returns
    -------
    outcube : SpectralCube object
        Input cube masked by the generated 'signal mask'.
    """

    if not cube.unit.is_equivalent(noisecube.unit):
        raise ValueError(
            "Incompatable units between 'cube' and 'noisecube'!")

    snr = cube.filled_data[:] / noisecube.filled_data[:]
    snr = snr.to(u.dimensionless_unscaled).value
    if mask is None:
        mask = np.ones_like(snr).astype('?')

    # generate core mask
    if verbose:
        print(
            "Generating core mask (S/N >= {} over {} consecutive "
            "channels)...".format(snr_hi, nchan_hi))
    mask_core = (snr > snr_hi)
    for iiter in range(nchan_hi-1):
        mask_core &= np.roll(mask_core, 1, 0)
    mask_core[:nchan_hi-1, :] = False
    for iiter in range(nchan_hi-1):
        mask_core |= np.roll(mask_core, -1, 0)
    mask_core &= mask

    # generate wing mask
    if verbose:
        print(
            "Generating wing mask (S/N >= {} over {} consecutive "
            "channels)...".format(snr_lo, nchan_lo))
    mask_wing = snr > snr_lo
    for iiter in range(nchan_lo-1):
        mask_wing &= np.roll(mask_wing, 1, 0)
    mask_wing[:nchan_lo-1, :] = False
    for iiter in range(nchan_lo-1):
        mask_wing |= np.roll(mask_wing, -1, 0)
    mask_wing &= mask

    # dilate core mask inside wing mask
    if verbose:
        print("Dilating core mask inside wing mask...")
    mask_signal = binary_dilation(
        mask_core, iterations=0, mask=mask_wing)

    # prune detections with small projected area on the sky
    if (prune_by_fracbeam > 0) or (prune_by_npix is not None):
        if verbose:
            print("Pruning detections with small projected area")
        if prune_by_npix is None:
            beamarea_pix = np.abs(
                cube.beam.sr.to(u.deg**2).value /
                cube.wcs.celestial.wcs.cdelt.prod())
            prune_by_npix = beamarea_pix * prune_by_fracbeam
        labels, count = label(mask_signal)
        for ind in np.arange(count)+1:
            if (labels == ind).any(axis=0).sum() < prune_by_npix:
                mask_signal[labels == ind] = False

    # expand along spatial dimensions by a fraction of beam FWHM
    if (expand_by_fracbeam > 0) or (expand_by_npix is not None):
        if verbose:
            print("Expanding signal mask along spatial dimensions")
        if expand_by_npix is None:
            beamHWHM_pix = np.ceil(
                cube.beam.major.to(u.deg).value / 2 /
                np.abs(cube.wcs.celestial.wcs.cdelt.min()))
            expand_by_npix = int(beamHWHM_pix * expand_by_fracbeam)
        structure = np.zeros(
            [3, expand_by_npix*2+1, expand_by_npix*2+1])
        Y, X = np.ogrid[:expand_by_npix*2+1, :expand_by_npix*2+1]
        R = np.sqrt((X - expand_by_npix)**2 + (Y-expand_by_npix)**2)
        structure[1, :] = (R <= expand_by_npix)
        mask_signal = binary_dilation(
            mask_signal, iterations=1, structure=structure, mask=mask)

    # expand along spectral dimension by a number of channels
    if expand_by_nchan > 0:
        if verbose:
            print("Expanding along spectral dimension by {} channels"
                  "".format(expand_by_nchan))
        for iiter in range(expand_by_nchan):
            tempmask = np.roll(mask_signal, 1, axis=0)
            tempmask[0, :] = False
            mask_signal |= tempmask
            tempmask = np.roll(mask_signal, -1, axis=0)
            tempmask[-1, :] = False
            mask_signal |= tempmask
        mask_signal &= mask

    return cube.with_mask(mask_signal)

def censoring_function(
        line_rms_width, channel_width, T_noise,
        spec_resp_kernel=[0, 1, 0],
        nchan_crit=2, snr_crit=2, scenario='optimistic'):
    """
    Calculate the shape of the censoring function.

    Parameters
    ----------
    line_rms_width: float or array-like
        The rms width of the Gaussian line profile
    channel_width: float or array-like
        The width of spectral channels, in the same unit as
        `line_rms_width`
    T_noise: float or array-like
        Noise temperature (1-sigma level)
    spec_resp_kernel: array-like with odd number of elements, optional
        A spectral response kernel designed to reproduce the
        appropriate channel-to-channel correlation
        (see Appendix A in Leroy et al. 2016, ApJ, 831, 16)
    nchan_crit: int, optional (default=2)
        # of channels used in determining significant detections
        (e.g., if the criterion is 2 consecutive channels above S/N=5,
        then `nchan_crit` should be set to 2)
    snr_crit: float, optional (default=2)
        S/R ratio used in determining significant detections
        (e.g., if the criterion is 2 consecutive channels above S/N=5,
        then `snr_crit` should be set to 5)
    scenario: {'optimistic', 'pessimistic'}, optional
        If 'optimistic' (default), this function will return the
        threshold below which the completeness is 0%.
        If 'pessimistic', this function will return the threshold
        above which the completeness is 100%.

    Returns
    -------
    line_intensity: float or array-like
        Completeness threshold in terms of integrated line intensity,
        carrying a unit of [T_noise] * [line_rms_width]
    """
    x = (np.asfarray(channel_width) /
         (2**0.5 * np.asfarray(line_rms_width)))

    if scenario == 'optimistic':
        # line center aligns perfectly with
        # the center of the `nchan_crit` consecutive channels
        uplim = nchan_crit / 2
        lolim = nchan_crit / 2 - 1
    elif scenario == 'pessimistic':
        # line center is off by half a channel width from
        # the center of the `nchan_crit` consecutive channels
        uplim = nchan_crit / 2 + 0.5
        lolim = nchan_crit / 2 - 0.5
    else:
        raise ValueError("Invalid `scenario`")

    kernel = np.asfarray(spec_resp_kernel).flatten()
    if (kernel.size % 2 == 0) or (kernel.sum() != 1):
        raise ValueError("Invalid spectral response kernel")

    ind = np.arange(-(kernel.size-1)/2, (kernel.size+1)/2)
    uplim = (uplim + ind) * x[..., np.newaxis]
    lolim = (lolim + ind) * x[..., np.newaxis]
    fraction = ((erf(uplim) - erf(lolim)) / 2 * kernel).sum(axis=-1)

    return np.asfarray(T_noise) * channel_width * snr_crit / fraction


#!venv/bin/python

# informal testing of concurrency using gevent

import random
import sys

import gevent

import lolog

log = lolog.get_logger('gevent-test')

def main():
    lolog.init(stream=sys.stdout)

    num_tasks = 10
    tasks = []
    log.info('starting gevent test', num_tasks=num_tasks)
    for idx in range(num_tasks):
        count = random.randint(10, 20)
        tasks.append(gevent.spawn(worker, idx, count))

    gevent.joinall(tasks)

def worker(worker_id, count):
    log.add_local_context('wid1', worker_id)
    log.info('starting worker', count=count)
    for idx in range(1, count + 1):
        log.info('doing some work', iter=idx, wid2=worker_id)
        gevent.sleep(random.uniform(0.001, 0.050))

main()


from django.core.mail import send_mail
from django.db import transaction

from caluma.caluma_core.events import filter_events, on
from caluma.caluma_form import api as caluma_form_api, models as caluma_form_models
from caluma.caluma_workflow import (
    api as caluma_workflow_api,
    models as caluma_workflow_models,
)
from caluma.caluma_workflow.events import (
    post_complete_work_item,
    post_create_work_item,
    post_redo_work_item,
    post_reopen_case,
    pre_complete_work_item,
)

from ..common import format_currency, get_users_for_case
from ..email_texts import (
    email_cost_approval,
    email_general,
    email_payout_amount,
    email_rejection,
    email_rejection_early_career_award,
)
from ..settings import settings

@on(post_create_work_item, raise_exception=True)
@filter_events(lambda sender: sender in ["post_complete_work_item", "case_post_create"])
@transaction.atomic
def set_assigned_user(sender, work_item, user, **kwargs):
    if work_item.task_id == "submit-document":
        work_item.assigned_users = [user.claims["sub"]]
    elif work_item.task_id in [
        "revise-document",
        "additional-data",
        "additional-data-form",
    ]:
        work_item.assigned_users = caluma_workflow_models.WorkItem.objects.get(
            task_id="submit-document",
            case=work_item.case,
        ).assigned_users

    work_item.save()

def _send_work_item_mail(work_item):
    """
    Send the work_item emails.

    This must reside in a separate function in order to be able to patch it in the
    tests.
    """
    link = f"{settings.SELF_URI}/cases/{work_item.case.pk}"

    try:
        dossier_nr = work_item.case.document.answers.get(question_id="dossier-nr").value
    except caluma_form_models.Answer.DoesNotExist:
        dossier_nr = "Not found"

    framework_credit = None
    payout_amount = None
    selected_email_texts = email_general

    if work_item.task.slug == "additional-data":
        define_amount_work_item = (
            work_item.case.work_items.filter(task__slug="define-amount")
            .order_by("-closed_at")
            .first()
        )
        if not (
            define_amount_work_item
            and define_amount_work_item.status == "completed"
            and define_amount_work_item.document.answers.filter(
                question_id="define-amount-decision",
                value="define-amount-decision-reject",
            ).exists()
        ):
            decision_and_credit_work_item = (
                work_item.case.work_items.filter(task__slug="decision-and-credit")
                .order_by("-created_at")
                .first()
            )
            framework_credit = decision_and_credit_work_item.document.answers.get(
                question__slug="gesprochener-rahmenkredit",
            ).value
            framework_credit = format_currency(framework_credit, "CHF")
            selected_email_texts = email_cost_approval
    elif work_item.task.slug == "complete-document":
        define_amount_work_item = (
            work_item.case.work_items.filter(task__slug="define-amount")
            .order_by("-created_at")
            .first()
        )
        payout_amount_answer = define_amount_work_item.document.answers.filter(
            question__slug="define-amount-amount-float",
        ).first()
        payout_amount = format_currency(
            payout_amount_answer.value if payout_amount_answer else 0,
            "CHF",
        )
        selected_email_texts = email_payout_amount
    elif work_item.task.slug in ["review-document", "decision-and-credit"]:
        selected_email_texts = email_rejection
        if (
            work_item.case.document.form.slug == settings.EARLY_CAREER_AWARD_FORM_SLUG
            and work_item.task.slug == "decision-and-credit"
        ):
            selected_email_texts = email_rejection_early_career_award

    users = get_users_for_case(work_item.case)

    for user in users:
        subject = selected_email_texts.EMAIL_SUBJECTS[user["language"]]

        subject = subject.format(dossier_nr=dossier_nr)

        body = selected_email_texts.EMAIL_BODIES[user["language"]]

        body = body.format(
            first_name=user["first-name"] or "",
            last_name=user["last-name"] or "",
            link=link,
            dossier_nr=dossier_nr,
            framework_credit=framework_credit,
            payout_amount=payout_amount,
        )

        send_mail(
            subject,
            body,
            settings.MAILING_SENDER,
            [user["email"]],
            fail_silently=True,
        )

@on(post_create_work_item, raise_exception=True)
@filter_events(
    lambda sender, work_item: sender == "post_complete_work_item"
    and work_item.task_id
    in [
        "revise-document",
        "additional-data",
        "complete-document",
    ],
)
def send_new_work_item_mail(sender, work_item, user, **kwargs):
    _send_work_item_mail(work_item)

@on(post_complete_work_item, raise_exception=True)
@filter_events(
    lambda sender, work_item: work_item.task_id
    in ["review-document", "decision-and-credit"]
    and (
        work_item.document.answers.filter(
            question_id="decision-and-credit-decision",
            value="decision-and-credit-decision-close",
        ).exists()
        or work_item.document.answers.filter(
            question_id="review-document-decision",
            value="review-document-decision-complete",
        ).exists()
    ),
)
def send_rejection_mail(sender, work_item, user, **kwargs):
    _send_work_item_mail(work_item)

@on(post_create_work_item, raise_exception=True)
@on(post_reopen_case, raise_exception=True)
@transaction.atomic
def set_case_status_post_create_work_item_reopen_case(
    sender,
    user,
    work_item=None,
    case=None,
    **kwargs,
):
    work_item = (
        work_item if work_item else case.work_items.filter(status="ready").first()
    )

    status = settings.CASE_STATUS.get(work_item.task_id)
    if status is None:
        return
    work_item.case.meta["status"] = status
    work_item.case.save()

@on(post_create_work_item, raise_exception=True)
@filter_events(
    lambda sender, work_item: sender
    in ["post_complete_work_item", "post_skip_work_item"]
    and work_item.task_id == "circulation"
    and not work_item.child_case,
)
@transaction.atomic
def create_circulation_child_case(sender, work_item, user, **kwargs):
    caluma_workflow_api.start_case(
        workflow=caluma_workflow_models.Workflow.objects.get(pk="circulation"),
        form=caluma_form_models.Form.objects.get(pk="circulation-form"),
        user=user,
        parent_work_item=work_item,
    )

@on(post_create_work_item, raise_exception=True)
@filter_events(
    lambda sender, work_item, context: sender == "post_complete_work_item"
    and work_item.task_id == "circulation-decision"
    and context is not None,
)
@transaction.atomic
def invite_to_circulation(sender, work_item, user, context, **kwargs):
    for index, assign_user in enumerate(context["assign_users"]):
        if index == 0:
            work_item.assigned_users = [assign_user["idpId"]]
            work_item.meta["assigneeName"] = assign_user["name"]
            work_item.meta["assigneeEmail"] = assign_user["email"]
            work_item.save()
            continue

        caluma_workflow_models.WorkItem.objects.create(
            name=work_item.task.name,
            description=work_item.task.description,
            task=work_item.task,
            status=caluma_workflow_models.WorkItem.STATUS_READY,
            assigned_users=[assign_user["idpId"]],
            meta={
                "assigneeName": assign_user["name"],
                "assigneeEmail": assign_user["email"],
            },
            case=work_item.case,
            document=caluma_form_models.Document.objects.create(
                form=work_item.document.form,
            ),
        )

@on(post_create_work_item, raise_exception=True)
@filter_events(
    lambda sender, work_item: sender == "post_complete_work_item"
    and work_item.task_id == "additional-data-form",
)
@transaction.atomic
def create_additional_data_form_document(sender, work_item, user, context, **kwargs):
    form_slug = settings.ADDITIONAL_DATA_FORM.get(
        work_item.case.document.form_id,
        "additional-data-form",
    )
    if work_item.case.document.form_id == settings.INTERNAL_APPLICATION_FORM_SLUG:
        answer = work_item.case.document.answers.filter(
            question__slug=settings.INTERNAL_APPLICATION_TYPE_QUESTION_SLUG,
        )
        if (
            answer
            and answer.first().value in settings.INTERNAL_APPLICATION_PERIODICS_CHOICES
        ):
            form_slug = settings.ADDITIONAL_DATA_FORM["periodika-antrag"]

    form = caluma_form_models.Form.objects.get(slug=form_slug)

    work_item.document = caluma_form_api.save_document(form=form, user=user)
    work_item.save()

@on(pre_complete_work_item, raise_exception=True)
@filter_events(
    lambda sender, work_item: sender == "pre_complete_work_item"
    and work_item.task_id == "finish-circulation",
)
@transaction.atomic
def finish_circulation(sender, work_item, user, **kwargs):
    caluma_workflow_api.cancel_work_item(
        work_item=caluma_workflow_models.WorkItem.objects.get(
            task_id="invite-to-circulation",
            case=work_item.case,
            status=caluma_workflow_models.WorkItem.STATUS_READY,
        ),
        user=user,
    )

@on(post_complete_work_item, raise_exception=True)
@filter_events(
    lambda sender, work_item: sender == "post_complete_work_item"
    and work_item.task_id == "additional-data",
)
@transaction.atomic
def finish_additional_data(sender, work_item, user, **kwargs):
    form_work_item = caluma_workflow_models.WorkItem.objects.filter(
        task_id="additional-data-form",
        case=work_item.case,
        status=caluma_workflow_models.WorkItem.STATUS_READY,
    ).first()

    if form_work_item:
        caluma_workflow_api.suspend_work_item(work_item=form_work_item, user=user)

@on(post_complete_work_item, raise_exception=True)
@filter_events(
    lambda sender, work_item: sender == "post_complete_work_item"
    and work_item.task_id == "additional-data-form",
)
@transaction.atomic
def finish_additional_data_form(sender, work_item, user, **kwargs):
    work_item = caluma_workflow_models.WorkItem.objects.filter(
        task_id="advance-credits",
        case=work_item.case,
        status=caluma_workflow_models.WorkItem.STATUS_READY,
    ).first()

    if work_item:
        caluma_workflow_api.complete_work_item(
            work_item=work_item,
            user=user,
        )

@on(post_complete_work_item, raise_exception=True)
@filter_events(
    lambda sender, work_item: sender == "post_complete_work_item"
    and work_item.task_id == "define-amount",
)
@transaction.atomic
def finish_define_amount(sender, work_item, user, **kwargs):
    decision = work_item.document.answers.get(
        question_id="define-amount-decision",
    )

    if any(state in decision.value for state in ["zurueckgezogen", "dismissed"]):
        for sibling in work_item.case.work_items.filter(
            status=caluma_workflow_models.WorkItem.STATUS_READY,
        ):
            caluma_workflow_api.complete_work_item(
                work_item=sibling,
                user=user,
            )
        return

    form_work_item = caluma_workflow_models.WorkItem.objects.filter(
        task_id="additional-data-form",
        case=work_item.case,
        status=caluma_workflow_models.WorkItem.STATUS_SUSPENDED,
    ).first()

    if form_work_item:
        caluma_workflow_api.resume_work_item(work_item=form_work_item, user=user)

@on(post_create_work_item, raise_exception=True)
@filter_events(lambda sender, work_item: work_item.task_id == "complete-document")
@transaction.atomic
def complete_additional_data_form(sender, work_item, user, **kwargs):
    form_work_item = caluma_workflow_models.WorkItem.objects.filter(
        task_id="additional-data-form",
        case=work_item.case,
        status=caluma_workflow_models.WorkItem.STATUS_READY,
    ).first()

    if form_work_item:
        caluma_workflow_api.complete_work_item(
            work_item=form_work_item,
            user=user,
        )

@on(post_redo_work_item, raise_exception=True)
@filter_events(lambda sender, work_item: work_item.task_id == "circulation")
@transaction.atomic
def redo_circulation(sender, work_item, user, **kwargs):
    work_item.child_case = caluma_workflow_api.start_case(
        workflow=caluma_workflow_models.Workflow.objects.get(pk="circulation"),
        form=caluma_form_models.Form.objects.get(pk="circulation-form"),
        user=user,
        parent_work_item=work_item,
    )
    work_item.save()

@on(post_redo_work_item, raise_exception=True)
@filter_events(lambda sender, work_item: work_item.task_id == "review-document")
@transaction.atomic
def redo_review_document(sender, work_item, user, **kwargs):
    circulation = work_item.case.work_items.get(task_id="circulation")
    circulation.child_case = caluma_workflow_api.start_case(
        workflow=caluma_workflow_models.Workflow.objects.get(pk="circulation"),
        form=caluma_form_models.Form.objects.get(pk="circulation-form"),
        user=user,
        parent_work_item=circulation,
    )
    circulation.save()

@on(post_redo_work_item, raise_exception=True)
@filter_events(lambda sender, work_item: work_item.task_id == "define-amount")
@transaction.atomic
def redo_define_amount(sender, work_item, user, **kwargs):
    credit_decision = (
        work_item.case.work_items.filter(task_id="decision-and-credit")
        .order_by("-created_at")
        .first()
        .document.answers.get(
            question_id="decision-and-credit-decision",
        )
    )

    if "additional-data" in credit_decision.value:
        advance_credits = (
            work_item.case.work_items.filter(task__slug="advance-credits")
            .order_by("-created_at")
            .first()
        )
        data_form = (
            work_item.case.work_items.filter(task__slug="additional-data-form")
            .order_by("-created_at")
            .first()
        )

        # This can't be done over caluma_workflow_api because they are the last work items of a "branch",
        # consequently they don't have another following work item which defines redoable for them.
        advance_credits.status = caluma_workflow_models.WorkItem.STATUS_READY
        advance_credits.save()
        data_form.status = caluma_workflow_models.WorkItem.STATUS_SUSPENDED
        data_form.save()

@on(post_complete_work_item, raise_exception=True)
@filter_events(
    lambda sender, work_item: sender == "post_complete_work_item"
    and work_item.task_id == "additional-data",
)
@transaction.atomic
def close_multiple_define_amount(sender, work_item, user, **kwargs):
    """
    Close all define-amount WorkItems.

    So the new that will open afterwards will be the only one.

    TODO: This is an ugly hack and we want to get rid of it.
    """
    define_amount = work_item.case.work_items.filter(
        task_id="define-amount",
        status=caluma_workflow_models.WorkItem.STATUS_REDO,
    )
    define_amount.update(status=caluma_workflow_models.WorkItem.STATUS_CANCELED)


def jewels(J,S):
    finalcount = 0
    for char in J:
        if char in S:
            scount = 0
            while(scount < len(S)):
                if S[scount] == char:
                    finalcount += 1
                    scount+= 1
                else:
                    scount += 1
    return finalcount

j = "aA"
s = "aAAbbbb"
j2 = 'b'
s2 = 'aaaaaaaaaajjjjjjj'
j3 = 'cFbh'
s3 = 'AFbcYYhh'
print(jewels(j,s))
print(jewels(j2,s2))
print(jewels(j3,s3))


# Soultion for Project Euler Problem #6 - https://projecteuler.net/problem=6
# (c) 2017 dpetker

n = 100

sum_of_squares = (n * (n + 1) * ((2 * n) + 1)) / 6
square_of_sum = pow(((n * (n + 1)) / 2), 2)

test_val = abs(sum_of_squares - square_of_sum)

print('The difference between the sum of the squares of the first 100 natural numbers and the square of the sum is {}'.format(test_val))


from flask import Flask
from flask_assets import Environment, Bundle
from flask.cli import AppGroup
import os

from PANEL.model import db

app = Flask(__name__)

def config_db(db_url='sqlite:///db/panel.db'):
    app.config['SQLALCHEMY_DATABASE_URI'] = db_url
    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
    if app.config.get('PROFILE', False): # pragma: no cover
        from werkzeug.contrib.profiler import ProfilerMiddleware
        app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30])
    app.secret_key = "#########"
    db.init_app(app)

def config_sass():
    #Converting Sass file to Css
    scss = Bundle('scss/main.scss', filters='pyscss', output='styles/css/main.css')
    #SCSS Connection
    assets = Environment(app)
    # assets.url= app.static_url_path
    assets.debug=True
    assets.register('scss_main', scss)

config_sass()
config_db()

from PANEL import routes

db_cli = AppGroup('db')

@db_cli.command('init')
def db_init():
    try: os.mkdir('PANEL/db')
    except: pass
    from PANEL.model import create_db
    create_db()

app.cli.add_command(db_cli)


import requests
from datetime import datetime
import logging

def available_symbols(headers):
    url = "https://fixer-fixer-currency-v1.p.rapidapi.com/symbols"
    symbols = requests.get(url=url, headers=headers).json().get('symbols').keys()
    for symbol in symbols:
        yield symbol

def remove_wrong_symbols(symbols, all_symbols):
    s1 = set(symbols)
    s2 = set(all_symbols)
    s_result = s1 & s2

    if len(s1 - s_result):
        print(f'Remove {s1 - s_result}, does not exists')

    return s_result

def request(base: str, symbols: list, headers: dict) -> dict:
    logger = logging.getLogger(__name__)

    url = "https://fixer-fixer-currency-v1.p.rapidapi.com/latest"

    availables = list(available_symbols(headers))
    symbols = ','.join(remove_wrong_symbols(symbols, availables))
    if base not in availables:
        raise ValueError(f'base {base} does not exists, replace by {availables}')

    logger.info(f"Fetching curreny of {symbols} use {base} as base")
    querystring = {"base": base, "symbols": symbols}
    return requests.get(url=url, headers=headers, params=querystring).json()

def clean_resquest(dict) -> dict:
    if dict.get('success') == True and dict.get('success') is not None:
        return {"timestamp": datetime.utcfromtimestamp(dict.get('timestamp')).strftime('%Y-%m-%d %H:%M:%S'),
                "base": dict.get('base'),
                "rates": dict.get('rates')
                }
    else:
        raise ValueError(f'Wrong dict {dict}')

def execute(base: str, symbols: list, headers: dict) -> dict:
    result = request(base, symbols, headers)
    return clean_resquest(result)


from future.utils import iteritems
from .models import Error, Results, Entry, from_string
import logging
logger = logging.getLogger(__name__)

boolean = [True, False]

processors = {
    'string_to_tuple': lambda p: p if isinstance(p, (tuple, list)) else tuple(p.split(','))
}

API_PARAMS = {
    "algorithm": {
        "aliases": ["algorithms", "a", "algo"],
        "required": True,
        "default": 'default',
        "processor": 'string_to_tuple',
        "description": ("Algorithms that will be used to process the request."
                        "It may be a list of comma-separated names."),
    },
    "expanded-jsonld": {
        "@id": "expanded-jsonld",
        "description": "use JSON-LD expansion to get full URIs",
        "aliases": ["expanded", "expanded_jsonld"],
        "options": boolean,
        "required": True,
        "default": False
    },
    "with-parameters": {
        "aliases": ['withparameters',
                    'with_parameters'],
        "description": "include initial parameters in the response",
        "options": boolean,
        "default": False,
        "required": True
    },
    "outformat": {
        "@id": "outformat",
        "aliases": ["o"],
        "default": "json-ld",
        "description": """The data can be semantically formatted (JSON-LD, turtle or n-triples),
given as a list of comma-separated fields (see the fields option) or constructed from a Jinja2
template (see the template option).""",
        "required": True,
        "options": ["json-ld", "turtle", "ntriples"],
    },
    "template": {
        "@id": "template",
        "required": False,
        "description": """Jinja2 template for the result. The input data for the template will
be the results as a dictionary.
For example:

Consider the results before templating:

```
[{
    "@type": "entry",
    "onyx:hasEmotionSet": [],
    "nif:isString": "testing the template",
    "marl:hasOpinion": [
        {
            "@type": "sentiment",
            "marl:hasPolarity": "marl:Positive"
        }
    ]
}]
```

And the template:

```
{% for entry in entries %}
{{ entry["nif:isString"] | upper }},{{entry.sentiments[0]["marl:hasPolarity"].split(":")[1]}}
{% endfor %}
```

The final result would be:

```
TESTING THE TEMPLATE,Positive
```
"""

    },
    "fields": {
        "@id": "fields",
        "required": False,
        "description": """A jmespath selector, that can be used to extract a new dictionary, array or value
from the results.
jmespath is a powerful query language for json and/or dictionaries.
It allows you to change the structure (and data) of your objects through queries.

e.g., the following expression gets a list of `[emotion label, intensity]` for each entry:
`entries[]."onyx:hasEmotionSet"[]."onyx:hasEmotion"[]["onyx:hasEmotionCategory","onyx:hasEmotionIntensity"]`

For more information, see: https://jmespath.org

"""
    },
    "help": {
        "@id": "help",
        "description": "Show additional help to know more about the possible parameters",
        "aliases": ["h"],
        "required": True,
        "options": boolean,
        "default": False
    },
    "verbose": {
        "@id": "verbose",
        "description": "Show all properties in the result",
        "aliases": ["v"],
        "required": True,
        "options": boolean,
        "default": False
    },
    "aliases": {
        "@id": "aliases",
        "description": "Replace JSON properties with their aliases",
        "aliases": [],
        "required": True,
        "options": boolean,
        "default": False
    },
    "emotion-model": {
        "@id": "emotionModel",
        "description": """Emotion model to use in the response.
Senpy will try to convert the output to this model automatically.

Examples: `wna:liking` and `emoml:big6`.
        """,
        "aliases": ["emoModel", "emotionModel"],
        "required": False
    },
    "conversion": {
        "@id": "conversion",
        "description": """How to show the elements that have (not) been converted.

* full: converted and original elements will appear side-by-side
* filtered: only converted elements will be shown
* nested: converted elements will be shown, and they will include a link to the original element
(using `prov:wasGeneratedBy`).
""",
        "required": True,
        "options": ["filtered", "nested", "full"],
        "default": "full"
    }
}

EVAL_PARAMS = {
    "algorithm": {
        "aliases": ["plug", "p", "plugins", "algorithms", 'algo', 'a', 'plugin'],
        "description": "Plugins to evaluate",
        "required": True,
        "help": "See activated plugins in /plugins",
        "processor": API_PARAMS['algorithm']['processor']
    },
    "dataset": {
        "aliases": ["datasets", "data", "d"],
        "description": "Datasets to be evaluated",
        "required": True,
        "help": "See avalaible datasets in /datasets"
    }
}

PLUGINS_PARAMS = {
    "plugin-type": {
        "@id": "pluginType",
        "description": 'What kind of plugins to list',
        "aliases": ["pluginType", "plugin_type"],
        "required": True,
        "default": 'analysisPlugin'
    }
}

WEB_PARAMS = {
    "in-headers": {
        "aliases": ["headers", "inheaders", "inHeaders", "in-headers", "in_headers"],
        "description": "Only include the JSON-LD context in the headers",
        "required": True,
        "default": False,
        "options": boolean
    },
}

CLI_PARAMS = {
    "plugin-folder": {
        "aliases": ["folder", "plugin_folder"],
        "required": True,
        "default": "."
    },
}

NIF_PARAMS = {
    "input": {
        "@id": "input",
        "aliases": ["i"],
        "required": True,
        "help": "Input text"
    },
    "intype": {
        "@id": "intype",
        "description": "input type",
        "aliases": ["t"],
        "required": False,
        "default": "direct",
        "options": ["direct", "url", "file"],
    },
    "informat": {
        "@id": "informat",
        "description": "input format",
        "aliases": ["f"],
        "required": False,
        "default": "text",
        "options": ["text", "json-ld"],
    },
    "language": {
        "@id": "language",
        "description": "language of the input",
        "aliases": ["l"],
        "required": False,
    },
    "prefix": {
        "@id": "prefix",
        "description": "prefix to use for new entities",
        "aliases": ["p"],
        "required": True,
        "default": "",
    },
    "urischeme": {
        "@id": "urischeme",
        "description": "scheme for NIF URIs",
        "aliases": ["u"],
        "required": False,
        "default": "RFC5147String",
        "options": ["RFC5147String", ]
    }
}

BUILTIN_PARAMS = {}

for d in [
        NIF_PARAMS, CLI_PARAMS, WEB_PARAMS, PLUGINS_PARAMS, EVAL_PARAMS,
        API_PARAMS
]:
    for k, v in d.items():
        BUILTIN_PARAMS[k] = v

def parse_params(indict, *specs):
    if not specs:
        specs = [NIF_PARAMS]
    logger.debug("Parsing: {}\n{}".format(indict, specs))
    outdict = indict.copy()
    wrong_params = {}
    for spec in specs:
        for param, options in iteritems(spec):
            for alias in options.get("aliases", []):
                # Replace each alias with the correct name of the parameter
                if alias in indict and alias != param:
                    outdict[param] = indict[alias]
                    del outdict[alias]
                    break
            if param not in outdict:
                if "default" in options:
                    # We assume the default is correct
                    outdict[param] = options["default"]
                elif options.get("required", False):
                    wrong_params[param] = spec[param]
                continue
            if 'processor' in options:
                outdict[param] = processors[options['processor']](outdict[param])
            if "options" in options:
                if options["options"] == boolean:
                    outdict[param] = str(outdict[param]).lower() in ['true', '1', '']
                elif outdict[param] not in options["options"]:
                    wrong_params[param] = spec[param]
    if wrong_params:
        logger.debug("Error parsing: %s", wrong_params)
        message = Error(
            status=400,
            message='Missing or invalid parameters',
            parameters=outdict,
            errors=wrong_params)
        raise message
    return outdict

def get_all_params(plugins, *specs):
    '''Return a list of parameters for a given set of specifications and plugins.'''
    dic = {}
    for s in specs:
        dic.update(s)
    dic.update(get_extra_params(plugins))
    return dic

def get_extra_params(plugins):
    '''Get a list of possible parameters given a list of plugins'''
    params = {}
    extra_params = {}
    for plugin in plugins:
        this_params = plugin.get('extra_params', {})
        for k, v in this_params.items():
            if k not in extra_params:
                extra_params[k] = {}
            extra_params[k][plugin.name] = v
    for k, v in extra_params.items():  # Resolve conflicts
        if len(v) == 1:  # Add the extra options that do not collide
            params[k] = list(v.values())[0]
        else:
            required = False
            aliases = None
            options = None
            default = None
            nodefault = False  # Set when defaults are not compatible

            for plugin, opt in v.items():
                params['{}.{}'.format(plugin, k)] = opt
                required = required or opt.get('required', False)
                newaliases = set(opt.get('aliases', []))
                if aliases is None:
                    aliases = newaliases
                else:
                    aliases = aliases & newaliases
                if 'options' in opt:
                    newoptions = set(opt['options'])
                    options = newoptions if options is None else options & newoptions
                if 'default' in opt:
                    newdefault = opt['default']
                    if newdefault:
                        if default is None and not nodefault:
                            default = newdefault
                        elif newdefault != default:
                            nodefault = True
                            default = None
            # Check for incompatibilities
            if options != set():
                params[k] = {
                    'default': default,
                    'aliases': list(aliases),
                    'required': required,
                    'options': list(options)
                }
    return params

def parse_analyses(params, plugins):
    '''
    Parse the given parameters individually for each plugin, and get a list of the parameters that
    belong to each of the plugins. Each item can then be used in the plugin.analyse_entries method.
    '''
    analysis_list = []
    for i, plugin in enumerate(plugins):
        if not plugin:
            continue
        this_params = filter_params(params, plugin, i)
        parsed = parse_params(this_params, plugin.get('extra_params', {}))
        analysis = plugin.activity(parsed)
        analysis_list.append(analysis)
    return analysis_list

def filter_params(params, plugin, ith=-1):
    '''
    Get the values within params that apply to a plugin.
    More specific names override more general names, in this order:

    <index_order>.parameter > <plugin.name>.parameter > parameter

    Example:

    >>> filter_params({'0.hello': True, 'hello': False}, Plugin(), 0)
    { '0.hello': True, 'hello': True}

    '''
    thisparams = {}
    if ith >= 0:
        ith = '{}.'.format(ith)
    else:
        ith = ""
    for k, v in params.items():
        if ith and k.startswith(str(ith)):
            thisparams[k[len(ith):]] = v
        elif k.startswith(plugin.name):
            thisparams[k[len(plugin.name) + 1:]] = v
        elif k not in thisparams:
            thisparams[k] = v
    return thisparams

def parse_call(params):
    '''
    Return a results object based on the parameters used in a call/request.
    '''
    params = parse_params(params, NIF_PARAMS)
    if params['informat'] == 'text':
        results = Results()
        entry = Entry(nif__isString=params['input'], id='prefix:')  # Use @base
        results.entries.append(entry)
    elif params['informat'] == 'json-ld':
        results = from_string(params['input'], cls=Results)
    else:  # pragma: no cover
        raise NotImplementedError('Informat {} is not implemented'.format(
            params['informat']))
    results.parameters = params
    return results


# 1. Write a program to count the number of strings
# where the length is two or more and
# the first and last character is same from the given list of strings.

List=[]
elements = int(input("Enter the number of elements you want in the your list:\n"))

print("Enter the",elements, "numbers(KEYS) for your List:")
for i in range(elements):
    num=input("Enter the List element:")
    List=List+[num]
print("Your entered list is:",List)

def Sample(List):
    count=0
    for la in List:
        if len(la) >= 2 and (la[0] == la[-1]):
            count=count+1
    return count

x=Sample(List)
print("The number of strings where the string length is two or more and the first and last char are same is:",x)

____

# 2. Write a program to remove duplicates from a list

List=[]
elements = int(input("Enter the number of elements you want in the your list:\n"))

print("Enter the",elements, "numbers for your List:")
for i in range(elements):
    num=int(input())
    List=List+[num]
print("Your entered list is:",List)

newList=[]
for x in List:
    if x not in newList:
        newList.append(x)

print("After removing duplicate elements:",newList)

____

#3. Write a program to print the numbers of a specified list after removing even numbers from the list.

sampleList=[]
elements = int(input("Enter the number of elements you want in the list\n"))
print("Enter the",elements, "numbers(E/O) for your List.")
for i in range(elements):
    num=int(input())
    sampleList=sampleList+[num]
print("Your entered list is.",sampleList)

for i in sampleList:
        if (i%2==0):
            sampleList.remove(i)

print("After removing even numbers, the remaining list is..",sampleList)