import colorsys
import logging
import math
import numpy as np
from enum import Enum, unique
import cv2
import matplotlib as mpl
import matplotlib.colors as mplc
import matplotlib.figure as mplfigure
import pdb
import torch
from matplotlib.backends.backend_agg import FigureCanvasAgg

from mseg.utils.fb_colormap import random_color, fixed_color
from mseg.utils.mask_utils import get_mask_from_polygon, get_most_populous_class

logger = logging.getLogger(__name__)

__all__ = ["ColorMode", "VisImage", "Visualizer"]

_SMALL_OBJECT_AREA_THRESH = 1000
_LARGE_MASK_AREA_THRESH = 120000
_OFF_WHITE = (1.0, 1.0, 240.0 / 255)
_BLACK = (0, 0, 0)
_RED = (1.0, 0, 0)

@unique
class ColorMode(Enum):
    """
    Enum of different color modes to use for instance visualizations.

    Attributes:
        IMAGE: Picks a random color for every instance and overlay segmentations with low opacity.
        SEGMENTATION: Let instances of the same category have similar colors, and overlay them with
            high opacity. This provides more attention on the quality of segmentation.
        IMAGE_BW: same as IMAGE, but convert all areas without masks to gray-scale.
            Only available for drawing per-instance mask predictions.
    """

    IMAGE = 0
    SEGMENTATION = 1
    IMAGE_BW = 2

class GenericMask:
    """
    Attribute:
        polygons (list[ndarray]): list[ndarray]: polygons for this mask.
            Each ndarray has format [x, y, x, y, ...]
        mask (ndarray): a binary mask
    """

    def __init__(self, mask_or_polygons, height, width):
        self._mask = self._polygons = self._has_holes = None
        self.height = height
        self.width = width

        m = mask_or_polygons
        if isinstance(m, dict):
            # RLEs
            assert "counts" in m and "size" in m
            if isinstance(m["counts"], list):  # uncompressed RLEs
                h, w = m["size"]
                assert h == height and w == width
                m = mask_util.frPyObjects(m, h, w)
            self._mask = mask_util.decode(m)[:, :]
            return

        if isinstance(m, list):  # list[ndarray]
            self._polygons = [np.asarray(x).reshape(-1) for x in m]
            return

        if isinstance(m, np.ndarray):  # assumed to be a binary mask
            assert m.shape[1] != 2, m.shape
            assert m.shape == (height, width), m.shape
            self._mask = m.astype("uint8")
            return

        raise ValueError("GenericMask cannot handle object {} of type '{}'".format(m, type(m)))

    @property
    def mask(self):
        if self._mask is None:
            self._mask = self.polygons_to_mask(self._polygons)
        return self._mask

    @property
    def polygons(self):
        if self._polygons is None:
            self._polygons, self._has_holes = self.mask_to_polygons(self._mask)
        return self._polygons

    @property
    def has_holes(self):
        if self._has_holes is None:
            if self._mask is not None:
                self._polygons, self._has_holes = self.mask_to_polygons(self._mask)
            else:
                self._has_holes = False  # if original format is polygon, does not have holes
        return self._has_holes

    def mask_to_polygons(self, mask):
        # cv2.RETR_CCOMP flag retrieves all the contours and arranges them to a 2-level
        # hierarchy. External contours (boundary) of the object are placed in hierarchy-1.
        # Internal contours (holes) are placed in hierarchy-2.
        # cv2.CHAIN_APPROX_NONE flag gets vertices of polygons from contours.
        res, hierarchy = cv2.findContours(mask.astype("uint8"), cv2.RETR_CCOMP, cv2.CHAIN_APPROX_NONE)

        if hierarchy is None:  # empty mask
            return [], False
        has_holes = (hierarchy.reshape(-1, 4)[:, 3] >= 0).sum() > 0

        res = [x.squeeze() for x in res]
        res = [x for x in res if x.size >= 6]  # should have at least 3 vertices to be valid
        return res, has_holes

    def polygons_to_mask(self, polygons):
        rle = mask_util.frPyObjects(polygons, self.height, self.width)
        rle = mask_util.merge(rle)
        return mask_util.decode(rle)[:, :]

    def area(self):
        return self.mask.sum()

    def bbox(self):
        p = mask_util.frPyObjects(self.polygons, self.height, self.width)
        p = mask_util.merge(p)
        bbox = mask_util.toBbox(p)
        bbox[2] += bbox[0]
        bbox[3] += bbox[1]
        return bbox

class _PanopticPrediction:
    def __init__(self, panoptic_seg, segments_info):
        self._seg = panoptic_seg

        self._sinfo = {s["id"]: s for s in segments_info}  # seg id -> seg info
        segment_ids, areas = torch.unique(panoptic_seg, sorted=True, return_counts=True)
        areas = areas.numpy()
        sorted_idxs = np.argsort(-areas)
        self._seg_ids, self._seg_areas = segment_ids[sorted_idxs], areas[sorted_idxs]
        self._seg_ids = self._seg_ids.tolist()
        for sid, area in zip(self._seg_ids, self._seg_areas):
            if sid in self._sinfo:
                self._sinfo[sid]["area"] = float(area)

    def non_empty_mask(self):
        """
        Returns:
            (H, W) array, a mask for all pixels that have a prediction
        """
        empty_ids = []
        for id in self._seg_ids:
            if id not in self._sinfo:
                empty_ids.append(id)
        if len(empty_ids) == 0:
            return np.zeros(self._seg.shape, dtype=np.uint8)
        assert len(empty_ids) == 1, ">1 ids corresponds to no labels. This is currently not supported"
        return (self._seg != empty_ids[0]).numpy().astype(np.bool)

    def semantic_masks(self):
        for sid in self._seg_ids:
            sinfo = self._sinfo.get(sid)
            if sinfo is None or sinfo["isthing"]:
                # Some pixels (e.g. id 0 in PanopticFPN) have no instance or semantic predictions.
                continue
            yield (self._seg == sid).numpy().astype(np.bool), sinfo

    def instance_masks(self):
        for sid in self._seg_ids:
            sinfo = self._sinfo.get(sid)
            if sinfo is None or not sinfo["isthing"]:
                continue
            mask = (self._seg == sid).numpy().astype(np.bool)
            if mask.sum() > 0:
                yield mask, sinfo

def _create_text_labels(classes, scores, class_names):
    """
    Args:
        classes (list[int] or None):
        scores (list[float] or None):
        class_names (list[str] or None):

    Returns:
        list[str] or None
    """
    labels = None
    if classes is not None and class_names is not None and len(class_names) > 1:
        labels = [class_names[i] for i in classes]
    if scores is not None:
        if labels is None:
            labels = ["{:.0f}%".format(s * 100) for s in scores]
        else:
            labels = ["{} {:.0f}%".format(l, s * 100) for l, s in zip(labels, scores)]
    return labels

class VisImage:
    def __init__(self, img, scale=1.0):
        """
        Args:
            img (ndarray): an RGB image of shape (H, W, 3).
            scale (float): scale the input image
        """
        self.img = img
        self.scale = scale
        self.width, self.height = img.shape[1], img.shape[0]
        self._setup_figure(img)

    def _setup_figure(self, img):
        """
        Args:
            Same as in :meth:`__init__()`.

        Returns:
            fig (matplotlib.pyplot.figure): top level container for all the image plot elements.
            ax (matplotlib.pyplot.Axes): contains figure elements and sets the coordinate system.
        """
        fig = mplfigure.Figure(frameon=False)
        self.dpi = fig.get_dpi()
        # add a small 1e-2 to avoid precision lost due to matplotlib's truncation
        # (https://github.com/matplotlib/matplotlib/issues/15363)
        fig.set_size_inches(
            (self.width * self.scale + 1e-2) / self.dpi,
            (self.height * self.scale + 1e-2) / self.dpi,
        )
        self.canvas = FigureCanvasAgg(fig)
        # self.canvas = mpl.backends.backend_cairo.FigureCanvasCairo(fig)
        ax = fig.add_axes([0.0, 0.0, 1.0, 1.0])
        ax.axis("off")
        ax.set_xlim(0.0, self.width)
        ax.set_ylim(self.height)

        self.fig = fig
        self.ax = ax

    def save(self, filepath):
        """
        Args:
            filepath (str): a string that contains the absolute path, including the file name, where
                the visualized image will be saved.
        """
        if filepath.endswith(".jpg") or filepath.endswith(".png"):
            # faster than matplotlib's imshow
            cv2.imwrite(filepath, self.get_image()[:, :, ::-1])
        else:
            # support general formats (e.g. pdf)
            self.ax.imshow(self.img, interpolation="nearest")
            self.fig.savefig(filepath)

    def get_image(self):
        """
        Returns:
            ndarray: the visualized image of shape (H, W, 3) (RGB) in uint8 type.
              The shape is scaled w.r.t the input image using the given `scale` argument.
        """
        canvas = self.canvas
        s, (width, height) = canvas.print_to_buffer()
        if (self.width, self.height) != (width, height):
            img = cv2.resize(self.img, (width, height))
        else:
            img = self.img

        # buf = io.BytesIO()  # works for cairo backend
        # canvas.print_rgba(buf)
        # width, height = self.width, self.height
        # s = buf.getvalue()

        buffer = np.frombuffer(s, dtype="uint8")

        # imshow is slow. blend manually (still quite slow)
        img_rgba = buffer.reshape(height, width, 4)
        rgb, alpha = np.split(img_rgba, [3], axis=2)

        try:
            import numexpr as ne  # fuse them with numexpr

            visualized_image = ne.evaluate("img * (1 - alpha / 255.0) + rgb * (alpha / 255.0)")
        except ImportError:
            alpha = alpha.astype("float32") / 255.0
            visualized_image = img * (1 - alpha) + rgb * alpha

        visualized_image = visualized_image.astype("uint8")

        return visualized_image

class Visualizer:
    def __init__(self, img_rgb, metadata, scale=1.0, instance_mode=ColorMode.IMAGE):
        """
        Args:
            img_rgb: a numpy array of shape (H, W, C), where H and W correspond to
                the height and width of the image respectively. C is the number of
                color channels. The image is required to be in RGB format since that
                is a requirement of the Matplotlib library. The image is also expected
                to be in the range [0, 255].
            metadata (MetadataCatalog): image metadata.
        """
        self.img = np.asarray(img_rgb).clip(0, 255).astype(np.uint8)
        self.metadata = metadata
        self.output = VisImage(self.img, scale=scale)
        self.cpu_device = torch.device("cpu")

        # too small texts are useless, therefore clamp to 9
        self._default_font_size = max(np.sqrt(self.output.height * self.output.width) // 90, 10 // scale)
        self._instance_mode = instance_mode

    def draw_sem_seg(self, sem_seg, area_threshold=None, alpha=0.8):
        """
        Draw semantic segmentation predictions/labels.

        Args:
            sem_seg (Tensor or ndarray): the segmentation of shape (H, W).
            area_threshold (int): segments with less than `area_threshold` are not drawn.
            alpha (float): the larger it is, the more opaque the segmentations are.

        Returns:
            output (VisImage): image object with visualizations.
        """
        if isinstance(sem_seg, torch.Tensor):
            sem_seg = sem_seg.numpy()
        labels, areas = np.unique(sem_seg, return_counts=True)
        sorted_idxs = np.argsort(-areas).tolist()
        labels = labels[sorted_idxs]
        for label in labels:
            try:
                mask_color = [x / 255 for x in self.metadata.stuff_colors[label]]
            except (AttributeError, IndexError):
                mask_color = None

            binary_mask = (sem_seg == label).astype(np.uint8)
            text = self.metadata.stuff_classes[label]
            self.draw_binary_mask(
                binary_mask,
                color=mask_color,
                edge_color=_OFF_WHITE,
                text=text,
                alpha=alpha,
                area_threshold=area_threshold,
            )
        return self.output

    def draw_panoptic_seg_predictions(self, panoptic_seg, segments_info, area_threshold=None, alpha=0.7):
        """
        Draw panoptic prediction results on an image.

        Args:
            panoptic_seg (Tensor): of shape (height, width) where the values are ids for each
                segment.
            segments_info (list[dict]): Describe each segment in `panoptic_seg`.
                Each dict contains keys "id", "category_id", "isthing".
            area_threshold (int): stuff segments with less than `area_threshold` are not drawn.

        Returns:
            output (VisImage): image object with visualizations.
        """
        pred = _PanopticPrediction(panoptic_seg, segments_info)

        if self._instance_mode == ColorMode.IMAGE_BW:
            self.output.img = self._create_grayscale_image(pred.non_empty_mask())

        # draw mask for all semantic segments first i.e. "stuff"
        for mask, sinfo in pred.semantic_masks():
            category_idx = sinfo["category_id"]
            try:
                mask_color = [x / 255 for x in self.metadata.stuff_colors[category_idx]]
            except AttributeError:
                mask_color = None

            text = self.metadata.stuff_classes[category_idx]
            self.draw_binary_mask(
                mask,
                color=mask_color,
                edge_color=_OFF_WHITE,
                text=text,
                alpha=alpha,
                area_threshold=area_threshold,
            )

        # draw mask for all instances second
        all_instances = list(pred.instance_masks())
        if len(all_instances) == 0:
            return self.output
        masks, sinfo = list(zip(*all_instances))
        category_ids = [x["category_id"] for x in sinfo]

        try:
            scores = [x["score"] for x in sinfo]
        except KeyError:
            scores = None
        labels = _create_text_labels(category_ids, scores, self.metadata.thing_classes)

        try:
            colors = [random_color(rgb=True, maximum=1) for k in category_ids]
        except AttributeError:
            colors = None
        self.overlay_instances(masks=masks, labels=labels, assigned_colors=colors, alpha=alpha)

        return self.output

    def overlay_instances(self, label_map, id_to_class_name_map, alpha=0.5):
        """
        Args:
            boxes (Boxes, RotatedBoxes or ndarray): either a :class:`Boxes`,
                or an Nx4 numpy array of XYXY_ABS format for the N objects in a single image,
                or a :class:`RotatedBoxes`,
                or an Nx5 numpy array of (x_center, y_center, width, height, angle_degrees) format
                for the N objects in a single image,
            labels (list[str]): the text to be displayed for each instance.
            masks (masks-like object): Supported types are:

                * `structures.masks.PolygonMasks`, `structures.masks.BitMasks`.
                * list[list[ndarray]]: contains the segmentation masks for all objects in one image.
                    The first level of the list corresponds to individual instances. The second
                    level to all the polygon that compose the instance, and the third level
                    to the polygon coordinates. The third level should have the format of
                    [x0, y0, x1, y1, ..., xn, yn] (n >= 3).
                * list[ndarray]: each ndarray is a binary mask of shape (H, W).
                * list[dict]: each dict is a COCO-style RLE.
            keypoints (Keypoint or array like): an array-like object of shape (N, K, 3),
                where the N is the number of instances and K is the number of keypoints.
                The last dimension corresponds to (x, y, visibility or score).
            assigned_colors (list[matplotlib.colors]): a list of colors, where each color
                corresponds to each mask or box in the image. Refer to 'matplotlib.colors'
                for full list of formats that the colors are accepted in.

        Returns:
            output (VisImage): image object with visualizations.
        """
        for label_idx in np.unique(label_map):
            color = fixed_color(rgb=True, maximum=1, idx=label_idx)
            text = id_to_class_name_map[label_idx]

            segment = label_map == label_idx

            # import pdb; pdb.set_trace()
            mask_obj = GenericMask(segment, self.output.height, self.output.width)
            polygons, _ = mask_obj.mask_to_polygons(segment)

            for polygon_verts in polygons:
                # print('Polygon verts: ', polygon_verts)

                # rasterize this specific polygon
                segment_mask = get_mask_from_polygon(polygon_verts, self.output.height, self.output.width)
                class_mode_idx = get_most_populous_class(segment_mask, label_map)
                if class_mode_idx != label_idx:
                    continue
                self.draw_polygon(polygon_verts, color, alpha=alpha)

                # import matplotlib.pyplot as plt
                # plt.imshow(segment_mask)
                # plt.show()

                # draw text in the center (defined by median) when box is not drawn
                # median is less sensitive to outliers.
                text_pos = np.median(segment_mask.nonzero(), axis=1)[::-1]
                horiz_align = "center"

                x0 = np.amin(polygon_verts[:, 0])
                x1 = np.amax(polygon_verts[:, 0])

                y0 = np.amin(polygon_verts[:, 1])
                y1 = np.amax(polygon_verts[:, 1])

                # else:
                #     continue  # drawing the box confidence for keypoints isn't very useful.
                # # for small objects, draw text at the side to avoid occlusion
                # instance_area = (y1 - y0) * (x1 - x0)
                # if (
                #     instance_area < _SMALL_OBJECT_AREA_THRESH * self.output.scale
                #     or y1 - y0 < 40 * self.output.scale
                # ):
                #     if y1 >= self.output.height - 5:
                #         text_pos = (x1, y0)
                #     else:
                #         text_pos = (x0, y1)

                height_ratio = (y1 - y0) / np.sqrt(self.output.height * self.output.width)
                lighter_color = self._change_color_brightness(color, brightness_factor=0.7)
                font_size = np.clip((height_ratio - 0.02) / 0.08 + 1, 1.2, 2) * 0.5 * self._default_font_size
                self.draw_text(
                    text,
                    text_pos,
                    color=lighter_color,
                    horizontal_alignment=horiz_align,
                    font_size=font_size,
                )
                # import matplotlib.pyplot as plt
                # plt.imshow(self.output.get_image())
                # plt.show()

        return self.output.get_image()

    """
    Primitive drawing functions:
    """

    def draw_text(self, text, position, *, font_size=None, color="g", horizontal_alignment="center", rotation=0):
        """
        Args:
            text (str): class label
            position (tuple): a tuple of the x and y coordinates to place text on image.
            font_size (int, optional): font of the text. If not provided, a font size
                proportional to the image width is calculated and used.
            color: color of the text. Refer to `matplotlib.colors` for full list
                of formats that are accepted.
            horizontal_alignment (str): see `matplotlib.text.Text`
            rotation: rotation angle in degrees CCW

        Returns:
            output (VisImage): image object with text drawn.
        """
        if not font_size:
            font_size = self._default_font_size

        # since the text background is dark, we don't want the text to be dark
        # color = np.maximum(list(mplc.to_rgb(color)), 0.2)
        # color[np.argmax(color)] = max(0.8, np.max(color))

        # force text color to be white!
        color = np.array([1.0, 1.0, 1.0])

        x, y = position
        self.output.ax.text(
            x,
            y,
            text,
            size=font_size * self.output.scale,
            family="sans-serif",
            bbox={"facecolor": "black", "alpha": 0.8, "pad": 0.7, "edgecolor": "none"},
            verticalalignment="top",
            horizontalalignment=horizontal_alignment,
            color=color,
            zorder=10,
            rotation=rotation,
        )
        return self.output

    def draw_binary_mask(self, binary_mask, color=None, *, edge_color=None, text=None, alpha=0.5, area_threshold=4096):
        """
        Args:
            binary_mask (ndarray): numpy array of shape (H, W), where H is the image height and
                W is the image width. Each value in the array is either a 0 or 1 value of uint8
                type.
            color: color of the mask. Refer to `matplotlib.colors` for a full list of
                formats that are accepted. If None, will pick a random color.
            edge_color: color of the polygon edges. Refer to `matplotlib.colors` for a
                full list of formats that are accepted.
            text (str): if None, will be drawn in the object's center of mass.
            alpha (float): blending efficient. Smaller values lead to more transparent masks.
            area_threshold (float): a connected component small than this will not be shown.

        Returns:
            output (VisImage): image object with mask drawn.
        """
        if color is None:
            color = random_color(rgb=True, maximum=1)
        if area_threshold is None:
            area_threshold = 4096

        has_valid_segment = False
        binary_mask = binary_mask.astype("uint8")  # opencv needs uint8
        mask = GenericMask(binary_mask, self.output.height, self.output.width)
        shape2d = (binary_mask.shape[0], binary_mask.shape[1])

        if not mask.has_holes:
            # draw polygons for regular masks
            for segment in mask.polygons:
                area = mask_util.area(mask_util.frPyObjects([segment], shape2d[0], shape2d[1]))
                if area < area_threshold:
                    continue
                has_valid_segment = True
                segment = segment.reshape(-1, 2)
                self.draw_polygon(segment, color=color, edge_color=edge_color, alpha=alpha)
        else:
            rgba = np.zeros(shape2d + (4,), dtype="float32")
            rgba[:, :, :3] = color
            rgba[:, :, 3] = (mask.mask == 1).astype("float32") * alpha
            has_valid_segment = True
            self.output.ax.imshow(rgba)

        if text is not None and has_valid_segment:
            # TODO sometimes drawn on wrong objects. the heuristics here can improve.
            lighter_color = self._change_color_brightness(color, brightness_factor=0.7)
            _num_cc, cc_labels, stats, centroids = cv2.connectedComponentsWithStats(binary_mask, 8)
            largest_component_id = np.argmax(stats[1:, -1]) + 1

            # draw text on the largest component, as well as other very large components.
            for cid in range(1, _num_cc):
                if cid == largest_component_id or stats[cid, -1] > _LARGE_MASK_AREA_THRESH:
                    # median is more stable than centroid
                    # center = centroids[largest_component_id]
                    center = np.median((cc_labels == cid).nonzero(), axis=1)[::-1]
                    self.draw_text(text, center, color=lighter_color)
        return self.output

    def draw_polygon(self, segment, color, edge_color=None, alpha=0.5):
        """
        Args:
            segment: numpy array of shape Nx2, containing all the points in the polygon.
            color: color of the polygon. Refer to `matplotlib.colors` for a full list of
                formats that are accepted.
            edge_color: color of the polygon edges. Refer to `matplotlib.colors` for a
                full list of formats that are accepted. If not provided, a darker shade
                of the polygon color will be used instead.
            alpha (float): blending efficient. Smaller values lead to more transparent masks.

        Returns:
            output (VisImage): image object with polygon drawn.
        """
        if edge_color is None:
            # make edge color darker than the polygon color
            if alpha > 0.8:
                edge_color = self._change_color_brightness(color, brightness_factor=-0.7)
            else:
                edge_color = color
        edge_color = mplc.to_rgb(edge_color) + (1,)

        polygon = mpl.patches.Polygon(
            segment,
            fill=True,
            facecolor=mplc.to_rgb(color) + (alpha,),
            edgecolor=edge_color,
            linewidth=max(self._default_font_size // 15 * self.output.scale, 1),
        )
        self.output.ax.add_patch(polygon)
        return self.output

    """
    Internal methods:
    """

    def _jitter(self, color):
        """
        Randomly modifies given color to produce a slightly different color than the color given.

        Args:
            color (tuple[double]): a tuple of 3 elements, containing the RGB values of the color
                picked. The values in the list are in the [0.0, 1.0] range.

        Returns:
            jittered_color (tuple[double]): a tuple of 3 elements, containing the RGB values of the
                color after being jittered. The values in the list are in the [0.0, 1.0] range.
        """
        color = mplc.to_rgb(color)
        vec = np.random.rand(3)
        # better to do it in another color space
        vec = vec / np.linalg.norm(vec) * 0.5
        res = np.clip(vec + color, 0, 1)
        return tuple(res)

    def _create_grayscale_image(self, mask=None):
        """
        Create a grayscale version of the original image.
        The colors in masked area, if given, will be kept.
        """
        img_bw = self.img.astype("f4").mean(axis=2)
        img_bw = np.stack([img_bw] * 3, axis=2)
        if mask is not None:
            img_bw[mask] = self.img[mask]
        return img_bw

    def _change_color_brightness(self, color, brightness_factor):
        """
        Depending on the brightness_factor, gives a lighter or darker color i.e. a color with
        less or more saturation than the original color.

        Args:
            color: color of the polygon. Refer to `matplotlib.colors` for a full list of
                formats that are accepted.
            brightness_factor (float): a value in [-1.0, 1.0] range. A lightness factor of
                0 will correspond to no change, a factor in [-1.0, 0) range will result in
                a darker color and a factor in (0, 1.0] range will result in a lighter color.

        Returns:
            modified_color (tuple[double]): a tuple containing the RGB values of the
                modified color. Each value in the tuple is in the [0.0, 1.0] range.
        """
        assert brightness_factor >= -1.0 and brightness_factor <= 1.0
        color = mplc.to_rgb(color)
        polygon_color = colorsys.rgb_to_hls(*mplc.to_rgb(color))
        modified_lightness = polygon_color[1] + (brightness_factor * polygon_color[1])
        modified_lightness = 0.0 if modified_lightness < 0.0 else modified_lightness
        modified_lightness = 1.0 if modified_lightness > 1.0 else modified_lightness
        modified_color = colorsys.hls_to_rgb(polygon_color[0], modified_lightness, polygon_color[2])
        return modified_color

    def get_output(self):
        """
        Returns:
            output (VisImage): the image output containing the visualizations added
            to the image.
        """
        return self.output


nums = [1, 0, -1, 0, -2, 2]
target = 0
answ = []
nums.sort()

for i in range(len(nums)-3):
    for j in range(i+1,len(nums)-2):
        l = j+1
        r = len(nums) - 1
        if j > i + 1 and nums[j] == nums[j - 1]:
            continue
        while l<r:
            sum = nums[i]+nums[j]+nums[l]+nums[r]
            if sum > target:
                r -= 1
            elif sum < target:
                l += 1
            else:
                answ_now = [nums[i],nums[j],nums[l],nums[r]]
                answ_now.sort()
                if answ_now not in answ:
                    answ.append(answ_now)
                while l<r and nums[l] == nums[l+1]:
                    l += 1
                while l<r and nums[r] == nums[r-1]:
                    r -= 1
                l += 1
                r -= 1

print(answ)


from oslo_policy import policy

from manila.policies import base

BASE_POLICY_NAME = 'share_access_rule:%s'

share_access_rule_policies = [
    policy.DocumentedRuleDefault(
        name=BASE_POLICY_NAME % 'get',
        check_str=base.RULE_DEFAULT,
        description="Get details of a share access rule.",
        operations=[
            {
                'method': 'GET',
                'path': '/share-access-rules/{share_access_id}'
            }
        ]),
    policy.DocumentedRuleDefault(
        name=BASE_POLICY_NAME % 'index',
        check_str=base.RULE_DEFAULT,
        description="List access rules of a given share.",
        operations=[
            {
                'method': 'GET',
                'path': ('/share-access-rules?share_id={share_id}'
                         '&key1=value1&key2=value2')
            }
        ]),
]

def list_rules():
    return share_access_rule_policies


# -*- coding: utf-8 -*-
import common.models as MODELS
import logging
from django.db.models import Q

logger = logging.getLogger('app')

def getUserList():
    """
    ユーザリストを取得する
    """
    logger.info('getUserList')

    return MODELS.User.objects.all().order_by('last_login').reverse()

def getUserProgressList(user):
    """
    ユーザの進捗一覧を取得する
    return progress_managent[]
    """
    logger.info('getUserProgressList: ' + user.username)
    progress_management_list = []

    result = MODELS.Progress_management.objects.filter(responsible_by=user.username).order_by('target', 'registered_at')
    if result:
        progress_management_list = result

    return progress_management_list

def getTeamProgressList(user):
    """
    ユーザからチームの別メンバの進捗を取得する
    return progress_management[]
    """
    logger.info('getTeamProgressList: ' + user.username)
    team_progress_lists = []

    # 自分のメンバーシップ取得
    my_membership = MODELS.Membership.objects.filter(user=user)

    # 1ユーザ1チームにしか対応していないため、とれた1件目のメンバーシップ取得
    if len(my_membership) == 1:
        my_membership = my_membership[0]

        #  チームが同一の自分以外のメンバーシップを取得
        team_membership_list = MODELS.Membership.objects.filter(team=my_membership.team).exclude(user=user)

        # 自分以外のチームのメンバーシップのユーザに紐づく進捗をリストに詰める
        for team_membership in team_membership_list:
            team_progress_list= MODELS.Progress_management.objects.filter(responsible_by=team_membership.user).order_by('target', 'registered_at')
            team_progress_lists.append(team_progress_list)

    else:
        # 自分のメンバーシップが1件以外の場合、何もしない
        pass

    return team_progress_lists

def getNumTarget(user):
    """
    ユーザが登録したターゲット数を返す
    return int
    """

    return  MODELS.Target.objects.filter(target_register__user=user).count()

def selectUserById(user_id):
    logger.info('selectUserById : ' + user_id)
    try:
        user = MODELS.User.objects.get(username=user_id)
        return user
    except MODELS.User.DoesNotExist:
        return None

def selectTeamsByUser(user):
    """
    ユーザが所属するチームのリストを返す
    """
    logger.info('selectTeamsByUser')
    team_list = []
    membership_list = MODELS.Membership.objects.filter(user=user)
    if len(membership_list) >= 1:
        for membership in membership_list:
            team_list.append(membership.team)
    return team_list

def selectRelatedTarget(user):
    """
    ユーザが登録した、もしくは担当しているターゲットを取得する
    return target[]
    """
    return MODELS.Target.objects.filter(Q(progress_management__responsible_by=user) | Q(target_register__user=user))

def selectActiveUser(target):
    """
    ターゲットの現担当者を帰す
    return user or None
    """
    try:
        if target.taken_flg:
            latest_progress_management = MODELS.Progress_management.objects.filter(target=target).latest('registered_at')
            return latest_progress_management.responsible_by
        else:
            return None
    except:
        return None

def deleteUser(user):
    """
    ユーザを削除する
    連動して削除するものは以下
    membership, invitation, progress_management, target_register、target、introducer、
    もしチーム管理者の場合、teamとそれに紐づくmembershipとinvitation全て

    もし削除対象ユーザの登録ターゲットの担当者が別ユーザである場合、ターゲットは担当者の登録ターゲットとなる

    削除順番(チーム関連)
    1.own membership or team's membership
    2.own invitation or team's invitation
    3.team if admin

    削除順番(ターゲット関連)
    1.もし登録ターゲットの担当が別ユーザだった場合、ターゲットは担当者の登録ターゲットとなる = target_registerをupdate
    2.削除対象ターゲットがintroducerのintroduceレコードを全て削除
    3.削除対象ターゲットがターゲットの全てのtarget_register
    4.削除対象がターゲットの全てのprogress_management
    5.削除対象ターゲット

    """

    #チーム関連項目削除
    team_list = selectTeamsByUser(user)

    for team in team_list:
        membership = MODELS.Membership.objects.get(user=user, team=team)
        if membership.team_admin_flg:
            MODELS.Membership.objects.filter(team = team).delete()
            MODELS.Invitation.objects.filter(team = team).delete()
            team.delete()
        else:
            membership.delete()
            MODELS.Invitation.objects.filter(invited_user=user).delete()

    #ターゲット関連項目削除
    target_list = selectRelatedTarget(user)

    for target in target_list:
        register_flg = False
        responsible_flg = False
        if MODELS.User.objects.get(target_register__target=target) == user:
            register_flg = True
        if user in MODELS.User.objects.filter(progress_management__target=target):
            responsible_flg = True

        if register_flg & responsible_flg:
            MODELS.Introduce.objects.filter(Q(target=target) | Q(introduced_by=target)).delete()
            MODELS.Progress_management.objects.filter(target=target).delete()
            MODELS.Target_register.objects.filter(target=target).delete()
            target.delete()
        elif responsible_flg:
            MODELS.Progress_management.objects.filter(target=target,responsible_by=user).delete()
        elif register_flg:
            active_user = selectActiveUser(target)
            if active_user:
                target_register = MODELS.Target_register.objects.get(target=target)
                target_register.user = active_user
                target_register.save()

                MODELS.Progress_management.objects.filter(target=target, responsible_by=user).delete()
            else:
                MODELS.Introduce.objects.filter(Q(target=target) | Q(introduced_by=target)).delete()
                MODELS.Progress_management.objects.filter(target=target).delete()
                MODELS.Target_register.objects.filter(target=target).delete()
                target.delete()

    user.is_active = False
    user.save()
    return "success"

def activateUser(user):
    """
    ユーザを再稼動させる
    return string
    """
    user.is_active = True
    user.save()

    return "success"

def deactivateUser(user):
    """
    ユーザを再稼動させる
    return string
    """
    user.is_active = False
    user.save()

    return "success"


import uvicorn
if __name__ == '__main__':
    uvicorn.run('nn_web:app', host='127.0.0.1', port=33334, log_level="info")

from os import listdir
from typing import Optional
import torch
from pydantic import BaseModel
from fastapi import FastAPI, File,Form, HTTPException, Response, status
import numpy as np
from PIL import Image
import torch
import timm
from torchvision import transforms
from pathlib import Path
from tqdm import tqdm

IMAGE_PATH="./../../../public/images"
device = "cuda" if torch.cuda.is_available() else "cpu"
model = timm.create_model('beit_base_patch16_224_in22k', pretrained=True)
model.head=torch.nn.Identity()
model.eval()
model.to(device)

import sqlite3
import io
conn = sqlite3.connect('NN_features.db')
conn_orig = sqlite3.connect('NN_features_orig.db')

dim=768
import faiss
sub_index = faiss.IndexFlatL2(dim)
index_id_map = faiss.IndexIDMap2(sub_index)
# import hnswlib
# index = hnswlib.Index(space='l2', dim=dim)
# index.init_index(max_elements=1_000_000, ef_construction=200, M=32)

import pickle
pca_w_file = Path("./pca_w.pkl")
pca = None
if pca_w_file.is_file():
    with open(pca_w_file, 'rb') as pickle_file:
        pca = pickle.load(pickle_file)

_transform=transforms.Compose([
                       transforms.ToTensor(),
                       transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])

def transform(im):
    desired_size = 224
    old_size = im.size  # old_size[0] is in (width, height) format
    ratio = float(desired_size)/max(old_size)
    new_size = tuple([int(x*ratio) for x in old_size])
    im = im.resize(new_size, Image.ANTIALIAS)
    new_im = Image.new("RGB", (desired_size, desired_size))
    new_im.paste(im, ((desired_size-new_size[0])//2, (desired_size-new_size[1])//2))
    return _transform(new_im)

def get_all_data_iterator(arraysize=10000):
    cursor = conn.cursor()
    query = '''
        SELECT id, nn_features
        FROM nn_table
        '''
    cursor.execute(query)
    while True:
        results = cursor.fetchmany(arraysize)
        if not results:
            break
        yield results

def init_index():
    for result in tqdm(get_all_data_iterator(10000)):
        ids = [x[0] for x in result]
        features = [convert_array(x[1]) for x in result]
        # features = Parallel(n_jobs=1)(delayed(convert_array)(feature) for feature in features)
        ids=np.int64(ids)
        features=np.array(features,dtype=np.float32)
        index_id_map.add_with_ids(features,ids)
    print("Index is ready")

def read_img_buffer(image_data):
    img = Image.open(io.BytesIO(image_data))
    img=img.convert('L').convert('RGB') #GREYSCALE
    # if img.mode != 'RGB':
    #     img = img.convert('RGB')
    return img

def get_features(image_buffer):
    image=read_img_buffer(image_buffer)
    image = transform(image).unsqueeze(0).to(device)
    with torch.no_grad():
        feature_vector = model(image).cpu().numpy()[0]
    feature_vector/=np.linalg.norm(feature_vector)
    return feature_vector

def create_table():
    cursor = conn.cursor()
    query = '''
	    CREATE TABLE IF NOT EXISTS nn_table(
	    	id INTEGER NOT NULL UNIQUE PRIMARY KEY,
	    	nn_features BLOB NOT NULL
	    )
	'''
    cursor.execute(query)
    conn.commit()

def check_if_exists_by_id(id):
    cursor = conn.cursor()
    query = '''SELECT EXISTS(SELECT 1 FROM nn_table WHERE id=(?))'''
    cursor.execute(query,(id,))
    all_rows = cursor.fetchone()
    return all_rows[0] == 1

def delete_descriptor_by_id(id):
    cursor = conn.cursor()
    query = '''DELETE FROM nn_table WHERE id=(?)'''
    cursor.execute(query,(id,))
    conn.commit()

def get_all_ids():
    cursor = conn.cursor()
    query = '''SELECT id FROM nn_table'''
    cursor.execute(query)
    all_rows = cursor.fetchall()
    return list(map(lambda el:el[0],all_rows))

def convert_array(text):
    out = io.BytesIO(text)
    out.seek(0)
    return np.load(out)

def adapt_array(arr):
    out = io.BytesIO()
    np.save(out, arr)
    out.seek(0)
    return sqlite3.Binary(out.read())

def add_descriptor(id,nn_features):
    cursor = conn.cursor()
    query = '''INSERT INTO nn_table(id, nn_features) VALUES (?,?)'''
    cursor.execute(query,(id,nn_features))
    conn.commit()

def add_descriptor_orig(id,nn_features):
    cursor = conn_orig.cursor()
    query = '''INSERT INTO nn_table(id, nn_features) VALUES (?,?)'''
    cursor.execute(query,(id,nn_features))
    conn_orig.commit()

def sync_db():
    ids_in_db=set(get_all_ids())
    file_names=listdir(IMAGE_PATH)
    for file_name in file_names:
        file_id=int(file_name[:file_name.index('.')])
        if file_id in ids_in_db:
            ids_in_db.remove(file_id)
    for id in ids_in_db:
        delete_descriptor_by_id(id)   #Fix this
        print(f"deleting {id}")
    print("db synced")

def get_aqe_vector(feature_vector):
    top_n_query=5
    _, I = index_id_map.search(feature_vector, top_n_query)
    alpha=1
    top_features=[]
    for i in range(top_n_query):
        top_features.append(index_id_map.reconstruct(int(list(I[0])[i])).flatten())
    new_feature=[]
    for i in range(dim):
        _sum=0
        for j in range(top_n_query):
            _sum+=top_features[j][i] * np.dot(feature_vector, top_features[j].T)**alpha
        new_feature.append(_sum)

    new_feature=np.array(new_feature)
    new_feature/=np.linalg.norm(new_feature)
    new_feature=new_feature.astype(np.float32).reshape(1,-1)
    return new_feature

def nn_find_similar(feature_vector, k, distance_threshold):
    new_feature_vector=get_aqe_vector(feature_vector)
    print(new_feature_vector)
    if k is not None:
        D, I = index_id_map.search(new_feature_vector, k)
        D = D.flatten()
        I = I.flatten()
    elif distance_threshold is not None:
        _, D, I = index_id_map.range_search(new_feature_vector, distance_threshold)

    res=[{"image_id":int(I[i]),"distance":float(D[i])} for i in range(len(D))]
    res = sorted(res, key=lambda x: x["distance"])
    return res

app = FastAPI()
@app.get("/")
async def read_root():
    return {"Hello": "World"}

@app.post("/calculate_nn_features")
async def calculate_NN_features_handler(image: bytes = File(...),image_id: str = Form(...)):
    try:
        image_id=int(image_id)
        features=get_features(image)
        add_descriptor_orig(image_id,adapt_array(features.astype(np.float32)))
        if pca:
            features=pca.transform(features.reshape(1,-1))[0]
            features/=np.linalg.norm(features)
        features=features.astype(np.float32)
        add_descriptor(image_id,adapt_array(features))
        # index.add_items(features,[image_id])
        index_id_map.add_with_ids(features.reshape(1,-1), np.int64([image_id]))
        return Response(status_code=status.HTTP_200_OK)
    except:
        raise HTTPException(status_code=500, detail="Can't calculate nn features")

class Item_image_id(BaseModel):
    image_id: int

@app.post("/delete_nn_features")
async def delete_nn_features_handler(item:Item_image_id):
    delete_descriptor_by_id(item.image_id)
    res = index_id_map.remove_ids(np.int64([item.image_id]))
    if res == 0: #nothing to delete
        print(f"err: no image with id {item.image_id}")
    return Response(status_code=status.HTTP_200_OK)
    # try:
    #     # index.mark_deleted(item.image_id)
    #     index_id_map.remove_ids(np.int64([item.image_id]))
    # except RuntimeError:
    #     print(f"err: no image with id {item.image_id}")
    # return Response(status_code=status.HTTP_200_OK)

class Item_image_id_k(BaseModel):
    image_id: int
    k: Optional[str] = None
    distance_threshold: Optional[str] = None

@app.post("/nn_get_similar_images_by_id")
async def get_similar_images_by_id_handler(item: Item_image_id_k):
    try:
        k=item.k
        distance_threshold=item.distance_threshold
        if item.k:
            k = int(k)
        if item.distance_threshold:
            distance_threshold = float(distance_threshold)
        if (k is None) == (distance_threshold is None):
            raise HTTPException(status_code=500, detail="both k and distance_threshold present")

        target_features = index_id_map.reconstruct(item.image_id).reshape(1,-1)
        print(target_features)
        #target_features = index.get_items([item.image_id])
        results = nn_find_similar(target_features,k, distance_threshold)
        return results
    except RuntimeError:
        raise HTTPException(status_code=500, detail="Can't get get_similar_images_by_id_handler")

@app.post("/nn_get_similar_images_by_image_buffer")
async def nn_get_similar_images_by_image_buffer_handler(image: bytes = File(...), k: Optional[str] = Form(None), distance_threshold: Optional[str] = Form(None)):
    try:
        if k:
            k = int(k)
        if distance_threshold:
            distance_threshold = float(distance_threshold)
        if (k is None) == (distance_threshold is None):
            raise HTTPException(status_code=500, detail="both k and distance_threshold present")

        target_features=get_features(image)
        if pca:
            target_features=pca.transform(target_features.reshape(1,-1))[0]
            target_features/=np.linalg.norm(target_features)
        target_features=target_features.astype(np.float32).reshape(1,-1)
        results = nn_find_similar(target_features, k, distance_threshold)
        return results
    except RuntimeError:
        raise HTTPException(status_code=500, detail="Can't get nn_get_similar_images_by_image_buffer_handler")

print(__name__)
if __name__ == 'nn_web':
    create_table()
    # sync_db()
    init_index()


class Solution:
    def minCost(self, colors: str, neededTime: List[int]) -> int:
        ans, left, right = 0, 0, 0

        while right < len(neededTime):
          cost = 0
          keep = 0

          while right < len(neededTime) and colors[left] == colors[right]:
            cost += neededTime[right]
            keep = max(keep, neededTime[right])
            right += 1

          left = right
          ans += cost - keep

        return ans


"""
The stochastic model of the observation.
"""

def calculate_likelihood(theta, a, obs):
    """
    Calculate the likelihood/probability of observing obs, if the parameter is theta.

    Input:
      theta:  a vector of values in [0,1], shape(K,)
      a:      an integer in [K] = {0, 1, ..., K-1}
      obs:    0 or 1.

    Output:
      lh:     a number in [0,1], the likelihood/probability.
    """

    if obs == 1:
        lh = theta[a]
    else:
        lh = 1-theta[a]

    return lh


import re
from canvasapi.assignment import Assignment

class AssignmentPlusFile:
    def __init__(self, assignment: Assignment, file_path: str):
        self.assignment = assignment
        self.file_path = file_path

class ProgressBar:
    """A utility progress bar class."""

    def __init__(
        self,
        total: int,
        percentage=True,
        left_text: str = "Progress:",
        right_text: str = None,
        bar_length: int = 20,
    ):
        """Constructor for the progress bar class."""
        self.total = total
        self.percentage = percentage
        self.left_text = left_text
        self.right_text = right_text
        self.bar_length = bar_length

    def update(self, current: int):
        """Updates and prints the progress bar."""
        fraction = current / self.total

        arrow = int(fraction * self.bar_length - 1) * "=" + ">"
        padding = int(self.bar_length - len(arrow)) * " "

        ending = "\n" if current == self.total else "\r"

        if not self.right_text:
            progress_text = (
                f"{int(fraction*100)}%"
                if self.percentage
                else f"{current}/{self.total}"
            )
        else:
            progress_text = (
                self.right_text.replace("(PERCENTAGE)", f"{int(fraction*100)}")
                if self.percentage
                else self.right_text.replace("(CURRENT)", f"{current}").replace(
                    "(TOTAL)", f"{self.total}"
                )
            )

        print(f"{self.left_text} [{arrow}{padding}] {progress_text}", end=ending)

def lower_remove_prefixes(filename: str, variations: list[str]):
    s = filename.lower()
    for variation in variations:
        s = s.removeprefix(variation)

    return s

def is_integer(n: int | float):
    if isinstance(n, int):
        return True
    if isinstance(n, float):
        return n.is_integer()
    return False

def to_num(s: str):
    if not s:
        return s

    if s.isdigit():
        return int(s)
    else:
        return float(s)

def number_from_end(s: str):
    try:
        return int(next(re.finditer(r"\d+$", s)).group(0))
    except Exception:
        raise ValueError("no numbers from end of string")


'''
This is an abstract example of Transforming in an ETL pipeline.

Inspired from the "Introduction to Data Engineering" course on Datacamp.com
Author: Alex Nakagawa
'''
# Get the rental rate column as a string
rental_rate_str = film_df.rental_rate.astype(str)

# Split up and expand the column
rental_rate_expanded = rental_rate_str.str.split('.', expand=True)

# Assign the columns to film_df
film_df = film_df.assign(
    rental_rate_dollar=rental_rate_expanded[0],
    rental_rate_cents=rental_rate_expanded[1],
)

# Use groupBy and mean to aggregate the column
ratings_per_film_df = rating_df.groupBy('film_id').mean('rating')

# Join the tables using the film_id column
film_df_with_ratings = film_df.join(
    ratings_per_film_df,
    film_df.film_id == ratings_per_film_df.film_id
)

# Show the 5 first results
print(film_df_with_ratings.show(5))


#!/usr/bin/python3.8

import threading
import signal
from time import sleep
from rf import YDSendPacketEvent
from rolling_keyfobs import RollingKeyFobs
from puck_bits_receiver import PuckBitsReceiverThread
from puck_bits_sender import PuckBitsYdSenderThread
from service_exit import ServiceExit

def sigint_handler(signal_, frame):
    print("received sigint, exiting gracefully")
    raise ServiceExit

if __name__ == "__main__":
    signal.signal(signal.SIGINT, sigint_handler)

    rolling_key_fobs = RollingKeyFobs()
    lock = threading.RLock()

    try:
        yd_sending = YDSendPacketEvent()
        thread1 = PuckBitsReceiverThread("thread1", lock, rolling_key_fobs, yd_sending)
        thread2 = PuckBitsYdSenderThread("thread2", lock, rolling_key_fobs, yd_sending)
        thread1.start()
        thread2.start()

        while True:
            sleep(0.5)

    except ServiceExit:

        # wait for both to end
        thread1.shutdown_thread()
        thread2.shutdown_thread()

        thread1.join()
        thread2.join()


import time

def get_number():
    while True:
        number_or_quit = input("Enter a starting number (greater than 0) or QUIT: ").lower()
        if number_or_quit.startswith('q'):
            return None
        if not number_or_quit.isdecimal() or number_or_quit == '0':
            continue
        else:
            number = int(number_or_quit)
            return number
def main():
    num = get_number()
    if not num:
        print("Goodbye!")
    else:
        print(f"{num} ", end='')
        while num != 1:
            if num % 2 == 0: # If n is even, the next number n is n / 2
                num //= 2
            else: # If n is odd, the next number n is n * 3 + 1
                num = 3 * num + 1
            time.sleep(0.09)
            print(f"{num} ", end='')

if __name__ == '__main__':
    main()


def getIsDeckSingleton(deck):

	violatingCards = []

	for deckCardName in deck:
		deckCard = deck[deckCardName]

		oracleText = deckCard.getFullOracleText()
		if ("A deck can have any number of cards named" in oracleText):
			continue

		if (deckCard.count > 1 and not deckCard.jsonData.get('type_line', '').startswith('Basic Land')):
			violatingCards.append(deckCardName)

	response = {'violatingCards': violatingCards, 'isDeckSingleton': len(violatingCards) == 0}

	return response

def printgetIsDeckSingletonToConsole(response):
	print ('Singleton status:')
	if (response['isDeckSingleton']):
		print ("Deck is singleton.")
	else:
		print ("Deck is not singleton, violating cards are:")
		for card in response['violatingCards']:
			print ('\t* ' + card)

def getDeckCardCount(deck):
	count = 0
	sideboardCount = 0
	commander = 0
	for deckCardName in deck:
		deckCard = deck[deckCardName]
		count += (deckCard.count - deckCard.sideboard)
		sideboardCount += deckCard.sideboard
		if (deckCard.commander):
			commander += deckCard.count
			count -= (deckCard.count - deckCard.sideboard)

	response = { "count": count, "sideboardCount": sideboardCount, "commander": commander }

	return response

def printGetDeckCardCountToConsole(response):
	print ('Card count:')
	print ("Commander:", response["commander"])
	print ("Maindeck:", response["count"])
	print ("Sideboard:", response["sideboardCount"])


import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler, MinMaxScaler

def read_data(paths, delimiter, skip_header):
    data = []
    for path in paths:
        subdata = np.genfromtxt(path, delimiter=delimiter, skip_header=skip_header)
        data.append(subdata)
    data = np.concatenate(data)
    return data

def read_data_pandas(paths, delimiter):
    data = []
    for path in paths:
        subdata = pd.read_csv(path, delimiter=delimiter)
        data.append(subdata)
    data = pd.concat(data)
    return data

def export_predictions(X, model, filename):
    y_pred = model.predict(X)
    ids = np.array(range(3000))
    df = {'Id': ids,
          'Bound': y_pred.astype(int)}
    df = pd.DataFrame(df).set_index('Id')
    df.to_csv(filename)

def getKmers(sequence, size):
    return [sequence[x:x + size] for x in range(len(sequence) - size + 1)]

def preprocess_tfidf(X_train, X_test, window_size=6, pca_components=100):
    n1 = X_train.shape[0]
    X = pd.concat([X_train, X_test])
    X['kMers'] = X['seq'].apply(lambda x: getKmers(x, size=window_size))
    X['sentences'] = X['kMers'].apply(lambda x: ' '.join(x))

    vectorizer = TfidfVectorizer(min_df=5)
    tfidf = vectorizer.fit_transform(X['sentences']).toarray()

    pca = PCA(n_components=pca_components)
    tfidf_ = pca.fit_transform(tfidf)

    scaler = MinMaxScaler()
    tfidf_ = scaler.fit_transform(tfidf_)

    return tfidf_[:n1], tfidf_[n1:]

import gudhi as gd
import gudhi.representations
from gudhi import plot_persistence_diagram
import tqdm.notebook as tq

def preprocess_TDA(X_train,X_test,base2idx = {'A':0,'C':1,'G':2,'T':3},e = np.eye(4)):

    print('Preprocess training samples : Compute Persistence Diagrams..')
    train_dgms = compute_persistence_diagrams(X_train['seq'].values)
    print('Preprocess test samples : Compute Persistence Diagrams..')
    test_dgms = compute_persistence_diagrams(X_test['seq'].values)
    return train_dgms, test_dgms

def compute_persistence_diagrams(X):
    ftrs = []
    for i,seq in tq.tqdm(enumerate(X),total = len(X)):
        dgm, dim_1_ftrs = seq2pd(seq) #compute persistence diagrams but only use 1-dimensional features for Sliced Wasserstein Kernel
        ftrs.append(dim_1_ftrs)
    return ftrs

def seq2pd(seq, base2idx = {'A':0,'C':1,'G':2,'T':3},e = np.eye(4)):

    b = seq2point_cloud(seq, base2idx = {'A':0,'C':1,'G':2,'T':3},e = np.eye(4))
    skeleton = gd.RipsComplex(points = b, max_edge_length = 0.8)
    Rips_simplex_tree_sample = skeleton.create_simplex_tree(max_dimension = 2)
    #compute persistence diagram for dimensions 0 and 1
    #but only 1-dimensional features will be used for the kernel
    dgm = Rips_simplex_tree_sample.persistence()
    dim_1_ftrs = Rips_simplex_tree_sample.persistence_intervals_in_dimension(1)
    return dgm, dim_1_ftrs

def seq2point_cloud(seq, base2idx = {'A':0,'C':1,'G':2,'T':3},e = np.eye(4)):
    b = np.zeros((len(seq),4))
    for k,base in enumerate(seq):
        if k==0:
            b[k] = e[base2idx[base]]
        else:
            b[k] = 0.5*(b[k-1]+e[base2idx[base]])
    return b


from PyQt5.QtCore import QPoint, QSettings, QSignalMapper, QSize, Qt, QFileInfo
from PyQt5.QtGui import QKeySequence
from PyQt5.QtWidgets import QAction, QApplication, QFileDialog, QMainWindow, QMdiArea, QMessageBox, qApp, QStyle

from plain_text_import_dialog import PlainTextImportDialog
from pyqtchild import MDIChildPlot

class MainWindow(QMainWindow):
    def __init__(self):
        super(MainWindow, self).__init__()

        self.mdiArea = QMdiArea()
        self.mdiArea.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded)
        self.mdiArea.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded)
        self.setCentralWidget(self.mdiArea)

        self.mdiArea.subWindowActivated.connect(self.update_menus)
        self.windowMapper = QSignalMapper(self)
        self.windowMapper.mapped.connect(self.set_active_sub_window)

        self.newAct = QAction(self)
        self.newAct.setIcon(self.style().standardIcon(QStyle.SP_FileIcon))
        # self.newAct.setIcon(QIcon(':/images/new.png'))
        self.newAct.setIconText('New')
        self.newAct.setShortcut(QKeySequence.New)
        self.newAct.setStatusTip('Create a new file')
        self.newAct.triggered.connect(self.new_file)

        self.openAct = QAction(self)
        self.openAct.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon))
        self.openAct.setIconText('Open...')
        self.openAct.setShortcut(QKeySequence.Open)
        self.openAct.setStatusTip('Open an existing file')
        self.openAct.triggered.connect(self.open)

        self.saveAct = QAction(self)
        self.saveAct.setIcon(self.style().standardIcon(QStyle.SP_DialogSaveButton))
        self.saveAct.setIconText('Save')
        self.saveAct.setShortcut(QKeySequence.Save)
        self.saveAct.setStatusTip('Save the document to disk')
        self.saveAct.triggered.connect(self.save)

        self.saveAsAct = QAction(self)
        self.saveAsAct.setIconText('Save As...')
        self.saveAsAct.setShortcut(QKeySequence.SaveAs)
        self.saveAsAct.setStatusTip('Save the document under a new name')
        self.saveAsAct.triggered.connect(self.save_as)

        self.exitAct = QAction(self)
        self.exitAct.setIconText('Exit')
        self.exitAct.setShortcut(QKeySequence.Quit)
        self.exitAct.setStatusTip('Exit the application')
        self.exitAct.triggered.connect(qApp.closeAllWindows)

        self.cutAct = QAction(self)
        # self.cutAct.setIcon(QIcon(':/images/cut.png'))
        self.cutAct.setIconText('Cut')
        self.cutAct.setShortcut(QKeySequence.Cut)
        self.cutAct.setStatusTip("Cut the current selection's contents to the clipboard")
        self.cutAct.triggered.connect(self.cut)

        self.copyAct = QAction(self)
        # self.copyAct.setIcon(QIcon(':/images/copy.png'))
        self.copyAct.setIconText('Copy')
        self.copyAct.setShortcut(QKeySequence.Copy)
        self.copyAct.setStatusTip("Copy the current selection's contents to the clipboard")
        self.copyAct.triggered.connect(self.copy)

        self.pasteAct = QAction(self)
        # self.pasteAct.setIcon(QIcon(':/images/paste.png'))
        self.pasteAct.setIconText('Paste')
        self.pasteAct.setShortcut(QKeySequence.Paste)
        self.pasteAct.setStatusTip("Paste the clipboard's contents into the current selection")
        self.pasteAct.triggered.connect(self.paste)

        self.closeAct = QAction(self)
        self.closeAct.setIconText('Close')
        self.closeAct.setShortcut('Ctrl+W')
        self.closeAct.setStatusTip("Close the active window")
        self.closeAct.triggered.connect(self.mdiArea.closeActiveSubWindow)

        self.closeAllAct = QAction(self)
        self.closeAllAct.setIconText('Close All')
        self.closeAllAct.setStatusTip('Close all the windows')
        self.closeAllAct.triggered.connect(self.mdiArea.closeAllSubWindows)

        self.tileAct = QAction(self)
        self.tileAct.setIconText('Tile')
        self.tileAct.setStatusTip('Tile the windows')
        self.tileAct.triggered.connect(self.mdiArea.tileSubWindows)

        self.cascadeAct = QAction(self)
        self.cascadeAct.setIconText('Cascade')
        self.cascadeAct.setStatusTip('Cascade the windows')
        self.cascadeAct.triggered.connect(self.mdiArea.cascadeSubWindows)

        self.nextAct = QAction(self)
        self.nextAct.setIconText('Next')
        self.nextAct.setShortcut(QKeySequence.NextChild)
        self.nextAct.setStatusTip('Move the focus to the next window')
        self.nextAct.triggered.connect(self.mdiArea.activateNextSubWindow)

        self.previousAct = QAction(self)
        self.previousAct.setIconText('Previous')
        self.previousAct.setShortcut(QKeySequence.PreviousChild)
        self.previousAct.setStatusTip('Move the focus to the previous window')
        self.previousAct.triggered.connect(self.mdiArea.activatePreviousSubWindow)

        self.separatorAct = QAction(self)
        self.separatorAct.setSeparator(True)

        self.aboutAct = QAction(self)
        self.aboutAct.setIconText('About')
        self.aboutAct.setStatusTip("Show the application's About box")
        self.aboutAct.triggered.connect(self.about)

        self.aboutQtAct = QAction(self)
        self.aboutQtAct.setIconText('About Qt')
        self.aboutQtAct.setStatusTip("Show the Qt library's About box")
        self.aboutQtAct.triggered.connect(qApp.aboutQt)

        self.fileMenu = self.menuBar().addMenu('File')
        self.fileMenu.addAction(self.newAct)
        self.fileMenu.addAction(self.openAct)
        self.fileMenu.addAction(self.saveAct)
        self.fileMenu.addAction(self.saveAsAct)
        self.fileMenu.addSeparator()
        action = self.fileMenu.addAction('Switch layout direction')
        action.triggered.connect(self.switch_layout_direction)
        self.fileMenu.addAction(self.exitAct)

        self.editMenu = self.menuBar().addMenu('Edit')
        self.editMenu.addAction(self.cutAct)
        self.editMenu.addAction(self.copyAct)
        self.editMenu.addAction(self.pasteAct)

        self.windowMenu = self.menuBar().addMenu('Window')
        self.update_window_menu()
        self.windowMenu.aboutToShow.connect(self.update_window_menu)

        self.menuBar().addSeparator()

        self.helpMenu = self.menuBar().addMenu('Help')
        self.helpMenu.addAction(self.aboutAct)
        self.helpMenu.addAction(self.aboutQtAct)

        self.file_tool_bar = self.addToolBar('File')
        self.file_tool_bar.addAction(self.newAct)
        self.file_tool_bar.addAction(self.openAct)
        self.file_tool_bar.addAction(self.saveAct)

        self.edit_tool_bar = self.addToolBar('Edit')
        self.edit_tool_bar.addAction(self.cutAct)
        self.edit_tool_bar.addAction(self.copyAct)
        self.edit_tool_bar.addAction(self.pasteAct)

        self.statusBar().showMessage('Ready')

        self.update_menus()

        self.settings = QSettings('SavSoft', 'Combiner')

        self.last_directory: str = ''
        self.read_settings()

        self.setWindowTitle('MDI')
        self.setUnifiedTitleAndToolBarOnMac(True)

    def closeEvent(self, event):
        self.mdiArea.closeAllSubWindows()
        if self.active_mdi_child():
            event.ignore()
        else:
            self.write_settings()
            event.accept()

    def new_file(self):
        child = self.create_mdi_child()
        child.new_file()
        child.show()

    def open(self):
        filters = ['IRTECON files (*.grd)', 'Plain text files (*.csv *.tsv *.dat *.txt)']
        file_name, _ = QFileDialog.getOpenFileName(self, filter=';;'.join(filters),
                                                   directory=self.last_directory,
                                                   options=QFileDialog.DontUseNativeDialog)
        if file_name:
            child = self.create_mdi_child()
            if QFileInfo(file_name).suffix() == 'grd':
                if child.load_irtecon_file(file_name):
                    self.statusBar().showMessage('File loaded', 2000)
                    self.last_directory = QFileInfo(file_name).dir().absolutePath()
                    child.show()
                else:
                    child.close()
            else:
                print(PlainTextImportDialog(file_name).exec())
                raise NotImplementedError

    def save(self):
        if self.active_mdi_child() and self.active_mdi_child().save():
            self.statusBar().showMessage('File saved', 2000)

    def save_as(self):
        if self.active_mdi_child() and self.active_mdi_child().save_as():
            self.statusBar().showMessage('File saved', 2000)

    def cut(self):
        if self.active_mdi_child():
            self.active_mdi_child().cut()

    def copy(self):
        if self.active_mdi_child():
            self.active_mdi_child().copy()

    def paste(self):
        if self.active_mdi_child():
            self.active_mdi_child().paste()

    def about(self):
        QMessageBox.about(self, 'About MDI',
                          'The <b>MDI</b> example demonstrates how to write multiple '
                          'document interface applications using Qt.')

    def update_menus(self):
        has_mdi_child = (self.active_mdi_child() is not None)
        self.saveAct.setEnabled(has_mdi_child)
        self.saveAsAct.setEnabled(has_mdi_child)
        self.pasteAct.setEnabled(has_mdi_child)
        self.closeAct.setEnabled(has_mdi_child)
        self.closeAllAct.setEnabled(has_mdi_child)
        self.tileAct.setEnabled(has_mdi_child)
        self.cascadeAct.setEnabled(has_mdi_child)
        self.nextAct.setEnabled(has_mdi_child)
        self.previousAct.setEnabled(has_mdi_child)
        self.separatorAct.setVisible(has_mdi_child)

    def update_window_menu(self):
        self.windowMenu.clear()
        self.windowMenu.addAction(self.closeAct)
        self.windowMenu.addAction(self.closeAllAct)
        self.windowMenu.addSeparator()
        self.windowMenu.addAction(self.tileAct)
        self.windowMenu.addAction(self.cascadeAct)
        self.windowMenu.addSeparator()
        self.windowMenu.addAction(self.nextAct)
        self.windowMenu.addAction(self.previousAct)
        self.windowMenu.addAction(self.separatorAct)

        windows = self.mdiArea.subWindowList()
        self.separatorAct.setVisible(len(windows) != 0)

        for i, window in enumerate(windows):
            child = window.widget()

            text = f'{i + 1:d} {child.user_friendly_current_file()}'
            if i < 9:
                text = '' + text

            action = self.windowMenu.addAction(text)
            action.setCheckable(True)
            action.setChecked(child == self.active_mdi_child())
            action.triggered.connect(self.windowMapper.map)
            self.windowMapper.setMapping(action, window)

    def create_mdi_child(self):
        child = MDIChildPlot()
        self.mdiArea.addSubWindow(child)
        return child

    def read_settings(self):
        pos = self.settings.value('pos', QPoint(200, 200))
        self.move(pos)
        size = self.settings.value('size', QSize(400, 400))
        self.resize(size)
        self.last_directory = self.settings.value('directory', '')

    def write_settings(self):
        self.settings.setValue('pos', self.pos())
        self.settings.setValue('size', self.size())
        self.settings.setValue('directory', self.last_directory)

    def active_mdi_child(self):
        active_sub_window = self.mdiArea.activeSubWindow()
        if active_sub_window:
            return active_sub_window.widget()
        return None

    def switch_layout_direction(self):
        if self.layoutDirection() == Qt.LeftToRight:
            qApp.setLayoutDirection(Qt.RightToLeft)
        else:
            qApp.setLayoutDirection(Qt.LeftToRight)

    def set_active_sub_window(self, window):
        if window:
            self.mdiArea.setActiveSubWindow(window)

if __name__ == '__main__':
    import sys

    app = QApplication(sys.argv)
    mainWin = MainWindow()
    mainWin.show()
    sys.exit(app.exec_())


import argparse
import datetime
import subprocess
import psutil
import time
import os
import threading
from tqdm import tqdm
from random import shuffle

FNULL = open(os.devnull, 'w')
facesPath = None
videosPath = None

def maybeCreateDir(path):
    if not os.path.exists(path):
        os.mkdir(path)

def getSize():
    tot = 0
    for f in os.listdir(facesPath):
        fp = os.path.join(facesPath, f)
        tot += os.path.getsize(fp)
    for f in os.listdir(videosPath):
        fp = os.path.join(videosPath, f)
        tot += os.path.getsize(fp)
    return tot

def runCommands(commands, idx):
    shuffle(commands)
    i = 0
    while i < len(commands):
        size = getSize()
        if size > 2 * (1024 ** 3):
            time.sleep(10)
            continue

        print("{}: ({}/{}), size = {}".format(idx, i + 1, len(commands), size))
        subprocess.run(commands[i], stdout=FNULL, stderr=subprocess.STDOUT)
        i += 1

parser = argparse.ArgumentParser()
parser.add_argument("-op", "--outputPath", required = True)
parser.add_argument("-t", "--numThreads", required = True)
args = parser.parse_args()

outputPath = args.outputPath
videosPath = os.path.join(outputPath, "videos")
facesPath = os.path.join(outputPath, "faces")
maybeCreateDir(outputPath)
maybeCreateDir(videosPath)
maybeCreateDir(facesPath)

alreadyThere = set()
for x in os.listdir(facesPath):
    alreadyThere.add(x.rsplit('_', 1)[0])

commands = []
with open("avspeech_train.csv", "r") as f:
    for line in tqdm(f.readlines()):
        line = line.split(',')
        videoId = line[0]

        if videoId in alreadyThere:
            continue

        start = float(line[1])
        end = min(float(line[2]), start + 60)

        start = str(datetime.timedelta(seconds=start))
        end = str(datetime.timedelta(seconds=end))

        command = [ "youtube-dl",
                    "https://www.youtube.com/watch?v={}".format(videoId),
                    "--external-downloader",
                    "ffmpeg",
                    "-f",
                    "webm",
                    "--external-downloader-args",
                    "-ss {} -to {}".format(start, end),
                    "-o",
                    os.path.join(videosPath, "{}.webm".format(videoId))
        ]

        commands.append(command)

numThreads = int(args.numThreads)
chunkSize = len(commands) // numThreads
splits = [[chunkSize * i, chunkSize * (i + 1)] for i in range(numThreads)]
splits[-1][-1] = len(commands)

for i, (l, r) in enumerate(splits):
    t = threading.Thread(target=runCommands, args=(commands[l:r], i))
    t.start()


from typing import Union, List, Callable, Optional, Dict
import urllib.parse
import os
import numpy as np
import torch
import torchvision
from .html_table import HTMLTableVisualizer
import logging

logger = logging.getLogger(__name__)
VCV_URL_PREFIX = "http://vcv.stanford.edu"
VCV_FILE_URL_PREFIX = f"{VCV_URL_PREFIX}/cgi-bin/file-explorer/?"

def setup_vi(path=None, exist_ok=True, title='default'):
    from ..common import vi_helper

    if path is None:
        from ..common import vi
        return vi, vi_helper
    if not exist_ok:
        assert not os.path.exists(path), (path, title)
    vi = HTMLTableVisualizer(path, title=title)
    vi.begin_html()
    vi_helper.print_url(vi)
    return vi, vi_helper

def tensor2im(image_tensor: Union[torch.Tensor, List[torch.Tensor], None]):
    """

    Args:
        image_tensor: (list of) shape [..., C, H, W]

    Returns:

    """
    if image_tensor is None:
        return None
    if isinstance(image_tensor, list):
        return [tensor2im(x) for x in image_tensor]  # keep the list structure
    # if image_tensor.dim() == 5 or image_tensor.dim() == 4:
    #     return [tensor2im(image_tensor[idx])
    #             for idx in range(image_tensor.size(0))]

    # n_dim = len(image_tensor.shape)  # (..., c, h, w)
    image_tensor = image_tensor.movedim(-3, -1)

    image_numpy = image_tensor.cpu().float().numpy()
    # assume range (0, 1)
    image_numpy = image_numpy * 255.0
    image_numpy = np.clip(image_numpy, 0, 255)  # (..., h, w, c)

    # to pil
    # output_img = Image.fromarray(np.uint8(image))
    # if width is not None and height is not None:
    #     output_img = output_img.resize((width, height), Image.BICUBIC)
    return image_numpy.astype(np.uint8)

def collect_tensor(tensor: torch.Tensor,
                   process_grid: Callable[[torch.Tensor], np.ndarray] = tensor2im,
                   pad_value=0, padding=0,
                   value_check: bool = True):
    """

    Args:
        tensor: expect range [0, 1]
        pad_value:
        padding:
        process_grid: e.g. tensor2im, tensor2flow
        value_check:

    Returns:

    """
    if value_check and (tensor.min() < -1e-3 or tensor.max() > 1 + 1e-3):
        print("checking min / max, not normalized? ")
        print(tensor.min(), tensor.max())
        # raise

    if len(tensor.shape) == 3:
        tensor = tensor[None]
    elif len(tensor.shape) == 4:  # (n, c, h, w)
        nrow_this = 1
    elif len(tensor.shape) == 5:  # (n, t, c, h, w)
        nrow_this = tensor.shape[1]

    tensor = tensor.flatten(end_dim=-4)
    if tensor.shape[0] == 1:
        # this is a hack since make_grid does not pad bs=1 tensors
        image_grid = torch.ones((tensor.shape[-3], tensor.shape[-2] + 2 * padding, tensor.shape[-1] + 2 * padding),
                                ) * pad_value
        image_grid.narrow(1, padding, tensor.shape[-2]).narrow(
            2, padding, tensor.shape[-1]
        ).copy_(tensor[0])

        if image_grid.shape[0] == 1:
            # output as rgb channels
            image_grid = image_grid.expand(3, image_grid.shape[1], image_grid.shape[2])

    else:
        image_grid = torchvision.utils.make_grid(tensor, nrow=nrow_this, pad_value=pad_value,
                                                 padding=padding)  # (b or b*d, c, h, w) -> (c, h, w)
        # make_grid will always output channel = 3
    image_grid_np = process_grid(image_grid)
    return image_grid_np

def print_viscam_url(path, verbose=True):
    path = os.path.abspath(path)
    # local file
    url = f"file:///{path}"
    if verbose:
        logger.info(f"[URL] {url}")
    return url

def print_vcv_url(path, verbose=True, nrow=4, query_kwargs: Optional[Dict] = None) -> str:
    path = os.path.abspath(path)

    filename = ''
    if os.path.isfile(path):
        path, filename = os.path.split(path)

    query = dict(
        dir=path,
        patterns_show=filename,
        patterns_highlight='',
        w=1500 // nrow, h=1500,
        n=nrow, autoplay=1, showmedia=1,
    )
    if query_kwargs is not None:
        query.update(query_kwargs)
    url_query = urllib.parse.urlencode(query)
    url = f'{VCV_FILE_URL_PREFIX}{url_query}'
    if verbose:
        logger.info(f"[URL] {url}")
    return url


from pydantic import BaseModel, Extra, Field

class Reference(BaseModel):
    """
    A simple object to allow referencing other components in the specification, internally and externally.

    The Reference Object is defined by [JSON Reference](https://tools.ietf.org/html/draft-pbryan-zyp-json-ref-03)
    and follows the same structure, behavior and rules.

    For this specification, reference resolution is accomplished as defined by the JSON Reference specification
    and not by the JSON Schema specification.
    """

    ref: str = Field(alias="$ref")
    """**REQUIRED**. The reference string."""

    class Config:
        extra = Extra.ignore
        allow_population_by_field_name = True
        schema_extra = {
            "examples": [{"$ref": "#/components/schemas/Pet"}, {"$ref": "Pet.json"}, {"$ref": "definitions.json#/Pet"}]
        }


# Stairs
# https://www.interviewbit.com/problems/stairs/
#
# You are climbing a stair case. It takes n steps to reach to the top.
#
# Each time you can either climb 1 or 2 steps. In how many distinct ways can you climb to the top?
#
# Example :
#
# Input : 3
# Return : 3
#
# Steps : [1 1 1], [1 2], [2 1]
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

class Solution:
    # @param A : integer
    # @return an integer
    def climbStairs(self, A):
        result = [0] * A

        if A < 2:
            return A

        result[0], result[1] = 1, 2

        for i in range(2, A):
            result[i] = result[i - 1] + result[i - 2]

        return result[-1]

# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #


from math import gcd
N, M = map(int, input().split())
AC = [list(map(int, input().split())) for _ in range(M)]
AC.sort(key=lambda x:x[1])

# クラスカル法 ... コストがちっちゃい path から貪欲に追加していく
ans = 0
for a, c in AC:
    a = gcd(N, a)
    # 幅 a の橋をできるだけ作る
    ans += (N-a) * c
    N = a # 状態数はa個にまとめられる

# 状態が一つになる <=> 全域木ができた
if N == 1:
    print(ans)
else:
    print(-1)


# -*- coding: utf-8 -*-
from flask import Flask
from flask import jsonify, request, make_response
from flask_sqlalchemy import SQLAlchemy

from data import db_session

# Importing all models to work correctly in app.py
from data.transport_type import TransportTypes
from data.regions_to_couriers import CouriersToRegions
from data.orders import Orders
from data.regions import Regions
from data.delivery import Delivery
from data.couriers import *

# Other packages
import os
from pprint import pprint
import datetime

app = Flask(__name__)

@app.route('/couriers', methods=['POST'])
def post_couriers():
    '''
    1) POST /couriers
    '''
    try:
        data = request.get_json()
        couriers_list = data['data']
    except BaseException as be:
        return make_response(
            jsonify({'validation_error': 'wrong json data'}), 400)

    db_sess = db_session.create_session()

    validation_error = {"validation_error": {
        "couriers": []
    }
    }

    valid_couriers = []
    for courier_json in couriers_list:

        if Couriers.validate_courier_json(courier_json, logger=app.logger):
            courier_json['working_hours'] = convert_wh_hours_to_str(
                courier_json['working_hours'])
            courier_json['courier_type'] = db_sess.query(
                TransportTypes).filter(
                    TransportTypes.type_name
                    == courier_json['courier_type']).first()
        else:
            validation_error["validation_error"]['couriers'].append(
                {'id': courier_json['courier_id']})
            continue

        added_regions = []
        regions = []
        for region_name in courier_json['regions']:
            if db_sess.query(Regions).filter(Regions.region_code ==
                                             region_name).first(
            ) is None and region_name not in added_regions:
                reg = Regions(region_name)
                db_sess.add(reg)
                # db_sess.commit()
                added_regions.append(reg)
            else:
                reg = db_sess.query(Regions).filter(
                    Regions.region_code == region_name).first()
                regions.append(reg)

        courier = Couriers()
        courier.courier_id = courier_json['courier_id']
        courier.courier_type_id = courier_json['courier_type'].type_id
        courier.regions = added_regions + regions
        courier.working_hours = courier_json['working_hours']
        valid_couriers.append(courier)

    if len(validation_error["validation_error"]['couriers']) != 0:
        return make_response(jsonify(validation_error), '400 Bad Request')

    response = {'couriers': []}
    app.logger.info(f'Adding the following Couriers:')
    for cour in valid_couriers:
        app.logger.info(cour)
        db_sess.add(cour)
        response['couriers'].append({"id": cour.courier_id})
        db_sess.commit()

    return make_response(jsonify(response), '201 Created')

@app.route('/couriers/<int:courier_id>', methods=['PATCH'])
def patch_couriers(courier_id):
    '''
    2) PATCH /couriers/$courier_id
    '''
    if isinstance(courier_id, int):
        if courier_id > 0:
            data = request.get_json()
            db_sess = db_session.create_session()

            if Couriers.validate_patch(data, logger=app.logger):

                cour = db_sess.query(Couriers).filter(
                    Couriers.courier_id.like(courier_id)).first()

                if cour:
                    app.logger.info(
                        f'Changing {cour} with following params:\n{data.keys()}')
                    for key in data:
                        # TODO Bad perfomance, change with getattr/setattr(?) or
                        # property function
                        if isinstance(data[key], str):

                            new_type = db_sess.query(TransportTypes).filter(
                                TransportTypes.type_name == data[key]).first()
                            if new_type:
                                cour.change_cour_type(new_type, db_sess)

                        elif isinstance(data[key][0], int):

                            new_regions = data[key]
                            cour.change_cour_regions(new_regions, db_sess)

                        elif isinstance(data[key][0], str):

                            new_working_hours = data[key]
                            cour.change_cour_work_hours(
                                new_working_hours, db_sess)

                    return make_response(
                        jsonify(
                            Couriers.make_courier_response(
                                cour)),
                        '200 OK')
                else:
                    app.logger.info(f'No courier with id {courier_id}')
                    return '', '404 Not found'

    app.logger.info(f'Requets courier with id {courier_id} is bad')
    return '', '400 Bad request'

@ app.route('/orders', methods=['POST'])
def set_orders():
    '''
    3) POST /orders
    '''
    try:
        data = request.get_json()
        orders_list = data['data']
    except BaseException as be:
        app.logger.exception(f'Wrong JSON data passed: {be}')
        return make_response('400 Bad request')

    db_sess = db_session.create_session()
    ve = False
    validation_error = {"validation_error": {
        "orders": []
    }
    }

    valid_orders = []
    for order_json in orders_list:

        if Orders.validate_order_json(order_json, logger=app.logger):
            if ve:
                continue
            order = Orders()
            order.order_id = order_json['order_id']
            order.weight = order_json['weight']

            query_region = db_sess.query(Regions).filter(
                Regions.region_code == order_json['region']).first()
            if query_region is None:
                reg = Regions(region_code=order_json['region'])
            else:
                reg = query_region

            db_sess.add(reg)

            order.region = reg
            order.delivery_time = convert_wh_hours_to_str(
                order_json['delivery_hours'])

            valid_orders.append(order)
        else:
            try:
                validation_error["validation_error"]['orders'].append(
                    {'id': order_json['order_id']}
                )
            except KeyError as ke:
                validation_error["validation_error"]['schema_error'] = order_json
            db_sess.rollback()
            ve = True
            continue

    if ve:
        app.logger.info(f'Passes not valid data: {validation_error}')
        return make_response(jsonify(validation_error), '400 Bad request')
    else:
        response = {'orders': []}

        for order in valid_orders:
            db_sess.add(order)
            response['orders'].append({"id": order.order_id})
            db_sess.commit()
        return make_response(jsonify(response), '201 Created')

@ app.route('/orders/assign', methods=['POST'])
def assign_orders():
    '''
    4) POST /orders/assign
    '''
    data = request.get_json()
    db_sess = db_session.create_session()

    if data is not None:
        if Couriers.validate_assigment(data):
            courier_id = data['courier_id']

            cour = db_sess.query(Couriers).filter(
                Couriers.courier_id == courier_id).first()

            if cour is not None:
                if not cour.is_working():

                    regions = [reg.region_id for reg in cour.regions]

                    orders = db_sess.query(Orders).filter(
                        Orders.order_complete_time == None).filter(
                            Orders.delivery_id == None).filter(
                                Orders.weight <= cour.courier_type.type_weight,
                                Orders.region_id.in_(regions)).all()

                    for_deliver = []
                    for order in orders:
                        if cour.could_he_take(order):
                            for_deliver.append(order)

                    if len(for_deliver) > 0:
                        response = {
                            "orders": [],
                            "assign_time": "2021-01-10T09:32:14.42Z"
                        }

                        delivery = Delivery()
                        for order in for_deliver:
                            response['orders'].append({'id': order.order_id})
                            # --- DB ---

                        delivery.delivery_courier = cour
                        delivery.assigned_courier_type_id = cour.courier_type_id
                        delivery.assigned_courier_type = cour.courier_type
                        assign_time = datetime.datetime.now()
                        delivery.assign_time = assign_time
                        delivery.orders_in_delivery = for_deliver
                        app.logger.info(
                            f'{cour} got {delivery} with {for_deliver}')
                        db_sess.add(delivery)
                        db_sess.commit()
                        # --- END ---

                        response['assign_time'] = assign_time.isoformat()[
                            :22] + "Z"
                        return make_response(jsonify(response), '200 OK')
                    return make_response(jsonify({'orders': []}), '200 OK')
                else:
                    try:
                        current_delivery = cour.get_current_delivery()
                        app.logger.info(
                            f'Courier {courier_id} have {current_delivery}')
                        orders_list = db_sess.query(
                            Orders.order_id).filter(
                                Orders.delivery_id
                                == current_delivery.delivery_id).filter(
                                    Orders.order_complete_time == None
                        ).all()
                        response = {
                            'orders':
                            [{'id': order.order_id} for order in orders_list],
                            'assign_time': current_delivery.get_str_assign_time()}
                        return make_response(jsonify(response), '200 OK')
                    except BaseException:
                        app.logger.exception(
                            '/assign can`t show any information.')
                        return '', '400 Bad request'
    return '', '400 Bad request'

@ app.route('/orders/complete', methods=['POST'])
def complete_order():
    '''
    5) POST /orders/complete
    '''
    data = request.get_json()
    db_sess = db_session.create_session()
    if data:
        if Orders.validate_complete(data, db_sess, logger=app.logger):
            order = db_sess.query(Orders).filter(
                Orders.order_id == data['order_id']).first()

            response = {'order_id': order.order_id}
            complete_time = datetime.datetime.fromisoformat(
                data['complete_time'][:22] + '0')

            if not order.is_completed():
                order.order_complete_time = complete_time
            else:
                return make_response(jsonify(response), '200 OK')

            db_sess.add(order)

            delivery = order.delivery
            if delivery.is_completed():
                delivery.delivery_complete_time = complete_time

            db_sess.add(delivery)
            db_sess.commit()
            return make_response(jsonify(response), '200 OK')
    return '', '400 Bad request'

@ app.route('/couriers/<int:courier_id>', methods=['GET'])
def get_courier_info(courier_id):
    '''
    6) GET /couriers/$courier_id
    '''
    db_sess = db_session.create_session()
    if courier_id >= 1:

        courier = db_sess.query(Couriers).filter(
            Couriers.courier_id == courier_id).first()

        if courier:
            delivery = db_sess.query(Delivery)\
                .filter(Delivery.delivery_complete_time is not None)\
                .filter(Delivery.courier_id == courier_id).all()

            # Counting Earnings of courier
            earnings = 0
            for d in delivery:
                earnings += d.count_earning()
                courier.earnings = earnings

            add_response = {'earnings': earnings}
            # Counting rationg of courier
            if len(delivery) > 0:
                regions_avg = Regions.count_avg_time_from_orders(delivery)
                if len(regions_avg) != 0:
                    rating = Couriers.count_rating_from_regions_avg(
                        regions_avg)

                    courier.rating = rating
                    add_response['rating'] = rating

            db_sess.add(courier)
            response = Couriers.make_courier_response(
                courier, **add_response)
            db_sess.commit()
            return make_response(jsonify(response), 200)
        else:
            app.logger.info(f'No courier with id {courier_id}')

    return '', '404 Not found'

def main(debug=False):
    # Preparing db and run app
    db_session.global_init_sqlite('db.sqlite')
    app.config.from_envvar('API_CONFIG')
    app.run(host='0.0.0.0', port=8080)

if __name__ == '__main__':
    main(debug=True)


DEGREE_SIGN= u'\N{DEGREE SIGN}'
HUMIDITY = 'Humidity: %d%%'

TEMPERATURE_TEXT_SIZE=140
DEGREE_UNIT_TEXT_SIZE=60
WEATHER_DETAILS_TEXT_SIZE=38

WEATHER_REFRESH_RATE=600000 # 10 Minutes
WEATHER_REQUEST_ENDPOINT="https://api.darksky.net/forecast/%s/%s,%s?lang=%s&units=%s"

KELVIN="K"
CELCIUS="C"
FAHRENHEIT="F"


import os
import subprocess
import psutil

from nodemgr.common.common_sys_cpu import SysCpuShare
from pysandesh.gen_py.sandesh.ttypes import SandeshLevel
from nodemgr.common.sandesh.nodeinfo.cpuinfo.ttypes import SysMemInfo, SysCpuInfo, CpuLoadAvg
from nodemgr.common.sandesh.nodeinfo.process_info.ttypes import DiskPartitionUsageStats

class LinuxSysData(object):
    def __init__(self, msg_log, corefile_path):
        self.msg_log = msg_log
        self.corefile_path = corefile_path
        self.core_dir_modified_time = 0
        self.num_socket = None
        self.num_cpu = None
        self.num_core_per_socket = None
        self.num_thread_per_core = None
        self.sys_cpu_share = SysCpuShare(self.get_num_cpu())

    @staticmethod
    def _run_cmd(cmd):
        proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, close_fds=True)
        return int(proc.communicate()[0])

    def get_num_socket(self):
        if self.num_socket is not None:
            return self.num_socket
        self.num_socket = LinuxSysData._run_cmd(
            'lscpu | grep "Socket(s):" | awk \'{print $2}\'')
        return self.num_socket

    def get_num_cpu(self):
        if self.num_cpu is not None:
            return self.num_cpu
        self.num_cpu = LinuxSysData._run_cmd(
            'lscpu | grep "^CPU(s):" | awk \'{print $2}\'')
        return self.num_cpu

    def get_num_core_per_socket(self):
        if self.num_core_per_socket is not None:
            return self.num_core_per_socket
        self.num_core_per_socket = LinuxSysData._run_cmd(
            'lscpu | grep "Core(s) per socket:" | awk \'{print $4}\'')
        return self.num_core_per_socket

    def get_num_thread_per_core(self):
        if self.num_thread_per_core is not None:
            return self.num_thread_per_core
        self.num_thread_per_core = LinuxSysData._run_cmd(
            'lscpu | grep "Thread(s) per core:" | awk \'{print $4}\'')
        return self.num_thread_per_core

    def get_sys_mem_info(self, node_type):
        virtmem_info = psutil.virtual_memory()
        sys_mem_info = SysMemInfo()
        sys_mem_info.total = virtmem_info.total // 1024
        sys_mem_info.used = virtmem_info.used // 1024
        sys_mem_info.free = virtmem_info.free // 1024
        sys_mem_info.buffers = virtmem_info.buffers // 1024
        sys_mem_info.cached = virtmem_info.cached // 1024
        sys_mem_info.node_type = node_type
        return sys_mem_info

    def get_sys_cpu_info(self, node_type):
        cpu_load_avg = self._get_sys_cpu_load_avg()
        sys_cpu_info = SysCpuInfo()
        sys_cpu_info.one_min_avg = cpu_load_avg.one_min_avg
        sys_cpu_info.five_min_avg = cpu_load_avg.five_min_avg
        sys_cpu_info.fifteen_min_avg = cpu_load_avg.fifteen_min_avg
        sys_cpu_info.cpu_share = self.sys_cpu_share.get()
        sys_cpu_info.node_type = node_type
        return sys_cpu_info

    def _get_sys_cpu_load_avg(self):
        load_avg = os.getloadavg()
        cpu_load_avg = CpuLoadAvg()
        cpu_load_avg.one_min_avg = load_avg[0]
        cpu_load_avg.five_min_avg = load_avg[1]
        cpu_load_avg.fifteen_min_avg = load_avg[2]
        return cpu_load_avg

    def check_ntp_status(self):
        # chronyc fails faster - it should be first
        ntp_status_cmds = [
            'chronyc -n sources | grep "^\^\*"', # noqa
            'ntpq -n -c pe | grep "^\*"', # noqa
        ]
        for cmd in ntp_status_cmds:
            # TODO: cache chosen method and use it later
            proc = subprocess.Popen(
                cmd, shell=True,
                stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
            (_, _) = proc.communicate()
            if proc.returncode == 0:
                return True
        return False

    def _get_corefile_path(self):
        result = self.corefile_path
        with open('/proc/sys/kernel/core_pattern', 'r') as fh:
            try:
                core_pattern = fh.readline().strip('\n')
                if core_pattern is not None and not core_pattern.startswith('|'):
                    result = os.path.dirname(core_pattern)
            except Exception:
                pass
        return result

    def get_corefiles(self):
        try:
            corefile_path = self._get_corefile_path()
            exception_set = {"lost+found"}
            files = [os.path.abspath(os.path.join(corefile_path, x)) for x in os.listdir(corefile_path)]
            files = filter(os.path.isfile, files)
            files = [os.path.join(corefile_path, f) for f in files if f not in exception_set]
            files.sort(key=lambda x: os.path.getmtime(x))
            return files
        except Exception as e:
            self.msg_log('Failed to get core files: %s' % (str(e)),
                         SandeshLevel.SYS_ERR)
            return list()

    def remove_corefiles(self, core_files):
        for core_file in core_files:
            self.msg_log('deleting core file: %s' % (core_file),
                         SandeshLevel.SYS_ERR)
            try:
                os.remove(core_file)
            except OSError as e:
                self.msg_log('Failed to delete core file: %s - %s'
                             % (core_file, str(e)),
                             SandeshLevel.SYS_ERR)

    def update_all_core_file(self):
        try:
            modified_time = os.path.getmtime(self._get_corefile_path())
        except OSError:
            # folder is not present - corefiles have not been changed
            return False
        if modified_time == self.core_dir_modified_time:
            return False
        self.core_dir_modified_time = modified_time
        self.all_core_file_list = self.get_corefiles()
        return True

    def get_disk_usage(self):
        disk_usage_info = {}
        partition = subprocess.Popen(
            "df -PT -t ext2 -t ext3 -t ext4 -t xfs",
            shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
            close_fds=True)
        for line in partition.stdout:
            line = line.decode()
            if 'Filesystem' in line:
                continue
            partition_name = line.rsplit()[0]
            partition_type = line.rsplit()[1]
            partition_space_used_1k = line.rsplit()[3]
            partition_space_available_1k = line.rsplit()[4]
            disk_usage_stat = DiskPartitionUsageStats()
            try:
                disk_usage_stat.partition_type = str(partition_type)
                disk_usage_stat.__key = str(partition_name)
                disk_usage_stat.partition_space_used_1k = \
                    int(partition_space_used_1k)
                disk_usage_stat.partition_space_available_1k = \
                    int(partition_space_available_1k)
                total_disk_space = \
                    disk_usage_stat.partition_space_used_1k + \
                    disk_usage_stat.partition_space_available_1k
                disk_usage_stat.percentage_partition_space_used = \
                    int(round((float(disk_usage_stat.partition_space_used_1k) /
                               float(total_disk_space)) * 100))
            except ValueError:
                self.msg_log('Failed to get local disk space usage',
                             SandeshLevel.SYS_ERR)
            else:
                disk_usage_info[partition_name] = disk_usage_stat
        return disk_usage_info