"""
Signal characteristics animation
"""

import os
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
from matplotlib import gridspec
matplotlib.use('TkAgg')

class sigCharacterAnimation:
    '''
    Animation for signal characteristics
    '''
    # signal attributes
    am1, am2 = None, None           # signal amplitude, should <=1
    f1, f2 = None, None             # signal frequency, should >=1, multiple of 1
    phase1, phase2 = None, None     # signal phase, [0, 2pi)
    t = None
    sin1, cos1 = None, None         # signal 1
    sin2, cos2 = None, None         # signal 2
    # animation attributes
    fig, axes = None, None
    lines, point = None, None
    frames = None
    transFigure = None

    def __init__(self, signal1=None, signal2=None):
        if signal1 is None:
            signal1 = [1, 1, 0]
        if signal2 is None:
            signal2 = [1, 2, np.pi / 2]
        self.am1, self.am2 = signal1[0], signal2[0]
        self.f1, self.f2 = signal1[1], signal2[1]
        self.phase1, self.phase2 = signal1[2], signal2[2]
        # generate signal
        min_freq = min(self.f1, self.f2)
        fs = 64 * min_freq
        self.t = np.arange(0, 1 / min_freq, 1 / fs)
        self.sin1 = self.am1 * np.sin(2 * np.pi * self.f1 * self.t + self.phase1)
        self.cos1 = self.am1 * np.cos(2 * np.pi * self.f1 * self.t + self.phase1)
        self.sin2 = self.am2 * np.sin(2 * np.pi * self.f2 * self.t + self.phase2)
        self.cos2 = self.am2 * np.cos(2 * np.pi * self.f2 * self.t + self.phase2)
        self.frames = len(self.t)

    def start(self, save_gif=False, f_name='correlation'):
        '''start animation or save as gif'''
        self._init_animation()
        anim = FuncAnimation(self.fig, self._animate, frames=self.frames, blit=False,
                             interval=100, init_func=self._init_canvas)
        if save_gif:
            if not os.path.exists('vis'):
                os.makedirs('vis')
            anim.save(os.path.join('vis', f_name + '.gif'), codec='png', writer='imagemagick')
        else:
            plt.show()

    def _init_animation(self):
        ''' initialize animation'''

        self.fig = plt.figure(figsize=(8, 6))
        self.axes = [matplotlib.axes.Axes] * 4
        gs = gridspec.GridSpec(2, 2, width_ratios=[1, 2])
        self.axes[0] = plt.subplot(gs[0, 0], aspect='equal')
        self.axes[1] = plt.subplot(gs[0, 1])
        self.axes[2] = plt.subplot(gs[1, 0], aspect='equal')
        self.axes[3] = plt.subplot(gs[1, 1])

        self.lines = [matplotlib.axes.Axes.plot] * 4
        self.points = [matplotlib.axes.Axes.plot] * 4
        self.connect = [matplotlib.lines.Line2D] * 2
        circ1 = plt.Circle((0, 0), self.am1, fill=False, linewidth=2, color='orange')
        circ2 = plt.Circle((0, 0), self.am2, fill=False, linewidth=2, color='green')
        self.axes[0].add_artist(circ1)
        self.axes[0].set_xlim(-1.1, 1.1)
        self.axes[0].set_ylim(-1.1, 1.1)
        self.axes[0].axis('off')
        self.axes[1].set_xlim(-0.1, 1)
        self.axes[1].set_ylim(-1.1, 1.1)
        self.axes[1].axis('off')
        self.axes[2].add_artist(circ2)
        self.axes[2].set_xlim(-1.1, 1.1)
        self.axes[2].set_ylim(-1.1, 1.1)
        self.axes[2].axis('off')
        self.axes[3].set_xlim(-0.1, 1)
        self.axes[3].set_ylim(-1.1, 1.1)
        self.axes[3].axis('off')

        self.lines[0], = self.axes[0].plot([], [], linewidth=2)
        self.lines[1], = self.axes[1].plot([], [], linewidth=2, color='orange')
        self.lines[2], = self.axes[2].plot([], [], linewidth=2)
        self.lines[3], = self.axes[3].plot([], [], linewidth=2, color='green')
        self.points[0], = self.axes[0].plot([], [], 'ro', markersize=6)
        self.points[1], = self.axes[1].plot([], [], 'ro', markersize=6)
        self.points[2], = self.axes[2].plot([], [], 'ro', markersize=6)
        self.points[3], = self.axes[3].plot([], [], 'ro', markersize=6)
        self.connect[0] = matplotlib.lines.Line2D([], [], color='r', transform=self.fig.transFigure)
        self.connect[1] = matplotlib.lines.Line2D([], [], color='r', transform=self.fig.transFigure)
        self.fig.lines.extend(self.connect)
        self.transFigure = self.fig.transFigure.inverted()

    def _init_canvas(self):
        '''do nothing, return artists to be re-drawn'''
        return self.lines + self.points + self.connect

    def _animate(self, i):
        '''perform animation step'''
        # update artists
        self.lines[0].set_data([0, self.cos1[-i]], [0, self.sin1[-i]])
        self.lines[1].set_data(self.t, np.roll(self.sin1, i))
        self.lines[2].set_data([0, self.cos2[-i]], [0, self.sin2[-i]])
        self.lines[3].set_data(self.t, np.roll(self.sin2, i))
        self.points[0].set_data([self.cos1[-i]], [self.sin1[-i]])
        self.points[1].set_data([0], [self.sin1[-i]])
        self.points[2].set_data([self.cos2[-i]], [self.sin2[-i]])
        self.points[3].set_data([0], [self.sin2[-i]])

        coord1 = self.transFigure.transform(self.axes[0].transData.transform([self.cos1[-i], self.sin1[-i]]))
        coord2 = self.transFigure.transform(self.axes[1].transData.transform([0, self.sin1[-i]]))
        self.connect[0].set_data((coord1[0], coord2[0]), (coord1[1], coord2[1]))
        coord1 = self.transFigure.transform(self.axes[2].transData.transform([self.cos2[-i], self.sin2[-i]]))
        coord2 = self.transFigure.transform(self.axes[3].transData.transform([0, self.sin2[-i]]))
        self.connect[1].set_data((coord1[0], coord2[0]), (coord1[1], coord2[1]))
        return self.lines + self.points + self.connect

if __name__ == '__main__':

    save_fig = True
    # amplitude
    # anim = sigCharacterAnimation(signal1=[1, 1, 0], signal2=[0.5, 1, 0])
    # anim.start(save_gif=save_fig, f_name='sig_char_amplitude')

    # # frequency
    # anim = sigCharacterAnimation(signal1=[1, 1, 0], signal2=[1, 2, 0])
    # anim.start(save_gif=save_fig, f_name='sig_char_frequency')

    # phase
    anim = sigCharacterAnimation(signal1=[1, 1, 0], signal2=[1, 1, np.pi / 2])
    anim.start(save_gif=save_fig, f_name='sig_char_phase')


""" Provides the different Bounds that are used by the Table to determine
the Cells that are adjacent to the Table. """

from __future__ import annotations

from _operator import attrgetter
from itertools import cycle
from typing import Callable, cast, Iterable, NamedTuple, Protocol, TypeVar

from pdf2gtfs.config import Config
from pdf2gtfs.datastructures.pdftable.bbox import BBox
from pdf2gtfs.datastructures.table.direction import Direction, E, N, S, W
from pdf2gtfs.datastructures.table.cell import C, Cs

B = TypeVar("B", bound="Bounds")

class F(Protocol):
    """ Used as a type to typecheck min/max functions correctly. """
    def __call__(self, cells: Iterable[C] | Iterable[BBox],
                 key: Callable[[C | BBox], float]) -> C:
        pass

# Arguments used by the N-/S-/W-/EBounds.
#  — func: The function (min / max) used to determine the correct limit.
#  — direction: The Direction of the limit.
BoundArg = NamedTuple("BoundArg", [("func", F), ("direction", Direction)])

class Bounds:
    """ Basic Bounds, where not all limits necessarily exist. """
    d: Direction = None

    def __init__(self, n: float | None, w: float | None,
                 s: float | None, e: float | None) -> None:
        self._n = n
        self._w = w
        self._s = s
        self._e = e
        self._update_hbox()
        self._update_vbox()

    @classmethod
    def from_bboxes(cls, bboxes: list[BBox], *,
                    n: BoundArg | None = None, w: BoundArg | None = None,
                    s: BoundArg | None = None, e: BoundArg | None = None
                    ) -> B:
        """ Create a new Bounds from the BBoxes, based on which args are given.

        :param bboxes: The BBoxes used for construction.
        :param n: The northern BoundArg. None for NBounds.
        :param w: The western BoundArg. None for WBounds.
        :param s: The southern BoundArg. None for SBounds.
        :param e: The eastern BoundArg. None for EBounds.
        :return: A new Bounds created from the given BBoxes,
            based on which BoundArgs are provided.
        """
        return cls(n=get_limit_from_cells(bboxes, n),
                   w=get_limit_from_cells(bboxes, w),
                   s=get_limit_from_cells(bboxes, s),
                   e=get_limit_from_cells(bboxes, e))

    @classmethod
    def overlaps_any(cls, cells: Cs, c2: C) -> bool:
        """ Check if the given Cell overlaps with any of the given Cells.

        The overlap function is determined by cls.

        :param cells: Use these Cells to check overlap.
        :param c2: The Cell that is checked.
        :return: True if the Cell overlaps. False, otherwise.
        """
        func = getattr(c2.bbox, cls.d.o.overlap_func)
        for c1 in cells:
            if func(c1.bbox, 0.8):
                return True
        return False

    @classmethod
    def select_adjacent_cells(cls, border: list[BBox], cells: Cs) -> Cs:
        """ Select those Cells that are adjacent to the border BBoxes.

        :param border: The row/col of a Table that is used to determine
            if a Cell is adjacent to the Table.
        :param cells: The Cells that are checked for adjacency.
        :return: Those Cells, which are adjacent to the Table.
        """
        def get_all_adjacent_cells() -> Cs:
            """ Get Cells that fit only three bounds,
            but are overlapping with Cells that overlap all four.

            If we are extra_greedy, also get those cells that recursively
            overlap with cells that overlap with other cells.
            """
            # Need to shallow copy for overlap_cells to be different.
            all_cells = list(min_cells)
            overlap_cells = all_cells if Config.extra_greedy else min_cells
            while True:
                new_cells = [c for c in cells
                             if cls.overlaps_any(overlap_cells, c)
                             and c not in all_cells]
                if not new_cells:
                    break
                all_cells += new_cells
            return all_cells

        # Get the three basic bounds, which are created from the border.
        bounds = cls.from_bboxes(border)
        cells = list(filter(bounds.within_bounds, cells))
        if not cells:
            return []

        bounds.update_missing_bound(cells)

        # These are the Cells that fit all bounds.
        min_cells = list(filter(bounds.within_bounds, cells))

        adjacent_cells = get_all_adjacent_cells()

        # Sort columns by y0 and rows by x0.
        lower_coord = attrgetter(f"bbox.{cls.d.o.normal.lower.coordinate}")
        return list(sorted(adjacent_cells, key=lower_coord))

    @property
    def n(self) -> float | None:
        """ The northern bound, i.e., y0/the lowest y coordinate. """
        return self._n

    @n.setter
    def n(self, value: float | None) -> None:
        self._n = value
        self._update_vbox()

    @property
    def s(self) -> float | None:
        """ The southern bound, i.e., y1/the largest y coordinate. """
        return self._s

    @s.setter
    def s(self, value: float | None) -> None:
        self._s = value
        self._update_vbox()

    @property
    def w(self) -> float | None:
        """ The western bound, i.e., x0/the lowest x coordinate. """
        return self._w

    @w.setter
    def w(self, value: float | None) -> None:
        self._w = value
        self._update_hbox()

    @property
    def e(self) -> float | None:
        """ The eastern bound, i.e., x1/the largest y coordinate. """
        return self._e

    @e.setter
    def e(self, value: float | None) -> None:
        self._e = value
        self._update_hbox()

    @property
    def hbox(self) -> BBox | None:
        """ The horizontal BBox, using only w/e. """
        return self._hbox

    @property
    def vbox(self) -> BBox | None:
        """ The vertical BBox, using only n/s. """
        return self._vbox

    def _update_hbox(self) -> None:
        if self.w is None or self.e is None:
            hbox = None
        else:
            hbox = BBox(self.w, -1, self.e, -1)
        self._hbox = hbox

    def _update_vbox(self) -> None:
        if self.n is None or self.s is None:
            vbox = None
        else:
            vbox = BBox(-1, self.n, -1, self.s)
        self._vbox = vbox

    def within_h_bounds(self, bbox: BBox) -> bool:
        """ Check if the given BBox is within the current Bounds, horizontally.

        :param bbox: The BBox that is checked.
        :return: True if the BBox is within Bounds. False, otherwise.
        """
        if self.hbox and self.hbox.is_h_overlap(bbox, 0.5):
            return True
        if self.hbox:
            return False
        if self.w is not None and bbox.x1 <= self.w:
            return False
        if self.e is not None and bbox.x0 >= self.e:
            return False
        return True

    def within_v_bounds(self, bbox: BBox) -> bool:
        """ Check if the given BBox is within the current Bounds, vertically.

        :param bbox: The BBox that is checked.
        :return: True if the BBox is within Bounds. False, otherwise.
        """
        if self.vbox and self.vbox.is_v_overlap(bbox, 0.5):
            return True
        if self.vbox:
            return False
        if self.n is not None and bbox.y1 <= self.n:
            return False
        if self.s is not None and bbox.y0 >= self.s:
            return False
        return True

    def within_bounds(self, cell: C) -> bool:
        """ Check if the Cell is within the bounds.

        If the hbox/vbox is None, that is, if at least one of the w/e or n/s
        coordinates is None, the check will not fail immediately.
        Instead, in that case, only the existing (if any) coordinate will
        be checked.

        :param cell: The Cell that is checked.
        :return: True, if obj is within both the hbox and the vbox.
            False, otherwise.
        """
        bbox = cell.bbox
        return self.within_h_bounds(bbox) and self.within_v_bounds(bbox)

    def merge(self, bounds: B) -> None:
        """ Merge the Bounds, such that the resulting Bounds contain both.

        :param bounds: The Bounds that is merged into this one.
        """
        # n/w use min for lower bound, s/e use max for larger bound.
        for coordinate, func in zip("nswe", cycle((min, max))):
            getter = attrgetter(coordinate)
            value = func(map(getter, (self, bounds)), default=None)
            setattr(self, coordinate, value)

    def _update_single_limit(
            self, which: str, arg: BoundArg, cells: list[C]) -> None:
        """ Update a single bound using the BoundArg and the Cells.

        :param which: Can be one of "n", "w", "s", "e".
        :param arg: The BoundArg that is used to determine the limit.
        :param cells: The Cells used to calculate the limit.
        """
        setattr(self, which, get_limit_from_cells(cells, arg))

    def __repr__(self) -> str:
        cls_name = self.__class__.__name__
        fmt = "{: >7.2f}"
        n = fmt.format(self.n) if self.n is not None else "None"
        w = fmt.format(self.w) if self.w is not None else "None"
        s = fmt.format(self.s) if self.s is not None else "None"
        e = fmt.format(self.e) if self.e is not None else "None"
        return f"{cls_name}(n={n}, w={w}, s={s}, e={e})"

class WBounds(Bounds):
    """ The western outer bounds of a Table. Used when expanding a Table. """
    d = W

    @classmethod
    def from_bboxes(cls, bboxes: list[BBox], **_) -> WBounds:
        n = BoundArg(min, N)
        s = BoundArg(max, S)
        # We use the opposite Direction here, because we want the outer Bounds.
        e = BoundArg(min, E.opposite)
        return super().from_bboxes(bboxes, n=n, s=s, e=e)

    def update_missing_bound(self, cells: list[C]) -> None:
        """ Add the missing bound (western) based on the given Cells. """
        args: BoundArg = BoundArg(max, W)
        self._update_single_limit("w", args, cells)

class EBounds(Bounds):
    """ The eastern outer bounds of a Table. Used when expanding a Table. """
    d = E

    @classmethod
    def from_bboxes(cls, bboxes: list[BBox], **_) -> EBounds:
        n = BoundArg(min, N)
        s = BoundArg(max, S)
        # We use the opposite Direction here, because we want the outer Bounds.
        w = BoundArg(max, W.opposite)
        return super().from_bboxes(bboxes, n=n, w=w, s=s)

    def update_missing_bound(self, cells: list[C]) -> None:
        """ Add the missing bound (eastern) based on the given ells. """
        args: BoundArg = BoundArg(min, E)
        self._update_single_limit("e", args, cells)

class NBounds(Bounds):
    """ The northern outer bounds of a Table. Used when expanding a Table. """
    d = N

    @classmethod
    def from_bboxes(cls, bboxes: list[BBox], **_) -> NBounds:
        w = BoundArg(min, W)
        # We use the opposite Direction here, because we want the outer Bounds.
        s = BoundArg(min, S.opposite)
        e = BoundArg(max, E)
        return super().from_bboxes(bboxes, w=w, s=s, e=e)

    def update_missing_bound(self, cells: list[C]) -> None:
        """ Add the missing bound (northern) based on the given Cells. """
        args: BoundArg = BoundArg(max, N)
        self._update_single_limit("n", args, cells)

class SBounds(Bounds):
    """ The southern outer bounds of a Table. Used when expanding a Table. """
    d = S

    @classmethod
    def from_bboxes(cls, bboxes: list[BBox], **_) -> SBounds:
        # We use the opposite Direction here, because we want the outer Bounds.
        n = BoundArg(max, N.opposite)
        w = BoundArg(min, W)
        e = BoundArg(max, E)
        return super().from_bboxes(bboxes, n=n, w=w, e=e)

    def update_missing_bound(self, cells: list[C]) -> None:
        """ Add the missing bound (southern) based on the given Cells. """
        args: BoundArg = BoundArg(min, S)
        self._update_single_limit("s", args, cells)

def get_limit_from_cells(objects: list[C] | list[BBox], arg: BoundArg | None
                         ) -> float | None:
    """ Calculate a limit from the Cells using the provided func and attr.

    :param objects: The Cells/BBoxes used to calculate the limit.
    :param arg: The BoundArg used to determine the limit.
    :return: The limit of the Cells, based on the given func and d.
    """
    if not objects or not arg:
        return None

    prefix = "bbox." if hasattr(objects[0], "bbox") else ""
    getter = attrgetter(prefix + arg.direction.coordinate)

    # Get the Cell/BBox that has the highest/lowest value for c.
    limit = arg.func(objects, key=getter)
    # Get the actual value.
    return cast(float, getter(limit))

def select_adjacent_cells(d: Direction, bboxes: list[BBox], cells: Cs) -> Cs:
    """ Get all Cells adjacent in d to the given reference Cells.

    :param d: The Direction to check for adjacency in.
    :param bboxes: The BBoxes used to check for adjacency.
    :param cells: The Cells that are checked for adjacency.
    :return: The Cells that are adjacent to ref_cells.
    """
    bound_cls = {N: NBounds, W: WBounds, S: SBounds, E: EBounds}[d]

    adjacent_cells: Cs = bound_cls.select_adjacent_cells(bboxes, cells)

    normal = d.o.normal
    # Remove Cells that are not overlapping with any reference Cell.
    starter_id = 0
    for adj_cell in adjacent_cells:
        for i, bbox in enumerate(bboxes[starter_id:], starter_id):
            if adj_cell.bbox.is_overlap(normal.name, bbox):
                break
        else:
            adjacent_cells.remove(adj_cell)
            break
        starter_id = i
    return adjacent_cells


SHAPE = "shapeBean"
COLOR = "colorBean"
SIZE_X = "sizeXBean"
SIZE_Y = "sizeYBean"
SIZE_Z = "sizeZBean"
RADIUS = "radiusBean"
POSITION_X = "positionXBean"
POSITION_Y = "positionYBean"
POSITION_Z = "positionZBean"
ROTATION_X = "rotationXBean"
ROTATION_Y = "rotationYBean"
ROTATION_Z = "rotationZBean"
GEOMETRY = "geometryFileBean"

# Bean attributes
VALUE = "value"
UUID = "uuid"
OVERRIDE = "override"
NAME = "name"
TYPE = "type"
CHILDREN = "children"

TYPE_VIS = 'de.dlr.sc.virsat.model.extension.visualisation.Visualisation'


import os
import sys
from sqlobject.compat import load_module_from_file

def load_module(module_name):
    mod = __import__(module_name)
    components = module_name.split('.')
    for comp in components[1:]:
        mod = getattr(mod, comp)
    return mod

def load_module_from_name(filename, module_name):
    if module_name in sys.modules:
        return sys.modules[module_name]
    init_filename = os.path.join(os.path.dirname(filename), '__init__.py')
    if not os.path.exists(init_filename):
        try:
            f = open(init_filename, 'w')
        except (OSError, IOError) as e:
            raise IOError(
                'Cannot write __init__.py file into directory %s (%s)\n'
                % (os.path.dirname(filename), e))
        f.write('#\n')
        f.close()
    if module_name in sys.modules:
        return sys.modules[module_name]
    if '.' in module_name:
        parent_name = '.'.join(module_name.split('.')[:-1])
        base_name = module_name.split('.')[-1]
        load_module_from_name(os.path.dirname(filename), parent_name)
    else:
        base_name = module_name
    return load_module_from_file(base_name, module_name, filename)


from django.urls import reverse
from rest_framework import status
from rest_framework.test import APITestCase

from api.serializers import BookSerializer
from books.models import Book, PublicationLanguage, Author
from django.test import Client

client = Client()

class TestBookList(APITestCase):
    def setUp(self) -> None:
        self.publication_language_id = PublicationLanguage.objects.get_or_create(
            language="en")[0]
        self.author_id = Author.objects.get_or_create(name="test_author")[0]

    def test_empty_list_books(self):
        response = client.get(reverse("api:book_list"))
        books = Book.objects.all()
        serializer = BookSerializer(books, many=True)

        self.assertEqual(response.data, serializer.data)
        self.assertEqual(response.status_code, status.HTTP_200_OK)

    def test_all_books(self):
        book = Book.objects.create(
            isbn="1234567891234",
            title="test_title",
            publication_year=2008,
            page_count=1000,
            cover="https://google.pl",
            publication_language=self.publication_language_id)
        book.author.set([self.author_id])

        response = client.get(reverse("api:book_list"))
        books = Book.objects.all()
        serializer = BookSerializer(books, many=True)

        self.assertEqual(response.data, serializer.data)
        self.assertEqual(response.status_code, status.HTTP_200_OK)

    def test_filter_publication_date(self):
        book_first = Book.objects.create(
            isbn="1234567891234",
            title="test_title",
            publication_year=2008,
            page_count=2008,
            cover="https://google.pl",
            publication_language=self.publication_language_id)
        book_first.author.set([self.author_id])

        book_second = Book.objects.create(
            isbn="1234567891235",
            title="test_title123",
            publication_year=2018,
            page_count=2008,
            cover="https://google.pl",
            publication_language=self.publication_language_id)
        book_second.author.set([self.author_id])

        url = f"{reverse('api:book_list')}?publication_year__gte=&publication_year__lte=2013"
        response = client.get(url)
        books = Book.objects.filter(publication_year__lte=2013)
        serializer = BookSerializer(books, many=True)

        self.assertEqual(response.data, serializer.data)
        self.assertEqual(response.status_code, status.HTTP_200_OK)


# Google Question

# Given an array = [2, 5, 1, 2, 3, 5, 1, 2, 4]
# It should return 2

# Given an array = [2, 1, 1, 2, 3, 5, 1, 2, 4]
# It should return 1

# Given an array = [2, 3, 4, 5]
# It should return undefined

# input:
#   array - always an array of integers
#       negative and positive
#       no size limit
#       can be empty or None

# output:
#   integer
#       find the first recurring element
#       if not -> None

# [1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 8, 7, 6, 5, 4, 3, 2, 1]
def first_recurring_character(arr):
    smaller_index_diff = len(arr) - 1
    element = None
    for i in range(len(arr)):
        for j in range(i + 1, len(arr)):
            if arr[i] == arr[j] and (j - i) < smaller_index_diff:
                smaller_index_diff = j - i
                element = arr[i]
    return element

print(first_recurring_character([2, 1, 1, 2, 3, 5, 1, 2, 4]))

# O(n) - Time Complexity
# O(n) - Space Complexity
def first_recurring_character2(arr):
    my_map = {}
    for i in range(len(arr)):
        if arr[i] in my_map.keys():
            return arr[i]
        else:
            my_map[arr[i]] = i
    return None

# print(first_recurring_character2([1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 8, 7, 6, 5, 4, 3, 2, 1]))


import asyncio
import json
from datetime import datetime

import aiohttp

from pydantic import BaseModel, Field, NonNegativeFloat

from faststream import ContextRepo, FastStream, Logger
from faststream.kafka import KafkaBroker

broker = KafkaBroker("localhost:9092")
app = FastStream(broker)

class CryptoPrice(BaseModel):
    price: NonNegativeFloat = Field(
        ..., examples=[50000.0], description="Current price of cryptocurrency in USD"
    )
    crypto_currency: str = Field(
        ..., examples=["BTC"], description="The cryptocurrency"
    )

publisher = broker.publisher("new_crypto_price")

async def fetch_crypto_price(
    url: str, crypto_currency: str, logger: Logger, context: ContextRepo, time_interval: int = 2
) -> None:
    # Always use context: ContextRepo for storing app_is_running variable
    while context.get("app_is_running"):
        async with aiohttp.ClientSession() as session:
            async with session.get(url) as response:
                if response.status == 200:
                    data = await response.json()
                    price = data["data"]["amount"]
                    new_crypto_price = CryptoPrice(
                        price=price, crypto_currency=crypto_currency
                    )
                    await publisher.publish(
                        new_crypto_price,
                        key=crypto_currency.encode("utf-8"),
                    )
                else:
                    logger.warning(
                        f"Failed API request {url} at time {datetime.now()}"
                    )

        await asyncio.sleep(time_interval)

@app.on_startup
async def app_setup(context: ContextRepo):
    context.set_global("app_is_running", True)

@app.on_shutdown
async def shutdown(context: ContextRepo):
    context.set_global("app_is_running", False)

    # Get all the running tasks and wait them to finish
    fetch_tasks = context.get("fetch_tasks")
    await asyncio.gather(*fetch_tasks)

@app.after_startup
async def publish_crypto_price(logger: Logger, context: ContextRepo):
    logger.info("Starting publishing:")

    cryptocurrencies = [("Bitcoin", "BTC"), ("Ethereum", "ETH")]
    fetch_tasks = [
        asyncio.create_task(
            fetch_crypto_price(
                f"https://api.coinbase.com/v2/prices/{crypto_currency}-USD/spot",
                crypto_currency,
                logger,
                context,
            )
        )
        for _, crypto_currency in cryptocurrencies
    ]
    # you need to save asyncio tasks so you can wait them to finish at app shutdown (the function with @app.on_shutdown function)
    context.set_global("fetch_tasks", fetch_tasks)


import numpy as np
import pandas as pd
import logging

logging.getLogger(__name__).addHandler(logging.NullHandler())
logger = logging.getLogger(__name__)

try:
    from sklearn.base import TransformerMixin, BaseEstimator
except ImportError:
    msg = "scikit-learn not installed"
    logger.warning(msg)

try:
    from fancyimpute import IterativeImputer, SoftImpute
except ImportError:
    msg = "fancyimpute not installed"
    logger.warning(msg)

class MultipleImputer(BaseEstimator, TransformerMixin):
    """
    Multiple Imputation via fancyimpute.IterativeImputer.
    """

    def __init__(self, multiple=5, n_iter=10, groupby=None, *args, **kwargs):
        self.multiple = multiple
        self.n_iter = n_iter
        self.args = args
        self.kwargs = kwargs
        self.groupby = groupby

    def transform(self, X, *args, **kwargs):
        assert isinstance(X, pd.DataFrame)
        df = pd.DataFrame(columns=X.columns, index=X.index)
        if isinstance(self.imputers, dict):
            for c, d in self.imputers.items():
                mask = d["mask"]
                imputers = d["impute"]
                imputed_data = np.array([imp.transform(X[mask, :]) for imp in imputers])
                mean = np.mean(imputed_data, axis=0)
                df.loc[mask, ~pd.isnull(X[mask, :]).all(axis=0)] = mean
            return df
        else:
            imputed_data = np.array([imp.transform(X) for imp in self.imputers])
            mean = np.mean(imputed_data, axis=0)
            df.loc[:, ~pd.isnull(X).all(axis=0)] = mean
            return df

    """
    def inverse_transform(self, Y, *args, **kwargs):
        # For non-compositional data, take the mask and reverting to nan
        # for compositional data, renormalisation would be needed
        pass
    """

    def fit(self, X, y=None):
        assert isinstance(X, pd.DataFrame)
        start = X
        y_present = y is not None
        groupby_present = self.groupby is not None
        self.imputers = []
        if y_present or groupby_present:
            assert not (groupby_present and y_present)
            if y_present:
                classes = np.unique(y)
                gen_mask = lambda c: y == c
            if groupby_present:
                classes = X[self.groupby].unique()
                gen_mask = lambda c: X[self.groupby] == c
            self.imputers = {
                c: {
                    "impute": [
                        IterativeImputer(
                            n_iter=self.n_iter,
                            sample_posterior=True,
                            random_state=ix,
                            **self.kwargs
                        )
                        for ix in range(self.multiple)
                    ],
                    "mask": gen_mask(c),
                }
                for c in classes
            }

            msg = """Imputation transformer: {} imputers x {} classes""".format(
                self.multiple, len(classes)
            )
            logger.info(msg)

            for c, d in self.imputers.items():
                for imp in d["impute"]:
                    imp.fit(X[d["mask"], :])

        else:
            for ix in range(self.multiple):
                self.imputers.append(
                    IterativeImputer(
                        n_iter=self.n_iter,
                        sample_posterior=True,
                        random_state=ix,
                        **self.kwargs
                    )
                )
            msg = """Imputation transformer: {} imputers""".format(self.multiple)
            logger.info(msg)
            for ix in range(self.multiple):
                self.imputers[ix].fit(X)

        return self

class PdSoftImputer(BaseEstimator, TransformerMixin):
    """
    Multiple Imputation via fancyimpute.SoftImpute.
    """

    def __init__(self, max_iters=100, groupby=None, donotimpute=[], *args, **kwargs):
        self.args = args
        self.kwargs = kwargs
        self.max_iters = max_iters
        self.groupby = groupby
        self.donotimpute = donotimpute

    def transform(self, X, *args, **kwargs):
        """
        Impute Missing Values

        Todo
        ------
            * Need to use masks to avoid :class:`fancyimpute.SoftImpute` returning 0. where it cannot impute.
        """
        assert isinstance(X, pd.DataFrame)
        df = pd.DataFrame(columns=X.columns, index=X.index)  # df of nans
        df.loc[:, self.donotimpute] = X.loc[:, self.donotimpute]
        to_impute = [i for i in X.columns if not i in self.donotimpute]
        imputable = ~pd.isnull(X.loc[:, to_impute]).all(axis=1)
        if isinstance(self.imputer, dict):
            for c, d in self.imputer.items():
                mask = d["mask"]
                mask = mask & imputable
                imputer = d["impute"]
                imputed_data = imputer.fit_transform(X.loc[mask, to_impute])
                assert imputed_data.shape[0] == X.loc[mask, :].index.size
                df.loc[mask, to_impute] = imputed_data

            return df
        else:
            imputed_data = self.imputer.fit_transform(X.loc[imputable, to_impute])
            assert imputed_data.shape[0] == X.loc[imputable, :].index.size
            df.loc[imputable, to_impute] = imputed_data
            return df

    """
    def inverse_transform(self, Y, *args, **kwargs):
        # For non-compositional data, take the mask and reverting to nan
        # for compositional data, renormalisation would be needed
        pass
    """

    def fit(self, X, y=None):
        assert isinstance(X, pd.DataFrame)
        start = X
        y_present = y is not None
        groupby_present = self.groupby is not None
        self.imputer = []
        if y_present or groupby_present:
            assert not (groupby_present and y_present)
            if y_present:
                classes = np.unique(y)
                gen_mask = lambda c: y == c
            if groupby_present:
                classes = X[self.groupby].unique()
                gen_mask = lambda c: X[self.groupby] == c
            self.imputer = {
                c: {
                    "impute": SoftImpute(max_iters=self.max_iters, **self.kwargs),
                    "mask": gen_mask(c),
                }
                for c in classes
            }

            msg = """Building Soft Imputation Transformers for {} classes""".format(
                len(classes)
            )
            logger.info(msg)

        else:
            self.imputer = SoftImpute(max_iters=self.max_iters, **self.kwargs)
            msg = """Building Soft Imputation Transformer"""
            logger.info(msg)

        return self


import pytest
import torch

from renate.benchmark.models.transformer import HuggingFaceSequenceClassificationTransformer

@pytest.mark.parametrize("model_name", ["distilbert-base-uncased", "bert-base-uncased"])
def test_init(model_name):
    HuggingFaceSequenceClassificationTransformer(
        pretrained_model_name_or_path=model_name, num_outputs=10
    )

@pytest.mark.parametrize(
    "model_name,input_dim",
    [
        ["distilbert-base-uncased", (128,)],
        ["bert-base-uncased", (256,)],
    ],
)
def test_text_transformer_fwd(model_name, input_dim):
    transformer = HuggingFaceSequenceClassificationTransformer(
        pretrained_model_name_or_path=model_name
    )

    x = {"input_ids": torch.randint(0, 30000, (5, *input_dim))}
    y_hat = transformer(x)

    assert y_hat.shape[0] == 5
    assert y_hat.shape[1] == 10


import torch
import torchaudio

from torchaudio.transforms import MelSpectrogram, Spectrogram

def load_wav_to_torch(full_path, hop_size=0, slice_train=False):
    wav, sampling_rate = torchaudio.load(full_path, normalize=True)
    if not slice_train:
        p = (wav.shape[-1] // hop_size + 1) * hop_size - wav.shape[-1]
        wav = torch.nn.functional.pad(wav, (0, p), mode="constant").data
    return wav.squeeze(0), sampling_rate

class SpectrogramFixed(torch.nn.Module):
    """In order to remove padding of torchaudio package + add log10 scale."""

    def __init__(self, **kwargs):
        super(SpectrogramFixed, self).__init__()
        self.torchaudio_backend = Spectrogram(**kwargs)

    def forward(self, x):
        outputs = self.torchaudio_backend(x)

        return outputs[..., :-1]

class MelSpectrogramFixed(torch.nn.Module):
    """In order to remove padding of torchaudio package + add log10 scale."""

    def __init__(self, **kwargs):
        super(MelSpectrogramFixed, self).__init__()
        self.torchaudio_backend = MelSpectrogram(**kwargs)

    def forward(self, x):
        outputs = torch.log(self.torchaudio_backend(x) + 0.001)

        return outputs[..., :-1]

def torch_wav2spec(wav_fn, fft_size, hop_size, win_length, num_mels, fmin, fmax, sample_rate):
    """ Waveform to linear-spectrogram and mel-sepctrogram. """
    # Read wavform
    wav, sr = load_wav_to_torch(wav_fn, hop_size, slice_train=False)
    if sr != sample_rate:
        raise ValueError(f"{sr} SR doesn't match target {sample_rate} SR")
    if torch.min(wav) < -1.:
        print('min value is ', torch.min(wav))
    if torch.max(wav) > 1.:
        print('max value is ', torch.max(wav))
    # Spectrogram process
    spec_fn = SpectrogramFixed(n_fft=fft_size, win_length=win_length, hop_length=hop_size,
                               window_fn=torch.hann_window).to(device=wav.device)
    spec = spec_fn(wav)
    # Mel-spectrogram
    mel_fn = MelSpectrogramFixed(sample_rate=sample_rate, n_fft=fft_size, win_length=win_length,
                                 hop_length=hop_size, f_min=fmin, f_max=fmax, n_mels=num_mels,
                                 window_fn=torch.hann_window).to(device=wav.device)
    mel = mel_fn(wav)
    # Wav-processing
    wav = wav.squeeze(0)[:mel.shape[-1]*hop_size]
    # Check wav and spectorgram
    assert wav.shape[-1] == mel.shape[-1] * hop_size, f"| wav: {wav.shape}, spec: {spec.shape}, mel: {mel.shape}"
    assert mel.shape[-1] == spec.shape[-1], f"| wav: {wav.shape}, spec: {spec.shape}, mel: {mel.shape}"
    return {"wav": wav.cpu().detach().numpy(), "linear": spec.squeeze(0).T.cpu().detach().numpy(),
            "mel": mel.squeeze(0).T.cpu().detach().numpy()}


#imports the os package and inotify
import os
from inotify_simple import INotify, flags
#pulls in the package
inotify = INotify()
#runs the below command so this script will keep running once it's finished
os.system("while :; do python3 File-Changes.py; done")
#creates the watch flags
watch_flags = flags.CREATE | flags.DELETE | flags.MODIFY | flags.DELETE_SELF
#creates watches for the below directory
wdVarLog = inotify.add_watch("/var/log", watch_flags)
wdETC = inotify.add_watch("/etc", watch_flags)

#for event in inotify prints to directorylogs
for event in inotify.read():
    fOpen = open("directorylogs.txt","a")
    fOpen.write(event)
    fOpen.close()
    #for flag in event prints out to the same file as the events
    for flag in flags.from_mask(event.mask):
        fstring = (" " + str(flag))
        fOpen = open("directorylogs.txt","a")
        fOpen.write(fstring)
        fOpen.close()


from flask import Blueprint, g
from flask_graphql import GraphQLView
from flask_cors import CORS
from .schema import schema

api = Blueprint('api', __name__)
CORS(api, supports_credentials=True)  # Enables CORS with cross origin cookies

class CustomGraphQlView(GraphQLView):
    def dispatch_request(self):
        response = super().dispatch_request()
        for cookie in g.get("cookies", []):
            response.set_cookie(cookie.key, cookie.value, **cookie.settings)

        return response

api.add_url_rule(
    "/graphql",
    view_func=CustomGraphQlView.as_view(
        "graphql",
        schema=schema,
        graphiql=True,
        middleware=[]
    )
)


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Fri Jan 12 15:36:15 2018

@author: chrelli
added unix time stamps to the first camera!

Ways to slim down the data:
    no unix time stamps?
    no color frame showing? - yes, helps a lot!
    no png compression? Totally fine at 30 fps!

Majow to do list:
    - use arduino to synchronize? Yes, could send out synchronization time code to another unit: Problem: doesn't account for delay of arriving frames

    - use depth roi to slim down writing footprint

    - use LED roi to get blinking time stamps

    -

## with connected device cam
from pyrealsense import offline
offline.save_depth_intrinsics(dev)
"""

#%% Import the nescessary stuff
# basic OS stuff
import time, os, sys, shutil
import json
# for math and plotting
import pandas as pd
import numpy as np
import scipy as sp
import matplotlib.pyplot as plt

# small utilities
import csv
from colour import Color
from itertools import compress # for list selection with logical
from tqdm import tqdm

# for image manipulation
import cv2

# for recording and connecting to the intel realsense librar
#import pyrealsense as pyrs
# add the realsense library
sys.path.append(r'/usr/local/lib')
# and load it!
import pyrealsense2 as rs

#import multiprocessing
from multiprocessing import Process

# import handy Functions
from utils.common_utils import *
from utils.recording_utils import *

#%% Parse some inputs

import argparse

parser = argparse.ArgumentParser(description='Records cad and d images with no roi cut to disk. Also records timestamps and led traces using the auto LED mask. Currently, with no ROI, the program maxes out disk write speed around 45 fps.',formatter_class=argparse.ArgumentDefaultsHelpFormatter)

parser.add_argument('--ncams', type=int, default = 4 , choices=[1,2,3,4],
                    help='number of cameras to stream')

parser.add_argument('--fps',type=int, default = 30 , choices=[30,60],
                    help='select fps to stream')

#parser.add_argument("--singlecore", help="disables mult.proc. for debugging on macbook, overrides ncams to 1",
#                    action="store_true")

parser.add_argument("--plots", help="shows the live video while recording",
                    action="store_true")

args = parser.parse_args()

#%% Constants
# frame_width,frame_height  = 848,480
frame_width,frame_height  = 640,480

fps_choice = args.fps
# number of padding digits for the frame numbers
n_padding_digits = 8

print('# cameras: '+str(args.ncams))
print('Frame size is '+str(frame_width)+'x'+str(frame_height)+' pixels.')
print('Grabbing frames at '+str(fps_choice)+' fps')

# get the current timestring
timestr = time.strftime("%Y%m%d-%H%M%S")

# reset the folder
#data_folder = '/media/chrelli/Data0'
#top_folder = data_folder + '/calibration_' + timestr
#reset_folder_if_present(top_folder)
#
#top_folder_0 = top_folder
#top_folder_1 = top_folder

# reset the folders
top_folder_0 = '/media/chrelli/Data0' + '/calibration_' + timestr
top_folder_1 = '/media/chrelli/Data1' + '/calibration_' + timestr

reset_folder_if_present(top_folder_0)
reset_folder_if_present(top_folder_1)

# also make the numpy folders
npy_folder_0 = top_folder_0+'/npy_raw'
npy_folder_1 = top_folder_1+'/npy_raw'

reset_folder_if_present(npy_folder_0)
reset_folder_if_present(npy_folder_1)

#%% 8 bit color setup
fps_color = (Color('White').rgb)
ts_color = (Color('Peru').rgb)

# convert to 8 bit color
fps_color=tuple(255*x for x in fps_color)
ts_color=tuple(255*x for x in ts_color)

#%% Block for running
# open the pyrealsense server
#serv = pyrs.Service()

# set the start time for the unix time stamp
start_time = time.time()

# open up a realsense context and get a list of the devices!
ctx = rs.context()

devices = [ctx.devices[i] for i in range(args.ncams)]
# sort the devices by their serial numbers
serials = [devices[i].get_info(rs.camera_info.serial_number) for i in range(args.ncams)]
devices = [x for _,x in sorted(zip(serials,devices))]

def sub_function_trick(which_device,top_folder):
    show_frames = args.plots

    ####################
    #
    # DEVICE SETUP BLOCK
    #
    #####################

    # get the serial of that device
    device =  devices[which_device]
    device_serial = device.get_info(rs.camera_info.serial_number)

    #set the preset
    advnc_mode = rs.rs400_advanced_mode(device)
    print("Advanced mode is", "enabled" if advnc_mode.is_enabled() else "disabled")
    # run like
    # advnc_mode.load_json(json_string)

    # load the preset here!
    preset_folder = '/home/chrelli/git/3d_sandbox/mycetrack0p8/presets/'
    if device_serial[:3] == '740':
        preset_name = 'master60pp'
    else:
        preset_name = 'slave60pp'

    jsonFile = preset_folder+preset_name+'.json'
    jsonObj = json.load(open(jsonFile))
    json_string = str(jsonObj).replace("'", '\"')
    print("Configuration " + jsonFile + " loaded");
    time.sleep(1.)
    advnc_mode.load_json(json_string)
    print("Configuration " + jsonFile + " applied!");

    if device_serial[:3] == '740':
        # master
        targetSyncMode = 1
    else:
        # slave
        targetSyncMode = 2
    device.first_depth_sensor().set_option(rs.option.inter_cam_sync_mode, targetSyncMode)

    # first, open up a  config
    config = rs.config()

    # then open a pipeline
    pipeline = rs.pipeline()

    # enable the selected device and streams # RGB SPACE HERE
    config.enable_device(device_serial);
    config.enable_stream(rs.stream.depth, frame_width,frame_height, rs.format.z16, fps_choice)
#        config.enable_stream(rs.stream.color, frame_width,frame_height, rs.format.rgb8, fps_choice)

    config.enable_stream(rs.stream.color, frame_width,frame_height, rs.format.rgb8, fps_choice)
    config.enable_stream(rs.stream.infrared,1, frame_width,frame_height, rs.format.y8, fps_choice)

    print("PING after enabling the sync mode is {}".format(device.first_depth_sensor().get_option(rs.option.inter_cam_sync_mode)))

    # Start streaming, call the stream 'cfg' for some reason, as pr example
    cfg = pipeline.start(config)

    # create an align object
    # alternative is to align to color, faster but less precise: align_to = rs.stream.color
    align_to = rs.stream.depth
    align = rs.align(align_to)

    print('dev '+str(which_device)+' serial is ' + device_serial)
    # Use the first three digits of the serial as a string to tag the device:
    device_tag = device_serial[0:3]

    if show_frames:
        # open a window for cv2
        window_title = "dev"+str(which_device)+"(#" + device_tag + ")"
        cv2.namedWindow(window_title+'cad')

        # block for setting up a low-level fps estimation,
        cnt = 0 # a counter
        last = time.time() # start_time
        fps = 0 # initial fps value

    # save the camera intrinsics
    # Getting the depth sensor's depth scale (see rs-align example for explanation)
    depth_sensor = cfg.get_device().first_depth_sensor()
    depth_scale = depth_sensor.get_depth_scale()
    print ("Depth Scale is: " , depth_scale)

    # this is how to get the intrinsics
    profile = cfg.get_stream(rs.stream.depth) # Fetch stream profile for depth stream
    intr = profile.as_video_stream_profile().get_intrinsics() # Downcast to video_stream_profile

    #% now make file and save time stamps and depth scaling and intrinsics etc
    # use the old naming convention

    parameternames = np.array(['cam_params.fx',
                       'cam_params.fy',
                       'cam_params.ppx',
                       'cam_params.ppy',
                       'd_scale',
                       'fps_choice',
                       'frame_width',
                       'frame_height'])
    parameters = np.array([intr.fx,
                       intr.fy,
                       intr.ppx,
                       intr.ppy,
                       depth_scale,
                       fps_choice,
                       intr.width,
                       intr.height])

    # open a file for writint the parameters
    with open(top_folder+'/parameters_'+str(which_device)+'.csv','w') as intrfile:
        writer = csv.writer(intrfile, delimiter=',')
        writer.writerow(parameternames)
        writer.writerow(parameters)

    # load the automatic led mask from the constants folder!
    led_mask,led_logic,led_centroid = load_auto_roi(which_device)

    # open a file for time stamps
    tsfile = open(top_folder+'/timestamps_'+str(which_device)+'.csv','w')

#    ## HF try to open an HF file
#    import h5py
#    #TODO input from somewhere
#    hf = h5py.File(top_folder+'/dev'+str(which_device)+'_d_'+'.h5', 'w')
#    # also open one for the cad
#    hf_cad = h5py.File(top_folder+'/dev'+str(which_device)+'_cad_'+'.h5', 'w')

    # NPY ADDITION
    npy_folder = top_folder+'/npy_raw'

    # open a file for led stamps
#        ledsfile = open(top_folder+'/ledstamps_'+str(which_device)+'.csv','w')

    print('starting to stream from device '+str(which_device)+'!')
    # wait for a bit for the cam to warm up
    # and loop over 30 frames
    warmup_time = 2 # seconds
    warmup = 0
    while warmup < fps_choice*warmup_time:
        frames = pipeline.wait_for_frames()
        warmup += 1
    print('device '+str(which_device)+' is warmed up!')

    # START A CLOCK FOR THE FRAMES!
    FRAME_CLOCK = 0
    try:
        while True:

            if show_frames:
                # for counting frame rate
                cnt += 1
                if (cnt % 10) == 0:
                    now = time.time() # after 10 frames
                    dt = now - last # how long did it take?
                    fps = 10/dt # calculate frame rate
                    last = now # assign a new value to the 'last time'

            #################################
            #
            # R E A D   B L O C K
            #
            #################################

            # Wait for a coherent pair of frames: depth and color
            frames = pipeline.wait_for_frames()

            # get the frame numbers and time stamps
#            ts = round(frames.get,2)
            ts = frames.get_timestamp()
            fn = frames.get_frame_number()

            # get the unix time stamp
            ts_unix = time.time()-start_time

            # run the alignment process
            aligned_frames = align.process(frames)
            depth_frame = aligned_frames.get_depth_frame() # aligned_depth_frame is a 640x480 depth image
            cad_frame = aligned_frames.get_color_frame()
            # also get one for the LED
            # depth_frame = frames.get_depth_frame()
            # color_frame = frames.get_color_frame()
            infrared_frame = frames.get_infrared_frame()

            # Convert images to numpy arrays
            depth = np.asanyarray(depth_frame.get_data())
            cad = np.asanyarray(cad_frame.get_data())
            c = np.asanyarray(infrared_frame.get_data())

            # get the LED value, round it a bit, could be profiled
            led_stamp = c[led_centroid[1],led_centroid[0]]

            # this is the writing block for the csv file, frame number and time stamp!
#            tsfile.write(str(FRAME_CLOCK)+','+str(fn)+','+str(ts)+','+str(ts_unix)+','+str(single_pixel_RGB2GRAY(led_stamp))+'\n')
            tsfile.write(str(FRAME_CLOCK)+','+str(fn)+','+str(ts)+','+str(ts_unix)+','+str(led_stamp)+'\n')

            # this is the writing block for the csv file, frame number and time stamp!
            #TODO put led with the others in same file?
#            ledsfile.write(str(single_pixel_RGB2GRAY(led_stamp))+'\n')

            # write the depth frames to tiff (replace: send to queue)
#            cv2.imwrite(top_folder+'/dev'+str(which_device)+'_d_'+str(FRAME_CLOCK).rjust(n_padding_digits,'0')+'.png', depth)
#            cv2.imwrite(top_folder+'/dev'+str(which_device)+'_cad_'+str(FRAME_CLOCK).rjust(n_padding_digits,'0')+'.png', cad)

#            hf.create_dataset(str(FRAME_CLOCK), data=depth)
#            hf_cad.create_dataset(str(FRAME_CLOCK), data=cad)

            np.save(npy_folder+'/dev'+str(which_device)+'_d_'+str(FRAME_CLOCK).rjust(n_padding_digits,'0')+'.npy',depth, allow_pickle = False)
            np.save(npy_folder+'/dev'+str(which_device)+'_cad_'+str(FRAME_CLOCK).rjust(n_padding_digits,'0')+'.npy',cad, allow_pickle = False)

            # UPDATE CLOCK
            FRAME_CLOCK += 1

            #
            if show_frames:
                # add text and show the CAD frames
                cv2.putText(cad, window_title+', fps: '+str(fps)[:4], (0, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, fps_color)
                cv2.putText(cad, str(round(ts)), (0, frame_height-20), cv2.FONT_HERSHEY_SIMPLEX, 1, ts_color)
                cv2.imshow(window_title+'cad', cad)

            if cv2.waitKey(1) & 0xFF == ord('q'):
                # looks for a small q to nbe pressed
                # close the time stamp file
                tsfile.close

                # close the hf file
#                hf.close()
#                hf_cad.close()
#                ledsfile.close

                # stop the device
                pipeline.stop()
                print('pipeline from device '+str(which_device)+' is now closed!')
                break

    finally:
        tsfile.close

        # close the hf file
#        hf.close()
#        hf_cad.close()

        # stop the device
        pipeline.stop()
        print('pipeline from device '+str(which_device)+' is now closed!')

#%% define helping funtions for the multiprocessing
# these functions have to not be iterable.

def read_device_0():
    print('starting camera 1!')
    which_device = 0
    top_folder = top_folder_0
    sub_function_trick(which_device,top_folder)

def read_device_1():
    print('starting camera 2!')
    which_device = 1
    top_folder = top_folder_0
    sub_function_trick(which_device,top_folder)

def read_device_2():
    print('starting camera 3!')
    which_device = 2
    top_folder = top_folder_1
    sub_function_trick(which_device,top_folder)

def read_device_3():
    print('starting camera 4!')
    which_device = 3
    top_folder = top_folder_1
    sub_function_trick(which_device,top_folder)

#%% run the processes on independent cores
from multiprocessing import Process
if __name__ == '__main__':
    if args.ncams == 4:
        print('starting 4 cams, with multiprocessing!')
        # start 4 worker processes
        Process(target=read_device_0).start()
        time.sleep(3.)
        Process(target=read_device_1).start()
        Process(target=read_device_2).start()
        Process(target=read_device_3).start()
        Process(target=blink_using_firmata_random).start()

    elif args.ncams == 3:
        print('starting 3 cams, with multiprocessing!')
        Process(target=read_device_0).start()
        Process(target=read_device_1).start()
        Process(target=read_device_2).start()
        Process(target=blink_using_firmata).start()

    elif args.ncams == 2:
        print('starting 2 cams, with multiprocessing!')
        Process(target=read_device_0).start()
        Process(target=read_device_1).start()
        Process(target=blink_using_firmata).start()

    elif args.ncams == 1:
        print('starting 1 cam, with multiprocessing!')
        Process(target=read_device_0).start()
        Process(target=blink_using_firmata).start()


import time
import tensorflow.compat.v1 as tf
# tf.disable_eager_execution()
tf.config.run_functions_eagerly(True)
tf.enable_eager_execution()
from utils import *
from models import RGCN
import random
import pandas as pd
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from gcn.models import GCN
# Set random seed
seed = 123
np.random.seed(seed)
tf.set_random_seed(seed)
random.seed(seed)

# Settings
flags = tf.app.flags
FLAGS = flags.FLAGS
flags.DEFINE_string('dataset', 'cora', 'Dataset string.')  # 'cora', 'citeseer', 'pubmed'
flags.DEFINE_float('learning_rate', 0.01, 'Initial learning rate.')
flags.DEFINE_integer('epochs', 200, 'Number of epochs to train.')
flags.DEFINE_integer('hidden1', 32, 'Number of units in hidden layer.')
flags.DEFINE_float('dropout', 0.6, 'Dropout rate (1 - keep probability).')
flags.DEFINE_float('weight_decay', 5e-4, 'Weight for L2 loss on embedding matrix.')
flags.DEFINE_float('para_var', 1, 'Parameter of variance-based attention')
flags.DEFINE_float('para_kl', 5e-4, 'Parameter of kl regularization')
flags.DEFINE_float('para_l2', 5e-4, 'Parameter for l2 loss.')
flags.DEFINE_integer('early_stopping', 20, 'Tolerance for early stopping (# of epochs).')
flags.DEFINE_integer('max_degree', 3, 'Maximum Chebyshev polynomial degree.')

# Load data
adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask, label = load_data(FLAGS.dataset)

# train_mask[641:700] = True

perturbed_features, y_train_gpc = perturb_features_gpc(features, 0.8)
gpc_idx, gpc_feature, gpc_y_train_gpc = get_gpc_train_data(perturbed_features, y_train_gpc, train_mask, test_mask, val_mask)

print(len(gpc_idx))

kernel = 1.0 * RBF(1.0)
gpc = GaussianProcessClassifier(kernel=kernel,
        random_state=0).fit(gpc_feature[:500], gpc_y_train_gpc[:500])
# gpc.score(perturbed_features, y_train_gpc)
gpc_res = gpc.predict(perturbed_features)
# print(type(gpc_res))
# gpc_res = np.asarray(y_train_gpc) # delete later, used for testing

# print(type(y_train_gpc))
gpc_predict_pert_idx = [i for i, x in enumerate(gpc_res==1) if x]

# print(all_idx_to_remove)

features = sp.csr_matrix(perturbed_features)
# features, y_train, train_mask, adj, label, y_val, val_mask = remove_pert(features, y_train, train_mask, adj, label, y_val, val_mask, gpc_predict_pert_idx)
features = modify_pert(features, gpc_predict_pert_idx) # for continuous features
# print(train_mask)

features = preprocess_features(features)

support = [preprocess_adj(adj, -0.5), preprocess_adj(adj, -1.0)]
placeholders = {
    'support': [tf.sparse_placeholder(tf.float32) for _ in range(2)],
    'features': tf.sparse_placeholder(tf.float32, shape=tf.constant(features[2], dtype=tf.int64)),
    'labels': tf.placeholder(tf.float32, shape=(None, y_train.shape[1])),
    'labels_mask': tf.placeholder(tf.int32),
    'dropout': tf.placeholder_with_default(0., shape=()),
    'num_features_nonzero': tf.placeholder(tf.int32),
}
model = GCN(placeholders, input_dim=features[2][1], logging=True)
sess = tf.Session()
def evaluate(features, support, labels, mask, placeholders, adj):
    t_test = time.time()
    feed_dict_val = construct_feed_dict(features, support, labels, mask, placeholders, adj)
    outs_val = sess.run([model.loss, model.accuracy], feed_dict=feed_dict_val)
    return outs_val[0], outs_val[1], (time.time() - t_test)

sess.run(tf.global_variables_initializer())
cost_val = []
var1 = []
for epoch in range(FLAGS.epochs):
    t = time.time()
    feed_dict = construct_feed_dict(features, support, y_train, train_mask, placeholders, adj)
    feed_dict.update({placeholders['dropout']: FLAGS.dropout})
    outs = sess.run([model.opt_op, model.loss, model.accuracy, model.vars], feed_dict=feed_dict)
    # print(outs[3].shape)
    # if epoch == (FLAGS.epochs-1):
    #     # print('%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    #     df = pd.DataFrame(data = outs[3])
    #     df.to_csv('/home/zihe-leon/Desktop/RobustGCN-master/src/var0.csv', index = False)
    cost, _, duration = evaluate(features, support, y_val, val_mask, placeholders, adj)
    cost_val.append(cost)
    # Print results
    print("Epoch:", '%04d' % (epoch + 1), "train_loss=", "{:.5f}".format(outs[1]),
          "train_acc=", "{:.5f}".format(outs[2]), "time=", "{:.5f}".format(time.time() - t))
    if epoch > FLAGS.early_stopping and cost_val[-1] > np.mean(cost_val[-(FLAGS.early_stopping+1):-1]):
        print("Early stopping...")
        break
print("Optimization Finished!")

adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask, label = load_data(FLAGS.dataset)
features = preprocess_features(features)
support = [preprocess_adj(adj, -0.5), preprocess_adj(adj, -1.0)]
test_cost, test_acc, test_duration = evaluate(features, support, y_test, test_mask, placeholders, adj)
print("Test set results:", "cost=", "{:.5f}".format(test_cost),
      "accuracy=", "{:.5f}".format(test_acc), "time=", "{:.5f}".format(test_duration))


def cleaning_digor_eng():
    with open('Дигорско-английский.txt', 'r', encoding='utf-8') as f:
        lines = list(map(lambda x: x.rstrip("\n").lower().replace('æ', 'ӕ'), f.readlines()))
    # bad_letters = 'eyopakxc'
    # good_letters = 'еуоракхс'
    # for i in range(len(bad_letters)):
    #     lines = [i.replace(bad_letters[i], good_letters[i]) for i in lines]
    with open('Дигорско-английский очищенный.txt', 'w', encoding='utf-8') as f:
        for line in lines:
            if line.strip():
                f.write(line + '\n')

def cleaning_digor_rus():
    # print('	' == ' ')
    # return
    with open('Дигорско-русский.txt', 'r', encoding='utf-8') as f:
        lines = list(map(lambda x: x.rstrip("\n").lower().replace('æ', 'ӕ'), f.readlines()))
    # bad_letters = 'eyopakxc'
    # good_letters = 'еуоракхс'
    # for i in range(len(bad_letters)):
    #     lines = [i.replace(bad_letters[i], good_letters[i]) for i in lines]
    with open('Дигорско-русский очищенный.txt', 'w', encoding='utf-8') as f:
        for line in lines:
            if line.find('[') == -1 or \
                    'm1' in line and ('см.' in line or 'тж.' in line or 'мн.' in line) or \
                    'm3' in line:
                if line.strip():
                    f.write(line + '\n')

def cleaning_pronouns():
    with open('Местоимения.txt', 'r', encoding='utf-8') as f:
        lines = list(map(lambda x: x.rstrip("\n").lower().replace('æ', 'ӕ'), f.readlines()))
    # bad_letters = 'eyopakxc'
    # good_letters = 'еуоракхс'
    # for i in range(len(bad_letters)):
    #     lines = [i.replace(bad_letters[i], good_letters[i]) for i in lines]
    with open('Местоимения очищенные.txt', 'w', encoding='utf-8') as f:
        for line in lines:
            if line.strip():
                f.write(line + '\n')

class Lexeme:
    def __init__(self, lex, transl_en, gramm):
        self.lex = lex
        self.transl_en = transl_en
        self.gramm = gramm

    def __str__(self):
        return f'''{self.lex}\n{self.transl_en}\n{self.gramm}\n'''

    def __repr__(self):
        return f'''{self.lex}\n{self.transl_en}\n{self.gramm}\n'''

def find_lexem(lexems, word):
    for i in range(len(lexems)):
        if word in lexems[i].lex:
            return i
    return -1

osetin_alphabet = {'а': 1, 'ӕ': 2, 'б': 3, 'в': 4, 'г': 5, 'гъ': 6, 'д': 7, 'дж': 8, 'дз': 9, 'е': 10,
                   'ё': 11, 'ж': 12, 'з': 13, 'и': 14, 'й': 15, 'к': 16, 'къ': 17, 'л': 18, 'м': 19,
                   'н': 20, 'о': 21, 'п': 22, 'пъ': 23, 'р': 24, 'с': 25, 'т': 26, 'тъ': 27, 'у': 28,
                   'ф': 29, 'х': 30, 'хъ': 31, 'ц': 32, 'цъ': 33, 'ч': 34, 'чъ': 35, 'ш': 36, 'щ': 37,
                   'ъ': 38, 'ы': 39, 'ь': 40, 'э': 41, 'ю': 42, 'я': 43}

def words_comparator(word1, word2):
    try:
        ossetian_alphabet = " а ӕ б в г гъ д дж дз е ё ж з и й к къ л м н о п пъ р с т тъ у ф х хъ ц цъ ч чъ ш щ ъ ы ь э ю я"
        i1 = 0
        i2 = 0
        while i1 < len(word1) and i2 < len(word2):
            letter1 = word1[i1]
            letter2 = word2[i2]
            if i1 + 1 < len(word1) and word1[i1: i1 + 2] in ossetian_alphabet:
                letter1 = word1[i1: i1 + 2]
                i1 += 1
            if i2 + 1 < len(word2) and word2[i2: i2 + 2] in ossetian_alphabet:
                letter2 = word2[i2: i2 + 2]
                i2 += 1
            if ossetian_alphabet.index(letter1) < ossetian_alphabet.index(letter2):
                return -1
            elif ossetian_alphabet.index(letter1) > ossetian_alphabet.index(letter2):
                return 1
            i1 += 1
            i2 += 1
        if len(word1) < len(word2):
            return -1
        elif len(word1) > len(word2):
            return 1
        return 0
    except Exception:
        if '-' not in word1 and '-' not in word2:
            print(word1, word2)
            print(Exception.args)
        if '-' in word1:
            return -1
        return 1

# print('c' == 'c')
# print(words_comparator("cтъона", 'дехгӕнӕг'))
# exit()

def union_dictionaries():
    lexems = []
    count_paired_words = 0
    all_ = 0
    with open('Дигорско-английский очищенный.txt', 'r', encoding='utf-8') as f:
        lines = list(map(lambda x: x.rstrip("\n"), f.readlines()))
        i = 0
        while i < len(lines):
            if '-lexeme' in lines[i]:
                lexems.append(Lexeme(
                    lex=[i.strip('/|').strip() for i in lines[i + 1].removeprefix(' lex:').strip('.').split('.')],
                    # lex=[lines[i + 1].strip(' lex:').strip('').split('.')],
                    transl_en=lines[i + 5].removeprefix(' transl_en:').strip(),
                    gramm=[i.strip() for i in lines[i + 6].removeprefix(' gramm:').split(',')]
                ))
                if '' in lexems[-1].lex:
                    lexems[-1].lex = lexems[-1].lex.remove('')
                i += 6
            i += 1
    with open('Местоимения очищенные.txt', 'r', encoding='utf-8') as f:
        lines = list(map(lambda x: x.rstrip("\n"), f.readlines()))
        i = 0
        while i < len(lines):
            if '-lexeme' in lines[i]:
                lexems.append(Lexeme(
                    lex=[i.strip('/|').strip() for i in lines[i + 2].removeprefix(' stem:').strip('.').split('.')],
                    transl_en=lines[i + 5].removeprefix(' transl_en:').strip(),
                    gramm=[i.strip() for i in lines[i + 6].removeprefix(' gramm:').split(',')]
                ))
                if '' in lexems[-1].lex:
                    lexems[-1].lex = lexems[-1].lex.remove('')
                i += 6
            i += 1

    with open('Дигорско-русский очищенный.txt', 'r', encoding='utf-8') as f:
        lines = list(map(lambda x: x.rstrip("\n"), f.readlines()))

    with open('unioned dictionary.txt', 'w', encoding='utf-8') as f:
        i = 0
        while i < len(lines):
            if lines[i].find('[') == -1:
                all_ += 1
                f.write(lines[i] + '\n')
                r = i + 1
                while r < len(lines) and lines[r].find('[') != -1:
                    if lines[r].strip():
                        f.write(lines[r] + '\n')
                    r += 1
                ind = find_lexem(lexems, lines[i])
                if ind != -1:
                    count_paired_words += 1
                    f.write('	[m1]')
                    for gr in lexems[ind].gramm[:-1]:
                        f.write(f'[p][i][c][com]{gr},[/com][/c][/i][/p]')
                    f.write(f'[p][i][c][com]{lexems[ind].gramm[-1]}[/com][/c][/i][/p]')
                    f.write('[/m]\n')
                    f.write(f'	[m1][trn]{lexems[ind].transl_en}[/trn][/m]\n')
                i = r
            else:
                i += 1
    print(all_, 'all_')
    print(count_paired_words, 'count_paired_words')
    print(len(lexems), 'lexems')
    used_lexems = [False] * len(lexems)
    with open('unioned dictionary.txt', 'r', encoding='utf-8') as f:
        lines = list(map(lambda x: x.rstrip("\n"), f.readlines()))
        for i in range(len(lines)):
            if lines[i].find('[') == -1:
                ind = find_lexem(lexems, lines[i])
                if ind != -1:
                    used_lexems[ind] = True
    inserted = 0
    for i in range(len(lexems)):
        if not used_lexems[i]:
            inserted += 1
            ind = 0
            error_list = []
            temp = lexems[i].lex
            if len(temp) > 1:
                error_list.append(lexems[i])
                continue
            temp = temp[0]
            r = 0
            while r < len(lines):
                if lines[r].find('[') == -1:
                    if words_comparator(temp, lines[r]) == 1:
                        ind = r
                        r += 1
                    else:
                        break
                else:
                    r += 1
            # if ind == 0:
            #     print(temp)
            gram = '	[m1]'
            for gr in lexems[i].gramm[:-1]:
                gram += f'[p][i][c][com]{gr},[/com][/c][/i][/p]'
            gram += f'[p][i][c][com]{lexems[i].gramm[-1]}[/com][/c][/i][/p]' + '[/m]'
            trn = f'	[m1][trn]{lexems[i].transl_en}[/trn][/m]'
            lines.insert(ind, lexems[i].lex[0])
            lines.insert(ind + 1, gram)
            lines.insert(ind + 2, trn)
    print(inserted, "inserted")
    print(len(error_list), 'errorlist')
    with open('unioned dictionary2.txt', 'w', encoding='utf-8') as f:
        f.writelines([i + '\n' for i in lines])
    with open('errorlist.txt', 'w', encoding='utf-8') as f:
        f.writelines([i + '\n' for i in error_list])

# with open('Дигорско-русский очищенный.txt', 'r', encoding='utf-8') as f:
#     a = sum(map(lambda x: x.rstrip("\n").strip() == "", f.readlines()))
#     print(a)
# cleaning_digor_rus()
# with open('Дигорско-русский очищенный.txt', 'r', encoding='utf-8') as f:
#     a = sum(map(lambda x: x.rstrip("\n").strip() == "", f.readlines()))
#     print(a)
# with open('Местоимения.txt', 'r', encoding='utf-8') as f:
#     a = sum(map(lambda x: x.rstrip("\n").strip() == "", f.readlines()))
#     print(a)
# cleaning_pronouns()
# with open('Местоимения очищенные.txt', 'r', encoding='utf-8') as f:
#     c  = 0
#     for i in f.readlines():
#         if i.startswith(' stem:'):
#             if len(i.strip(' stem:').strip().split('.')) > 1:
#                 print(i)
#                 c += 1
#     print(c)
#
# with open('Дигорско-английский очищенный.txt', 'r', encoding='utf-8') as f:
#     c = 0
#     for i in f.readlines():
#         if i.startswith(' lex:'):
#             if len(i.strip(' lex:').strip().split('.')) > 1:
#                 print(i)
#                 c += 1
#     print(c)
# print(ord('æ'), ord('æ'))
# cleaning_digor_eng()
# print(1)
# cleaning_digor_rus()
# print(2)
# cleaning_pronouns()
# print(3)
# union_dictionaries()
# print(4)
# print('ӕ' < 'б')

def final_cleaning():
    with open('Английский.txt', 'r', encoding='utf-8') as f:
        lines = list(map(lambda x: x.rstrip("\n"), f.readlines()))
    # print(len(lines))
    # i = 0
    # while i < len(lines):
    #     while i < len(lines) and not lines[i].strip():
    #         print(1)
    #         del lines[i]
    #     i += 1
    # print(len(lines))
    fixed_lines = []
    words_without_trn = []
    i = 0
    while i < len(lines):
        fixed_lines.append(lines[i])
        if i == len(lines) - 1 or '[' not in lines[i + 1]:
            words_without_trn.append(lines[i])
            i += 1
        else:
            i += 1
            temp = []
            while i < len(lines) and '[' in lines[i]:
                temp.append(lines[i])
                i += 1
            temp.sort()
            fixed_lines.extend(temp)

    with open('слова без перевода.txt', 'w', encoding='utf-8') as f:
        f.writelines([i + '\n' for i in words_without_trn])
    with open('Английский исправленный.txt', 'w', encoding='utf-8') as f:
        f.writelines([i + '\n' for i in fixed_lines])

def extract_word(s, start_ind, char_break):
    """
    не спрашивайте зачем она нужна. так надо
    :param s:
    :param start_ind:
    :param char_break:
    :return: слово и индекс, где остановились
    """
    temp = ''
    while s[start_ind] != char_break:
        temp += s[start_ind]
        start_ind += 1
    return temp.strip(), start_ind

def clear_string_m1():
    kw = ['см.', 'тж.', 'мн.']  # keywords
    with open('Английский исправленный.txt', 'r', encoding='utf-8') as f:
        lines = list(map(lambda x: x.strip().rstrip("\n"), f.readlines()))
    with open('temp.txt', 'w', encoding='utf-8') as f:
        for i in range(len(lines)):
            line = lines[i]
            if 'm1' in line and kw[0] in line or kw[1] in line or kw[2] in line:
                dick = {
                    kw[0]: [],
                    kw[1]: [],
                    kw[2]: []
                }
                cur_kw = 0
                r = 0
                lenn = len(line)
                while r < lenn:
                    if line.startswith('[p][i][c][com]', r):
                        temp, r = extract_word(line, r + 14, '[')
                        r += 14  # minimum 10 forward
                        if temp in kw:
                            cur_kw = kw.index(temp)
                    if line.startswith('[ref]', r):
                        temp, r = extract_word(line, r + 5, '[')
                        r += 3
                        dick[kw[cur_kw]].append(temp)
                    # try: заигнорю слова на которые нет ссылок потому что он указывал только тег ref
                    #     if line.startswith('[i][com]', r) and not line.startswith('[i][com](', r):
                    #         temp, r = extract_word(line, r + 4, '')
                    #         dick[kw[cur_kw]].append(temp)
                    # except:
                    #     pass
                    if line.startswith('[i][com]([p][c]', r):
                        temp, r = extract_word(line, r + 15, '[')  # temp always == 'мн.'
                        r += 8
                        temp, r = extract_word(line, r, ')')
                        dick[kw[2]].append(temp)
                    r += 1
                part1 = ''
                if len(dick[kw[0]]) > 0:
                    part1 = f'[p][i][c][com]{kw[0]}[/com][/c][/i][/p] {", ".join([f"[ref]{ref}[/ref]" for ref in dick[kw[0]]])}'
                part2 = ''
                if len(dick[kw[1]]) > 0:
                    part2 = f'[p][i][c][com]{kw[1]}[/com][/c][/i][/p] {", ".join([f"[ref]{ref}[/ref]" for ref in dick[kw[1]]])}'
                part3 = ''
                if len(dick[kw[2]]) > 0:
                    part3 = f'[i][com]([p][c]{kw[2]}[/c][/p] {dick[kw[2]][0]})[/com][/i]'

                if len(part1) > 0 or len(part2) > 0 or len(part3) > 0:
                    line = f'[m1]{part1} {part2} {part3}[/m]'
                else:
                    continue
            if '[' in line:
                line = '	' + line
            while '  ' in line:
                line = line.replace('  ', ' ')
            f.write(line + '\n')

def testing():
    kw = ['см.', 'тж.', 'мн.']  # keywords
    with open('test.txt', 'r', encoding='utf-8') as f:
        lines = list(map(lambda x: x.strip().rstrip("\n"), f.readlines()))
        for i in range(len(lines)):
            line = lines[i]
            # if line.count('.') > 1:
            #     print(line)
    with open('test_out.txt', 'w', encoding='utf-8') as f:
        for i in range(len(lines)):
            line = lines[i]
            if 'm1' in line and kw[0] in line or kw[1] in line or kw[2] in line:
                dick = {
                    kw[0]: [],
                    kw[1]: [],
                    kw[2]: []
                }
                cur_kw = 0
                r = 0
                lenn = len(line)
                while r < lenn:
                    if line.startswith('[p][i][c][com]', r):
                        temp, r = extract_word(line, r + 14, '[')
                        r += 14  # minimum 10 forward
                        if temp in kw:
                            cur_kw = kw.index(temp)
                    if line.startswith('[ref]', r):
                        temp, r = extract_word(line, r + 5, '[')
                        r += 2
                        dick[kw[cur_kw]].append(temp)
                    # try: заигнорю слова на которые нет ссылок потому что он указывал только тег ref
                    #     if line.startswith('[i][com]', r) and not line.startswith('[i][com](', r):
                    #         temp, r = extract_word(line, r + 4, '')
                    #         dick[kw[cur_kw]].append(temp)
                    # except:
                    #     pass
                    if line.startswith('[i][com]([p][c]', r):
                        temp, r = extract_word(line, r + 15, '[')  # temp always == 'мн.'
                        r += 8
                        temp, r = extract_word(line, r, ')')
                        dick[kw[2]].append(temp)
                    r += 1
                part1 = ''
                if len(dick[kw[0]]) > 0:
                    part1 = f'[p][i][c][com]{kw[0]}[/com][/c][/i][/p] {", ".join([f"[ref]{ref}[/ref]" for ref in dick[kw[0]]])}'
                part2 = ''
                if len(dick[kw[1]]) > 0:
                    part2 = f'[p][i][c][com]{kw[1]}[/com][/c][/i][/p] {", ".join([f"[ref]{ref}[/ref]" for ref in dick[kw[1]]])}'
                part3 = ''
                if len(dick[kw[2]]) > 0:
                    part3 = f'[i][com]([p][c]{kw[2]}[/c][/p] {dick[kw[2]][0]})[/com][/i]'

                if len(part1) > 0 or len(part2) > 0 or len(part3) > 0:
                    line = f'[m1]{part1} {part2} {part3}[/m]'
                else:
                    continue
            if '[' in line:
                line = '	' + line
            while '  ' in line:
                line = line.replace('  ', ' ')
            f.write(line + '\n')

def replace_m1():
    kw = ['см.', 'тж.', 'мн.']  # keywords
    with open('temp.txt', 'r', encoding='utf-8') as f:
        lines = list(map(lambda x: x.strip().rstrip("\n"), f.readlines()))
    with open('temp2.txt', 'w', encoding='utf-8') as f:
        i = 0
        while i < len(lines):
            word = lines[i]
            r = i + 1
            space = '	'
            gr = []
            other = []
            trn = []
            m3 = []
            while r < len(lines) and '[' in lines[r]:
                if 'm3' in lines[r]:
                    m3.append(lines[r])
                elif 'trn' in lines[r]:
                    trn.append(lines[r])
                elif kw[0] in lines[r] or kw[1] in lines[r] or kw[2] in lines[r]:
                    other.append(lines[r])
                else:
                    gr.append(lines[r])
                r += 1

            i = r
            f.write(word + '\n')
            for m in other:
                f.write('	' + m + '\n')
            for m in gr:
                f.write('	' + m + '\n')
            for m in trn:
                f.write('	' + m + '\n')
            for m in m3:
                f.write('	' + m + '\n')

def transfer_duplicates():
    with open('temp2.txt', 'r', encoding='utf-8') as f:
        lines = list(map(lambda x: x.strip().rstrip("\n"), f.readlines()))
    space = '	'
    used = []
    temp3_lines = []
    duplicated_lines = []
    i = 0
    while i < len(lines):
        word = lines[i]

        if word in used:
            duplicated_lines.append(word)
            r = i + 1
            while r < len(lines) and '[' in lines[r]:
                duplicated_lines.append(space + lines[r])
                r += 1
            i = r
        else:
            temp3_lines.append(word)
            r = i + 1
            while r < len(lines) and '[' in lines[r]:
                temp3_lines.append(space + lines[r])
                r += 1
            i = r
            used.append(word)

    with open('итоговый вариант.txt', 'w', encoding='utf-8') as f:
        for line in temp3_lines:
            f.write(line + '\n')
    with open('duplicated words.txt', 'w', encoding='utf-8') as f:
        for line in duplicated_lines:
            f.write(line + '\n')

transfer_duplicates()
# if 'см' in line or 'тж' in line:
#     sign = ''
#     refs = []
#     r = 0
#     while r < len(line):
#         if line[r] == 'p' and line[r:r + 13] == 'p][i][c][com]':
#             temp = ''
#             t = r + 13
#             while line[t] != '[':
#                 temp += line[t]
#                 t += 1
#             temp = temp.strip()
#             if temp in ['см', 'тж']:
#                 sign = temp
#         if line[r] == 'r' and line[r:r + 4] == 'ref]':
#             temp = ''
#             t = r + 4
#             while line[t] != '[':
#                 temp += line[t]
#                 t += 1
#             refs.append(temp)
#     line = f'	[m1][p][i][c][com]{sign}[/com][/c][/i][/p] {", ".join([f"[ref]{ref}[/ref]" for ref in refs])}[/m]'
# elif 'мн' in line:
#     pass
# line = line.removeprefix('[m1]').removesuffix('[/m]')
# line, ref = line[:line.find('[ref]')], line[line.find('[ref]'):]
# words = [''.strip(' ,.:|/;').removeprefix('[p][i][c][com]') for i in line.strip().split('[/com][/c][/i][/p]')]
# print(words)
# if i == 20:
#     return
# ref = ''


from __future__ import absolute_import, division, print_function

# Local Imports
from modeling import ArgStrModel
from arguments import TrainingArguments
from training import ArgStrTrainer
from processors import get_datasets
from utils import Data_Collator, set_seed

# Standard Imports
import os
import random

# Third Party Imports
import json
import torch
import mlflow

from transformers import BertConfig

if __name__ == '__main__':
    # load the best trained model and config
    config_available = False
    os.environ["CUDA_VISIBLE_DEVICES"] = "3"
    # define the experiment parameters
    bert_arch = "bert-base-uncased"
    task_name = "STLAS_randomized_LOO_gretz_topic"
    exp_name = task_name + "_v1_bb"
    mlflow_exp_name = "ASL_randomized_v2"
    NUM_OF_SEEDS = 5

    # define the path from where the task data is loaded from.
    task_data_dir = "/mnt/data2/Sid/arg_quality/pytorch/task4_hpo/data/*.csv"

    if config_available:
        config_file = open("/mnt/data2/Sid/arg_quality/pytorch/task4_hpo/best_model_details_MTLAS_LOO_swanson_v2_bb"
                           ".json", )
        config_data = json.load(config_file)
    else:
        # define config
        config_data = {
            "bert_arch": bert_arch,
            "task_name": task_name,
            "exp_name": exp_name,
            "device": torch.device('cuda' if torch.cuda.is_available() else 'cpu'),
            "is_distributed": False,
            "resources_per_trial": {
                'cpu': 4,
                'gpu': 1
            },

            "data_dir": task_data_dir,
            "split_by_topic": True if "randomized" in task_name else False,
            "eval_batch_size": 128,
            "train_batch_size": 64,
            "max_seq_length": None,
            "max_seq_length_perc": 0.95,
            "data_collator": Data_Collator,

            "dropout_prob": 0.1,
            "bert_hidden_layers": 4,
            "nn_config": 1,
            "dataset_loss_method": "unweighted",

            "learning_rate": 4.8672041684500765e-06,
            "weight_decay": 0.1936871758204528,
            "num_epochs": 50,
            "max_steps": -1,  # We use num_epochs instead.

            "mlflow": {
                "experiment_name": mlflow_exp_name,
                "tracking_uri": "http://mlflow.dbs.ifi.lmu.de:5000"
            }
        }

    if mlflow.get_experiment_by_name(mlflow_exp_name) is None:
        mlflow.create_experiment(mlflow_exp_name)
    mlflow.set_experiment(experiment_name=mlflow_exp_name)
    mlflow.set_tracking_uri("http://mlflow.dbs.ifi.lmu.de:5000")

    # define 10 seeds to run for training
    seed_list = random.sample(range(0, 10000), NUM_OF_SEEDS)
    for seed_value in seed_list:
        set_seed(seed_value)
        with mlflow.start_run(run_name="asl_randomized_v1"):
            mlflow.log_param("seed", seed_value)
            mlflow.log_param("Task Name", task_name)
            print("Seed:", seed_value)
            train_dataset, eval_dataset, test_dataset, task_dict = get_datasets(config_data)
            # Load model setup.
            if not config_available:
                bert_config = BertConfig.from_pretrained(
                    config_data["bert_arch"],
                    finetuning_task=task_name,
                    output_hidden_states=True
                )

                model = ArgStrModel.from_pretrained(
                    config_data["bert_arch"],
                    config=bert_config,
                    dropout_prob=config_data["dropout_prob"],
                    bert_hidden_layers=config_data["bert_hidden_layers"],
                    mlp_config=config_data["nn_config"],
                    task_dict=task_dict,
                    device=config_data["device"]
                )
            else:
                bert_config = BertConfig.from_pretrained(
                    config_data["bert_arch"],
                    finetuning_task=task_name,
                    output_hidden_states=True
                )
                model = ArgStrModel(
                    config=bert_config,
                    dropout_prob=config_data["dropout_prob"],
                    bert_hidden_layers=config_data["bert_hidden_layers"],
                    mlp_config=config_data["nn_config"],
                    task_dict=task_dict,
                    device=config_data["device"])

                with open(os.path.join(config_data["best_checkpoint_path"], "best_model.pt"), 'rb') as checkpt:
                    model_state, optimizer_state = torch.load(checkpt)
                model.load_state_dict(model_state)

            training_args = TrainingArguments(
                learning_rate=config_data["learning_rate"],
                train_model=True,
                evaluate_during_training=True,
                save_steps=0,
                max_num_train_epochs=25,
                train_batch_size=config_data["train_batch_size"],
                eval_batch_size=config_data["eval_batch_size"],
                weight_decay=config_data["weight_decay"],
                weighted_dataset_loss=config_data["dataset_loss_method"]
            )

            retrain_runner = ArgStrTrainer(
                model=model,
                args=training_args,
                train_dataset=train_dataset,
                eval_dataset=eval_dataset,
                test_dataset=test_dataset,
                data_collator=Data_Collator,
                task_name=config_data["task_name"]
            )

            retrain_runner.train_model(device=config_data["device"],
                                       mlflow_logging=True,
                                       retraining=True,
                                       seed_value=seed_value)

            if "MTLAS" in task_name:
                retrain_runner.infer_model(
                    infer_dataset=test_dataset,
                    device=config_data["device"],
                    exp_name=config_data["exp_name"],
                    task_dict=task_dict,
                    exp_seed=seed_value,
                )


import csv
import json

# Save .csv file from dict List [{}]
def save_file(results, filename, format):

        if(format=='csv'):
            if(len(results) > 0):
                with open(f'{filename}.csv', 'w', encoding='utf8', newline='') as output_file:
                    output_file.write('sep=,\n')
                    fc = csv.DictWriter(output_file, fieldnames=results[0].keys())
                    fc.writeheader()
                    fc.writerows(results)
        elif(format=='json'):
            with open(f'{filename}.json', 'w') as f:
                json.dump(results, f)

        print(f'file saved to {filename}.{format}')


import streamlit as st
import pandas as pd
import numpy as np
import re
import emoji
import io
from collections import Counter
import datetime
import plotly.express as px
from numpy import random
from multiprocessing.dummy import Pool as ThreadPool
from wordcloud import WordCloud, STOPWORDS
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
import matplotlib.pyplot as plt
from deep_translator import GoogleTranslator
from helper import *

st.set_page_config(
    page_title="Chat Analytics Dashboard",
    page_icon="🔍",
    layout="wide")

#Styling the Structure
#https://share.streamlit.io/rawrapter/chat-analytics-dashboard/main/chat_analyze.py
hide_streamlit_style = """

            """
st.markdown(hide_streamlit_style, unsafe_allow_html=True)

st.title("Chat Analytics Dashboard")
st.markdown('Made with ♥ in India. © Anant Arun',unsafe_allow_html=True)

# translator = GoogleTranslator(source='auto', target='en')

#Calling Vader to extract out all sentiment analysis
sid_obj = SentimentIntensityAnalyzer()
pool = ThreadPool(8)
stopwords = set(STOPWORDS)

with st.expander("How to export your Conversation"):
    st.write("""To export a copy of the history of an individual chat or group:
  \n1. Open the conversation or group.
  \n2. For Android: Click on three vertical dots on top right corner and select More.  \nFor iOS: Tap on Contact/Group Name.
  \n3. Select Export chat.
  \n4. Choose Without Media.
  \n5. You will asked how to save your chat history attached as a .txt document.  \nSave it wherever you like. Then download the .txt file and upload it below.""")

#File uploader from streamlit to upload file
chat_file = st.file_uploader("Upload chat file (Don't worry your data is safe. Analysis is done in your browser only.)", type=["txt"])

chat_content = []

if chat_file != None:
    raw_text = io.TextIOWrapper(chat_file,encoding='utf-8')
    chat_content = raw_text.readlines()

def translate_request(text):
        translate_text = GoogleTranslator(target='en').translate(text.strip().lower())
        if translate_text != None:
            translate_text = " ".join(word for word in translate_text.split(" ") if word not in stopwords)
        return translate_text

def list_to_DF(list,format=0):

    date_format=['%d/%m/%Y, %I:%M %p','%d/%m/%y, %I:%M %p','%m/%d/%y, %I:%M %p']
    date=re.compile('\d{1,2}/\d{1,2}/\d{2,4}')

    df=pd.DataFrame(columns=['date_time','author','message'])
    for chat in list:
        if date.match(chat):
            dat_time,conversation=re.split(' - ',chat,maxsplit=1)
            try:
                aut,msg=re.split(':',conversation,maxsplit=1)
            except ValueError:
                aut=np.nan
                msg=str.strip(conversation)
            d=str.strip(dat_time)
            try:
                dt=datetime.datetime.strptime(str.strip(dat_time),date_format[format])
            except ValueError:
                return list_to_DF(list,format+1)
            df=df.append({'date_time':dt,'author':aut,'message':str.strip(msg)},ignore_index=True)
        else:
            df.iloc[-1].message=df.iloc[-1].message+' '+chat

    return df

def data_preperation(df):

    year = lambda x:x.year
    emoji_extract = lambda x:''.join(re.findall(emoji.get_emoji_regexp(),x))
    url_pattern = r'(https?://\S+)'

    df.dropna(inplace=True)
    df['date'] = df['date_time'].apply(pd.Timestamp.date)
    df['day'] = df['date_time'].apply(pd.Timestamp.day_name)
    df['month'] = df['date_time'].apply(pd.Timestamp.month_name)
    df['year'] = df['date_time'].apply(year)
    df['time'] = df['date_time'].apply(pd.Timestamp.time).apply(lambda x: datetime.datetime.strptime(str(x), "%H:%M:%S")).apply(lambda x: x.strftime("%I:%M %p"))
    df['emoji_used'] = df.message.apply(emoji_extract)
    df['Media'] = df.message.str.contains('<Media omitted>')
    df['urlcount'] = df.message.apply(lambda x: re.findall(url_pattern, x)).str.len()
    return df

if chat_content!=[]:
    df=list_to_DF(chat_content)
    df=data_preperation(df)

    st.subheader("Conversation Stats")
    st.write("\n")
    st.write("Total Text Messages: ", df.shape[0])
    st.write("Total Media Messages: ", df[df['Media']].shape[0])
    st.write("Total Emojis: ", sum(df['emoji_used'].str.len()))
    st.write("Total Links/URLs: ", np.sum(df.urlcount))

    media_messages_df = df[df['message'] == '<Media omitted>']
    messages_df = df.drop(media_messages_df.index)

    author_value_counts = df['author'].value_counts().to_frame()
    fig0 = px.bar(author_value_counts, y='author', x=author_value_counts.index,color='author',color_continuous_scale='Tealgrn' ,labels={'index':'Employees','author':'Overall Participation'}, title="Employees Team Interaction")
    st.plotly_chart(fig0)

    sort_type = st.selectbox("Sort By:",["Date","Day","Time","Month"])
    if sort_type=="Date":
        keyword="date"
    elif sort_type=="Day":
        keyword="day"
    elif sort_type=="Time":
        keyword = "time"
    elif sort_type=="Month":
        keyword = "month"

    sort_df = messages_df.groupby(keyword).sum()
    sort_df['MessageCount'] = messages_df.groupby(keyword).size().values
    sort_df.reset_index(inplace=True)
    fig = px.line(sort_df, x=keyword, y="MessageCount", title=f"Overall Number of Messages according to {keyword}",)
    fig.update_xaxes(nticks=20,showgrid=False)
    st.plotly_chart(fig)

    author_df = df["author"].value_counts().reset_index()
    author_df.rename(columns={"index":"author", "author":"Number of messages"}, inplace=True)
    author_df["Total %"] = round(author_df["Number of messages"]*100/df.shape[0], 2)
    author_df["Involvement"] = author_df["Total %"].apply(lambda x: talkativeness(x, df["author"].nunique()))
    t_author_df = df.copy()
    t_author_df["year"] = t_author_df["date"].apply(lambda x: x.year)
    t_author_df["month"] = t_author_df["date"].apply(lambda x: x.strftime("%b"))
    months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
    t_author_df['month'] = pd.Categorical(t_author_df['month'], months)
    analysis_1_df = t_author_df.pivot_table(index=["month", "year"], columns=["author"], values=["message"], aggfunc="count", fill_value=0)
    analysis_1_df.columns = [col_[1] for col_ in analysis_1_df.columns]
    analysis_1_df = analysis_1_df.reset_index().sort_values(["year", "month"])
    analysis_1_df["month_year"] = analysis_1_df.apply(lambda x: x["month"] + " " + str(x["year"]), axis=1)
    analysis_1_df.drop(["month", "year"], axis=1, inplace=True)
    analysis_1_df.set_index('month_year',inplace=True)
    author_df["Trend"] = author_df["author"].apply(lambda x: trendline(analysis_1_df[x]))
    st.write('Overall Team Involvement Trend',author_df)

   #emoji distribution
    senders = st.selectbox("Select participant:",messages_df.author.unique())
    dummy_df = messages_df[messages_df['author'] == senders]
    #Individual Line chart
    dummy_df1 = dummy_df.groupby(keyword).sum()
    dummy_df1['MessageCount'] = dummy_df.groupby(keyword).size().values
    dummy_df1.reset_index(inplace=True)
    fig2 = px.line(sort_df, x=keyword, y="MessageCount", title=f"Overall Involvement of {senders} in {keyword} wise",)
    fig2.update_xaxes(nticks=20,showgrid=False)
    st.plotly_chart(fig2)

    total_emojis_list = list([a for b in dummy_df.emoji_used for a in b])
    emoji_dict = dict(Counter(total_emojis_list))
    emoji_dict = sorted(emoji_dict.items(), key=lambda x: x[1], reverse=True)
    author_emoji_df = pd.DataFrame(emoji_dict, columns=['emoji', 'count'])
    fig5 = px.pie(author_emoji_df, values='count', names='emoji', title=f'Emoji Distribution for {senders}')
    fig5.update_traces(textposition='inside', textinfo='percent+label',showlegend=False)
    st.plotly_chart(fig5)

    comment_words = ''

    for val in dummy_df.message:
        # typecaste each val to string
        val = str(val)
        # split the value
        tokens = val.split()
        # Converts each token into lowercase
        for i in range(len(tokens)):
            tokens[i] = tokens[i].lower()
        comment_words += " ".join(tokens)+" "

    wordcloud = WordCloud(width = 800, height = 800,
                background_color ='black',
                stopwords = stopwords,min_font_size=6).generate(comment_words)

    # plot the WordCloud image
    with st.expander("Tap to View Wordcloud"):
        fig, ax = plt.subplots(figsize = (10, 10),facecolor = 'k')
        ax.imshow(wordcloud,interpolation='bilinear')
        ax.axis("off")
        plt.tight_layout(pad=0)
        st.pyplot(fig)

    senti = []
    with st.spinner(f'Analyzing Sentiment for {senders}.. (This may take some time depending on size of data)'):
        try:
            translation = pool.map(translate_request, dummy_df["message"].values)
        except Exception as e:
            raise e
        pool.close()
        pool.join()

        for i in translation:
            if i!=None:
                sentiment_dict = sid_obj.polarity_scores(i)

                if sentiment_dict['compound'] >= 0.05 :
                    senti.append("Positive")
                elif sentiment_dict['compound'] <= - 0.05 :
                    senti.append("Negative")
                else :
                    senti.append("Neutral")

    all_sents = Counter(senti)
    fig6 = px.bar(y=all_sents.values(), x=all_sents.keys(),color=all_sents.keys(),color_discrete_sequence=['green','blue','red'] ,labels={'x':'Sentiment','y':'Interaction'},title=f"Sentiments for {senders}")
    fig6.update_layout(showlegend=False)
    st.plotly_chart(fig6)
    result = max(all_sents,key=all_sents.get)
    st.info(f"{senders} mostly conveys {result} behaviour")
    st.write("\n")

    """
    # This code was made for testing purpose only
    if st.checkbox(f"Click to check score for the {senders} (Out of 100)"):
        score_df = messages_df[messages_df['author'] == senders]
        score_df['MessageCount'] = score_df.shape[0]
        if score_df[(score_df['MessageCount'] > 400)].shape[0] > 0:
            st.write(f"Score for {senders}: ",random.randint(80,100))
        elif score_df[(score_df['MessageCount'] > 300) & (score_df['MessageCount'] < 400)].shape[0] > 0:
            st.write(f"Score for {senders}: ",random.randint(70,80))
        elif score_df[(score_df['MessageCount'] > 200) & (score_df['MessageCount'] < 300)].shape[0] > 0:
            st.write(f"Score for {senders}: ",random.randint(60,70))
        elif score_df[(score_df['MessageCount'] > 100) & (score_df['MessageCount'] < 200)].shape[0] > 0:
            st.write(f"Score for {senders}: ",random.randint(50,60))
        else:
            st.write(f"Score for {senders}: ",random.randint(40,50))
    """

st.markdown('

Developed and Maintained by Anant Arun',unsafe_allow_html=True)


import sys

sol_list = {} #to reduce the recursive computation #really nice trick

def sol(num):
	if num <=	11:
		return num
	if num in sol_list.keys():
		return sol_list[num]
	else:
		sol_list[num] = sol(int(num/2)) + sol(int(num/3)) + sol(int(num/4))
		return sol_list[num]

try:
	while True:
	    i = int(input())
	    print(sol(i))
except :
    sys.exit(0)


from models.video_model import VideoModel

class PublicationModel:
    def __init__(self, publication_url: str, publication_id: str, author_unique_id: str,
                 desc: str, like_count: int, comment_count: int,view_count: int, share_count: int,
                 category: int, created_at: int, hashtags: list[str], video: VideoModel):
        self.publication_url = publication_url
        self.publication_id = publication_id
        self.author_unique_id = author_unique_id
        self.desc = desc
        self.like_count = like_count
        self.comment_count = comment_count
        self.view_count = view_count
        self.share_count = share_count
        self.hashtags = hashtags
        self.category = category
        self.created_at = created_at
        self.video = video

    def __str__(self):
        return f"<Publication publication_url:{self.publication_url}, publication_id:{self.publication_id}, " \
               f"author_unique_id:{self.author_unique_id}, desc:{self.desc}, like_count:{self.like_count}, " \
               f"comment_count:{self.comment_count}, view_count:{self.view_count}," \
               f"share_count:{self.share_count}, hashtags:{self.hashtags}, " \
               f"category:{self.category}, created_at:{self.created_at}, video:{self.video}"