#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Base track file data.
"""
from    typing      import (
    Type, Optional, Union, Dict, Tuple, Any, List,
    Sequence, Iterable, Iterator, cast, overload
)
from    datetime    import datetime
from    pathlib     import Path
from    copy        import deepcopy, copy
from    enum        import Enum
import  numpy       as     np

from    taskmodel       import levelprop, Level, InstrumentType
from    taskmodel.level import PHASE
from    utils           import initdefaults, NoArgs
from   .views           import Beads, Cycles, isellipsis, TrackView
from   .trackio         import opentrack, PATHTYPES, instrumentinfo
from   .beadstats       import (
    RawPrecisionCache  as _RawPrecisionCache,
    beadextension as _beadextension,
    phaseposition as _phaseposition,
)

IDTYPE       = Union[None, int, range]  # missing Ellipsys as mypy won't accept it
PIDTYPE      = Union[IDTYPE, slice, Sequence[int]]

DATA         = Dict[int, np.ndarray]
BEADS        = Dict[int, 'Bead']
DIMENSIONS   = Tuple[Tuple[float, float], Tuple[float, float]]

def _doc(tpe: type) -> str:
    if tpe.__doc__:
        doc = cast(str, tpe.__doc__).strip()
        return doc[0].lower()+doc[1:].replace('\n', '\n    ')+"\n"
    return ''

def _lazies():
    return ('_data', '_rawprecisions') + tuple(LazyProperty.LIST)

class Axis(Enum):
    "which axis to look at"
    Xaxis = 'Xaxis'
    Yaxis = 'Yaxis'
    Zaxis = 'Zaxis'
    @classmethod
    def _missing_(cls, name):
        if name in 'xyz':
            name = name.upper()
        if name in 'XYZ':
            name += 'axis'
        return cls(name)

class Bead:
    """
    Characteristics of a bead:

    * `position` is the bead's (X, Y, Z) position
    * `image` is the bead's calibration image
    """
    position: Tuple[float, float, float] = (0., 0., 0.)
    image:    np.ndarray                 = np.zeros(0, dtype = np.uint8)

    @initdefaults(locals())
    def __init__(self, **_):
        pass

    def thumbnail(self, size, fov):
        "extracts a thumbnail around the bead position"
        pos  = fov.topixel(np.array(list(self.position[:2])))
        ind  = np.int32(np.round(pos))-size//2  # type: ignore
        return fov.image[ind[1]:ind[1]+size,ind[0]:ind[0]+size]

class FoV:
    """
    The data concerning the field of view:

    * `image` is one image of the field of view
    * `dim` are conversion factors from pixel to nm in the format:
    "(X slope, X bias), (Y slope, Y bias)".
    * `beads` is a dictionnary of information per bead:
    """
    if __doc__:
        __doc__ += ''.join(f'    {i}\n' for i in cast(str, Bead.__doc__).split('\n')[-4:])

    image                       = np.empty((0,0), dtype = np.uint8)
    beads:         BEADS        = {}
    dim:           DIMENSIONS   = ((1., 0.), (1., 0.))

    @initdefaults(locals())
    def __init__(self, **kwa):
        pass

    def bounds(self, pixel = False):
        "image bounds in nm (*pixel == False*) or pixels"
        rng = self.image.shape[1], self.image.shape[0]
        return (0, 0) + rng if pixel else self.tonm((0,0)) + self.tonm(rng)

    def size(self, pixel = False):
        "image size in nm (*pixel == False*) or pixels"
        if self.image is None:
            xpos = [i.position[0] for i in self.beads.values()]
            ypos = [i.position[1] for i in self.beads.values()]
            if len(xpos) and len(ypos):
                return (max(1., np.nanmax(ypos)-np.nanmin(ypos)),
                        max(1., np.nanmax(xpos)-np.nanmin(xpos)))
            return 1., 1.

        rng = self.image.shape[1], self.image.shape[0]
        return rng if pixel else self.tonm(rng)

    def tonm(self, arr):
        "converts pixels to nm"
        return self.__convert(arr, self.dim)

    def topixel(self, arr):
        "converts pixels to nm"
        return self.__convert(arr, tuple((1./i, -j/i) for i, j in self.dim))

    @property
    def scale(self):
        "The pixel scale: error occurs if the pixel is not square"
        if abs(self.dim[0][0]-self.dim[1][0]) > 1e-6:
            raise ValueError("Pixel is not square")
        return self.dim[0][0]

    @staticmethod
    def __convert(arr, dim):
        if len(arr) == 0:
            return arr

        (sl1, int1), (sl2, int2) = dim
        if isinstance(arr, np.ndarray):
            return [sl1, sl2] * arr + [int1, int2]

        if isinstance(arr, tuple) and len(arr) == 2 and np.isscalar(arr[0]):
            return tuple(i*k+j for (i, j), k in zip(dim, arr))

        tpe = iter if hasattr(arr, '__next__') else type(arr)
        return tpe([(sl1*i+int1, sl2*j+int2) for i, j in arr])  # type: ignore

class Secondaries:
    """
    Consists in arrays of sparse measures:

    * `track.secondaries.tservo` is the servo temperature
    * `track.secondaries.tsample` is the sample temperature
    * `track.secondaries.tsink` is the heat sink temperature
    * `track.secondaries.vcap` is a measure of magnet altitude using voltages
    * `track.secondaries.zmag` is a measure of magnet altitude provided by its motor
    * `track.secondaries.seconds` is the time axis
    """
    def __init__(self, track):
        self.__track = track

    data    = property(lambda self: self.__track._secondaries,
                       doc = "returns all the data")
    tservo  = cast(np.ndarray, property(lambda self: self.__value("Tservo"),
                                        doc = "the servo temperature"))
    tsample = cast(np.ndarray, property(lambda self: self.__value("Tsample"),
                                        doc = "the sample temperature"))
    tsink   = cast(np.ndarray, property(lambda self: self.__value("Tsink"),
                                        doc = "the sink temperature"))
    vcap    = cast(np.ndarray, property(lambda self: self.data.get("vcap"),
                                        doc = "the magnet position: vcap"))
    frames  = cast(np.ndarray, property(lambda self: self.__track._secondaries["t"],
                                        doc = "the time axis (frame count)"))
    seconds = cast(
        np.ndarray,
        property(
            lambda self: (self.__track._secondaries["t"] / self.__track.framerate),
            doc = "the time axis (s)"
        )
    )
    zmag    = cast(np.ndarray, property(lambda self: self.__track._secondaries["zmag"],
                                        doc = "the magnet altitude sampled at frame rate"))

    @property
    def cid(self) -> np.ndarray:
        "return the cycle per frame"
        arr = np.zeros(self.__track.nframes, dtype = 'i4')
        for i, j  in enumerate(np.split(
                arr,
                self.__track.phases[:,0]-self.__track.phases[0,0]
        )[1:]):
            j[:] = i
        return arr

    @property
    def cycleframe(self) -> np.ndarray:
        "return the frame-number per cycle, with each cycle starting at 0"
        arr = np.zeros(self.__track.nframes, dtype='i4')
        for j in np.split(
                arr,
                self.__track.phases[:, 0]-self.__track.phases[0, 0]
        )[1:]:
            j[:] = np.arange(len(j))
        return arr

    @property
    def cidcycles(self) -> Cycles:
        "return the phases per frame in cycles"
        return self.__track.cycles.withdata({"cid": self.cid})

    @property
    def phase(self) -> np.ndarray:
        "return the phases per frame"
        arr = np.zeros(self.__track.nframes, dtype = 'i1')
        nph = self.__track.nphases
        for i, j  in enumerate(np.split(
                arr,
                self.__track.phases.ravel()-self.__track.phases[0,0]
        )[1:]):
            j[:] = i % nph
        return arr

    @property
    def phasecycles(self) -> Cycles:
        "return the phases per frame in cycles"
        return self.__track.cycles.withdata({"phase": self.phase})

    @property
    def zmagcycles(self) -> Cycles:
        "the magnet altitude sampled at frame rate"
        return self.__track.cycles.withdata({"zmag": self.zmag})

    @property
    def cycles(self) -> Cycles:
        "return zmag, phase and cid per cycle"
        return self.__track.cycles.withdata({
            i: getattr(self, i) for i in ('cid', 'zmag', 'phase')
        })

    def keys(self):
        "return the available secondaries"
        return set(self.data.keys()) | {"tservo", "tsample", "tsink", "vcap", "seconds", "zmag"}

    def __getitem__(self, name):
        "returns a secondary value"
        if hasattr(self, name):
            return getattr(self, name)

        return self.__value(name) if name.startswith("T") else self.data[name]

    def __value(self, name):
        val = getattr(self.__track, '_secondaries')
        if val is None or name not in val:
            return None
        arr           = np.copy(val[name])
        arr['index'] -= self.__track.phases[0,0]
        arr           = arr[arr['index'] >= 0]
        arr           = arr[arr['index'] < self.__track.nframes]
        return arr

class LazyProperty:
    "Checks whether the file was opened prior to returning a value"
    LIST: List[str] = []

    def __init__(self, name: str = '', tpe: type = None) -> None:
        self._name = ''
        self._type = tpe
        if tpe and getattr(tpe, '__doc__', None):
            self.__doc__ = tpe.__doc__
        if name:
            self._name = '_'+name
            self.LIST.append(self._name)

    def __set_name__(self, _, name):
        self.__init__(name, self._type)

    @staticmethod
    def _load(inst):
        inst.load()

    def __get__(self, inst: 'Track', owner):
        if inst is not None:
            self._load(inst)

        return (self._type(inst) if self._type and inst else
                getattr(owner if inst is None else inst, self._name))

    def __set__(self, obj: 'Track', val):
        self._load(obj)
        setattr(obj, self._name, val)
        return getattr(obj, self._name)

class InstrumentProperty(LazyProperty):
    "Checks whether the file was opened prior to returning a value"
    def _load(self, inst):
        if not inst.isloaded:
            info = instrumentinfo(inst)
            info['type'] = InstrumentType(info['type'])
            setattr(inst, self._name, info)

class ResettingProperty:
    "Resets all if this attribute is changed"
    def __init__(self):
        self._name = ''

    def __set_name__(self, _, name):
        self._name = '_'+name

    def __get__(self, obj: 'Track', _):
        return getattr(obj, self._name) if obj else self

    def __set__(self, obj: 'Track', val):
        setattr(obj, self._name, val)
        obj.unload()
        return getattr(obj, self._name)

class ViewDescriptor:
    "Access to views"
    tpe:  Optional[type] = None
    args: Dict[str, Any] = dict()

    def __get__(self, instance, owner):
        return self if instance is None else instance.view(self.tpe, **self.args)

    def __set_name__(self, _, name):
        self.tpe  = Cycles if name.startswith('cycles') else Beads
        self.args = dict(copy = False)
        setattr(self, '__doc__', getattr(self.tpe, '__doc__', None))

class PhaseManipulator:
    """
    Helper class for manipulating phases.
    """
    def __init__(self, track):
        self._track = track

    def __getitem__(self, value):
        return PHASE[value]

    def cut(self, cid:PIDTYPE = None) -> Tuple[np.ndarray, np.ndarray]:
        """
        Returns a selection of phases, *reindexed* to zero, with a list of
        frame ids corresponding to theses phases.

        This can be used to create a track containing a fraction of the original data.
        """
        trk = self._track
        if isellipsis(cid):
            cycs = slice(None, None)

        if isinstance(cid, (slice, range)):
            cycs = slice(0               if cid.start is None else cid.start,
                         len(trk.phases) if cid.stop  is None else cid.stop,
                         1               if cid.step  is None else cid.step)
        else:
            cycs = np.array(cid, dtype = 'i4')

        phases = trk.phases[cycs]
        first  = phases[:,0]
        if isinstance(cycs, slice):
            last = trk.phases[cycs.start+1:cycs.stop+1:cycs.step,0]  # type: ignore
        else:
            tmp  = cycs+1
            last = trk.phases[tmp[tmp < len(trk.phases)],0]
        if len(last) < len(first):
            last = np.append(last, trk.nframes+trk.phases[0,0])

        inds   = np.concatenate([np.arange(j, dtype = 'i4')+i for i, j in zip(first, last-first)])
        inds  -= self._track.phases[0, 0]
        phases = (np.insert(np.cumsum(np.diff(np.hstack([phases, last[:,None]]))), 0, 0)
                  [:-1].reshape((-1, phases.shape[1])))
        return inds, phases

    def duration(self, cid:PIDTYPE = None, pid:IDTYPE = None) -> Union[np.ndarray, int]:
        """
        Returns the duration of a phase per cycle.
        """
        if isinstance(pid, (tuple, list, np.ndarray)):
            return np.vstack([self.__duration(cid, i) for i in cast(list, pid)]).T
        return self.__duration(cid, pid)

    def select(self, cid:PIDTYPE = None, pid:PIDTYPE = None) -> Union[np.ndarray, int]:
        """
        Returns the start time of the cycle and phase.

        If pid >= nphases, the end time of cycles is returned.
        if pid is a sequence of ints, a table is returned.
        """
        if isinstance(pid, (tuple, list, np.ndarray)):
            return np.vstack([self.__select(cid, i) for i in cast(list, pid)]).T
        return self.__select(cid, pid)

    nframes  = cast(int, property(lambda self: self._track.nframes))
    ncycles  = cast(int, property(lambda self: self._track.ncycles))
    nphases  = cast(int, property(lambda self: self._track.nphases))
    if __doc__:
        __doc__ += "   * `cut`: "      + cast(str, cut.__doc__)     .strip()+"\n"
        __doc__ += "   * `duration`: " + cast(str, duration.__doc__).strip()+"\n"
        __doc__ += "   * `select`: "   + cast(str, select.__doc__)  .strip()+"\n"

    def __duration(self, cid:PIDTYPE = None, pid:IDTYPE = None) -> Union[np.ndarray, int]:
        phases = self._track.phases
        if isinstance(pid, str):
            pid = self[pid]

        if isinstance(pid, int):
            pid  = range(pid, cast(int, None if pid == -1 else pid+1))
        elif isellipsis(pid):
            pid  = range(phases.shape[1])
        elif isinstance(pid, (slice, range)):
            pid = range(
                None if pid.start is None else self[pid.start],
                None if pid.stop  is None else self[pid.stop]
            )
        else:
            raise TypeError()

        start = 0 if pid.start is None else pid.start
        if pid.stop == start:
            return np.zeros(len(phases), dtype = 'i4')[cid]
        return self.select(cid, pid.stop) - self.select(cid, pid.start)

    def __select(self, cid:PIDTYPE = None, pid:PIDTYPE = None) -> Union[np.ndarray, int]:
        phases = self._track.phases
        ells   = isellipsis(cid), isellipsis(pid)
        if not ells[1]:
            pid = self[pid]
        if np.isscalar(pid) and pid >= self._track.nphases:
            if np.isscalar(cid):
                return (self._track.nframes if cid >= self._track.ncycles-1 else
                        phases[1+cast(int, cid),0]-phases[0,0])

            tmp = np.append(phases[1:,0]-phases[0,0], np.int32(self._track.nframes))
            return tmp[None if ells[0] else cid]

        return (phases        if all(ells) else
                phases[:,pid] if ells[0]   else
                phases[cid,:] if ells[1]   else
                phases[cid,pid]) - phases[0,0]

class PathInfo:
    """
    Provides information on the path itself:

    * `paths`: a tuple of paths
    * `trackpath`: the main path, i.e. not the grs
    * `size` (*megabytes*) is the size in bytes (megabytes) of *trackpath*
    * `stat`: stats on the *trackpath*
    * `modification`: the date oflast modification. This is basically the
    time of experiment.
    * `creation`: the creation date. **DISCARD** when using PicoTwist tracks.
    """
    track: 'Track'

    def __get__(self, inst, tpe):
        if inst is None:
            return self
        cpy       = PathInfo()
        cpy.track = inst
        return cpy

    @property
    def paths(self) -> List[Path]:
        "returns all paths"
        path = self.track.path
        return (
            [Path(path)] if isinstance(path, str)  else
            [path]       if isinstance(path, Path) else
            []           if path is None           else
            [Path(str(i)) for i in cast(Iterable, path)]
        )

    @property
    def trackpath(self) -> Path:
        "returns all paths"
        path = self.track.path
        return Path(str(path[0])) if isinstance(path, (list, tuple)) else Path(str(path))

    pathcount    = property(lambda self: len(self.paths))
    stat         = property(lambda self: self.trackpath.stat())
    size         = property(lambda self: self.stat.st_size)
    megabytes    = property(lambda self: self.size >> 20)
    creation     = property(lambda self: datetime.fromtimestamp(self.stat.st_ctime))

    @property
    def modification(self):
        "return the modification date of the **original** track file."
        date = getattr(self.track, '_modificationdate', None)
        if date is None and self.trackpath.exists():
            date = self.stat.st_mtime
        else:
            date = 0
        return datetime.fromtimestamp(date)

@levelprop(Level.project)
class Track:
    """
    The data from a track file, accessed lazily (only upon request).

    The data can be read as:

    ```python
    >>> raw = Track(path =  "/path/to/a/file.trk")
    >>> grs = Track(path =  ("/path/to/a/file.trk",
    ...                      "/path/to/a/gr/directory",
    ...                      "/path/to/a/specific/gr"))
    ```

    The data can then be accessed as follows:

    * for the *time* axis: `raw.beads['t']`
    * for the magnet altitude: `raw.beads['zmag']`
    * specific beads: `raw.beads[0]` where 0 can be any bead number
    * specific cycles: `raw.cycles[1,5]` where 1 and 5 can be any bead or cycle number.

    Some slicing is possible:

    * `raw.cycles[:,range(5,10)]` accesses cycles 5 though 10 for all beads.
    * `raw.cycles[[2,5],...]` accesses all cycles for beads 5 and 5.

    Only data for the Z axis is available. Use the `axis = 'X'` or `axis = 'Y'`
    options in the constructor to access other data.

    Other attributes are:

    * `framerate` is this experiment's frame rate
    * `phases` is a 2D array with one row per cycle and one column per phase
    containing the first index value of each cycle and phase.
    * `path` is the path(s) to the data
    * `axis` (Є {{'X', 'Y', 'Z'}}) is the data axis
    * `ncycles` is the number of cycles
    * `nphases` is the number of phases
    * `secondaries` {secondaries}
    * `fov` {fov}
    * `pathinfo` {pathinfo}
    """
    if __doc__:
        __doc__ = __doc__.format(
            secondaries = _doc(Secondaries),
            fov         = _doc(FoV),
            pathinfo    = _doc(PathInfo)
        )

    key: Optional[str] = None
    instrument         = cast(Dict[str, Any],      InstrumentProperty())
    phases             = cast(np.ndarray,          LazyProperty())
    framerate          = cast(float,               LazyProperty())
    fov                = cast(FoV,                 LazyProperty())
    secondaries        = cast(Secondaries,         LazyProperty(tpe = Secondaries))
    path               = cast(Optional[PATHTYPES], ResettingProperty())
    axis               = cast(Axis,                ResettingProperty())
    data               = cast(
        DATA,
        property(lambda self: self.getdata(), lambda self, val: self.setdata(val))
    )

    @initdefaults('key',
                  **{i: '_' for i in locals() if i != 'key' and i[0] != '_'})
    def __init__(self, **kwa):
        self._rawprecisions: _RawPrecisionCache = _RawPrecisionCache()
        if 'rawprecisions' in kwa and isinstance(kwa['rawprecisions'], _RawPrecisionCache):
            self._rawprecisions = kwa['rawprecisions']
        elif 'rawprecisions' in kwa:
            self._rawprecisions.computer = kwa['rawprecisions']

    ncycles  = cast(int,                 property(lambda self: len(self.phases)))
    nphases  = cast(int,                 property(lambda self: self.phases.shape[1]))
    beads    = cast(Beads,               ViewDescriptor())
    cycles   = cast(Cycles,              ViewDescriptor())
    phase    = property(PhaseManipulator, doc = PhaseManipulator.__doc__)
    pathinfo = PathInfo()

    def getdata(self) -> DATA:
        "returns the dataframe with all bead info"
        self.load()
        return cast(DATA, self._data)

    def setdata(self, data: Optional[Dict[int, np.ndarray]]):
        "sets the dataframe"
        if data is None:
            self.unload()
        else:
            self._data = data

    @property
    def nframes(self) -> int:
        "returns the number of frames"
        return len(next(iter(self.data.values()), []))

    @property
    def isloaded(self) -> bool:
        "returns whether the data was already acccessed"
        return self._data is not None

    def load(self, cycles: Optional[slice] = None) -> 'Track':
        "Loads the data"
        if self._data is None and self._path is not None:
            opentrack(self, cycles)
        return self

    def unload(self):
        "Unloads the data"
        for name in _lazies():
            setattr(self, name, deepcopy(getattr(type(self), name)))

    def view(self, tpe:Union[Type[TrackView], str], **kwa):
        "Creates a view of the suggested type"
        viewtype = (tpe     if isinstance(tpe, type) else
                    Cycles  if tpe.lower() == 'cycles' else
                    Beads)
        kwa.setdefault('parents', (self.key,) if self.key else (self.path,))
        kwa.setdefault('track',   self)
        return viewtype(**kwa)

    @overload       # noqa: F811
    def rawprecision(
            self,
            ibead:  int,
            phases: Union[None, Dict[int, float], Tuple[int, int]],
    ):
        "Obtain the raw precision for a given bead"

    @overload       # noqa: F811
    def rawprecision(self, computertype: str) -> None:
        "Set the raw precision computer"

    @overload       # noqa: F811
    def rawprecision(self) -> str:
        "Obtain the raw precision computer"

    @overload       # noqa: F811
    def rawprecision(
            self,
            ibead:  Optional[Iterable[int]],
            phases: Union[None, Dict[int, float], Tuple[int, int]],
    ) -> Iterator[Tuple[int,float]]:
        "Obtain the raw precision for a number of beads"

    def rawprecision(self, ibead = NoArgs, phases = None):   # noqa: F811
        "Obtain the raw precision for a number of beads"
        if ibead is NoArgs:
            return self._rawprecisions.computer
        if isinstance(ibead, (type, str)):
            self._rawprecisions.computer = ibead
            return None
        return self._rawprecisions.get(self, ibead, phases)

    if __doc__ is not None:
        setattr(rawprecision, '__doc__', getattr(_RawPrecisionCache.get, '__doc__', None))

    beadextension = _beadextension
    phaseposition = _phaseposition

    def shallowcopy(self):
        "make a shallow copy of the track: different containers but for the true data"
        cpy = self.__class__()
        cpy.__dict__.update({i: copy(j) for i, j in self.__dict__.items()})
        return cpy

    def __getstate__(self):
        keys = set(_lazies()+('_path', '_axis'))

        test = dict.fromkeys(keys, lambda i, j: j != getattr(type(self), i))  # type: ignore
        test.update(_phases = lambda _, i: len(i),
                    key     = lambda _, i: i is not None)

        cnv  = dict.fromkeys(keys | {'key'}, lambda i: i)  # type: ignore
        cnv.update(_secondaries = lambda i: getattr(i, 'data', None),
                   _axis        = lambda i: getattr(i, 'value', i)[0])

        info = self.__dict__.copy()
        if self._lazydata_:
            for i in ('_data', '_secondaries', '_fov'):
                info.pop(i, None)

        for name in set(cnv) & set(info):
            val = info.pop(name)
            if test[name](name, val):
                info[name[1:] if name[0] == '_' else name] = cnv[name](val)
        return info

    def __setstate__(self, values):
        if isinstance(values.get('fov', None), dict):
            fov           = values['fov']
            fov["beads"]  = {i: Bead(**j) for i, j in fov.get('beads', {}).items()}
            values['fov'] = FoV(**fov)

        if isinstance(values.get('instrument', {}).get("type", None), str):
            values['instrument']['type'] = InstrumentType(values['instrument']['type'])

        self.__init__(**values)
        keys = frozenset(self.__getstate__().keys()) | frozenset(('data', 'secondaries'))
        self.__dict__.update({i: j for i, j in values.items() if i not in keys})

    @property
    def _lazydata_(self):
        """
        Used internally to discard the data from __getstate__, or not
        """
        return self.__dict__.get('_lazydata_', self.path is not None)

    @_lazydata_.setter
    def _lazydata_(self, val):
        if val is None:
            self.__dict__.pop('_lazydata_', None)
        else:
            self.__dict__['_lazydata_'] = val

    _framerate:       float                = 30.
    _fov:             Optional[FoV]        = None
    _instrument:      Dict[str, Any]       = {
        "type": InstrumentType.picotwist.name,
        "name": None
    }
    _phases:          np.ndarray           = np.empty((0,9), dtype = 'i4')
    _data:            Optional[DATA]       = None  # type: ignore
    _secondaries:     Optional[DATA]       = None
    _rawprecisions:   _RawPrecisionCache   = _RawPrecisionCache()
    _path:            Optional[PATHTYPES]  = None
    _axis:            Axis                 = Axis.Zaxis


# -*- coding: utf-8-*-
import re
import requests
import os
from client.plugins.utilities import jasperpath

WORDS = ["WEATHER"]

def findWholeWord(w):
    return re.compile(r'\b({0})\b'.format(w), flags=re.IGNORECASE).search

def handle(text, mic, speaker, profile, visionProcess):
    """
        Responds to user-input, typically speech text, by relaying the
        meaning of life.

        Arguments:
        text -- user-input, typically transcribed speech
        mic -- used to interact with the user (for both input and output)
        profile -- contains information related to the user (e.g., phone
                   number)
    """
    #message = "Hold on, checking the weather for you..."
    # mic.say(message)

    location_name = "Bangalore"

    if re.search('(?<=TODAY)(.*)', text):
        weather_for = 'today'
    elif re.search('(?<=TOMORROW)(.*)', text):
        weather_for = 'tomorrow'
    elif re.search('(?<=THIS WEEK)(.*)', text):
        weather_for = 'this_week'
    elif re.search('(?<=NEXT WEEK)(.*)', text):
        weather_for = 'next_week'
    else:
        weather_for = 'today'

    if re.search('(?<=IN)(.*)', text, re.IGNORECASE):
        print("Found word in.. that means location is specified")
        location_temp = re.search("IN (\w+)", text, re.IGNORECASE)
        location_name = str(location_temp.group(1))
        name_start = ['NEW', 'SAN', 'ABU', 'PORT', 'KUALA', 'LOS']
        if location_name.upper() in name_start:
            print("Complex city name")
            location_temp2 = re.search(location_name + " (\w+)", text)
            location_name = location_name + " " + str(location_temp2.group(1))

    openWeatherCurrentUrl = 'http://api.openweathermap.org/data/2.5/weather?q=' + \
        location_name + '&units=metric&appid=4f95de0f661f4bb718a10067a7f27b60'
    openWeatherForeCastUrl = "api.openweathermap.org/data/2.5/forecast?q=" + \
        location_name + '&units=metric'

    responseObject = requests.get(openWeatherCurrentUrl)
    responseData = responseObject.json()

    temp_now = responseData['main']['temp']
    humidity = responseData['main']['humidity']
    description_short = responseData['weather'][0]['main']

    """
    sunset = responseData['sys']['sunset']
    sunrise = responseData['sys']['sunrise']
    temp_max = responseData['main']['temp_max']
    temp_min = responseData['main']['temp_min']
    temp_now = responseData['main']['temp']
    humidity = responseData['main']['humidity']
    description_short = responseData['weather'][0]['main']
    description_long = responseData['weather'][0]['description']
    print temp_max
    print temp_min
    print temp_now
    """

    iconCode = responseData['weather'][0]['icon']
    iconURL = 'http://openweathermap.org/img/w/' + iconCode + '.png'
    icon_path = jasperpath.WEATHER_ICONS + "/" + iconCode + '.png'

    if not os.path.isfile(icon_path):
        resp = requests.get(iconURL)
        if resp.status_code == 200:
            with open(icon_path, 'wb') as f:
                for chunk in resp.iter_content(1024):
                    f.write(chunk)

    weather_message = "It is currently " + str(temp_now) + " degrees with " + str(
        humidity) + " percent humidity. The weather type is " + description_short
    # Need to modify message according to weather_for

    visionProcess['image_path'] = icon_path
    # visionProcess['message']=weather_message
    # visionProcess['message_timeout']=int(5)
    speaker.clean_and_say(weather_message)

def isValid(text):
    """
        Returns True if the input is related to the meaning of life.

        Arguments:
        text -- user-input, typically transcribed speech
    """
    return bool(re.search(r'\bWhat is the weather', text, re.IGNORECASE)
                or re.search(r'\bWhat\'s the weather', text, re.IGNORECASE)
                or re.search(r'\bHow\'s the weather', text, re.IGNORECASE)
                or re.search(r'\bHow is the weather', text, re.IGNORECASE))


from django.test import TestCase
from django.urls import reverse
from .models import Tarefa

class TarefaTestCase(TestCase):
    def setUp(self):
        self.tarefa1 = Tarefa.objects.create(titulo='Tarefa 1', texto='Texto da tarefa 1')
        self.tarefa2 = Tarefa.objects.create(titulo='Tarefa 2', texto='Texto da tarefa 2')

    def test_criar_tarefa(self):
        # Fazer uma requisição POST para criar uma nova tarefa
        response = self.client.post(reverse('criar_tarefa'), {'titulo': 'Nova Tarefa', 'texto': 'Texto da nova tarefa'})
        # Verificar se a tarefa foi criada no banco de dados
        self.assertTrue(Tarefa.objects.filter(titulo='Nova Tarefa', texto='Texto da nova tarefa').exists())
        # Verificar se a tarefa criada está sendo exibida na página de listagem de tarefas
        response = self.client.get(reverse('criar_tarefa'))
        self.assertContains(response, 'Nova Tarefa')

    def test_excluir_tarefa(self):
        # Verificar se a tarefa 1 existe antes da exclusão
        self.assertTrue(Tarefa.objects.filter(id=self.tarefa1.id).exists())
        # Fazer uma requisição POST para excluir a tarefa 1
        response = self.client.post(reverse('excluir_tarefa', args=[self.tarefa1.id]))
        # Verificar se a resposta é um redirecionamento para a página de criação de tarefas
        self.assertRedirects(response, reverse('criar_tarefa'))
        # Verificar se a tarefa 1 foi removida do banco de dados
        self.assertFalse(Tarefa.objects.filter(id=self.tarefa1.id).exists())
        # Verificar se a tarefa 2 ainda existe
        self.assertTrue(Tarefa.objects.filter(id=self.tarefa2.id).exists())


import RPi.GPIO as pin
from time import sleep

pin.setmode(pin.BCM)
pin.setwarnings(False)

class Motor():

	def __init__(self,lft1,lft2,lft_pwm,rht1,rht2,rht_pwm):

		self.lft1 = lft1
		self.lft2 = lft2
		self.rht1 = rht1
		self.rht2 = rht2
		self.lft_pwm = lft_pwm
		self.rht_pwm = rht_pwm

		pin.setup(self.lft1, pin.OUT)
		pin.setup(self.lft2, pin.OUT)
		pin.setup(self.rht1, pin.OUT)
		pin.setup(self.rht2, pin.OUT)
		pin.setup(self.lft_pwm, pin.OUT)
		pin.setup(self.rht_pwm, pin.OUT)

		self.lft_pwm = pin.PWM(self.lft_pwm , 100)
		self.rht_pwm = pin.PWM(self.rht_pwm, 100)

		self.lft_pwm.start(0)
		self.rht_pwm.start(0)

	def motor_control(self, left_speed, right_speed):

		if left_speed > 0 :
			pin.output(self.lft1, pin.LOW)
			pin.output(self.lft2, pin.HIGH)
		else:
			pin.output(self.lft2, pin.LOW)
			pin.output(self.lft1, pin.HIGH)

		if right_speed > 0 :
			pin.output(self.rht1, pin.HIGH)
			pin.output(self.rht2, pin.LOW)
		else:
			pin.output(self.rht2, pin.HIGH)
			pin.output(self.rht1, pin.LOW)

		if left_speed > 100: left_speed =100
		if left_speed < -100: left_speed = -100
		if right_speed > 100: right_speed = 100
		if right_speed < -100: right_speed = -100

		self.lft_pwm.ChangeDutyCycle(abs(left_speed))
		self.rht_pwm.ChangeDutyCycle(abs(right_speed))

if __name__ == '__main__':
	motor = Motor(17,4,27,22,10,9)
	while True:
		motor.motor_control(50,50)
		sleep(10)
		motor.motor_control(-50,-50)
		sleep(10)


import pandas as pd
from collections import Counter
import matplotlib.pyplot as plt
import numpy as np

df = pd.read_csv('survey_results_responses.csv')

# print(df.head())

ids = df['ResponseId']
lang_responses = df['LanguageHaveWorkedWith']

languages_counter = Counter()

for response in lang_responses:
    if isinstance(response, str):
        languages_counter.update(response.split(';'))

languages = []
popularity = []

for item in languages_counter.most_common(15):
    languages.append(item[0])
    popularity.append(item[1])

print(languages)
print(popularity)

# plt.bar(languages, popularity)
plt.barh(languages, popularity)
plt.show()


from fractions import Fraction
from re import search, findall

from converter import get_absolute_path_for_file, text_files_folder_path

analysis_folder_path = 'analysis'
resolution = 480, 4 # 480/8 o 960/16
exp_pitch = 'pitch=(\d+) '
exp_vol = 'vol=(\d+)'
exp_time = 'Time=(\d+) '
exp_division = 'division=(\d+)'
exp_signature = 'signature=(\d+)\/(\d+)'

def get_notes(text_file_name: str) -> list:
    note = []
    with open(text_file_name, mode='r') as fhandler:

        # div = 1440  # ticks for 3/8 bar
        # div = 1920  # ticks for 4/4 bar

        for i in fhandler:

            if 'division' in i:
                if search(exp_division, i):
                    division = int(search(exp_division, i).groups()[0])
                    if division != 480:
                        print(f"WARNING Division: {division}")

            if 'signature' in i:
                if search(exp_signature, i):
                    numerator, denominator = int(search(exp_signature, i).groups()[0]), \
                                             int(search(exp_signature, i).groups()[1])
                    signature = Fraction(numerator, denominator)
                    div = signature * division * resolution[1]

                    print(div)

            if 'Note on' in i:
                if search(exp_pitch, i) and search(exp_vol, i) and search(exp_time, i):
                    n = int(findall(exp_pitch, i)[0])
                    v = int(findall(exp_vol, i)[0])
                    start = int(findall(exp_time, i)[0])
                    note.append(
                        [n, v, -start, False, start, int(start / div), int(int((start % div) / resolution[0]))])

            if 'Note off' in i:
                if search(exp_pitch, i) and search(exp_vol, i) and search(exp_time, i):
                    n = int(findall(exp_pitch, i)[0])
                    end = int(findall(exp_time, i)[0])
                    res = [item for item in note if item[0] == n and item[3] == False]
                    if len(res) > 1:
                        print('error')
                        continue
                    res[0][2] += end
                    res[0][3] = True
    return note

def generate_file_a(notes: list, analysis_a_file: str):
    with open(analysis_a_file, 'w') as fhandler_output:
        last_bar = 0

        last_sub_bar = 0

        for n in notes:
            bar = n[5]
            sub_bar = n[6]

            if last_bar != bar:
                fhandler_output.write('------------- {0} -------------\n'.format(bar))
                last_bar = bar

            if last_sub_bar != sub_bar:
                fhandler_output.write('--- {0} ---\n'.format(sub_bar))
                last_sub_bar = sub_bar

            fhandler_output.write('N={0} D={2} V={1} P={3}\n'.format(n[0], n[1], n[2], n[4]))

def analysis_a(text_file: str):
    text_file_absolute_path = get_absolute_path_for_file(text_file, text_files_folder_path, 'txt')
    notes = get_notes(text_file_absolute_path)
    print(notes)
    analysis_file_absolute_path = get_absolute_path_for_file(text_file, analysis_folder_path, 'txt')
    generate_file_a(notes, analysis_file_absolute_path)


#!/usr/bin/python3

from PyQt5 import QtCore, QtGui, QtWidgets
import webbrowser
import os
import subprocess

class Ui_WelcomeScreen(object):
    ######################### CUSTOM ACTIONS ##########################
    def forumButtonAction(self):
        webbrowser.open("https://forum.namiblinux.org/categories")
    def chatButtonAction(self):
        webbrowser.open("https://www.namiblinux.org/support/chat/")
    def donateButtonAction(self):
        webbrowser.open("https://www.namiblinux.org/donate/")
    def wikiButtonAction(self):
        webbrowser.open("https://wiki.namiblinux.org/")
    def newsButtonAction(self):
        webbrowser.open("https://forum.namiblinux.org/c/announcements")
    def helpButtonAction(self):
        webbrowser.open("https://github.com/namiblinux")
    def installButtonAction(self):
        subprocess.Popen(["calamares_polkit"])
    def startCheckAction(self):
        if self.launchAtStartCheck.isChecked():
            homedir = os.path.expanduser('~')
            autostartfile = os.path.join(homedir, ".config/autostart/namib-welcome.desktop")
            subprocess.Popen(["cp", "/usr/share/applications/namib-welcome.desktop", autostartfile])
        else:
            homedir = os.path.expanduser('~')
            autostartfile = os.path.join(homedir, ".config/autostart/namib-welcome.desktop")
            subprocess.Popen(["rm", autostartfile])
    #####################################################################

    def setupUi(self, WelcomeScreen):
        WelcomeScreen.setObjectName("WelcomeScreen")
        WelcomeScreen.resize(640, 480)
        WelcomeScreen.setMinimumSize(640, 480)
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap("/usr/share/namib-welcome/img/namib-welcome.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        WelcomeScreen.setWindowIcon(icon)
        self.MainWidget = QtWidgets.QWidget(WelcomeScreen)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Expanding)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.MainWidget.sizePolicy().hasHeightForWidth())
        self.MainWidget.setSizePolicy(sizePolicy)
        self.MainWidget.setAutoFillBackground(False)
        self.MainWidget.setObjectName("MainWidget")
        self.gridLayout = QtWidgets.QGridLayout(self.MainWidget)
        self.gridLayout.setObjectName("gridLayout")
        self.mainGrid = QtWidgets.QGridLayout()
        self.mainGrid.setSizeConstraint(QtWidgets.QLayout.SetMaximumSize)
        self.mainGrid.setObjectName("mainGrid")
        self.wikiButton = QtWidgets.QPushButton(self.MainWidget)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.wikiButton.sizePolicy().hasHeightForWidth())
        self.wikiButton.setSizePolicy(sizePolicy)
        icon1 = QtGui.QIcon()
        icon1.addPixmap(QtGui.QPixmap("/usr/share/namib-welcome/img/wiki.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.wikiButton.setIcon(icon1)
        self.wikiButton.setObjectName("wikiButton")

        ######################### WIKI BUTTON ##########################
        self.wikiButton.clicked.connect(self.wikiButtonAction)
        ################################################################

        self.mainGrid.addWidget(self.wikiButton, 7, 0, 1, 1)
        self.lineTop = QtWidgets.QFrame(self.MainWidget)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.lineTop.sizePolicy().hasHeightForWidth())
        self.lineTop.setSizePolicy(sizePolicy)
        self.lineTop.setFrameShape(QtWidgets.QFrame.HLine)
        self.lineTop.setFrameShadow(QtWidgets.QFrame.Sunken)
        self.lineTop.setObjectName("lineTop")
        self.mainGrid.addWidget(self.lineTop, 4, 0, 1, 3)
        self.logoLayout = QtWidgets.QHBoxLayout()
        self.logoLayout.setSizeConstraint(QtWidgets.QLayout.SetMaximumSize)
        self.logoLayout.setObjectName("logoLayout")
        spacerItem = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum)
        self.logoLayout.addItem(spacerItem)
        self.Logo = QtWidgets.QLabel(self.MainWidget)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.Logo.sizePolicy().hasHeightForWidth())
        self.Logo.setSizePolicy(sizePolicy)
        self.Logo.setMaximumSize(QtCore.QSize(64, 64))
        self.Logo.setText("")
        self.Logo.setPixmap(QtGui.QPixmap("/usr/share/namib-welcome/img/logo.png"))
        self.Logo.setScaledContents(True)
        self.Logo.setWordWrap(False)
        self.Logo.setObjectName("Logo")
        self.logoLayout.addWidget(self.Logo)
        spacerItem1 = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum)
        self.logoLayout.addItem(spacerItem1)
        self.mainGrid.addLayout(self.logoLayout, 0, 0, 1, 3)
        self.linksLabel = QtWidgets.QLabel(self.MainWidget)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.linksLabel.sizePolicy().hasHeightForWidth())
        self.linksLabel.setSizePolicy(sizePolicy)
        self.linksLabel.setScaledContents(True)
        self.linksLabel.setWordWrap(True)
        self.linksLabel.setObjectName("linksLabel")
        self.mainGrid.addWidget(self.linksLabel, 5, 0, 1, 3)
        self.donateButton = QtWidgets.QPushButton(self.MainWidget)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.donateButton.sizePolicy().hasHeightForWidth())
        self.donateButton.setSizePolicy(sizePolicy)
        icon2 = QtGui.QIcon()
        icon2.addPixmap(QtGui.QPixmap("/usr/share/namib-welcome/img/donate.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.donateButton.setIcon(icon2)
        self.donateButton.setObjectName("donateButton")

        ######################### DONATE BUTTON ########################
        self.donateButton.clicked.connect(self.donateButtonAction)
        ################################################################

        self.mainGrid.addWidget(self.donateButton, 7, 2, 1, 1)
        self.chatButton = QtWidgets.QPushButton(self.MainWidget)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.chatButton.sizePolicy().hasHeightForWidth())
        self.chatButton.setSizePolicy(sizePolicy)
        icon3 = QtGui.QIcon()
        icon3.addPixmap(QtGui.QPixmap("/usr/share/namib-welcome/img/chat.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.chatButton.setIcon(icon3)
        self.chatButton.setObjectName("chatButton")

        ######################### CHAT BUTTON ##########################
        self.chatButton.clicked.connect(self.chatButtonAction)
        ###############################################################

        self.mainGrid.addWidget(self.chatButton, 7, 1, 1, 1)
        self.installationLabel = QtWidgets.QLabel(self.MainWidget)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.installationLabel.sizePolicy().hasHeightForWidth())
        self.installationLabel.setSizePolicy(sizePolicy)
        self.installationLabel.setObjectName("installationLabel")

        ######################### INSTALL LABEL ##########################
        self.installationLabel.setVisible(False)
        ##################################################################

        self.mainGrid.addWidget(self.installationLabel, 8, 1, 1, 1)
        self.installButton = QtWidgets.QPushButton(self.MainWidget)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.installButton.sizePolicy().hasHeightForWidth())
        self.installButton.setSizePolicy(sizePolicy)
        icon4 = QtGui.QIcon()
        icon4.addPixmap(QtGui.QPixmap("/usr/share/namib-welcome/img/install.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.installButton.setIcon(icon4)
        self.installButton.setObjectName("installButton")

        ######################### INSTALL BUTTON ##########################
        self.installButton.clicked.connect(self.installButtonAction)
        self.installButton.setVisible(False)
        ###################################################################

        self.mainGrid.addWidget(self.installButton, 9, 1, 1, 1)
        self.welcomeLabel = QtWidgets.QLabel(self.MainWidget)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.welcomeLabel.sizePolicy().hasHeightForWidth())
        self.welcomeLabel.setSizePolicy(sizePolicy)
        self.welcomeLabel.setObjectName("welcomeLabel")
        self.mainGrid.addWidget(self.welcomeLabel, 1, 0, 1, 3)
        self.lineBottom = QtWidgets.QFrame(self.MainWidget)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.lineBottom.sizePolicy().hasHeightForWidth())
        self.lineBottom.setSizePolicy(sizePolicy)
        self.lineBottom.setFrameShape(QtWidgets.QFrame.HLine)
        self.lineBottom.setFrameShadow(QtWidgets.QFrame.Sunken)
        self.lineBottom.setObjectName("lineBottom")
        self.mainGrid.addWidget(self.lineBottom, 10, 0, 1, 3)
        self.forumsButton = QtWidgets.QPushButton(self.MainWidget)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.forumsButton.sizePolicy().hasHeightForWidth())
        self.forumsButton.setSizePolicy(sizePolicy)
        icon5 = QtGui.QIcon()
        icon5.addPixmap(QtGui.QPixmap("/usr/share/namib-welcome/img/forums.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.forumsButton.setIcon(icon5)
        self.forumsButton.setObjectName("forumsButton")

        ######################### FORUM BUTTON #########################
        self.forumsButton.clicked.connect(self.forumButtonAction)
        ################################################################

        self.mainGrid.addWidget(self.forumsButton, 6, 1, 1, 1)
        self.settingsLayout = QtWidgets.QHBoxLayout()
        self.settingsLayout.setSizeConstraint(QtWidgets.QLayout.SetMaximumSize)
        self.settingsLayout.setObjectName("settingsLayout")
        self.languageSelector = QtWidgets.QComboBox(self.MainWidget)
        self.languageSelector.setObjectName("languageSelector")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.languageSelector.addItem("")
        self.settingsLayout.addWidget(self.languageSelector)
        self.languageSelector.hide()
        spacerItem2 = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum)
        self.settingsLayout.addItem(spacerItem2)
        self.launchAtStartCheck = QtWidgets.QCheckBox(self.MainWidget)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.launchAtStartCheck.sizePolicy().hasHeightForWidth())
        self.launchAtStartCheck.setSizePolicy(sizePolicy)
        self.launchAtStartCheck.setLayoutDirection(QtCore.Qt.LeftToRight)
        self.launchAtStartCheck.setObjectName("launchAtStartCheck")

        ######################### LAUNCH AT START BUTTON #########################
        self.launchAtStartCheck.clicked.connect(self.startCheckAction)
        ##########################################################################

        self.settingsLayout.addWidget(self.launchAtStartCheck)
        self.mainGrid.addLayout(self.settingsLayout, 11, 0, 1, 3)
        self.helpButton = QtWidgets.QPushButton(self.MainWidget)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.helpButton.sizePolicy().hasHeightForWidth())
        self.helpButton.setSizePolicy(sizePolicy)
        icon6 = QtGui.QIcon()
        icon6.addPixmap(QtGui.QPixmap("/usr/share/namib-welcome/img/helpus.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.helpButton.setIcon(icon6)
        self.helpButton.setObjectName("helpButton")

        ######################### HELP BUTTON #########################
        self.helpButton.clicked.connect(self.helpButtonAction)
        ###############################################################

        self.mainGrid.addWidget(self.helpButton, 6, 2, 1, 1)
        self.newsButton = QtWidgets.QPushButton(self.MainWidget)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.newsButton.sizePolicy().hasHeightForWidth())
        self.newsButton.setSizePolicy(sizePolicy)
        icon7 = QtGui.QIcon()
        icon7.addPixmap(QtGui.QPixmap("/usr/share/namib-welcome/img/news.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.newsButton.setIcon(icon7)
        self.newsButton.setObjectName("newsButton")

        ######################### NEWS BUTTON ##########################
        self.newsButton.clicked.connect(self.newsButtonAction)
        ################################################################

        self.mainGrid.addWidget(self.newsButton, 6, 0, 1, 1)
        self.infoLabel = QtWidgets.QLabel(self.MainWidget)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.infoLabel.sizePolicy().hasHeightForWidth())
        self.infoLabel.setSizePolicy(sizePolicy)
        self.infoLabel.setScaledContents(True)
        self.infoLabel.setWordWrap(True)
        self.infoLabel.setObjectName("infoLabel")
        self.mainGrid.addWidget(self.infoLabel, 2, 0, 1, 3)
        self.gridLayout.addLayout(self.mainGrid, 0, 0, 1, 1)
        WelcomeScreen.setCentralWidget(self.MainWidget)

        self.retranslateUi(WelcomeScreen)
        self.languageSelector.setCurrentIndex(12)
        QtCore.QMetaObject.connectSlotsByName(WelcomeScreen)

        ######################### LIVEUSER ? ##########################
        if os.path.isfile("/usr/bin/calamares_polkit"):
            self.installButton.setVisible(True)
            self.installationLabel.setVisible(True)
        ###############################################################

        ######################### IS AUTOSTART ? ######################
        homedir = os.path.expanduser('~')
        autostartfile = os.path.join(homedir, ".config/autostart/namib-welcome.desktop")
        if os.path.isfile(autostartfile):
            self.launchAtStartCheck.setChecked(True)
        ###############################################################

    def retranslateUi(self, WelcomeScreen):
        _translate = QtCore.QCoreApplication.translate
        WelcomeScreen.setWindowTitle(_translate("WelcomeScreen", "Welcome Screen"))
        self.wikiButton.setText(_translate("WelcomeScreen", "Wiki"))
        self.linksLabel.setText(_translate("WelcomeScreen", "
LINKS :
"))
        self.donateButton.setText(_translate("WelcomeScreen", "Donate"))
        self.chatButton.setText(_translate("WelcomeScreen", "Chat room"))
        self.installationLabel.setText(_translate("WelcomeScreen", "
INSTALLATION :
"))
        self.installButton.setText(_translate("WelcomeScreen", "Install Now"))
        self.welcomeLabel.setText(_translate("WelcomeScreen", "
Welcome to Namib GNU/Linux !
"))
        self.forumsButton.setText(_translate("WelcomeScreen", "Forums"))
        self.languageSelector.setCurrentText(_translate("WelcomeScreen", "English"))
        self.languageSelector.setItemText(0, _translate("WelcomeScreen", "Albanian"))
        self.languageSelector.setItemText(1, _translate("WelcomeScreen", "Arabic"))
        self.languageSelector.setItemText(2, _translate("WelcomeScreen", "Asturian (Spain)"))
        self.languageSelector.setItemText(3, _translate("WelcomeScreen", "Belarusian"))
        self.languageSelector.setItemText(4, _translate("WelcomeScreen", "Bulgarian"))
        self.languageSelector.setItemText(5, _translate("WelcomeScreen", "Catalan"))
        self.languageSelector.setItemText(6, _translate("WelcomeScreen", "Chinese (China)"))
        self.languageSelector.setItemText(7, _translate("WelcomeScreen", "Chinese (Taiwan)"))
        self.languageSelector.setItemText(8, _translate("WelcomeScreen", "Croatian"))
        self.languageSelector.setItemText(9, _translate("WelcomeScreen", "Czech"))
        self.languageSelector.setItemText(10, _translate("WelcomeScreen", "Danish"))
        self.languageSelector.setItemText(11, _translate("WelcomeScreen", "Dutch"))
        self.languageSelector.setItemText(12, _translate("WelcomeScreen", "English"))
        self.languageSelector.setItemText(13, _translate("WelcomeScreen", "French"))
        self.languageSelector.setItemText(14, _translate("WelcomeScreen", "German"))
        self.languageSelector.setItemText(15, _translate("WelcomeScreen", "Georgian"))
        self.languageSelector.setItemText(16, _translate("WelcomeScreen", "Greek (Greece)"))
        self.languageSelector.setItemText(17, _translate("WelcomeScreen", "Hebrew"))
        self.languageSelector.setItemText(18, _translate("WelcomeScreen", "Hindi (India)"))
        self.languageSelector.setItemText(19, _translate("WelcomeScreen", "New Item"))
        self.languageSelector.setItemText(20, _translate("WelcomeScreen", "Hungarian"))
        self.languageSelector.setItemText(21, _translate("WelcomeScreen", "Icelandic"))
        self.languageSelector.setItemText(22, _translate("WelcomeScreen", "Indonesian (Indonesia)"))
        self.languageSelector.setItemText(23, _translate("WelcomeScreen", "Italian"))
        self.languageSelector.setItemText(24, _translate("WelcomeScreen", "Japanese"))
        self.languageSelector.setItemText(25, _translate("WelcomeScreen", "Korean (Korea)"))
        self.languageSelector.setItemText(26, _translate("WelcomeScreen", "Lithuanian"))
        self.languageSelector.setItemText(27, _translate("WelcomeScreen", "Norwegian Bokmål"))
        self.languageSelector.setItemText(28, _translate("WelcomeScreen", "Persian (Iran)"))
        self.languageSelector.setItemText(29, _translate("WelcomeScreen", "Polish"))
        self.languageSelector.setItemText(30, _translate("WelcomeScreen", "Portuguese (Brazil)"))
        self.languageSelector.setItemText(31, _translate("WelcomeScreen", "Portuguese (Portugal)"))
        self.languageSelector.setItemText(32, _translate("WelcomeScreen", "Romanian (Romania)"))
        self.languageSelector.setItemText(33, _translate("WelcomeScreen", "Russian"))
        self.languageSelector.setItemText(34, _translate("WelcomeScreen", "Slovak"))
        self.languageSelector.setItemText(35, _translate("WelcomeScreen", "Slovenian (Slovenia)"))
        self.languageSelector.setItemText(36, _translate("WelcomeScreen", "Slovenian"))
        self.languageSelector.setItemText(37, _translate("WelcomeScreen", "Spanish"))
        self.languageSelector.setItemText(38, _translate("WelcomeScreen", "Serbian (Serbia)"))
        self.languageSelector.setItemText(39, _translate("WelcomeScreen", "Serbian"))
        self.languageSelector.setItemText(40, _translate("WelcomeScreen", "Swedish"))
        self.languageSelector.setItemText(41, _translate("WelcomeScreen", "THai"))
        self.languageSelector.setItemText(42, _translate("WelcomeScreen", "Turkish (Turkey)"))
        self.languageSelector.setItemText(43, _translate("WelcomeScreen", "Turkish"))
        self.languageSelector.setItemText(44, _translate("WelcomeScreen", "Ukrainian"))
        self.languageSelector.setItemText(45, _translate("WelcomeScreen", "Vietnamese (Viet Nam)"))
        self.launchAtStartCheck.setText(_translate("WelcomeScreen", "Launch at start"))
        self.helpButton.setText(_translate("WelcomeScreen", "Help us"))
        self.newsButton.setText(_translate("WelcomeScreen", "News"))
        self.infoLabel.setText(_translate("WelcomeScreen", "
Welcome to Namib GNU/Linux. The links below will help you get started with Namib. So enjoy the experience, and don\'t hesitate to send us your feedback.
"))

if __name__ == "__main__":
    import sys
    app = QtWidgets.QApplication(sys.argv)
    WelcomeScreen = QtWidgets.QMainWindow()
    ui = Ui_WelcomeScreen()
    ui.setupUi(WelcomeScreen)
    WelcomeScreen.show()
    sys.exit(app.exec_())


import datetime
import sys
import time
import traceback

# logger
from pandacommon.pandalogger.PandaLogger import PandaLogger
from pandajedi.jediconfig import jedi_config
from pandajedi.jedicore import Interaction, JediException
from pandajedi.jedicore.FactoryBase import FactoryBase
from pandajedi.jedicore.JediDatasetSpec import JediDatasetSpec
from pandajedi.jedicore.JediTaskSpec import JediTaskSpec
from pandajedi.jedicore.MsgWrapper import MsgWrapper
from pandajedi.jedicore.ThreadUtils import ListWithLock, ThreadPool, WorkerThread
from pandajedi.jedirefine import RefinerUtils

from .JediKnight import JediKnight

logger = PandaLogger().getLogger(__name__.split(".")[-1])

# worker class to refine TASK_PARAM to fill JEDI tables
class TaskRefiner(JediKnight, FactoryBase):
    # constructor
    def __init__(self, commuChannel, taskBufferIF, ddmIF, vos, prodSourceLabels):
        self.vos = self.parseInit(vos)
        self.prodSourceLabels = self.parseInit(prodSourceLabels)
        JediKnight.__init__(self, commuChannel, taskBufferIF, ddmIF, logger)
        FactoryBase.__init__(self, self.vos, self.prodSourceLabels, logger, jedi_config.taskrefine.modConfig)

    # main
    def start(self):
        # start base classes
        JediKnight.start(self)
        FactoryBase.initializeMods(self, self.taskBufferIF, self.ddmIF)
        # go into main loop
        while True:
            startTime = datetime.datetime.utcnow()
            try:
                # get logger
                tmpLog = MsgWrapper(logger)
                tmpLog.debug("start")
                # loop over all vos
                for vo in self.vos:
                    # loop over all sourceLabels
                    for prodSourceLabel in self.prodSourceLabels:
                        # get the list of tasks to refine
                        tmpList = self.taskBufferIF.getTasksToRefine_JEDI(vo, prodSourceLabel)
                        if tmpList is None:
                            # failed
                            tmpLog.error("failed to get the list of tasks to refine")
                        else:
                            tmpLog.debug(f"got {len(tmpList)} tasks")
                            # put to a locked list
                            taskList = ListWithLock(tmpList)
                            # make thread pool
                            threadPool = ThreadPool()
                            # get work queue mapper
                            workQueueMapper = self.taskBufferIF.getWorkQueueMap()
                            # make workers
                            nWorker = jedi_config.taskrefine.nWorkers
                            for iWorker in range(nWorker):
                                thr = TaskRefinerThread(taskList, threadPool, self.taskBufferIF, self.ddmIF, self, workQueueMapper)
                                thr.start()
                            # join
                            threadPool.join()
            except Exception:
                errtype, errvalue = sys.exc_info()[:2]
                tmpLog.error(f"failed in {self.__class__.__name__}.start() with {errtype.__name__} {errvalue}")
                tmpLog.error(f"Traceback: {traceback.format_exc()}")
            # sleep if needed
            loopCycle = jedi_config.taskrefine.loopCycle
            timeDelta = datetime.datetime.utcnow() - startTime
            sleepPeriod = loopCycle - timeDelta.seconds
            if sleepPeriod > 0:
                time.sleep(sleepPeriod)
            # randomize cycle
            self.randomSleep(max_val=loopCycle)

# thread for real worker
class TaskRefinerThread(WorkerThread):
    # constructor
    def __init__(self, taskList, threadPool, taskbufferIF, ddmIF, implFactory, workQueueMapper):
        # initialize woker with no semaphore
        WorkerThread.__init__(self, None, threadPool, logger)
        # attributres
        self.taskList = taskList
        self.taskBufferIF = taskbufferIF
        self.ddmIF = ddmIF
        self.implFactory = implFactory
        self.workQueueMapper = workQueueMapper
        self.msgType = "taskrefiner"

    # main
    def runImpl(self):
        while True:
            try:
                # get a part of list
                nTasks = 10
                taskList = self.taskList.get(nTasks)
                # no more datasets
                if len(taskList) == 0:
                    self.logger.info(f"{self.__class__.__name__} terminating since no more items")
                    return
                # loop over all tasks
                for jediTaskID, splitRule, taskStatus, parent_tid in taskList:
                    # make logger
                    tmpLog = MsgWrapper(self.logger, f"< jediTaskID={jediTaskID} >")
                    tmpLog.debug("start")
                    tmpStat = Interaction.SC_SUCCEEDED
                    errStr = ""
                    # read task parameters
                    try:
                        taskParam = None
                        taskParam = self.taskBufferIF.getTaskParamsWithID_JEDI(jediTaskID)
                        taskParamMap = RefinerUtils.decodeJSON(taskParam)
                    except Exception:
                        errtype, errvalue = sys.exc_info()[:2]
                        errStr = f"conversion to map from json failed with {errtype.__name__}:{errvalue}"
                        tmpLog.debug(taskParam)
                        tmpLog.error(errStr)
                        continue
                        tmpStat = Interaction.SC_FAILED
                    # get impl
                    if tmpStat == Interaction.SC_SUCCEEDED:
                        tmpLog.info("getting Impl")
                        try:
                            # get VO and sourceLabel
                            vo = taskParamMap["vo"]
                            prodSourceLabel = taskParamMap["prodSourceLabel"]
                            taskType = taskParamMap["taskType"]
                            tmpLog.info(f"vo={vo} sourceLabel={prodSourceLabel} taskType={taskType}")
                            # get impl
                            impl = self.implFactory.instantiateImpl(vo, prodSourceLabel, taskType, self.taskBufferIF, self.ddmIF)
                            if impl is None:
                                # task refiner is undefined
                                errStr = f"task refiner is undefined for vo={vo} sourceLabel={prodSourceLabel}"
                                tmpLog.error(errStr)
                                tmpStat = Interaction.SC_FAILED
                        except Exception:
                            errtype, errvalue = sys.exc_info()[:2]
                            errStr = f"failed to get task refiner with {errtype.__name__}:{errvalue}"
                            tmpLog.error(errStr)
                            tmpStat = Interaction.SC_FAILED
                    # extract common parameters
                    if tmpStat == Interaction.SC_SUCCEEDED:
                        tmpLog.info("extracting common")
                        try:
                            # initalize impl
                            impl.initializeRefiner(tmpLog)
                            impl.oldTaskStatus = taskStatus
                            # extract common parameters
                            impl.extractCommon(jediTaskID, taskParamMap, self.workQueueMapper, splitRule)
                            # set parent tid
                            if parent_tid not in [None, jediTaskID]:
                                impl.taskSpec.parent_tid = parent_tid
                        except Exception:
                            errtype, errvalue = sys.exc_info()[:2]
                            # on hold in case of external error
                            if errtype == JediException.ExternalTempError:
                                tmpErrStr = f"pending due to external problem. {errvalue}"
                                setFrozenTime = True
                                impl.taskSpec.status = taskStatus
                                impl.taskSpec.setOnHold()
                                impl.taskSpec.setErrDiag(tmpErrStr)
                                # not to update some task attributes
                                impl.taskSpec.resetRefinedAttrs()
                                tmpLog.info(tmpErrStr)
                                self.taskBufferIF.updateTask_JEDI(
                                    impl.taskSpec,
                                    {"jediTaskID": impl.taskSpec.jediTaskID},
                                    oldStatus=[taskStatus],
                                    insertUnknown=impl.unknownDatasetList,
                                    setFrozenTime=setFrozenTime,
                                )
                                continue
                            errStr = f"failed to extract common parameters with {errtype.__name__}:{errvalue} {traceback.format_exc()}"
                            tmpLog.error(errStr)
                            tmpStat = Interaction.SC_FAILED
                    # check attribute length
                    if tmpStat == Interaction.SC_SUCCEEDED:
                        tmpLog.info("checking attribute length")
                        if not impl.taskSpec.checkAttrLength():
                            tmpLog.error(impl.taskSpec.errorDialog)
                            tmpStat = Interaction.SC_FAILED
                    # staging
                    if tmpStat == Interaction.SC_SUCCEEDED:
                        if "toStaging" in taskParamMap and taskStatus not in ["staged", "rerefine"]:
                            errStr = "wait until staging is done"
                            impl.taskSpec.status = "staging"
                            impl.taskSpec.oldStatus = taskStatus
                            impl.taskSpec.setErrDiag(errStr)
                            # not to update some task attributes
                            impl.taskSpec.resetRefinedAttrs()
                            tmpLog.info(errStr)
                            self.taskBufferIF.updateTask_JEDI(
                                impl.taskSpec, {"jediTaskID": impl.taskSpec.jediTaskID}, oldStatus=[taskStatus], updateDEFT=False, setFrozenTime=False
                            )
                            continue
                    # check parent
                    noWaitParent = False
                    parentState = None
                    if tmpStat == Interaction.SC_SUCCEEDED:
                        if parent_tid not in [None, jediTaskID]:
                            tmpLog.info("check parent task")
                            try:
                                tmpStat = self.taskBufferIF.checkParentTask_JEDI(parent_tid)
                                parentState = tmpStat
                                if tmpStat == "completed":
                                    # parent is done
                                    tmpStat = Interaction.SC_SUCCEEDED
                                elif tmpStat is None or tmpStat == "running":
                                    if not impl.taskSpec.noWaitParent():
                                        # parent is running
                                        errStr = f"pending until parent task {parent_tid} is done"
                                        impl.taskSpec.status = taskStatus
                                        impl.taskSpec.setOnHold()
                                        impl.taskSpec.setErrDiag(errStr)
                                        # not to update some task attributes
                                        impl.taskSpec.resetRefinedAttrs()
                                        tmpLog.info(errStr)
                                        self.taskBufferIF.updateTask_JEDI(
                                            impl.taskSpec, {"jediTaskID": impl.taskSpec.jediTaskID}, oldStatus=[taskStatus], setFrozenTime=False
                                        )
                                        continue
                                    else:
                                        # not wait for parent
                                        tmpStat = Interaction.SC_SUCCEEDED
                                        noWaitParent = True
                                else:
                                    # parent is corrupted
                                    tmpStat = Interaction.SC_FAILED
                                    tmpErrStr = f"parent task {parent_tid} failed to complete"
                                    impl.taskSpec.setErrDiag(tmpErrStr)
                            except Exception:
                                errtype, errvalue = sys.exc_info()[:2]
                                errStr = f"failed to check parent task with {errtype.__name__}:{errvalue}"
                                tmpLog.error(errStr)
                                tmpStat = Interaction.SC_FAILED
                    # refine
                    if tmpStat == Interaction.SC_SUCCEEDED:
                        tmpLog.info(f"refining with {impl.__class__.__name__}")
                        try:
                            tmpStat = impl.doRefine(jediTaskID, taskParamMap)
                        except Exception:
                            errtype, errvalue = sys.exc_info()[:2]
                            # wait unknown input if noWaitParent or waitInput
                            toFinish = False
                            if (
                                ((impl.taskSpec.noWaitParent() or impl.taskSpec.waitInput()) and errtype == JediException.UnknownDatasetError)
                                or parentState == "running"
                                or errtype == Interaction.JEDITemporaryError
                            ):
                                if impl.taskSpec.noWaitParent() and errtype == JediException.UnknownDatasetError and parentState != "running":
                                    if impl.taskSpec.allowEmptyInput():
                                        tmpErrStr = f"finishing due to missing input while parent is {parentState}"
                                        toFinish = True
                                        setFrozenTime = False
                                    else:
                                        tmpErrStr = f"pending due to missing input while parent is {parentState}"
                                        setFrozenTime = True
                                elif impl.taskSpec.noWaitParent() or parentState == "running":
                                    tmpErrStr = f"pending until parent produces input. parent is {parentState}"
                                    setFrozenTime = False
                                elif errtype == Interaction.JEDITemporaryError:
                                    tmpErrStr = f"pending due to DDM problem. {errvalue}"
                                    setFrozenTime = True
                                else:
                                    tmpErrStr = "pending until input is staged"
                                    setFrozenTime = True
                                if toFinish:
                                    impl.taskSpec.status = "finishing"
                                else:
                                    impl.taskSpec.status = taskStatus
                                    impl.taskSpec.setOnHold()
                                impl.taskSpec.setErrDiag(tmpErrStr)
                                # not to update some task attributes
                                impl.taskSpec.resetRefinedAttrs()
                                tmpLog.info(tmpErrStr)
                                self.taskBufferIF.updateTask_JEDI(
                                    impl.taskSpec,
                                    {"jediTaskID": impl.taskSpec.jediTaskID},
                                    oldStatus=[taskStatus],
                                    insertUnknown=impl.unknownDatasetList,
                                    setFrozenTime=setFrozenTime,
                                )
                                continue
                            elif (
                                not (impl.taskSpec.noWaitParent() or impl.taskSpec.waitInput())
                                and errtype == JediException.UnknownDatasetError
                                and impl.taskSpec.allowEmptyInput()
                            ):
                                impl.taskSpec.status = "finishing"
                                tmpErrStr = f"finishing due to missing input after parent is {parentState}"
                                impl.taskSpec.setErrDiag(tmpErrStr)
                                # not to update some task attributes
                                impl.taskSpec.resetRefinedAttrs()
                                tmpLog.info(tmpErrStr)
                                self.taskBufferIF.updateTask_JEDI(
                                    impl.taskSpec, {"jediTaskID": impl.taskSpec.jediTaskID}, oldStatus=[taskStatus], insertUnknown=impl.unknownDatasetList
                                )
                                continue
                            else:
                                errStr = f"failed to refine task with {errtype.__name__}:{errvalue}"
                                tmpLog.error(errStr)
                                tmpStat = Interaction.SC_FAILED
                    # register
                    if tmpStat != Interaction.SC_SUCCEEDED:
                        tmpLog.error("failed to refine the task")
                        if impl is None or impl.taskSpec is None:
                            tmpTaskSpec = JediTaskSpec()
                            tmpTaskSpec.jediTaskID = jediTaskID
                        else:
                            tmpTaskSpec = impl.taskSpec
                        tmpTaskSpec.status = "tobroken"
                        if errStr != "":
                            tmpTaskSpec.setErrDiag(errStr, True)
                        self.taskBufferIF.updateTask_JEDI(tmpTaskSpec, {"jediTaskID": tmpTaskSpec.jediTaskID}, oldStatus=[taskStatus])
                    else:
                        tmpLog.info("registering")
                        # fill JEDI tables
                        try:
                            # enable protection against task duplication
                            if "uniqueTaskName" in taskParamMap and taskParamMap["uniqueTaskName"] and not impl.taskSpec.checkPreProcessed():
                                uniqueTaskName = True
                            else:
                                uniqueTaskName = False
                            strTaskParams = None
                            if impl.updatedTaskParams is not None:
                                strTaskParams = RefinerUtils.encodeJSON(impl.updatedTaskParams)
                            if taskStatus in ["registered", "staged"]:
                                # unset pre-process flag
                                if impl.taskSpec.checkPreProcessed():
                                    impl.taskSpec.setPostPreProcess()
                                # full registration
                                tmpStat, newTaskStatus = self.taskBufferIF.registerTaskInOneShot_JEDI(
                                    jediTaskID,
                                    impl.taskSpec,
                                    impl.inMasterDatasetSpec,
                                    impl.inSecDatasetSpecList,
                                    impl.outDatasetSpecList,
                                    impl.outputTemplateMap,
                                    impl.jobParamsTemplate,
                                    strTaskParams,
                                    impl.unmergeMasterDatasetSpec,
                                    impl.unmergeDatasetSpecMap,
                                    uniqueTaskName,
                                    taskStatus,
                                )
                                if not tmpStat:
                                    tmpErrStr = "failed to register the task to JEDI in a single shot"
                                    tmpLog.error(tmpErrStr)
                                    tmpTaskSpec = JediTaskSpec()
                                    tmpTaskSpec.status = newTaskStatus
                                    tmpTaskSpec.errorDialog = impl.taskSpec.errorDialog
                                    tmpTaskSpec.setErrDiag(tmpErrStr, True)
                                    self.taskBufferIF.updateTask_JEDI(tmpTaskSpec, {"jediTaskID": impl.taskSpec.jediTaskID}, oldStatus=[taskStatus])
                                tmpMsg = f"set task_status={newTaskStatus}"
                                tmpLog.info(tmpMsg)
                                tmpLog.sendMsg(tmpMsg, self.msgType)
                                # send message to contents feeder if the task is registered
                                if tmpStat and impl.taskSpec.is_msg_driven():
                                    push_ret = self.taskBufferIF.push_task_trigger_message("jedi_contents_feeder", jediTaskID)
                                    if push_ret:
                                        tmpLog.debug("pushed trigger message to jedi_contents_feeder")
                                    else:
                                        tmpLog.warning("failed to push trigger message to jedi_contents_feeder")
                            else:
                                # disable scouts if previous attempt didn't use it
                                if not impl.taskSpec.useScout(splitRule):
                                    impl.taskSpec.setUseScout(False)
                                # disallow to reset some attributes
                                impl.taskSpec.reserve_old_attributes()
                                # update task with new params
                                self.taskBufferIF.updateTask_JEDI(impl.taskSpec, {"jediTaskID": impl.taskSpec.jediTaskID}, oldStatus=[taskStatus])
                                # appending for incremental execution
                                tmpStat = self.taskBufferIF.appendDatasets_JEDI(jediTaskID, impl.inMasterDatasetSpec, impl.inSecDatasetSpecList)
                                if not tmpStat:
                                    tmpLog.error("failed to append datasets for incexec")
                        except Exception:
                            errtype, errvalue = sys.exc_info()[:2]
                            tmpErrStr = f"failed to register the task to JEDI with {errtype.__name__}:{errvalue}"
                            tmpLog.error(tmpErrStr)
                        else:
                            tmpLog.info("done")
            except Exception:
                errtype, errvalue = sys.exc_info()[:2]
                logger.error(f"{self.__class__.__name__} failed in runImpl() with {errtype.__name__}:{errvalue}")

# lauch

def launcher(commuChannel, taskBufferIF, ddmIF, vos=None, prodSourceLabels=None):
    p = TaskRefiner(commuChannel, taskBufferIF, ddmIF, vos, prodSourceLabels)
    p.start()


#!/usr/bin/python
# -*- coding: utf-8 -*-
# author: beimingmaster@gmail.com

import os
import time
import json
import requests
from optparse import OptionParser

from common import config

def do_download(task_id):
    print('do download work for task id: %s ...' % task_id)
    file_url = None
    url_json_file = '%s/%s/url.json' % (config.data_path, task_id)
    data_json_file = '%s/%s/data.json' % (config.data_path, task_id)
    print('url json file: ', url_json_file)
    print('data json file: ', data_json_file)
    if not os.path.exists(url_json_file):
        print('url json file does not exist: ', url_json_file)
        return False
    elif os.path.exists(data_json_file):
        print('data json file does exist: ', data_json_file)
        return False
    with open(url_json_file, 'r', encoding='utf8') as f:
        url_json_data = json.load(f)
        if 'file_url' in url_json_data:
            file_url = url_json_data['file_url']
    print('file url: ', file_url)
    if file_url:
        try:
            r = requests.get(file_url, timeout=10, stream=True, verify=None)
            if r.status_code == 200:
                with open(data_json_file, 'wb') as f:
                    #f.write(r.text)
                    for chunk in r.iter_content(chunk_size=1024*1024):
                        if chunk:
                            f.write(chunk)
            else:
                print('downloading task data has error: %s' % file_url)
        except Exception as ex:
            print('requests has error', ex)
    else:
        print('file url is no exists for task_id : %s' % task_id)
        return False

if __name__ == '__main__':
    parser = OptionParser()
    parser.add_option('-t', '--task', type='string', dest='task_id', help='task id to analysis')
    (options, args) = parser.parse_args()
    print('options: ', options)
    print('args: ', args)
    task_id = options.task_id
    if task_id:
        start = time.time()
        do_download(task_id)
        end = time.time()
        print('doing download work takes %.2fs!' % (end-start))
    else:
        print('task id not provided')


#signature verification

import gpg
import time

asc_file = "ex13out.txt.asc"

c = gpg.Context()

try:
    data, result = c.verify(open(asc_file))
    verified = True
except gpg.errors.BadSignatures as e:
    verified = False
    print(e)

if verified is True:
    for i in range(len(result.signatures)):
        sign = result.signatures[i]
        print("""Good signatrue from:
{0}
with key {1}
made at {2}""".format(c.get_key(sign.fpr).uids[0].uid,
        sign.fpr, time.ctime(sign.timestamp)))
else:
    pass


"""Adapted from:
https://github.com/hugovk/random-street-view/blob/main/random_street_view.py
"""

import argparse
import json
import os
import random
import sys
import requests
from urllib.request import urlopen, urlretrieve
from datetime import datetime
import contextlib
import yaml
import traceback

import shapefile  # pip install pyshp
import mapillary.interface as mly  # pip install mapillary

# Optional, http://stackoverflow.com/a/1557906/724176
try:
    import timing
    assert timing  # avoid flake8 warning
except ImportError:
    pass

IMG_SUFFIX = "jpg"
MAX_TRIES = 10  # Used to set number of maximum attempts at finding a non-filtered image

with open("api_key.yaml", "r") as ymlfile:
    key = yaml.load(ymlfile, Loader=yaml.FullLoader)
token = key['mly_token']
mly.set_access_token(token)

parser = argparse.ArgumentParser(
    description="Get random Street View images from within the borders of a given country. http://bboxfinder.com may "
    "be helpful for creating box coordinates and https://www.mapillary.com/app may be helpful for checking those boxes "
    "contain any images. By default, images are filtered out if they have any traffic signs as detected by Mapillary's "
    "systems; this should not be trusted absolutely, images should be manually checked for signs.",
    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("country", help="ISO 3166-1 Alpha-3 Country Code, 'none', or 'near_global'")
help_str = "For default from country borders, enter 0. Ignored if 'near_global'."
parser.add_argument("min_lon", type=float, help=help_str)
parser.add_argument("min_lat", type=float, help=help_str)
parser.add_argument("max_lon", type=float, help=help_str)
parser.add_argument("max_lat", type=float, help=help_str)
parser.add_argument("-n", "--images-wanted", type=int, default=100, help="Total number of images wanted.")
parser.add_argument("-b", "--burst", type=int, default=10, help="The maximum number of nearby images downloaded from "
                    "any random geographical point that hits. >1 (e.g. 10) is recommended if using 'near_global'. Note "
                    "that nearby images may be captured by the same camera on the same day, so there is a trade-off "
                    "between speed of image retrieval and diversity of images to chosen here. I have so far been "
                    "unable to determine the definition of 'nearby' from the Mapillary SDK documentation, but each "
                    "point hit will often return 100s-1000s of nearby images as shown by '--save-to-json'.")
parser.add_argument("-j", "--save-to-json", action="store_true", help="Save to a JSON file metadata of images found "
                    "near a point.")
parser.add_argument("-N", "--no-filter", action="store_true", help="Turn off filtering of images with traffic signs.")
args = parser.parse_args()

# TODO: "--all-in-box", '-A' mode where every single image in the box is downloaded, not just random ones; ADD A WARNING

if args.images_wanted < 1:
    raise ValueError("Number of images wanted must be at least 1.")
# TODO: Allow for -n 0 for only downloading metadata for images near point (require that -j must also be specified)

# Determine if a point is inside a given polygon or not
# Polygon is a list of (x,y) pairs.
# http://www.ariel.com.au/a/python-point-int-poly.html
def point_inside_polygon(x, y, poly):
    n = len(poly)
    inside = False
    p1x, p1y = poly[0]
    for i in range(n + 1):
        p2x, p2y = poly[i % n]
        if y > min(p1y, p2y):
            if y <= max(p1y, p2y):
                if x <= max(p1x, p2x):
                    if p1y != p2y:
                        xinters = (y - p1y) * (p2x - p1x) / (p2y - p1y) + p1x
                    if p1x == p2x or x <= xinters:
                        inside = not inside
        p1x, p1y = p2x, p2y
    return inside

print("Loading borders...")
shape_file = "TM_WORLD_BORDERS-0.3.shp"
if not os.path.exists(shape_file):
    sys.exit(
        f"Cannot find {shape_file}. Please download it from "
        "http://thematicmapping.org/downloads/world_borders.php and try again."
    )

sf = shapefile.Reader(shape_file, encoding="latin1")
shapes = sf.shapes()

if args.country.lower() == "none":
    if args.min_lon == 0 or args.min_lat == 0 or args.max_lon == 0 or args.max_lat == 0:
        sys.exit("A valid bounding box must be entered if no country is specified.")
    min_lon = args.min_lon
    min_lat = args.min_lat
    max_lon = args.max_lon
    max_lat = args.max_lat
    borders = []
elif args.country.lower() == "near_global":
    min_lon = -160
    min_lat = -56
    max_lon = 180
    max_lat = 71
    borders = []
else:
    print("Finding country...")
    for i, record in enumerate(sf.records()):
        if record[2] == args.country.upper():
            print(record[2], record[4])
            print(shapes[i].bbox)
            min_lon = shapes[i].bbox[0] if args.min_lon == 0 else args.min_lon
            min_lat = shapes[i].bbox[1] if args.min_lat == 0 else args.min_lat
            max_lon = shapes[i].bbox[2] if args.max_lon == 0 else args.max_lon
            max_lat = shapes[i].bbox[3] if args.max_lat == 0 else args.max_lat
            borders = shapes[i].points
            break

print("Getting images...")
attempts, country_hits, point_hits, point_misses, imagery_hits, imagery_misses, imagery_filtered = 0, 0, 0, 0, 0, 0, 0

dtime = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
outdir = f"mly_random_{args.country.upper()}_{dtime}"

img_ids = []
try:
    while True:
        attempts += 1
        rand_lat = random.uniform(min_lat, max_lat)
        rand_lon = random.uniform(min_lon, max_lon)

        # Check if (lon, lat) is inside country borders
        point_inside = True
        if borders != []:
            point_inside = point_inside_polygon(rand_lon, rand_lat, borders)

        if point_inside:
            # print("In country")
            country_hits += 1

            try:
                # We will only retrieve flat images, although the MTSD contains 1138 flattened 360° panorama images
                # NOTE: I couldn't figure out how to suppress the GET prints from the Mapillary SDK
                print()
                images = mly.get_image_close_to(longitude=rand_lon, latitude=rand_lat, image_type='flat',
                                                fields=['thumb_original_url']).to_dict()
                point_hits += 1
            except IndexError:
                point_misses += 1
                print(f"  No images found close to point")
                continue

            hit_dir = f"hits_{args.country.upper()}_{dtime}"
            if args.save_to_json:
                os.makedirs(hit_dir, exist_ok=True)
                with open(os.path.join(hit_dir, f"hit_#{point_hits}_with_{len(images['features'])}_imgs_"
                                       f"{rand_lon}_{rand_lat}.json"), mode="w") as f:
                    json.dump(images, f, indent=4)

            print('\n\n  lat,lon: ' + str(rand_lat) + ',' + str(rand_lon))
            ii = 0
            found = 0
            while found < args.burst and ii < (found + 1) * MAX_TRIES:
                if ii >= len(images['features']):
                    break
                img_id = images['features'][ii]['properties']['id']
                if img_id in img_ids:
                    imagery_misses += 1
                    ii += 1
                    continue
                url_request = f"https://graph.mapillary.com/{img_id}?access_token={token}&fields=thumb_original_url"
                response = requests.get(url_request).json()  # Query the API for the original image URL
                try:
                    url = response['thumb_original_url']
                except KeyError:
                    print(f"  Error retrieving image URL for {img_id}")
                    imagery_misses += 1
                    ii += 1
                    continue

                # Filter out images with traffic signs detected by Mapillary
                passed_filter = True
                if not args.no_filter:
                    # NOTE: I couldn't figure out how to suppress the GET prints from the Mapillary SDK
                    detections = mly.get_detections_with_image_id(img_id).to_dict()
                    def is_sign(detection):
                        v = detection['properties']['value']
                        return (
                            "complementary" in v or
                            "information" in v or
                            "regulatory" in v or
                            "warning" in v
                        )
                    signs = [d['properties']['value'] for d in detections['features'] if not is_sign(d)]
                    if signs != []:
                        imagery_filtered += 1
                        passed_filter = False
                        print("  ----- Skipped image with traffic sign detections -----")

                if passed_filter:
                    os.makedirs(outdir, exist_ok=True)
                    outfile = os.path.join(outdir, f"{img_id}.{IMG_SUFFIX}")
                    try:
                        # Download the image
                        data = requests.get(url)
                        with open(outfile, "wb") as f:
                            f.write(data.content)
                    except KeyboardInterrupt:
                        sys.exit("exit")

                    if os.path.isfile(outfile):
                        print(f"  ========== Got one! Taken from this point: {found + 1} "
                              f"(from {ii + 1} attempts) ==========")
                        img_ids.append(img_id)
                        imagery_hits += 1
                        found += 1
                        if imagery_hits >= args.images_wanted:
                            break
                    else:
                        imagery_misses += 1
                ii += 1
            if imagery_hits >= args.images_wanted:
                break
        else:
            # print("  Point outside country")
            pass
except KeyboardInterrupt:
    print("Keyboard interrupt")
except Exception:
    # Make sure that stats are still printed and saved
    traceback.print_exc()

stats_str = f"Attempts:\t{attempts}\n"
if borders != []:
    stats_str += f"Country hits:\t{country_hits}\n"
stats_str += f"Point misses:\t{point_misses}\n"
stats_str += f"Point hits:\t{point_hits}\n"
stats_str += f"Imagery misses:\t{imagery_misses}\n"
if not args.no_filter:
    stats_str += f"Filtered out:\t{imagery_filtered}\n"
stats_str += f"Imagery hits:\t{imagery_hits}"
print(f"\n{stats_str}")
with open(os.path.join(outdir, "stats.txt"), mode="w") as f:
    f.write(stats_str)


import os
from pkgutil import extend_path
import numpy as np
import PIL.Image as pil
import torch
import json

from .mono_dataset import MonoDataset

class Virtual_Kitti(MonoDataset):
    def __init__(self, *args, **kwargs):
        super(Virtual_Kitti, self).__init__(*args, **kwargs)
        self.K = np.array([[0.58, 0, 0.5, 0],
                           [0, 1.92, 0.5, 0],
                           [0, 0, 1, 0],
                           [0, 0, 0, 1]], dtype=np.float32)

        self.full_res_shape = (1242, 375)

        self.rgb_head = 'rgb_'
        self.depth_head = 'depth_'
        self.depth_img_ext ='.png'

    def check_depth(self):
        return True

    def get_color(self, folder, frame_index, side, do_flip=False):
        color = self.loader(os.path.join(self.data_path, 'rgb', folder, 'frames/rgb/Camera_0',
                                 (self.rgb_head + str(frame_index).zfill(5) + self.img_ext)))

        if do_flip:
            color = color.transpose(pil.FLIP_LEFT_RIGHT)

        return color

    def get_depth(self, folder, frame_index, side, do_flip=False):
        depth_path = os.path.join(self.data_path, 'depth', folder, 'frames/depth/Camera_0',
                                 (self.depth_head + str(frame_index).zfill(5) + self.depth_img_ext))
        depth_gt = pil.open(depth_path)
        depth_gt = np.array(depth_gt).astype(np.float32) * 80.0 / 65535.0

        if do_flip:
            depth_gt = np.fliplr(depth_gt)

        return depth_gt