import h5py
import numpy as np
import os
import sys
import io

from . import filewriter
# from .Types import nptype

try:
    sver = h5py.__version__.split(".", 2)
    h5maj = int(sver[0])
    h5min = int(sver[1])
    h5ver = h5maj * 1000 + h5min
except Exception:
    h5ver = 1000

if sys.version_info > (3,):
    unicode = str
else:
    bytes = str

def _slice2selection(t, shape):
    """ converts slice(s) to selection

    :param t: slice tuple
    :type t: :obj:`tuple`
    :return shape: field shape
    :type shape: :obj:`list` < :obj:`int` >
    :returns: hyperslab selection
    :rtype: :class:`h5cpp.dataspace.Hyperslab`
    """
    if isinstance(t, filewriter.FTHyperslab):
        offset = list(t.offset or [])
        block = list(t.block or [])
        count = list(t.count or [])
        stride = list(t.stride or [])
        slices = []
        for dm, sz in enumerate(shape):
            if len(offset) > dm:
                if offset[dm] is None:
                    offset[dm] = 0
            else:
                offset.append(0)
            if len(block) > dm:
                if block[dm] is None:
                    block[dm] = 1
            else:
                block.append(1)
            if len(count) > dm:
                if count[dm] is None:
                    count[dm] = sz
            else:
                count.append(sz)
            if len(stride) > dm:
                if stride[dm] is None:
                    stride[dm] = 1
            else:
                stride.append(1)
            slices.append(
                h5py.MultiBlockSlice(
                    start=offset[dm], count=count[dm],
                    stride=stride[dm], block=block[dm]))
        return tuple(slices)
    return t

def unlimited_selection(sel, shape):
    """ checks if hyperslab is unlimited

    :param sel: hyperslab selection
    :type sel: :class:`filewriter.FTHyperslab`
    :param shape: give shape
    :type shape: :obj:`list`
    :returns: if hyperslab is unlimited list
    :rtype: :obj:`list` <:obj:`bool`>
    """
    res = None
    if isinstance(sel, tuple):
        res = []
        for sl in sel:
            if hasattr(sl, "stop"):
                res.append(
                    True if sl.stop in [unlimited()]
                    else False)
            elif hasattr(sl, "count"):
                res.append(
                    True if sl.count in [unlimited()]
                    else False)
            else:
                res.append(
                    True if sl in [unlimited()]
                    else False)

    elif hasattr(sel, "count"):
        res = []
        for ct in sel.count():
            res.append(
                True if ct in [unlimited()]
                else False)
    elif isinstance(sel, slice):
        res = [True if sel.stop in [unlimited()]
               else False]

    elif sel in [unlimited()]:
        res = [True]
    lsh = len(shape)
    lct = len(res)
    ln = max(lsh, lct)
    if res and any(t is True for t in res):
        slices = []
        for ii in range(ln):
            offset = 0
            # block = 1
            # stride = 1
            if ii < lsh:
                count = shape[ii]
            else:
                count = 1
            if ii < lct and res[ii]:
                count = unlimited()
            # print("Hyperslab %s %s %s %s" % (offset, block, count, stride))
            slices.append(
                slice(offset, count, None)
                # h5py.MultiBlockSlice(
                #     start=offset, count=count,
                #     stride=stride, block=block)
            )
        return tuple(slices)
    else:
        return None

def _selection2slice(t, shape):
    """ converts selection to slice(s)

    :param t: slice tuple
    :type t: :obj:`tuple`
    :return shape: field shape
    :type shape: :obj:`list` < :obj:`int` >
    :returns: tuple of slices
    :rtype: :obj:`tuple`<>
    """
    if isinstance(t, filewriter.FTHyperslab):
        offset = list(t.offset or [])
        block = list(t.block or [])
        count = list(t.count or [])
        stride = list(t.stride or [])
        slices = []
        for dm, sz in enumerate(shape):
            if len(offset) > dm:
                if offset[dm] is None:
                    offset[dm] = 0
            else:
                offset.append(0)
            if len(block) > dm:
                if block[dm] is None:
                    block[dm] = 1
            else:
                block.append(1)
            if len(count) > dm:
                if count[dm] is None:
                    count[dm] = sz
            else:
                count.append(sz)
            if len(stride) > dm:
                if stride[dm] is None:
                    stride[dm] = 1
            else:
                block.append(1)
            if len(stride) > dm:
                if stride[dm] is None:
                    stride[dm] = 1
            else:
                stride.append(1)
            if block[dm] == 1 and count[dm] == 1 and stride[dm] == 1:
                slices.append(offset[dm])
            elif stride[dm] == 1:
                slices.append(slice(
                    offset[dm], offset[dm] + block[dm] * count[dm], None))
            elif stride[dm] != 1 and block[dm] == 1:
                slices.append(slice(offset[dm],
                                    offset[dm] + count[dm] * stride[dm],
                                    stride[dm]))
            elif stride[dm] != 1 and count[dm] == 1:
                slices.append(slice(offset[dm],
                                    offset[dm] + block[dm] * stride[dm],
                                    stride[dm]))
            else:
                slices.append(Ellipsis)
        return tuple(slices)
    return t

def nptype(dtype):
    """ converts to numpy types

    :param dtype: h5 writer type type
    :type dtype: :obj:`str`
    :returns: nupy type
    :rtype: :obj:`str`
    """
    if str(dtype) in ['string', b'string']:
        return 'str'
    return dtype

def is_image_file_supported():
    """ provides if loading of image files are supported

    :retruns: if loading of image files are supported
    :rtype: :obj:`bool`
    """
    return h5ver >= 2009

def is_vds_supported():
    """ provides if VDS are supported

    :retruns: if VDS are supported
    :rtype: :obj:`bool`
    """
    return h5ver >= 2009

def is_mbs_supported():
    """ provides if MultiBlockSlice are supported

    :retruns: if MultiBlockSlice are supported
    :rtype: :obj:`bool`
    """
    return h5ver >= 3000

def is_unlimited_vds_supported():
    """ provides if unlimited vds are supported

    :retruns: if unlimited vds are supported
    :rtype: :obj:`bool`
    """
    return h5ver >= 3000

def is_strings_as_bytes():
    """ provides if string read to bytes

    :retruns: if string read to bytes
    :rtype: :obj:`bool`
    """
    return h5ver >= 3000

def unlimited(parent=None):
    """ return dataspace UNLIMITED variable for the current writer module

    :param parent: parent object
    :type parent: :class:`FTObject`
    :returns:  dataspace UNLIMITED variable
    :rtype: :class:`h5py.h5s.UNLIMITED`
    """
    try:
        return h5py.h5s.UNLIMITED
    except Exception:
        return h5py.UNLIMITED

def load_file(membuffer, filename=None, readonly=False, **pars):
    """ load a file from memory byte buffer

    :param membuffer: memory buffer
    :type membuffer: :obj:`bytes` or :obj:`io.BytesIO`
    :param filename: file name
    :type filename: :obj:`str`
    :param readonly: readonly flag
    :type readonly: :obj:`bool`
    :param pars: parameters
    :type pars: :obj:`dict` < :obj:`str`, :obj:`str`>
    :returns: file object
    :rtype: :class:`H5PYFile`
    """
    if not is_image_file_supported():
        raise Exception("Loading a file from a memory buffer not supported")
    if not hasattr(membuffer, 'read') or not hasattr(membuffer, 'seek'):
        if hasattr(membuffer, "tobytes"):
            membuffer = membuffer.tobytes()
        membuffer = io.BytesIO(membuffer)
    if readonly:
        fobj = h5py.File(membuffer, "r", **pars)
    else:
        fobj = h5py.File(membuffer, "r+", **pars)
    return H5PYFile(fobj, filename)

def open_file(filename, readonly=False, **pars):
    """ open the new file

    :param filename: file name
    :type filename: :obj:`str`
    :param readonly: readonly flag
    :type readonly: :obj:`bool`
    :param pars: parameters
    :type pars: :obj:`dict` < :obj:`str`, :obj:`str`>
    :returns: file object
    :rtype: :class:`H5PYFile`
    """
    if readonly:
        return H5PYFile(h5py.File(filename, "r", **pars), filename)
    else:
        return H5PYFile(h5py.File(filename, "r+", **pars), filename)

def create_file(filename, overwrite=False, **pars):
    """ create a new file

    :param filename: file name
    :type filename: :obj:`str`
    :param overwrite: overwrite flag
    :type overwrite: :obj:`bool`
    :param pars: parameters
    :type pars: :obj:`dict` < :obj:`str`, :obj:`str`>
    :returns: file object
    :rtype: :class:`H5PYFile`
    """
    fl = h5py.File(filename, "w" if overwrite else "w-", **pars)
    fl.attrs["file_time"] = unicode(H5PYFile.currenttime())
    fl.attrs["HDF5_Version"] = str(h5py.version.hdf5_version)
    fl.attrs["NX_class"] = u"NXroot"
    # fl.attrs["NeXus_version"] = u"4.3.0"
    fl.attrs["file_name"] = unicode(filename)
    fl.attrs["file_update_time"] = unicode(H5PYFile.currenttime())
    return H5PYFile(fl, filename)

def link(target, parent, name):
    """ create link

    :param target: file name
    :type target: :obj:`str`
    :param parent: parent object
    :type parent: :class:`FTObject`
    :param name: link name
    :type name: :obj:`str`
    :returns: link object
    :rtype: :class:`H5PYLink`
    """
    localfname = H5PYLink.getfilename(parent)
    if ":/" in target:
        filename, path = target.split(":/")

        if os.path.abspath(filename) != os.path.abspath(localfname):
            parent.h5object[name] = h5py.ExternalLink(filename, path)
        else:
            parent.h5object[name] = h5py.SoftLink(path)

    else:
        parent.h5object[name] = h5py.SoftLink(target)
    return H5PYLink(
        parent.h5object.get(name, getlink=True), parent).setname(name)

def get_links(parent):
    """ get links

    :param parent: parent object
    :type parent: :class:`FTObject`
    :returns: list of link objects
    :returns: link object
    :rtype: :obj: `list` <:class:`H5PYLink`>
    """

    return [H5PYLink(
        parent.h5object.get(name, getlink=True), parent).setname(name)
        for name in parent.names()]

def data_filter(filterid=None, name=None, options=None, availability=None,
                shuffle=None, rate=None):
    """ create data filter

    :param filterid: hdf5 filter id
    :type filterid: :obj:`int`
    :param name: filter name
    :type name: :obj:`str`
    :param options: filter cd values
    :type options: :obj:`tuple` <:obj:`int`>
    :param availability: filter availability i.e. 'optional' or 'mandatory'
    :type availability: :obj:`str`
    :param shuffle: filter shuffle
    :type shuffle: :obj:`bool`
    :param rate: filter shuffle
    :type rate: :obj:`bool`
    :returns: deflate filter object
    :rtype: :class:`H5PYDataFilter`
    """
    dtf = H5PYDataFilter()
    if filterid:
        dtf.filterid = filterid
    if name:
        dtf.name = name
    if shuffle:
        dtf.shuffle = shuffle
    if rate:
        dtf.rate = rate
    if options:
        dtf.options = options
    if availability:
        dtf.availability = availability
    return dtf

def deflate_filter(rate=None, shuffle=None, availability=None):
    """ create data filter

    :param rate: filter shuffle
    :type rate: :obj:`bool`
    :param shuffle: filter shuffle
    :type shuffle: :obj:`bool`
    :returns: deflate filter object
    :rtype: :class:`H5CppDataFilter`
    """
    dtf = H5PYDataFilter()
    dtf.filterid = 1
    dtf.name = "deflate"
    if shuffle:
        dtf.shuffle = shuffle
    dtf.rate = rate or 2
    if availability:
        dtf.availability = availability
    return dtf

def target_field_view(filename, fieldpath, shape,
                      dtype=None, maxshape=None):
    """ create target field view for VDS

    :param filename: file name
    :type filename: :obj:`str`
    :param fieldpath: nexus field path
    :type fieldpath: :obj:`str`
    :param shape: shape
    :type shape: :obj:`list` < :obj:`int` >
    :param dtype: attribute type
    :type dtype: :obj:`str`
    :param maxshape: shape
    :type maxshape: :obj:`list` < :obj:`int` >
    :returns: target field view object
    :rtype: :class:`FTTargetFieldView`
    """
    if not is_vds_supported():
        raise Exception("VDS not supported")
    if shape:
        maxshape = maxshape or [None for _ in shape]
    vs = h5py.VirtualSource(
        filename, fieldpath,
        tuple(shape or []), dtype, tuple(maxshape or []))
    return H5PYTargetFieldView(vs, tuple(shape or []))

def virtual_field_layout(shape, dtype, maxshape=None):
    """ creates a virtual field layout for a VDS file

    :param shape: shape
    :type shape: :obj:`list` < :obj:`int` >
    :param dtype: attribute type
    :type dtype: :obj:`str`
    :param maxshape: shape
    :type maxshape: :obj:`list` < :obj:`int` >
    :returns: virtual layout
    :rtype: :class:`FTVirtualFieldLayout`
    """
    if not is_vds_supported():
        raise Exception("VDS not supported")
    maxshape = maxshape or [None for _ in shape]
    return H5PYVirtualFieldLayout(
        h5py.VirtualLayout(tuple(shape), dtype, tuple(maxshape or [])),
        tuple(shape)
    )

class H5PYFile(filewriter.FTFile):

    """ file tree file
    """

    def __init__(self, h5object, filename):
        """ constructor

        :param h5object: h5 object
        :type h5object: :obj:`any`
        :param filename:  file name
        :type filename: :obj:`str`
        """
        filewriter.FTFile.__init__(self, h5object, filename)
        #: (:obj:`str`) object nexus path
        self.path = None

    def root(self):
        """ root object

        :returns: parent object
        :rtype: :class:`H5PYGroup`
        """
        g = H5PYGroup(self._h5object, self)
        g.name = u"/"
        g.path = u"/"
        return g

    def flush(self):
        """ flash the data
        """
        if self._h5object.mode in ["r+"]:
            self._h5object.attrs["file_update_time"] = \
                unicode(self.currenttime())
        return self._h5object.flush()

    def close(self):
        """ close file
        """
        filewriter.FTFile.close(self)
        if self._h5object.mode in ["r+"]:
            self._h5object.attrs["file_update_time"] = \
                unicode(self.currenttime())
        return self._h5object.close()

    @property
    def is_valid(self):
        """ check if group is valid

        :returns: valid flag
        :rtype: :obj:`bool`
        """
        return self._h5object.name is not None

    @property
    def readonly(self):
        """ check if file is readonly

        :returns: readonly flag
        :rtype: :obj:`bool`
        """
        isvalid = self.is_valid
        return self._h5object.mode in ["r"] if isvalid else None

    def reopen(self, readonly=False, swmr=False, libver=None):
        """ reopen file

        :param readonly: readonly flag
        :type readonly: :obj:`bool`
        :param swmr: swmr flag
        :type swmr: :obj:`bool`
        :param libver:  library version, default: 'latest'
        :type libver: :obj:`str`
        """
        libver = libver or 'latest'
        isvalid = self.is_valid
        lreadonly = self._h5object.mode in ["r"] if isvalid else None

        if (not isvalid or lreadonly != readonly or
           self._h5object.libver != libver):
            if isvalid:
                self.close()
            self._h5object = h5py.File(
                self.name, "r" if readonly else "r+", libver=libver)
            filewriter.FTFile.reopen(self)
        if hasattr(self._h5object, "swmr_mode") and swmr:
            self._h5object.swmr_mode = swmr
        elif swmr:
            raise Exception("SWMR not supported")

    def hasswmr(self):
        """ if has swmr_mode

        :returns: has swmr_mode
        :rtype: :obj:`bool`
        """
        return hasattr(self._h5object, "swmr_mode")

class H5PYGroup(filewriter.FTGroup):

    """ file tree group
    """

    def __init__(self, h5object, tparent=None):
        """ constructor

        :param h5object: h5 object
        :type h5object: :obj:`any`
        :param tparent: treee parent
        :type tparent: :obj:`FTObject`
        """
        filewriter.FTGroup.__init__(self, h5object, tparent)
        self.path = u""
        self.name = None
        if hasattr(h5object, "name"):
            name = h5object.name
            self.name = name.split("/")[-1]
            if tparent and tparent.path:
                if tparent.path == u"/":
                    self.path = u"/" + self.name
                else:
                    self.path = tparent.path + u"/" + self.name
            if ":" not in self.name:
                if u"NX_class" in h5object.attrs:
                    clss = filewriter.first(h5object.attrs["NX_class"])
                else:
                    clss = ""
                if clss:
                    self.path += u":" + str(clss)

    def open(self, name):
        """ open a file tree element

        :param name: element name
        :type name: :obj:`str`
        :returns: file tree object
        :rtype: :class:`FTObject`
        """
        if name not in self._h5object:
            at = self._h5object.attrs[name]
            if at is None:
                raise Exception("Empty attriibute")
            return H5PYAttribute((self._h5object.attrs, name), self)

        itm = self._h5object.get(name)
        if isinstance(itm, h5py._hl.dataset.Dataset):
            el = H5PYField(itm, self)
        elif isinstance(itm, h5py._hl.group.Group):
            el = H5PYGroup(itm, self)
        else:
            itm = self._h5object.get(name, getlink=True)
            el = H5PYLink(itm, self).setname(name)
        return el

    def open_link(self, name):
        """ open a file tree element as link

        :param name: element name
        :type name: :obj:`str`
        :returns: file tree object
        :rtype: :class:`FTObject`
        """
        itm = self._h5object.get(name, getlink=True)
        return H5PYLink(itm, self).setname(name)

    class H5PYGroupIter(object):

        def __init__(self, group):
            """ constructor

            :param group: group object
            :type manager: :obj:`H5PYGroup`
            """

            self.__group = group
            self.__names = sorted(self.__group._h5object.keys()) or []

        def __next__(self):
            """ the next attribute

            :returns: attribute object
            :rtype: :class:`FTAtribute`
            """
            if self.__names:
                return self.__group.open(self.__names.pop(0))
            else:
                raise StopIteration()

        next = __next__

        def __iter__(self):
            """ attribute iterator

            :returns: attribute iterator
            :rtype: :class:`H5PYAttrIter`
            """
            return self

    def __iter__(self):
        """ attribute iterator

        :returns: attribute iterator
        :rtype: :class:`H5PYAttrIter`
        """
        return self.H5PYGroupIter(self)

    def close(self):
        """ close group
        """
        filewriter.FTGroup.close(self)
        self._h5object = None

    def create_group(self, n, nxclass=""):
        """ open a file tree element

        :param n: group name
        :type n: :obj:`str`
        :param nxclass: group type
        :type nxclass: :obj:`str`
        :returns: file tree group
        :rtype: :class:`H5PYGroup`
        """
        grp = self._h5object.create_group(n)
        if nxclass:
            grp.attrs["NX_class"] = unicode(nxclass)
        return H5PYGroup(grp, self)

    def create_virtual_field(self, name, layout, fillvalue=None):
        """ creates a virtual filed tres element

        :param name: group name
        :type name: :obj:`str`
        :param layout: virual field layout
        :type layout: :class:`H5PYFieldLayout`
        :param fillvalue:  fill value
        :type fillvalue: :obj:`int`
        """
        if not is_vds_supported():
            raise Exception("VDS not supported")
        return H5PYField(
            self._h5object.create_virtual_dataset(
                name, layout._h5object, fillvalue),
            self)

    def create_field(self, name, type_code,
                     shape=None, chunk=None, dfilter=None):
        """ creates a field tree element

        :param name: group name
        :type name: :obj:`str`
        :param type_code: nexus field type
        :type type_code: :obj:`str`
        :param shape: shape
        :type shape: :obj:`list` < :obj:`int` >
        :param chunk: chunk
        :type chunk: :obj:`list` < :obj:`int` >
        :param dfilter: filter deflater
        :type dfilter: :class:`H5PYDataFilter`
        :returns: file tree field
        :rtype: :class:`H5PYField`
        """
        if type_code in ['string', b'string']:
            type_code = h5py.special_dtype(vlen=unicode)
            # type_code = h5py.special_dtype(vlen=bytes)
        if type_code == h5py.special_dtype(vlen=unicode) and \
           shape is None and chunk is None:
            return H5PYField(
                self._h5object.create_dataset(name,  (), type_code), self)

        shape = shape or [1]
        f = None
        mshape = [None for _ in shape] or (None,)
        if dfilter:
            if isinstance(dfilter, list):
                if len(dfilter) == 2 and dfilter[0] and \
                   (dfilter[0].shuffle or dfilter[0].name == "shuffle"
                        or dfilter[0].filterid == 4) \
                        and not dfilter[0].rate:
                    dfilter = dfilter[1]
                    dfilter.shuffle = True
                elif len(dfilter) == 1:
                    dfilter = dfilter[0]
                else:
                    raise Exception("Filter pipes not supported by h5py. "
                                    "Please change to h5cpp")
            if dfilter.filterid == 1 or dfilter.name == "deflate" or \
               dfilter.rate:
                if dfilter.options and dfilter.options[0]:
                    dfilter.rate = dfilter.options[0]
                f = H5PYField(
                    self._h5object.create_dataset(
                        name, shape, type_code,
                        chunks=(tuple(chunk)
                                if chunk is not None else None),
                        compression="gzip",
                        compression_opts=(
                            dfilter.options[0]
                            if dfilter.options
                            else dfilter.rate),
                        shuffle=dfilter.shuffle, maxshape=mshape
                    ),
                    self)
            elif dfilter.filterid > 0 or dfilter.name:
                f = H5PYField(
                    self._h5object.create_dataset(
                        name, shape, type_code,
                        chunks=(tuple(chunk)
                                if chunk is not None else None),
                        compression=(dfilter.filterid or dfilter.name),
                        compression_opts=tuple(dfilter.options),
                        shuffle=dfilter.shuffle, maxshape=mshape
                    ),
                    self)
        if not f:
            f = H5PYField(
                self._h5object.create_dataset(
                    name, shape, type_code,
                    chunks=(tuple(chunk)
                            if chunk is not None else None),
                    maxshape=mshape
                ),
                self)
        return f

    @property
    def attributes(self):
        """ return the attribute manager

        :returns: attribute manager
        :rtype: :class:`H5PYAttributeManager`
        """
        return H5PYAttributeManager(self._h5object.attrs, self)

    @property
    def size(self):
        """ group size

        :returns: group size
        :rtype: :obj:`int`
        """
        return len(list(self._h5object.keys()))

    def exists(self, name):
        """ if child exists

        :param name: child name
        :type name: :obj:`str`
        :returns: existing flag
        :rtype: :obj:`bool`
        """
        return name in self._h5object.keys()

    def names(self):
        """ read the child names

        :returns: h5 object
        :rtype: :obj:`list` <`str`>
        """
        return list(self._h5object.keys())

    @property
    def is_valid(self):
        """ check if group is valid

        :returns: valid flag
        :rtype: :obj:`bool`
        """
        try:
            return self._h5object.name is not None
        except Exception:
            return False

    def reopen(self):
        """ reopen file
        """
        if isinstance(self._tparent, H5PYFile):
            self._h5object = self._tparent.h5object
        else:
            self._h5object = self._tparent.h5object.get(self.name)
        filewriter.FTGroup.reopen(self)

class H5PYField(filewriter.FTField):

    """ file writer field
    """

    def __init__(self, h5object, tparent=None):
        """ constructor

        :param h5object: h5 object
        :type h5object: :obj:`any`
        :param tparent: treee parent
        :type tparent: :obj:`FTObject`
        """
        filewriter.FTField.__init__(self, h5object, tparent)
        self.path = ''
        self.name = None
        if hasattr(h5object, "name"):
            name = h5object.name
            self.name = name.split("/")[-1]
            if tparent and tparent.path:
                if tparent.path == "/":
                    self.path = "/" + self.name
                else:
                    self.path = tparent.path + "/" + self.name

    @property
    def attributes(self):
        """ return the attribute manager

        :returns: attribute manager
        :rtype: :class:`H5PYAttributeManager`
        """
        return H5PYAttributeManager(self._h5object.attrs, self)

    def reopen(self):
        """ reopen field
        """
        self._h5object = self._tparent.h5object.get(self.name)
        filewriter.FTField.reopen(self)

    def refresh(self):
        """ refresh the field

        :returns: refreshed
        :rtype: :obj:`bool`
        """
        if hasattr(self._h5object, "id"):
            if hasattr(self._h5object.id, "refresh"):
                self._h5object.id.refresh()
                return True
        return False

    def grow(self, dim=0, ext=1):
        """ grow the field

        :param dim: growing dimension
        :type dim: :obj:`int`
        :param dim: size of the grow
        :type dim: :obj:`int`
        """
        shape = list(self._h5object.shape)
        if shape:
            shape[dim] += ext
            return self._h5object.resize(shape)
        else:
            return self._h5object

    def read(self):
        """ read the field value

        :returns: h5 object
        :rtype: :obj:`any`
        """
        fl = self._h5object[...]
        if hasattr(fl, "decode") and not isinstance(fl, unicode):
            fl = fl.decode(encoding="utf-8")
        if is_strings_as_bytes() and hasattr(fl, "astype") and \
           self.dtype in ['string', b'string']:
            try:
                fl = fl.astype('str')
            except Exception:
                # print(str(e))
                pass
        return fl

    def write(self, o):
        """ write the field value

        :param o: h5 object
        :type o: :obj:`any`
        """
        self._h5object[...] = o

    def __setitem__(self, t, o):
        """ set value

        :param t: slice tuple
        :type t: :obj:`tuple`
        :param o: h5 object
        :type o: :obj:`any`
        """
        if isinstance(o, np.ndarray):
            hsh = self._h5object.shape
            if t is Ellipsis:
                tsz = [i for i in range(len(hsh))]
            elif isinstance(t, slice):
                tsz = [i for i in range(len(hsh))]
            else:
                tsz = [i for (i, s) in enumerate(t) if isinstance(s, slice)]
            osz = len(o.shape)
            if len(tsz) > osz and len(hsh) > max(tsz):
                shape = tuple([hsh[e] for e in tsz])
                o = o.reshape(shape)
        return self._h5object.__setitem__(t, o)

    def __getitem__(self, t):
        """ get value

        :param t: slice tuple
        :type t: :obj:`tuple`
        :returns: h5 object
        :rtype: :obj:`any`
        """
        fl = self._h5object.__getitem__(t)
        if hasattr(fl, "decode") and not isinstance(fl, unicode):
            fl = fl.decode(encoding="utf-8")
        if is_strings_as_bytes() and hasattr(fl, "astype") and \
           self.dtype in ['string', b'string']:
            try:
                fl = fl.astype('str')
            except Exception:
                # print(str(e))
                pass
        return fl

    @property
    def is_valid(self):
        """ check if group is valid

        :returns: valid flag
        :rtype: :obj:`bool`
        """
        try:
            return self._h5object.name is not None
        except Exception:
            return False

    def close(self):
        """ close field
        """
        filewriter.FTField.close(self)
        self._h5object = None

    @property
    def dtype(self):
        """ field data type

        :returns: field data type
        :rtype: :obj:`str`
        """

        if self._h5object.dtype.kind == 'O':
            return "string"

        return str(self._h5object.dtype)

    @property
    def shape(self):
        """ field shape

        :returns: field shape
        :rtype: :obj:`list` < :obj:`int` >
        """
        return self._h5object.shape

    @property
    def size(self):
        """ field size

        :returns: field size
        :rtype: :obj:`int`
        """
        return self._h5object.size

class H5PYLink(filewriter.FTLink):

    """ file tree link
    """

    def __init__(self, h5object, tparent=None):
        """ constructor

        :param h5object: h5 object
        :type h5object: :obj:`any`
        :param tparent: treee parent
        :type tparent: :obj:`FTObject`
        """
        filewriter.FTLink.__init__(self, h5object, tparent)
        self.path = ''
        self.name = None
        if tparent and tparent.path:
            self.path = tparent.path
        if not self.path.endswith("/"):
            self.path += "/"

    def setname(self, name):
        self.name = name
        self.path += self.name
        return self

    @property
    def is_valid(self):
        """ check if link is valid

        :returns: valid flag
        :rtype: :obj:`bool`
        """
        try:
            obj = self._h5object
            if obj is None:
                raise Exception("Empty object")

            self.parent.h5object[self.name]
            return True
        except Exception:
            return False

    def refresh(self):
        """ refresh the field

        :returns: refreshed
        :rtype: :obj:`bool`
        """
        if hasattr(self._h5object, "id"):
            if hasattr(self._h5object.id, "refresh"):
                self._h5object.id.refresh()
                return True
        return False

    def read(self):
        """ read object value

        :returns: valid flag
        :rtype: :obj:`bool`
        """
        fl = self.parent.h5object[self.name][...]
        if hasattr(fl, "decode") and not isinstance(fl, unicode):
            return fl.decode(encoding="utf-8")
        else:
            return fl

    @classmethod
    def getfilename(cls, obj):
        """ provides a filename from h5 node

        :param obj: h5 node
        :type obj: :class:`FTObject`
        :returns: file name
        :rtype: :obj:`str`
        """
        filename = ""
        while not filename:
            par = obj.parent
            if par is None:
                break
            if isinstance(par, H5PYFile):
                filename = par.name
                break
            else:
                obj = par
        return filename

    @property
    def target_path(self):
        """ target path

        :returns: target path
        :rtype: :obj:`str`
        """
        filename = self.getfilename(self)
        try:
            path = self.h5object.path
        except Exception:
            path = self.path

        if filename and ":/" not in path:
            path = "/".join([gr.split(":")[0] for gr in path.split("/")])
            path = filename + ":/" + path
        return path

    def reopen(self):
        """ reopen field
        """
        self._h5object = self._tparent.h5object.get(self.name, getlink=True)
        filewriter.FTLink.reopen(self)

    def close(self):
        """ close group
        """
        filewriter.FTLink.close(self)
        self._h5object = None

class H5PYDataFilter(filewriter.FTDataFilter):

    """ file tree data filter
    """

class H5PYVirtualFieldLayout(filewriter.FTVirtualFieldLayout):

    """ virtual field layout """

    def __init__(self, h5object, shape):
        """ constructor

        :param h5object: h5 object
        :type h5object: :obj:`any`
        :param shape: shape
        :type shape: :obj:`list` < :obj:`int` >
        """
        filewriter.FTVirtualFieldLayout.__init__(self, h5object)
        #: (:obj:`list` < :obj:`int` >) shape
        self.shape = shape

    def __setitem__(self, key, source):
        """ add target field to layout

        :param key: slide
        :type key: :obj:`tuple`
        :param source: target field view
        :type source: :class:`H5PYTargetFieldView`
        """
        #: (:obj:`list` < :obj:`int` >) shape
        self._h5object.__setitem__(key, source._h5object)

    def add(self, key, source, sourcekey=None, shape=None):
        """ add target field to layout

        :param key: slide
        :type key: :obj:`tuple`
        :param source: target field view
        :type source: :class:`H5PYTargetFieldView`
        :param sourcekey: slide or selection
        :type sourcekey: :obj:`tuple`
        :param shape: target shape in the layout
        :type shape: :obj:`tuple`
        """
        if shape is None:
            shape = list(source.shape or [])
            if hasattr(key, "__len__"):
                size = len(key)
                while len(shape) < size:
                    shape.insert(0, 1)
        # # if is_mbs_supported():
        #     key = _slice2selection(key, tuple(shape))
        # else:
        key = _selection2slice(key, tuple(shape))
        if sourcekey is not None and sourcekey != filewriter.FTHyperslab():
            if is_mbs_supported():
                sourcekey = _slice2selection(sourcekey, source.shape)
            else:
                sourcekey = _selection2slice(sourcekey, source.shape)
            self._h5object.__setitem__(key, source._h5object[sourcekey])
        elif key is not None:
            usel = unlimited_selection(key, shape)
            if usel is not None:
                self._h5object[key] = source._h5object[usel]
            else:
                self._h5object.__setitem__(key, source._h5object)
        else:
            self._h5object.__setitem__(key, source._h5object)

class H5PYTargetFieldView(filewriter.FTTargetFieldView):

    """ target field view for VDS """

    def __init__(self, h5object, shape):
        """ constructor

        :param h5object: h5 object
        :type h5object: :obj:`any`
        :param shape: shape
        :type shape: :obj:`list` < :obj:`int` >
        """
        filewriter.FTTargetFieldView.__init__(self, h5object)
        #: (:obj:`list` < :obj:`int` >) shape
        self.shape = shape

class H5PYDeflate(H5PYDataFilter):
    pass

class H5PYAttributeManager(filewriter.FTAttributeManager):

    """ file tree attribute
    """

    def __init__(self, h5object, tparent=None):
        """ constructor

        :param h5object: h5 object
        :type h5object: :obj:`any`
        :param tparent: treee parent
        :type tparent: :obj:`FTObject`
        """
        filewriter.FTAttributeManager.__init__(self, h5object, tparent)
        #: (:obj:`str`) object nexus path
        self.path = ''
        #: (:obj:`str`) object name
        self.name = None
        if hasattr(h5object, "name"):
            self.path = h5object.name
            self.name = self.path.split("/")[-1]

    def create(self, name, dtype, shape=None, overwrite=False):
        """ create a new attribute

        :param name: attribute name
        :type name: :obj:`str`
        :param dtype: attribute type
        :type dtype: :obj:`str`
        :param shape: attribute shape
        :type shape: :obj:`list` < :obj:`int` >
        :param overwrite: overwrite flag
        :type overwrite: :obj:`bool`
        :returns: attribute object
        :rtype: :class:`H5PYAtribute`
        """

        if not overwrite and name in self.h5object.keys():
            raise Exception("Attribute %s exists" % name)

        shape = shape or []
        if shape:
            if isinstance(shape, list):
                shape = tuple(shape)
            if dtype in ['string', b'string']:
                dtype = h5py.special_dtype(vlen=unicode)
                if is_strings_as_bytes():
                    etype = 'str'
                else:
                    etype = dtype
                self._h5object.create(
                    name, np.empty(shape, dtype=etype),
                    shape=shape, dtype=nptype(dtype))
            else:
                self._h5object.create(
                    name, np.zeros(shape, dtype=dtype),
                    shape, dtype)
        else:
            if dtype in ['string', b'string']:
                dtype = h5py.special_dtype(vlen=unicode)
                self._h5object.create(
                    name, np.array(u"", dtype=dtype),
                    dtype=dtype)
            else:
                self._h5object.create(
                    name, np.array(0, dtype=dtype), (1,), dtype)
        at = H5PYAttribute((self._h5object, name), self.parent)
        return at

    def __len__(self):
        """ number of attributes

        :returns: number of attributes
        :rtype: :obj:`int`
        """
        return len(list(self._h5object.keys()))

    class H5PYAttrIter(object):

        def __init__(self, manager):
            """ constructor

            :param manager: attribute manager
            :type manager: :obj:`H5PYAttributeManager`
            """

            self.__manager = manager
            self.__iter = self.__manager._h5object.__iter__()

        def __next__(self):
            """ the next attribute

            :returns: attribute object
            :rtype: :class:`FTAtribute`
            """
            name = next(self.__iter)
            if name is None:
                return None
            return H5PYAttribute((self.__manager._h5object, name),
                                 self.__manager.parent)

        next = __next__

        def __iter__(self):
            """ attribute iterator

            :returns: attribute iterator
            :rtype: :class:`H5PYAttrIter`
            """
            return self

    def __iter__(self):
        """ attribute iterator

        :returns: attribute iterator
        :rtype: :class:`H5PYAttrIter`
        """
        return self.H5PYAttrIter(self)

    def __getitem__(self, name):
        """ get value

        :param name: attribute name
        :type name: :obj:`str`
        :returns: attribute object
        :rtype: :class:`FTAtribute`
        """
        return H5PYAttribute((self._h5object, name), self.parent)

    def names(self):
        """ key values

        :returns: attribute names
        :rtype: :obj:`list` <:obj:`str`>
        """
        return self._h5object.keys()

    def reopen(self):
        """ reopen field
        """
        self._h5object = self._tparent.h5object.attrs
        filewriter.FTAttributeManager.reopen(self)

    def close(self):
        """ close attribure manager
        """
        filewriter.FTAttributeManager.close(self)

    @property
    def is_valid(self):
        """ check if link is valid

        :returns: valid flag
        :rtype: :obj:`bool`
        """
        return self.parent.is_valid

class H5PYAttribute(filewriter.FTAttribute):

    """ file tree attribute
    """

    def __init__(self, h5object, tparent=None):
        """ constructor

        :param h5object: h5 object
        :type h5object: :obj:`any`
        :param tparent: treee parent
        :type tparent: :obj:`FTObject`
        """
        filewriter.FTAttribute.__init__(self, h5object, tparent)
        self.name = h5object[1]

        self.path = tparent.path
        self.path += "@%s" % self.name

    def read(self):
        """ read attribute value

        :returns: python object
        :rtype: :obj:`any`
        """
        at = self._h5object[0][self.name]
        if hasattr(at, "decode") and not isinstance(at, unicode):
            return at.decode(encoding="utf-8")
        else:
            return at

    def write(self, o):
        """ write attribute value

        :param o: python object
        :type o: :obj:`any`
        """
        if self.dtype in ['string', b'string']:
            if isinstance(o, str):
                self._h5object[0][self.name] = unicode(o)
            else:
                dtype = h5py.special_dtype(vlen=unicode)
                self._h5object[0][self.name] = np.array(o, dtype=dtype)
        else:
            self._h5object[0][self.name] = np.array(o, dtype=self.dtype)

    def __setitem__(self, t, o):
        """ write attribute value

        :param t: slice tuple
        :type t: :obj:`tuple`
        :param o: python object
        :type o: :obj:`any`
        """
        if t is Ellipsis or t == slice(None, None, None) or \
           t == (slice(None, None, None), slice(None, None, None)) or \
           (hasattr(o, "__len__") and t == slice(0, len(o), None)):
            if self.dtype in ['string', b'string']:
                if isinstance(o, str):
                    self._h5object[0][self.name] = unicode(o)
                else:
                    dtype = h5py.special_dtype(vlen=unicode)
                    self._h5object[0][self.name] = np.array(o, dtype=dtype)
            else:
                self._h5object[0][self.name] = np.array(o, dtype=self.dtype)
        elif isinstance(t, slice):
            var = self._h5object[0][self.name]
            if self.dtype not in ['string', b'string']:
                var[t] = np.array(o, dtype=self.dtype)
            else:
                dtype = h5py.special_dtype(vlen=unicode)
                var[t] = np.array(o, dtype=dtype)
                var = var.astype(dtype)
            try:
                self._h5object[0][self.name] = var
            except Exception:
                dtype = h5py.special_dtype(vlen=unicode)
                tvar = np.array(var, dtype=dtype)
                self._h5object[0][self.name] = tvar

        elif isinstance(t, tuple):
            var = self._h5object[0][self.name]
            if self.dtype not in ['string', b'string']:
                var[t] = np.array(o, dtype=self.dtype)
            else:
                dtype = h5py.special_dtype(vlen=unicode)
                if hasattr(var, "flatten"):
                    vv = var.flatten().tolist() + \
                        np.array(o, dtype=dtype).flatten().tolist()
                    nt = np.array(vv, dtype=dtype)
                    var = np.array(var, dtype=nt.dtype)
                    var[t] = np.array(o, dtype=dtype)
                elif hasattr(var, "tolist"):
                    var = var.tolist()
                    var[t] = np.array(o, dtype=self.dtype).tolist()
                else:
                    var[t] = np.array(o, dtype=self.dtype).tolist()
                var = var.astype(dtype)
            self._h5object[0][self.name] = var
        else:
            if isinstance(o, str) or isinstance(o, unicode):
                self._h5object[0][self.name] = unicode(o)
            else:
                self._h5object[0][self.name] = np.array(o, dtype=self.dtype)

    def __getitem__(self, t):
        """ read attribute value

        :param t: slice tuple
        :type t: :obj:`tuple`
        :returns: python object
        :rtype: :obj:`any`
        """
        if not isinstance(t, int):
            if t is Ellipsis:
                at = self._h5object[0][self.name]
            else:
                at = self._h5object[0][self.name][t]
        else:
            at = self._h5object[0][self.name].__getitem__(t)
        if hasattr(at, "decode") and not isinstance(at, unicode):
            return at.decode(encoding="utf-8")
        else:
            return at

    @property
    def is_valid(self):
        """ check if field is valid

        :returns: valid flag
        :rtype: :obj:`bool`
        """
        try:
            return self.name in self._h5object[0].keys()
        except Exception:
            return False

    @property
    def dtype(self):
        """ attribute data type

        :returns: attribute data type
        :rtype: :obj:`str`
        """
        dt = type(self._h5object[0][self.name]).__name__
        if dt == "ndarray":
            dt = str(self._h5object[0][self.name].dtype)
        if dt.endswith("_"):
            dt = dt[:-1]
        if dt == "bytes":
            dt = "string"
        if dt == "unicode":
            dt = "string"
        if dt == "str":
            dt = "string"
        if dt == "object":
            dt = "string"
        if dt.startswith("|S"):
            dt = "string"
        return dt

    @property
    def shape(self):
        """ attribute shape

        :returns: attribute shape
        :rtype: :obj:`list` < :obj:`int` >
        """
        if hasattr(self._h5object[0][self.name], "shape"):
            return self._h5object[0][self.name].shape
        else:
            return ()

    def reopen(self):
        """ reopen attribute
        """
        self._h5object = (self._tparent.h5object.attrs, self.name)
        filewriter.FTAttribute.reopen(self)

    def close(self):
        """ close attribute
        """
        filewriter.FTAttribute.close(self)
        self._h5object = None


# -*- coding: utf-8 -*-

import os
import time
import json
import logging

import gevent
import gevent.event
import gevent.monkey
import pytest
import requests
from mock import MagicMock

from huskar_sdk_v2.http.ioloops import IOLoop
from huskar_sdk_v2.http.ioloops.http import HuskarApiIOLoop
from huskar_sdk_v2.http.ioloops.file import FileCacheIOLoop
from huskar_sdk_v2.http.ioloops.entity import Component
from huskar_sdk_v2.http.components.config import Config
from huskar_sdk_v2.http.components.service import Service
from huskar_sdk_v2.http.components.switch import Switch

logging.basicConfig()
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))

@pytest.fixture(autouse=True)
def gevent_monkey_patch():
    gevent.monkey.patch_all()

@pytest.fixture
def requests_mock(monkeypatch):
    Session = MagicMock()

    class MockResponse(object):
        wait_time = 0
        stop_exception = None
        changed_data_processed = gevent.event.Event()
        pending_requests = []
        first_line = open(os.path.join(
            CURRENT_DIR, "test_data.txt")).read().strip()
        ok = True
        status_code = 200

        def __init__(self, *args, **kwds):
            self.changed_data_processed.clear()
            while self.pending_requests:
                self.pending_requests.pop()

        @classmethod
        def set_result_file(cls, filename):
            with open(os.path.join(CURRENT_DIR, filename)) as f:
                cls.pending_requests.extend(f.read().splitlines())
            cls.changed_data_processed.clear()

        def should_raise_exception(self):
            if self.stop_exception is not None:
                raise self.stop_exception

        @classmethod
        def set_error_mode(cls, status_code=401, message=''):
            cls.ok = False
            cls.status_code = status_code
            cls.text = message or (
                u'{"data": null, "message": "The token is missing",'
                u' "status": "Unauthorized"}')
            cls.content = cls.text
            cls.json = lambda cls: json.loads(cls.text)

        @classmethod
        def raise_for_status(cls):
            if cls.ok:
                return
            raise requests.HTTPError(response=cls)

        @classmethod
        def add_response(cls, line):
            cls.pending_requests.append(line)
            cls.changed_data_processed.clear()

        @classmethod
        def set_first_line(cls, line):
            cls.first_line = line

        @classmethod
        def wait_processed(cls, timeout=3.0):
            return cls.changed_data_processed.wait(timeout)

        def iter_lines(self, *args, **kwds):
            gevent.sleep(self.wait_time)
            self.should_raise_exception()
            yield self.first_line
            while True:
                self.should_raise_exception()
                while self.pending_requests:
                    req = self.pending_requests.pop()
                    yield req
                else:
                    if not self.changed_data_processed.is_set():
                        self.changed_data_processed.set()
                yield '{"body": {}, "message": "ping"}'
                time.sleep(1)

    session = Session()
    session.headers = {
        'Connection': 'keep-alive',
        'Accept-Encoding': 'gzip, deflate',
        'Accept': '*/*',
        'User-Agent': 'mocked-requests/0.0.0'
    }
    session.post = MockResponse
    monkeypatch.setattr(requests, 'Session', Session)
    return MockResponse

@pytest.fixture
def cache_dir(worker_id, tmpdir):
    return str(tmpdir.mkdir('huskar_{}'.format(worker_id)))

@pytest.fixture
def clear_ioloop_instance(request):
    request.addfinalizer(IOLoop.clear_instance)
    request.addfinalizer(IOLoop.clear_configure)

@pytest.fixture
def empty_client(request, requests_mock, cache_dir, clear_ioloop_instance):
    client = HuskarApiIOLoop('test_url', 'test_token', cache_dir=cache_dir)
    client.install()

    request.addfinalizer(lambda: client.stop(3))
    request.addfinalizer(Component.value_processors.clear)
    return client

@pytest.fixture
def client(empty_client):
    empty_client.watched_configs.add_watch("arch.test", 'overall')
    empty_client.watched_switches.add_watch("arch.test", 'another-cluster')
    empty_client.watched_services.add_watch("arch.test", 'alpha-stable')
    return empty_client

@pytest.fixture
def no_cache_initial_client(request, clear_ioloop_instance):
    client = HuskarApiIOLoop('test_url', 'test_token', cache_dir=None)
    client.install()
    request.addfinalizer(lambda: client.stop(3))
    return client

@pytest.fixture
def no_cache_client(no_cache_initial_client):
    no_cache_initial_client.run()
    return no_cache_initial_client

@pytest.fixture
def started_client(client):
    client.run()
    return client

# you can treat the file_cache_client is a HuskarClient run in another process,
# and it only read and update data from file cache.
@pytest.fixture
def file_cache_client(request, monkeypatch, cache_dir):
    monkeypatch.setattr(FileCacheIOLoop, 'try_to_be_writer', lambda self: None)
    ioloop = FileCacheIOLoop('test_url', 'test_token', cache_dir=cache_dir)
    ioloop.check_file_stat_gap = 0.2
    request.addfinalizer(ioloop.stop)
    return ioloop

@pytest.fixture
def started_file_cache_client(file_cache_client):
    file_cache_client.run()
    return file_cache_client

@pytest.fixture
def fake_config_with_file_cache_client(file_cache_client):
    class FakeConfig(Config):
        @property
        def client(self):
            return file_cache_client.watched_configs
    return FakeConfig

@pytest.fixture
def fake_service_with_file_cache_client(file_cache_client):
    class FakeService(Service):
        @property
        def client(self):
            return file_cache_client.watched_services
    return FakeService

@pytest.fixture
def fake_switch_with_file_cache_client(file_cache_client):
    class FakeSwitch(Switch):
        @property
        def client(self):
            return file_cache_client.watched_switches
    return FakeSwitch

@pytest.fixture
def test_data():
    with open(os.path.join(CURRENT_DIR, 'test_data.txt'), 'r') as f:
        content = json.load(f)

    class singleton(object):
        config_content = content['body']['config']
        service_content = content['body']['service']
        switch_content = content['body']['switch']

    return singleton()

@pytest.fixture
def wait_huskar_api_ioloop_connected():
    def _(timeout):
        ioloop = IOLoop.current()
        assert isinstance(ioloop, HuskarApiIOLoop)
        assert ioloop.connected.wait(timeout)
    return _

@pytest.fixture
def sleep_ops(monkeypatch):
    origin_sleep = gevent.sleep

    class SleepOps(object):
        def __init__(self, time=None):
            self.time = time

        def sleep(self, time):
            if self.time is not None:
                origin_sleep(self.time)
            else:
                origin_sleep(time)

        def set_constant_sleep_time(self, time):
            self.time = time

    sleep_ops = SleepOps()
    monkeypatch.setattr("gevent.sleep", sleep_ops.sleep)
    return sleep_ops


from node import *
from neomodels import *
import re

class Gene(Node):
    '''
    Class for gene nodes on neo4j
    identifier: STRING
    level: [ 0 | 1 | 2 | 3 | 4 | 5 ]
    nvariants: INT
    gene_disease:
        0 (non-driver)
        1 (Syndromic)
        2 (Non-Syndromic)
        3 (Both)
        4 (Unknown)

    '''
    def __init__(self, identifier):
        label = "GENE"
        super(Gene, self).__init__(identifier, label)
        self.gene_disease = None
        self.level = 0
        self.expression = 'NA'
        self.nvariants = 0
        self.gos = list()
        self.inheritance_pattern = 0
        self.summary = None
        self.summary_source = None
        self.aliases = None
        self.color = None

    def check(self):
        '''
        Queries Gene on neo4j and fills the attributes.
        If not in database: NodeNotFound
        '''
        self.normalize_identifier()
        self.unalias()
        NEO.query_by_id(self)
        return self

    def normalize_identifier(self):
        '''
        Removes strange characters from identifiers
        '''
        self.identifier = re.sub(r'\W+', '_', self.identifier)

    def unalias(self):
        '''
        Unalias
        '''
        NEO.query_unalias(self)

    def fill_attributes(self, genedict, prefix=None):
        '''
        Fills the attributes of the object. Avoids querying db
        '''
        if prefix is None:
            self.level = genedict['level']
            self.nvariants = genedict['nvariants']
            self.gene_disease = genedict['gene_disease']
            self.inheritance_pattern = genedict['inheritance_pattern']
        else:
            self.level = genedict[prefix + '_level']
            self.nvariants = genedict[prefix + '_nvariants']
            self.gene_disease = genedict[prefix + '_gene_disease']
            self.inheritance_pattern = genedict[prefix + '_inheritance_pattern']

    def is_driver(self):
        '''
        Checks if gene is driver or not
            # gene_disease = 0 -> No-driver
            # gene_disease = 1 -> Syndromic
            # gene_disease = 2 -> Non-syndromic
            # gene_disease = 3 -> Both
        '''
        if self.gene_disease is None:
            try:
                self.check()
            except:
                return False
        if int(self.gene_disease) == 0:
            return False
        else:
            return True

    def get_gene_disease_class(self):
        '''
        Returns gene-disease class string
        '''
        gd_class = None
        if self.gene_disease == 1:
            gd_class = "syndromic"
        elif self.gene_disease == 2:
            gd_class = "non-syndromic"
        elif self.gene_disease == 3:
            gd_class = "both"
        return gd_class

    def level_to_class(self):
        '''
        Returns level string for cytoscape class
        '''
        level_class = None
        if self.level == 0:
            level_class = "skeleton"
        else:
            level_class = "lvl" + str(self.level)
        return level_class

    def get_expression(self, exp_id):
        '''
        Gets desired expression value for Gene
        '''
        self.expression = NEO.query_expression(self, exp_id)
        return self.expression

    def get_neighbours(self, level, dist, exp_id):
        '''
        Gets all the neighbours to Gene within 'level' interactions and at distance
        'dist' at most. All child nodes will have expression of 'exp_id'
        Returns a GraphCyt object.
        '''
        ngraph = NEO.query_get_neighbours(self, level, dist, exp_id)
        return ngraph

    def get_go(self):
        '''
        Gets GeneOntologies of the gene
        '''
        self.gos = NEO.query_gos(self)
        return self.gos

    def to_json_dict(self, positions=None):
        '''
        Returns dictionary ready to convert to json
        '''
        element = dict()
        element['data'] = dict()
        element['data']['id'] = self.identifier
        element['data']['name'] = self.identifier
        element['data']['level'] = self.level
        element['data']['exp'] = self.expression
        element['data']['gene_disease'] = self.gene_disease
        element['data']['nvariants'] = self.nvariants
        element['data']['inheritance_pattern'] = self.inheritance_pattern
        element['data']['color'] = self.color
        if self.is_driver():
            element['classes'] = "driver"
        else:
            element['classes'] = self.level_to_class()
        if self.get_gene_disease_class() is not None:
            element['classes'] += " " + self.get_gene_disease_class()
        if positions is not None:
            element['position'] = dict()
            element['position']['x'] = positions[0]
            element['position']['y'] = positions[1]
        return element

    def path_to_level(self, level):
        '''
        Returns a dictionary of GraphCytoscape object with all shortest paths to all drivers
        Key = target gene identifier
        '''
        paths = NEO.query_path_to_level(self, level)
        return paths

    def path_to_gene(self, cobj):
        '''
        Shortest path to Gene Object
        '''
        path = NEO.query_shortest_path(self, cobj)
        return path

    def get_summary(self):
        '''
        Returns summary for gene
        '''
        self.summary, self.summary_source = NEO.query_get_summary(self)
        return self.summary

    def get_aliases(self):
        '''
        Returns aliases for gene
        '''
        def alias_key(a):
            '''
            Sorts aliases based on our arbitrary needs:
                First Symbols, then ENS and finally Entrez.
                Within each group, sort by length of the symbol.
            '''
            order = { 'sym': (1, len(a)), 'ens': (2, len(a)), 'num': (3, len(a)) }
            key = 'sym'
            if re.match('^ENS[GTP]', a):
                key = 'ens'
            elif re.match('^\d+$', a):
                key = 'num'
            return order[key]

        self.aliases = sorted(NEO.query_all_aliases(self), key=alias_key)
        return self.aliases

    def __hash__(self):
        return hash((self.identifier, self.gene_disease, self.level))


import numpy as np

def compute_mean_and_variance(data):
    mean = np.mean(data, axis=0)
    variance = np.var(data, axis=0)
    return mean, variance

# This function reorders randomly the input data
def shuffle_data(input_data):
    # The seed is fixed in order to have always the same reorder
    # This is only for testing purposes
    np.random.seed(0)

    data = np.copy(input_data)
    np.random.shuffle(data)

    return data

# Also known as linear scaling
# x' = (x - min(x)) / (max(x) - min(x))
# where x is data and x' is the normalized data,
#
# The function can have both 1 or 2 arguments:
#   - With only 1 argument the scaling is applied to all
#     the columns of the training data (1st argument)
#   - With 2 arguments the scaling is applied to a subset
#     of columns. The first argument is the training data,
#     the second is the subset of columns to scale
#
# In case the function has more than 2 arguments, the default
# behavior is to scale all the columns
def min_max_scaling(*args):
    training_input = np.array(args[0])
    normalized_dataset = []

    if len(args) == 2:
        columns = args[1]

        # Retrieves the minimum and maximum value in the columns
        min_val = np.amin(training_input[:, columns], axis=0)
        max_val = np.amax(training_input[:, columns], axis=0)

        # Normalizes the dataset
        normalized_dataset = (training_input[:, columns] - min_val) / (max_val - min_val)

        # Substitute in each scaled column the new values
        for column, i in zip(range(normalized_dataset.shape[1]), columns):
            training_input[:, i] = normalized_dataset[:, column]

        normalized_dataset = training_input
    else:
        # Retrieves the minimum and maximum value in the columns
        min_val = np.amin(training_input, axis=0)
        max_val = np.amax(training_input, axis=0)

        # Normalizes the dataset
        normalized_dataset = (training_input - min_val) / (max_val - min_val)

    return normalized_dataset

# x' = (x - mean) / standard deviation
# x is the dataset
# x' is the normalized data set
#
# The function can have both 1 or 2 arguments:
#   - With only 1 argument the scaling is applied to all
#     the columns of the training data (1st argument)
#   - With 2 arguments the scaling is applied to a subset
#     of columns. The first argument is the training data,
#     the second is the subset of columns to scale
#
# In case the function has more than 2 arguments, the default
# behavior is to scale all the columns
def z_score_scaling(*args):
    training_input = np.array(args[0])
    normalized_dataset = []

    if len(args) == 2:
        columns = args[1]

        # Computes the mean over the affected columns
        mu = np.mean(training_input[:, columns], axis=0)

        # Computes the standard deviation over the affected columns
        sigma = np.std(training_input[:, columns], axis=0)

        # Normalizes the dataset
        normalized_dataset = np.divide(np.subtract(training_input[:, columns], mu), sigma)

        # Substitute in each scaled column the new values
        for column, i in zip(range(normalized_dataset.shape[1]), columns):
            training_input[:, i] = normalized_dataset[:, column]

        normalized_dataset = training_input
    else:
        # Computes the mean over the affected columns
        mu = np.mean(training_input, axis=0)

        # Computes the standard deviation over the affected columns
        sigma = np.std(training_input, axis=0)

        # Normalizes the dataset
        normalized_dataset = np.divide(np.subtract(training_input, mu), sigma)

    return normalized_dataset

# This function implements the one-hot encoding on the input data
#
# Note that this function works only when the input data is composed
# only by integers
def one_hot_encoding(input_data):
    # Computes the maximum value on each column in the input data
    max_values = np.max(input_data, axis=0)

    dictionaries = []

    for value in max_values:
        # Creates an array filled with zero of length 'value'
        encoding = np.zeros(value, dtype=max_values.dtype)

        dictionary_entry = {}

        # Fills the dictionary entry with all the encodings
        # of values in the range [1, value]
        for i in range(1, value + 1):
            tmp = encoding.copy()
            tmp[i - 1] = 1
            dictionary_entry[i] = tmp

        # Appends the dictionary entry to the list
        dictionaries.append(dictionary_entry)

    result_data = []
    # For each row of the input data,
    # encodes the values
    for row in input_data:
        new_row = []

        # Creates new row by concatenating the encoding of each value in the row
        for column, i in zip(row, range(len(row))):
            # Takes the correct encoding for that column
            dic = dictionaries[i]

            new_row = np.concatenate((new_row, dic[column]), axis=None)

        result_data.append(new_row)

    return np.array(result_data, dtype=max_values.dtype)


import os
import torch
import time

from models.model import ViTPose
from configs.ViTPose_base_coco_256x192 import model as model_cfg

CKPT_PATH = "/workspace/ViTPose_pytorch/vitpose-b-multi-coco.pth"

C, H, W = (3, 256, 192)

model = ViTPose(model_cfg)
ckpt = torch.load(CKPT_PATH)
model.load_state_dict(ckpt['state_dict'])
model.eval()

output_onnx = 'vitpose.onnx'
input_names = ["input_0"]
output_names = ["output_0"]

device = next(model.parameters()).device
inputs = torch.randn(1, C, H, W).to(device)

dynamic_axes = {'input_0' : {0 : 'batch_size'},
                'output_0' : {0 : 'batch_size'}}
start_time = time.time()

torch_out = torch.onnx.export(model, inputs, output_onnx, export_params=True, verbose=False,
                              input_names=input_names, output_names=output_names,
                              opset_version=11, dynamic_axes = dynamic_axes)
print(f">>> Saved at: {os.path.abspath(output_onnx)}")


from db import db
from sqlalchemy.dialects import postgresql

class RecommendModel (db.Model):
    __tablename__ = 'recommend'

    user_id = db.Column(postgresql.UUID, primary_key=True)
    news_ids = db.Column(postgresql.ARRAY(postgresql.VARCHAR))

    @ classmethod
    def recommend_news_by_user_id(cls, user_id):
        news_ids = cls.query.filter_by(user_id=user_id).first().news_ids
        return [] if news_ids is None else news_ids


# Comparative modeling with ligand transfer from the template
from modeller import *              # Load standard Modeller classes
from modeller.automodel import *    # Load the automodel class

log.verbose()    # request verbose output
env = environ()  # create a new MODELLER environment to build this model in

# directories for input atom files
env.io.atom_files_directory = ['.', '../atom_files']

# Read in HETATM records from template PDBs
env.io.hetatm = True

a = automodel(env,
              alnfile  = 'Taget_rcsb_pdb_101M_Template_101m.ali',  # alignment filename
              knowns   = '101mA',              # codes of the templates
              sequence = '101M_1',              # code of the target
              assess_methods=(assess.DOPE,
                              #soap_protein_od.Scorer(),
                              assess.GA341))
a.starting_model= 1                 # index of the first model
a.ending_model  = 5                 # index of the last model
                                    # (determines how many models to calculate)
a.make()


#!/usr/bin/python3

from pyrob.api import *

@task
def task_5_4():
    f = 0
    d = 0
    move_down()
    while 1:
        if wall_is_above():
            move_left()
            if wall_is_on_the_left():
                break
        if f == 0 and wall_is_beneath() == False and wall_is_above() == False:
            move_down()
        if wall_is_beneath():
            move_right()
            f = 1
        if wall_is_beneath() == False and f == 1:
            move_down()
            move_left()
            f = 0

if __name__ == "__main__":
    run_tasks()


import numpy as np
import pandas as pd

from collections import defaultdict
from project.utils import get_performances, plot_performances

class Base:
    def __init__(self, perform_logging: bool = True) -> None:
        self.data = None
        self.gts = None
        self.performance_history = defaultdict(list)
        self.perform_logging = perform_logging

    def set_data(self, df: pd.DataFrame) -> None:
        raise NotImplementedError

    def train(self) -> None:
        raise NotImplementedError

    def predict(self, x: pd.DataFrame) -> np.ndarray:
        raise NotImplementedError

    def log(self, T: int) -> None:
        gts = self.gts.loc[:T].to_numpy().squeeze()
        preds = self.predict(self.data.loc[:T])

        performances = get_performances(preds, gts)
        for key, value in performances.items():
            self.performance_history[key].append(value)

    def plot_performances(self) -> None:
        class_name = self.__class__.__name__

        for key, history in self.performance_history.items():
            history = np.array(history)
            if key == 'fraction of incorrect doses':
                plot_performances(history, class_name, 'Time', 'Fraction', 'Fraction of incorrect doses')
            elif key == 'cumulative regret':
                plot_performances(history, class_name, 'Time', 'Regret', 'Cumulative regret')
            elif key == 'average regret':
                plot_performances(history, class_name, 'Time', 'Regret', 'Average regret')
            else:
                raise NotImplementedError

    def get_final_performances(self) -> dict:
        final_performances = {}
        for key, values in self.performance_history.items():
            final_performances[key] = values[-1]
        return final_performances


# Definition for a binary tree node.
class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

class Solution:
    def merge(self, t1: TreeNode, t2: TreeNode) -> TreeNode:
        if not t1 and not t2:
            return None
        if t1 and t2:
            newNode = TreeNode(t1.val + t2.val)
            newNode.left = self.merge(t1.left, t2.left)
            newNode.right = self.merge(t1.right, t2.right)
            return newNode
        if t1 and not t2:
            newNode = TreeNode(t1.val)
            newNode.left = self.merge(t1.left, None)
            newNode.right = self.merge(t1.right, None)
            return newNode
        if t2 and not t1:
            newNode = TreeNode(t2.val)
            newNode.left = self.merge(None, t2.left)
            newNode.right = self.merge(None, t2.right)
            return newNode

    def mergeTrees(self, t1: TreeNode, t2: TreeNode) -> TreeNode:
        return self.merge(t1, t2)


# -----------------------------------------------------------------------------------------------------
'''
&usage:		HMM-GMM的孤立词识别模型
@author:	hongwen sun
'''
# -----------------------------------------------------------------------------------------------------
# 导入依赖库，特别需要注意hmmlearn
from python_speech_features import mfcc
from scipy.io import wavfile
from hmmlearn import hmm
from sklearn.externals import joblib
import numpy as np
import os

# -----------------------------------------------------------------------------------------------------
'''
&usage:		准备所需数据
'''
# -----------------------------------------------------------------------------------------------------
# 生成wavdict，key=wavid，value=wavfile
def gen_wavlist(wavpath):
	wavdict = {}
	labeldict = {}
	for (dirpath, dirnames, filenames) in os.walk(wavpath):
		for filename in filenames:
			if filename.endswith('.wav'):
				filepath = os.sep.join([dirpath, filename])
				fileid = filename.strip('.wav')
				wavdict[fileid] = filepath
				label = fileid.split('_')[1]
				labeldict[fileid] = label
	return wavdict, labeldict

# 特征提取，feat = compute_mfcc(wadict[wavid])
def compute_mfcc(file):
	fs, audio = wavfile.read(file)
	# 这里我故意fs/2,有些类似减小step，不建议这样做，投机取巧做法
	mfcc_feat = mfcc(audio, samplerate=(fs/2), numcep=26)
	return mfcc_feat

# -----------------------------------------------------------------------------------------------------
'''
&usage:		搭建HMM-GMM的孤立词识别模型
参数意义：
	CATEGORY:	所有标签的列表
	n_comp:		每个孤立词中的状态数
	n_mix:		每个状态包含的混合高斯数量
	cov_type:	协方差矩阵的类型
	n_iter:		训练迭代次数
'''
# -----------------------------------------------------------------------------------------------------

class Model():
	"""docstring for Model"""
	def __init__(self, CATEGORY=None, n_comp=3, n_mix = 3, cov_type='diag', n_iter=1000):
		super(Model, self).__init__()
		self.CATEGORY = CATEGORY
		self.category = len(CATEGORY)
		self.n_comp = n_comp
		self.n_mix = n_mix
		self.cov_type = cov_type
		self.n_iter = n_iter
		# 关键步骤，初始化models，返回特定参数的模型的列表
		self.models = []
		for k in range(self.category):
			model = hmm.GMMHMM(n_components=self.n_comp, n_mix = self.n_mix,
								covariance_type=self.cov_type, n_iter=self.n_iter)
			self.models.append(model)

	# 模型训练
	def train(self, wavdict=None, labeldict=None):
		for k in range(10):
			subdata = []
			model = self.models[k]
			for x in wavdict:
				if labeldict[x] == self.CATEGORY[k]:
					mfcc_feat = compute_mfcc(wavdict[x])
					model.fit(mfcc_feat)

	# 使用特定的测试集合进行测试
	def test(self, wavdict=None, labeldict=None):
		result = []
		for k in range(self.category):
			subre = []
			label = []
			model = self.models[k]
			for x in wavdict:
				mfcc_feat = compute_mfcc(wavdict[x])
				# 生成每个数据在当前模型下的得分情况
				re = model.score(mfcc_feat)
				subre.append(re)
				label.append(labeldict[x])
			# 汇总得分情况
			result.append(subre)
		# 选取得分最高的种类
		result = np.vstack(result).argmax(axis=0)
		# 返回种类的类别标签
		result = [self.CATEGORY[label] for label in result]
		print('识别得到结果：\n',result)
		print('原始标签类别：\n',label)
		# 检查识别率，为：正确识别的个数/总数
		totalnum = len(label)
		correctnum = 0
		for i in range(totalnum):
		 	if result[i] == label[i]:
		 	 	correctnum += 1
		print('识别率:', correctnum/totalnum)

	def save(self, path="models.pkl"):
		# 利用external joblib保存生成的hmm模型
		joblib.dump(self.models, path)

	def load(self, path="models.pkl"):
		# 导入hmm模型
		self.models = joblib.load(path)

# -----------------------------------------------------------------------------------------------------
'''
&usage:		使用模型进行训练和识别
'''
# -----------------------------------------------------------------------------------------------------
# 准备训练所需数据
CATEGORY = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10']
wavdict, labeldict = gen_wavlist('training_data')
testdict, testlabel = gen_wavlist('test_data')
# 进行训练
models = Model(CATEGORY=CATEGORY)
models.train(wavdict=wavdict, labeldict=labeldict)
models.save()
models.load()
models.test(wavdict=wavdict, labeldict=labeldict)
models.test(wavdict=testdict, labeldict=testlabel)


from email.policy import default
import sys
from collections import deque, defaultdict
from typing import Any, Callable, Dict, List, Optional, Tuple, Union

from sympy import Q

sys.path.insert(0, "/home/jerrymengxiao/dev/LC")

from utils.TreeNode import TreeNode, list_to_tree

"""
987. Vertical Order Traversal of a Binary Tree
HARD

Given the root of a binary tree, calculate the vertical order traversal of the binary tree.

For each node at position (row, col), its left and right children will be at positions (row + 1, col - 1) and (row + 1, col + 1) respectively. The root of the tree is at (0, 0).

The vertical order traversal of a binary tree is a list of top-to-bottom orderings for each column index starting from the leftmost column and ending on the rightmost column. There may be multiple nodes in the same row and same column. In such a case, sort these nodes by their values.

Return the vertical order traversal of the binary tree.
"""

class Solution:
    """
    BFS
    """

    def verticalTraversal(self, root: TreeNode) -> List[List[int]]:
        # g = defaultdict(list)
        # q = deque([(root, 0)]) #node, index
        # while q:
        #     temp_g = defaultdict(list)
        #     for i in range(len(q)):
        #         node, index = q.popleft()
        #         temp_g[index].append(node.val)
        #         if node.left:
        #             q.append((node.left, index-1))
        #         if node.right:
        #             q.append((node.right, index+1))
        #     for index in temp_g:
        #         g[index].extend(sorted(temp_g[index]))
        # return [g[index] for index in sorted(g)]
        g = defaultdict(list)
        q = [(root, 0)]
        while q:
            temp_g = defaultdict(list)
            for i in range(len(q)):
                node, index = q.pop(0)
                temp_g[index].append(node.val)
                if node.left:
                    q.append((node.left, index - 1))
                if node.right:
                    q.append((node.right, index + 1))
            for index in temp_g:
                g[index].extend(sorted(temp_g[index]))
        return [g[i] for i in sorted(g)]

def main():
    tests = [
        {"vals": [3, 9, 20, None, None, 15, 7], "traversal": [[9], [3, 15], [20], [7]]},
        {"vals": [1, 2, 3, 4, 5, 6, 7], "traversal": [[4], [2], [1, 5, 6], [3], [7]]},
    ]

    i = 0
    for test in tests:
        i += 1
        vals = test["vals"]
        root = list_to_tree(vals)
        solver = Solution()
        print(solver.verticalTraversal(root))
        assert solver.verticalTraversal(root) == test["traversal"]
        print(f"Passed test case {i}")

if __name__ == "__main__":
    main()


from messages_pb2 import PrinterStatus, DripRecorded, SetDripCount, MoveToDripCount, SetCurrentHeight

class ProtoBuffableMessage(object):
    TYPE_ID = 0

    def get_bytes(self):
        raise NotImplementedError()

    @classmethod
    def from_bytes(cls, proto_bytes):
        raise NotImplementedError()

class PrinterStatusMessage(ProtoBuffableMessage):
    TYPE_ID = 51

    def __init__(self, targetHeightMicrometer, currentHeightMicrometer, waitingForNextLayerHeight, status):
        self._target_height_micrometer = targetHeightMicrometer
        self._current_height_micrometer = currentHeightMicrometer
        self._waiting_for_next_layer_height = waitingForNextLayerHeight
        self._status = status

    @property
    def target_height_micrometer():
        return self._target_height_micrometer

    @property
    def current_height_micrometer():
        return self._current_height_micrometer

    @property
    def waiting_for_next_layer_height():
        return self._waiting_for_next_layer_height

    @property
    def status():
        return self._status

    def get_bytes(self):
        encoded = PrinterStatus()
        encoded.targetHeightMicrometer = self._target_height_micrometer
        encoded.currentHeightMicrometer = self._current_height_micrometer
        encoded.waitingForNextLayerHeight = self._waiting_for_next_layer_height
        encoded.status = self._status

        if encoded.IsInitialized():
            return encoded.SerializeToString()
        else:
            raise Exception("Protobuf Message encoding incomplete")

    @classmethod
    def from_bytes(cls, proto_bytes):
        decoded = PrinterStatus()
        decoded.ParseFromString(proto_bytes)
        return cls(
            decoded.targetHeightMicrometer,
            decoded.currentHeightMicrometer,
            decoded.waitingForNextLayerHeight,
            decoded.status,
            )

    def __eq__(self, other):
        if (self.__class__ == other.__class__ and
                self._target_height_micrometer == other._target_height_micrometer and
                self._current_height_micrometer == other._current_height_micrometer and
                self._waiting_for_next_layer_height == other._waiting_for_next_layer_height and
                self._status == other._status
                ):
            return True
        else:
            return False

    def __repr__(self):
        return ("Target Height            : {}\n"
               "Current Height           : {}\n"
               "Waiting for next height  : {}\n"
               "Status                   : {}").format(
                self._target_height_micrometer,
                self._current_height_micrometer,
                self._waiting_for_next_layer_height,
                self._status)

class SetCurrentHeightMessage(ProtoBuffableMessage):
    TYPE_ID = 52

    def __init__(self, heightMicrometers):
        self._height_micrometers = heightMicrometers

    @property
    def drips(self):
        return self._height_micrometers

    def get_bytes(self):
        encoded = SetCurrentHeight()
        encoded.heightMicrometers = self._height_micrometers
        if encoded.IsInitialized():
            return encoded.SerializeToString()
        else:
            raise Exception("Protobuf Message encoding incomplete")

    @classmethod
    def from_bytes(cls, proto_bytes):
        decoded = SetCurrentHeight()
        decoded.ParseFromString(proto_bytes)
        return cls(decoded.heightMicrometers)

    def __eq__(self, other):
        if (self.__class__ == other.__class__ and
                self._height_micrometers == other._height_micrometers):
            return True
        else:
            return False

    def __repr__(self):
        return "HeightMicrometers={}".format(self._height_micrometers)

class DripRecordedMessage(ProtoBuffableMessage):
    TYPE_ID = 3

    def __init__(self, drips):
        self._drips = drips

    @property
    def drips(self):
        return self._drips

    def get_bytes(self):
        encoded = DripRecorded()
        encoded.drips = self._drips
        if encoded.IsInitialized():
            return encoded.SerializeToString()
        else:
            logger.error("Protobuf Message encoding incomplete. Did the spec change? Have you compiled your proto files?")
            raise Exception("Protobuf Message encoding incomplete")

    @classmethod
    def from_bytes(cls, proto_bytes):
        decoded = DripRecorded()
        decoded.ParseFromString(proto_bytes)
        return cls(decoded.drips)

    def __eq__(self, other):
        if (self.__class__ == other.__class__ and
                self._drips == other._drips):
            return True
        else:
            return False

    def __repr__(self):
        return "drips={}".format(self._drips)

class SetDripCountMessage(ProtoBuffableMessage):
    TYPE_ID = 4

    def __init__(self, drips):
        self._drips = drips

    @property
    def drips(self):
        return self._drips

    def get_bytes(self):
        encoded = SetDripCount()
        encoded.drips = self._drips
        if encoded.IsInitialized():
            return encoded.SerializeToString()
        else:
            logger.error("Protobuf Message encoding incomplete. Did the spec change? Have you compiled your proto files?")
            raise Exception("Protobuf Message encoding incomplete")

    @classmethod
    def from_bytes(cls, proto_bytes):
        decoded = SetDripCount()
        decoded.ParseFromString(proto_bytes)
        return cls(decoded.drips)

    def __eq__(self, other):
        if (self.__class__ == other.__class__ and
                self._drips == other._drips):
            return True
        else:
            return False

    def __repr__(self):
        return "drips={}".format(self._drips)

class MoveToDripCountMessage(ProtoBuffableMessage):
    TYPE_ID = 5

    def __init__(self, drips):
        self._drips = drips

    @property
    def drips(self):
        return self._drips

    def get_bytes(self):
        encoded = MoveToDripCount()
        encoded.drips = self._drips
        if encoded.IsInitialized():
            return encoded.SerializeToString()
        else:
            logger.error("Protobuf Message encoding incomplete. Did the spec change? Have you compiled your proto files?")
            raise Exception("Protobuf Message encoding incomplete")

    @classmethod
    def from_bytes(cls, proto_bytes):
        decoded = MoveToDripCount()
        decoded.ParseFromString(proto_bytes)
        return cls(decoded.drips)

    def __eq__(self, other):
        if (self.__class__ == other.__class__ and
                self._drips == other._drips):
            return True
        else:
            return False

    def __repr__(self):
        return "drips={}".format(self._drips)


"""
-u'' + u^3 = pi^2 cos(pi x) + cos^3(pi x)    on (-1, 1)

with pure Neumann BC and solution u_star(x) = cos(pi x)

Implemented with Deep Ritz Approach.

"""

import jax
import jax.numpy as jnp
from jax import random, grad, vmap, jit

from natgrad.domains import Interval
import natgrad.mlp as mlp
from natgrad.integrators import TrapezoidalIntegrator
from natgrad.gram import gram_factory, nat_grad_factory
from natgrad.utility import grid_line_search_factory, model_nonlinear
from natgrad.inner import model_del_i_factory

jax.config.update("jax_enable_x64", True)

seed = 0

# integration
interval = Interval(-1., 1.)
integrator = TrapezoidalIntegrator(interval, 18000)
eval_integrator = TrapezoidalIntegrator(interval, 80000)

# model
activation = lambda x : jnp.tanh(x)
layer_sizes = [1, 16, 1]
params = mlp.init_params(layer_sizes, random.PRNGKey(seed))
model = mlp.mlp(activation)
v_model = vmap(model, (None, 0))

# solution and right-hand side
u_star = lambda x: jnp.reshape(jnp.cos(jnp.pi * x), ())
f = lambda x: (jnp.pi**2) * u_star(x) + u_star(x)**3

# gramians
model_derivative = model_del_i_factory()
gram_grad = gram_factory(
    model = model,
    trafo = model_derivative,
    integrator = integrator,
)

gram_nonlinear = gram_factory(
    model = model,
    trafo = model_nonlinear,
    integrator = integrator,
)

@jit
def gram(params):
    return gram_grad(params) + gram_nonlinear(params)

nat_grad = nat_grad_factory(gram)

# loss function
grad_model = vmap(grad(lambda params, x: model(params, x), 1), (None, 0))

@jit
def loss_gradient(params):
    grad_squared = lambda x: 0.5 * jnp.reshape(grad_model(params, x)**2, (len(x)))
    return integrator(grad_squared)

@jit
def loss_lower_order_term(params):
    model_to_four = lambda x: 0.25 * v_model(params, x)**4
    return integrator(model_to_four)

@jit
def loss_rhs(params):
    rhs = lambda x: vmap(f, (0))(x) * v_model(params, x)
    return integrator(rhs)

@jit
def loss(params):
    return loss_gradient(params) + loss_lower_order_term(params) - loss_rhs(params)

# set up grid line search
grid = jnp.linspace(0, 30, 31)
steps = 0.5**grid
ls_update = grid_line_search_factory(loss, steps)

# errors
error = lambda x: model(params, x) - u_star(x)
v_error = vmap(error, (0))
v_error_abs_grad = vmap(
        lambda x: jnp.dot(grad(error)(x), grad(error)(x))**0.5
        )

def l2_norm(f, integrator):
    return integrator(lambda x: (f(x))**2)**0.5

for iteration in range(1000):
    grads = grad(loss)(params)
    nat_grads = nat_grad(params, grads)
    params, actual_step = ls_update(params, nat_grads)

    if iteration % 10 == 0:
        # errors
        l2_error = l2_norm(v_error, eval_integrator)
        h1_error = l2_error + l2_norm(v_error_abs_grad, eval_integrator)

        print(
            f'NG Iteration: {iteration} with loss: {loss(params)} with error '
            f'L2: {l2_error} and error H1: {h1_error} and step: {actual_step}'
        )


import json

data = json.load(open('vt_minute_output.json', 'r'))

class Counter:
	def __init__(self):
		self.num_appear = 0
		self.num_detected = 0
		self.percent = None

	def __str__(self):
		return '{}/{} ({})'.format(self.num_detected, self.num_appear, self.percent)

	def appear(self):
		self.num_appear += 1
		self.update_percent()

	def detected(self):
		self.num_detected += 1
		self.update_percent()

	def update_percent(self):
		self.percent = self.num_detected / self.num_appear

cnt = {}

for record in data:
	for name, result in record['scans'].items():
		if name not in cnt:
			cnt[name] = Counter()

		cnt[name].appear()
		if result['detected']:
			cnt[name].detected()

cnt_sorted = sorted(cnt.items(), key=lambda x: x[1].percent, reverse=True)

for key, item in cnt_sorted:
	print(key, item)


from catboost import CatBoostClassifier
from sklearn.metrics import classification_report

from utils.logger import get_logger

class CatBoost:
    def __init__(self, args):
        self.args = args
        self.logger = get_logger(args.log_dir, args.model)
        self.params = {
            'iterations': 1000,
            'learning_rate': 0.4,
            'depth': 3,
            'loss_function': 'MultiClass'
        }

    def fit(self, train_dataset, test_dataset):
        X, y, sample_ids = train_dataset
        clf = CatBoostClassifier(**self.params)
        clf.fit(X, y)
        pred_labels = clf.predict(X)
        self.logger.debug("\n{}".format(
            classification_report(y, pred_labels, digits=4)
        ))
        pred_labels = clf.predict(test_dataset[0])
        self.logger.debug("\n{}".format(
            classification_report(test_dataset[1], pred_labels, digits=4)
        ))


def shell_sort(arr):
    sublistcount = len(arr)//2

    # While we still have sub lists
    while sublistcount > 0:
        for start in range(sublistcount):
            # Use a gap insertion
            gap_insertion_sort(arr, start, sublistcount)
        sublistcount = sublistcount // 2

def gap_insertion_sort(arr, start, gap):

    for i in range(start+gap, len(arr), gap):
        currentvalue = arr[i]
        position = i
        # Using the Gap
        while position >= start+gap and arr[position-gap] > currentvalue:
            # swap the elements at these indexes
            arr[position] = arr[position-gap]
            position = position-gap

        # Set current value
        arr[position] = currentvalue

arr = [7, 3, 4, 10, 9, 8, 1, 6]
shell_sort(arr)
print(arr)


#!/usr/bin/python3
"""
Contains the MyInt class
"""

class MyInt(int):
    """
    A subclass of the int python builtin class, which overload the
    __eq__ and __nq__ class methods
    """

    def __init__(self, value):
        """
        Conistructor methood
        """

        self.__value = value

    def __ne__(self, other):
        """
        A class methos for the `==` logical operation
        """
        if type(other) not in [MyInt, int, float]:
            return False
        if self.__value > other or self.__value < other:
            return False
        return True

    def __eq__(self, other):
        """
        A class methos for the `==` logical operation
        """
        return not (self.__ne__(other))

if __name__ == "__main__":
    my_i = MyInt(3)
    print(my_i)
    print("my_i == 3 : {}".format(my_i == 3))
    print("my_i != 3 : {}".format(my_i != 3))
    print("my_i == 'a' : {}".format(my_i == 'a'))
    print("my_i != 'a' : {}".format(my_i != 'a'))


from .. import Provider as SsnProvider

def checksum(sin):
    """
    Determine validity of a Canadian Social Insurance Number.
    Validation is performed using a modified Luhn Algorithm.  To check
    the Every second digit of the SIN is doubled and the result is
    summed.  If the result is a multiple of ten, the Social Insurance
    Number is considered valid.

    https://en.wikipedia.org/wiki/Social_Insurance_Number
    """

    # Remove spaces and create a list of digits.
    checksumCollection = list(sin.replace(" ", ""))
    checksumCollection = [int(i) for i in checksumCollection]

    # Discard the last digit, we will be calculating it later.
    checksumCollection[-1] = 0

    # Iterate over the provided SIN and double every second digit.
    # In the case that doubling that digit results in a two-digit
    # number, then add the two digits together and keep that sum.

    for i in range(1, len(checksumCollection), 2):
        result = checksumCollection[i] * 2
        if result < 10:
            checksumCollection[i] = result
        else:
            checksumCollection[i] = result - 10 + 1

    # The appropriate checksum digit is the value that, when summed
    # with the first eight values, results in a value divisible by 10

    check_digit = 10 - (sum(checksumCollection) % 10)
    check_digit = 0 if check_digit == 10 else check_digit

    return check_digit

class Provider(SsnProvider):
    # In order to create a valid SIN we need to provide a number that
    # passes a simple modified Luhn Algorithm checksum.
    #
    # This function reverses the checksum steps to create a random
    # valid nine-digit Canadian SIN (Social Insurance Number) in the
    # format '### ### ###'.
    def ssn(self) -> str:
        # Create an array of 8 elements initialized randomly.
        digits = self.generator.random.sample(range(9), 8)

        # The final step of the validation requires that all of the
        # digits sum to a multiple of 10. First, sum the first 8 and
        # set the 9th to the value that results in a multiple of 10.
        check_digit = 10 - (sum(digits) % 10)
        check_digit = 0 if check_digit == 10 else check_digit

        digits.append(check_digit)

        # digits is now the digital root of the number we want
        # multiplied by the magic number 121 212 121. The next step is
        # to reverse the multiplication which occurred on every other
        # element.
        for i in range(1, len(digits), 2):
            if digits[i] % 2 == 0:
                digits[i] = digits[i] // 2
            else:
                digits[i] = (digits[i] + 9) // 2

        # Build the resulting SIN string.
        sin = ""
        for i in range(0, len(digits)):
            sin += str(digits[i])
            # Add a space to make it conform to Canadian formatting.
            if i in (2, 5):
                sin += " "

        # Finally return our random but valid SIN.
        return sin


bl_info = {
 "name": "AN templates",
 "author": "Samy Tichadou (tonton)",
 "version": (0, 3, 0),
 "blender": (2, 83, 7),
 "location": "Animation Nodes Editor > Side Panel",
 "description": "Animation Nodes Templates Online Library",
 "wiki_url": "https://github.com/samytichadou/an_template_addon/blob/master/README.md",
 "tracker_url": "https://github.com/samytichadou/an_template_addon/issues/new",
 "category": "Node",
 "warning": "Beta version, use at your own risks"
 }

import bpy

# IMPORT SPECIFICS
##################################

from .startup_handler import ant_startup_handler

from .properties import *

from .op_open_url import ANTEMPLATES_OT_open_url, ANTEMPLATES_OT_open_url_image, ANTEMPLATES_OT_open_url_video, ANTEMPLATES_OT_open_url_readme
from .op_empty_download_dir import ANTEMPLATES_OT_clear_downloads
from .op_create_manifest import ANTEMPLATES_OT_create_manifest
from .op_create_nodetree_info import ANTEMPLATES_OT_create_nodetree_info
from .op_create_newsfeed import ANTEMPLATES_OT_news_temp_actions, ANTEMPLATES_OT_create_newsfeed
from .op_edit_nodetree_info import ANTEMPLATES_OT_edit_nodetree_info
from .op_import_nodetree import ANTEMPLATES_OT_import_nodetree
from .op_tag_search import ANTEMPLATES_MT_search_tag_menu, ANTEMPLATES_OT_search_tag_menu_caller, ANTEMPLATES_OT_search_tag
from .op_tag_add import ANTEMPLATES_MT_add_tag_menu, ANTEMPLATES_OT_add_tag_menu_caller, ANTEMPLATES_OT_add_tag
from .op_refresh_templates import ANTEMPLATES_OT_refresh_templates
from .op_submit_template import ANTEMPLATES_OT_submit_template
from .op_refresh_addon_version import ANTEMPLATES_OT_refresh_addon_version
from .op_submission_guidelines import ANTEMPLATES_OT_submission_guidelines
from .op_newsfeed_actions import ANTEMPLATES_OT_newsfeed_actions
from .op_search_select_nodetree import ANTEMPLATES_OT_search_select_nodetree

from .gui import (
                    ANTEMPLATES_PT_templates_panel,
                    ANTEMPLATES_PT_import_options_subpanel,
                    ANTEMPLATES_PT_nodetree_infos_subpanel,
                    ANTEMPLATES_PT_news_panel,
                    ANTEMPLATES_PT_utilities_panel,
                    ANTEMPLATES_PT_submission_panel,
                    ANTEMPLATES_PT_updater_subpanel,
                )
from .ui_list import ANTEMPLATES_UL_panel_ui_list

from .addon_prefs import ANTEMPLATESAddonPrefs

# register
##################################

classes = (
            ANTemplatesNodetrees,
            ANTemplatesBlenderVersions,
            ANTemplatesANVersions,
            ANTemplatesCategories,
            ANTemplatesTags,
            ANTemplatesNews,
            ANTemplatesProperties,

            ANTEMPLATES_OT_open_url,
            ANTEMPLATES_OT_open_url_image,
            ANTEMPLATES_OT_open_url_video,
            ANTEMPLATES_OT_open_url_readme,
            ANTEMPLATES_OT_clear_downloads,
            ANTEMPLATES_OT_create_manifest,
            ANTEMPLATES_OT_create_nodetree_info,
            ANTEMPLATES_OT_news_temp_actions,
            ANTEMPLATES_OT_create_newsfeed,
            ANTEMPLATES_OT_edit_nodetree_info,
            ANTEMPLATES_OT_import_nodetree,
            ANTEMPLATES_MT_search_tag_menu,
            ANTEMPLATES_OT_search_tag_menu_caller,
            ANTEMPLATES_OT_search_tag,
            ANTEMPLATES_MT_add_tag_menu,
            ANTEMPLATES_OT_add_tag_menu_caller,
            ANTEMPLATES_OT_add_tag,
            ANTEMPLATES_OT_refresh_templates,
            ANTEMPLATES_OT_submit_template,
            ANTEMPLATES_OT_refresh_addon_version,
            ANTEMPLATES_OT_submission_guidelines,
            ANTEMPLATES_OT_newsfeed_actions,
            ANTEMPLATES_OT_search_select_nodetree,

            ANTEMPLATES_PT_templates_panel,
            ANTEMPLATES_PT_import_options_subpanel,
            ANTEMPLATES_PT_nodetree_infos_subpanel,
            ANTEMPLATES_PT_news_panel,
            ANTEMPLATES_PT_utilities_panel,
            ANTEMPLATES_UL_panel_ui_list,
            ANTEMPLATES_PT_submission_panel,
            ANTEMPLATES_PT_updater_subpanel,

            ANTEMPLATESAddonPrefs,
        )

def register():

    ### OPERATORS ###
    from bpy.utils import register_class
    for cls in classes :
        register_class(cls)

    ### PROPERTIES ###
    bpy.types.WindowManager.an_templates_nodetrees = \
        bpy.props.CollectionProperty(type = ANTemplatesNodetrees, name="AN Template Nodetrees")

    bpy.types.WindowManager.an_templates_properties = \
        bpy.props.PointerProperty(type = ANTemplatesProperties, name="AN Template Properties")

    ### HANDLER ###
    bpy.app.handlers.load_post.append(ant_startup_handler)

def unregister():

    ### OPERATORS ###
    from bpy.utils import unregister_class
    for cls in reversed(classes) :
        unregister_class(cls)

    ### PROPERTIES ###
    del bpy.types.WindowManager.an_templates_nodetrees
    del bpy.types.WindowManager.an_templates_properties

    ### HANDLER ###
    bpy.app.handlers.load_post.remove(ant_startup_handler)