# -*- coding: utf-8 -*-
###########################################################
# Send data from a simple device (raspberry, gpu, watch) to another device
# version 0.5
# syntaxe:
#       python this_script_name remote_ip dataname [value] or python this_script_name start_server
#       eg: python this_script_name 192.168.0.4 HeadTouched
#           will return "1"
#       eg: python this_script_name 192.168.0.4 step 5
#           will return set the data step to "5"
# Aldebaran Robotics (c) 2008 All Rights Reserved
# NB:
# To kill python processus on windows:
# taskkill /im python.exe /f
###########################################################

"""
cd dev\git\protolab_group\scripts\versatile_almemory\
python versatile.py start_server
"""

try: import cv2
except: print( "WRN: OpenCV2 not found, using cv2 methods will fail" )
try: import numpy as np
except: print( "WRN: numpy not found, using numpy methods will fail" )
import socket
import struct
import sys
import time

def myOrd( a ):
    if sys.version_info[0] < 3:
        return ord(a)
    return a

def stringToBytes(s):
    """
    convert string to bytes, whatever the python version
    """

    # due to recursing into list, if the string is already bytified, do nothing !
    if not isinstance( s, str ):
        return s

    if sys.version_info[0] < 3:
        return s
    data = bytes()
    for i in range(len(s)):
        data += struct.pack( 'c', s[i].encode('ascii') ) # 'UTF-8'
    return data

def _waitSizedData( socket_object, bVerbose = False ):
    """
    wait for an int little endian formatted, followed by this amount of data
    return (data (without size field), size)
    data can be None, meaning the connection has been closed
    """
    try:
        data = socket_object.recv(4) # [Errorno 104]: Connection reset by peer !
    except Exception as err:
        print( "ERR: Versatile._waitSizedData: exception(1): %s" % str(err) )
        return None, 0

    nCptNoData = 0
    while( len(data) < 4 ):
        #~ print( "DBG: versatile._waitSizeData. data too short and receiving zero ?!? (end cnx?)" )
        time.sleep( 0.4) # warning, all server is blocked at this point !!! (at least on windows)
        data = socket_object.recv(4)
        nCptNoData += 1
        if nCptNoData > 2:
            print( "WRN: versatile._waitSizeData. client '%s' disappear, closing..." % str(socket_object.getpeername() ) )
            return None, 0
    #~ print( "DBG: Versatile._waitSizedData: 4 bytes received: '%s' (len:%d)" % (str(data),len(data)) )
    nWaitedSize = struct.unpack( "<I", data )[0]
    #print( "DBG: Versatile._waitSizedData: packet waited size: %s" % str(nWaitedSize) )
    nSize = 0
    if 0:
        data = []
        while nSize < nWaitedSize:
            data.append( socket_object.recv(1) )
            nSize += 1
            #~ print( "DBG: Versatile._waitSizedData: packet current size: %d/%d" % (nSize,nWaitedSize) )
    else:
        if nWaitedSize > 3*1024*1024: # max 3 Mo
            print( "ERR: packet to big, error reading? (size:%s)" % str(nWaitedSize) )
            print( "data: %s" % str(data) )
            if 1:
                databuf = socket_object.recv(1024*1024) # flushing a fair amount #recvall ?
                print( "DBG: flushed: %s" % len(databuf) )
            time.sleep(1)
            return None,0
        try:
            data = socket_object.recv(nWaitedSize) # TODO: is there a timeout that we can increase? (to prevent the patch below)
            if len(data) != nWaitedSize:
                if 0: print( "WRN: Versatile._waitSizedData: size is different %s!=%s (will continue to read, until ok)" % (nWaitedSize, len(data)))
                if 1:
                    while len(data)< nWaitedSize:
                        data += socket_object.recv(nWaitedSize-len(data))
                        time.sleep(0.001)

        except Exception as err:
            print( "ERR: Versatile._waitSizedData: exception: %s" % str(err) )
            print( "DBG: Versatile._waitSizedData: packet waited size: %s" % str(nWaitedSize) )
            if 0:
                # try to open close
                ip, port = socket_object.getpeername()
                print( "ip: %s" % ip )
                print( "port: %s" % str(port) )
                print( "WRN: versatile._waitSizeData. client '%s' error, reopening..." % str( ) )
                socket_object.shutdown(socket.SHUT_RDWR);
                socket_object.close()
                socket_object.connect((ip,port))
            return None, 0 # Alma 2018-10-23: adding this return, not so sure...

    #~ print( "DBG: Versatile._waitSizedData: data: %s (len:%d)" % (str(data),len(data) ) )
    #~ print(dir(socket_object))

    return ( data, nSize )

class Versatile:
    """
    Send data from a simple device (raspberry, gpu, watch) to a robot thru ALMemory.

    Datas are passed thru socket (default port is 10001) using this packet format:
    (using little endian format)

    0: Packet size: size of the packet excluding this field - DWORD
    4: Command or type - CHAR
                            0: ping - check working - will always return a value 1
                            1: value. followed by a value. eg, an answer to some command...
                            2: get value from a named variables container (eg a STM). followed by a string (cf string)
                            3: set value to a named variables container (eg a STM). followed by a string and a value (cf value multi type style) - will return True or False if error

                            4: ask for a new client ID  - (no param) - return a string
                            5: set client ID (for future call) - (param: id value (string))
                            6: set a parameter related to this client - (param: parameter_name, parameter_value)
                            7: get this parameter  - (param: parameter_name)

                            10: register for an image stream. followed by a camera index (0..255), a resolution (see VersatileResolution), an image format (see VersatileImageFormat) and a required fps (1..1000).
                            11: unregister from an image stream. followed by the camera index, resolution and format (NB: you can't ask twice for the same (camera,res,and format)
                            20: register for a sound stream
                            21: unregister from the sound stream

                            30: get a broadcasted image - followed by a broadcaster id

    5: Command parameters
        length dependant of the command

    # all packet send will be acknowledged with at least a 1

    """
    nCommandType_EOC   = -1
    nCommandType_Ping   = 0
    nCommandType_Value = 1
    nCommandType_Get    = 2
    nCommandType_Set    = 3

    nCommandType_CreateClientID       = 4
    nCommandType_SetClientID            = 5
    nCommandType_SetClientParam     = 6
    nCommandType_GetClientParam     = 7

    nCommandType_SubscribeCamera    = 10
    nCommandType_UnsubscribeCamera    = 11
    nCommandType_SubscribeSound    = 20
    nCommandType_UnsubscribeSound    = 21

    nCommandType_GetBroadcasted    = 30

    @staticmethod
    def  isCommandRequiringDataName( nNumCommand ):
        return nNumCommand == Versatile.nCommandType_Get \
            or nNumCommand == Versatile.nCommandType_Set \
            or nNumCommand == Versatile.nCommandType_GetClientParam \
            or nNumCommand == Versatile.nCommandType_SetClientParam \
            or nNumCommand == Versatile.nCommandType_GetBroadcasted

    @staticmethod
    def  isCommandRequiringValue( nNumCommand ):
        return nNumCommand == Versatile.nCommandType_Value \
            or nNumCommand == Versatile.nCommandType_Set \
            or nNumCommand == Versatile.nCommandType_SetClientID \
            or nNumCommand == Versatile.nCommandType_SetClientParam

    @staticmethod
    def  commandNumToLibelle( nNumCommand ):
        listLibelle = [
            "Ping",
            "Value",
            "Get",
            "Set",
            "CreateClientID",
            "SetClientID",
            "SetClientParam",
            "GetClientParam",
            "SubscribeCamera",
            "UnsubscribeCamera",
            "SubscribeSound",
            "UnsubscribeSound",
            "GetBroadcasted",
        ]
        if nNumCommand == -1:
            return "EOC"
        if nNumCommand >= len(listLibelle):
            return "Unknown command"
        return listLibelle[nNumCommand]

    class VersatileString:
        """
        a string stored in a string starting by the length of the string.
        0: length of the stringin byte (DWORD)
        4: string (...)
        """
        def __init__( self, s = "" ):
            self.s = s

        def set( self, s ):
            self.s = s

        def toPacket( self ):
            """
            convert object to string
            """
            data = struct.pack('<I', len(self.s)) # '<' => little endian
            #~ data += self.s
            # python 3 want explicit conversion (that's not so silly)
            #~ data += struct.pack("s", self.s.encode('ascii'))
            #~ for i in range(len(self.s)):
                #~ data += struct.pack( 'c', self.s[i].encode('ascii') ) # 'UTF-8'
            data += stringToBytes( self.s )

            #~ print( "DBG: len data: %s" % len(data) )

            return data

        @staticmethod
        def fromPacket( data, nOptionnalSize = -1 ):
            """
            decode a string from a packet
            return value, size used by the value in data
            """
            if nOptionnalSize == -1:
                nSize = struct.unpack_from("<I",data)[0]
                nOffset = 4
            else:
                nSize = nOptionnalSize
                nOffset = 0

            if sys.version_info[0] < 3:
                s = data[nOffset:nOffset+nSize]
            else:
                s = ""
                #~ print(" nSize: %s" % nSize )
                for i in range(nSize):
                    c = struct.unpack_from("<c",data[nOffset+i:nOffset+nSize+i])[0]
                    #~ print( "c: %s" % c )
                    c = c.decode('UTF-8')
                    s += str(c)

            return s, nSize+4
    # class VersatileString - end

    @staticmethod
    def autodetectType( v ):
        #~ print( "DBG: type v: %s" % str(type(v)) )
        if v == None:
            return 0
        if isinstance( v, int ) or isinstance( v, bool ):
            return 1
        if sys.version_info[0] < 3: # in python3 int type has now an unlimited length
            if isinstance( v, long ):
                if v > 0x7FFFFFFF:
                    print( "WRN: Versatile: truncation of long to int, value: %d" % v )
                return 1
        if isinstance( v, float ): # NB: True for both Unicode and byte strings
            return 2
        if isinstance( v, str ): # NB: True for both Unicode and byte strings # basestring change so str to switch to python3
            return 3
        #if isinstance( v, Versatile.VersatileImage ): #v est de type instance !
        #~ print(dir(v))
        #~ print(hasattr(v,"bIsGrey"))
        #~ print("str: %s" % str(str(v)) )
        if( hasattr(v,"bIsGrey") or hasattr(v,"aCommand") ): # CRADO, but only way to check it's a VersatileImage
            return 4

        if isinstance( v, list ) or isinstance( v, tuple ) :
            return 9

        print( "WRN: Versatile.autodetectType: unable to detect type of '%s' (current type: %s)" % ( str(v), str(type(v)) ) )
        return 0

    class VersatileValue:
        """
        a value stored in a string compound of a length+type+data.
        0: length the length taken by the data (without the type) in byte (DWORD)
        4: type (BYTE)
            1: int (or boolean converted to int)
            2: float
            3: string
            4: image
            5: audio chunk
            9: list of versatile values
            ...
        5: data (...)
        """
        nValueType_None   = 0
        nValueType_Int      = 1
        nValueType_Float   = 2
        nValueType_String = 3
        nValueType_Image = 4 # (see VersatileImage)
        nValueType_AudioChunk = 5 # (see VersatileAudioChunk)
        nValueType_List = 9 # list of VersatileValue

        def __init__( self, value = None, nNumEnsureType = None ):
            self._set( value, nNumEnsureType )

        def _set( self, value, nNumEnsureType ):
            if isinstance( value, Versatile.VersatileValue ):
                # construct from a VersatileValue
                self.v = value.v
                assert( nNumEnsureType == None ) # can't force a type when passing a VersatileValue
                self.nType = value.nType
                return

            self.v = value

            if nNumEnsureType == None:
                # auto detect
                nNumEnsureType = Versatile.autodetectType(value)
            self.nType = nNumEnsureType

            if isinstance( self.v, bool ):
                if self.v: self.v = 1
                else: self.v = 0

        def set( self, value, nNumEnsureType = None ):
            self._set( value, nNumEnsureType )

        def toPacket( self ):
            """
            convert object value to string
            """
            if 0: print( "DBG: VersatileValue.toPacket: packeting a value of type %s, value: '%s'" % ( str(self.nType), str(self.v) ) )

            #print struct.calcsize('B') #1
            #~ print(str(self))
            #~ print(str(self.v))
            #~ print(str(self.nType))
            if self.v == None:
                data = struct.pack('<I', 0) + struct.pack('B', self.nType)
            else:
                # TODO: perhaps explictly encode data using struct.pack?
                if self.nType == Versatile.VersatileValue.nValueType_Image:
                    vAsString = self.v.toPacket()

                elif self.nType == Versatile.VersatileValue.nValueType_List:
                    # we encode in the data the len of element then elements packetised
                    vAsString = struct.pack('<I', len(self.v) )
                    for i in range(len(self.v)):
                        valueToEncode = self.v[i]
                        # sometims VersatileValue can be lazy constructed with some base type included in them (like a list of int, instead of a list of versatilevalue of type int), as it's more efficient, it does'nt hurt...
                        if not isinstance( valueToEncode, Versatile.VersatileValue ):
                            valueToEncode = Versatile.VersatileValue(valueToEncode)
                        vAsString += valueToEncode.toPacket()
                else:
                    # TODO: explicit encoding for string ?
                    vAsString = str(self.v)

                data = struct.pack('<I', len(vAsString)) + struct.pack('B', self.nType) + stringToBytes(vAsString)
            return data

        @staticmethod # comme staticmethod, but permits using static class variable nValueType_Int # but some stuffs are then changed.
        def fromPacket( data ):
            """
            decode a value from a packet
            return value, size used by the value in data
            """
            #~ for i in range(min(len(data),8)):
                #~ print( "DBG: data[%d] = %x" % ( i, myOrd(data[i])) )
            nSize = struct.unpack_from('<I',data)[0]
            nType = struct.unpack_from('B',data[4:4+1])[0]
            nIdxStart = 5
            nIdxEnd = 5+nSize

            # TODO: perhaps explictly decode data using struct.unpack?

            if nType == Versatile.VersatileValue.nValueType_None:
                value = None
            if nType == Versatile.VersatileValue.nValueType_Int:
                value = int(data[nIdxStart:nIdxEnd])
            elif nType == Versatile.VersatileValue.nValueType_Float:
                value = float(data[nIdxStart:nIdxEnd])
            elif nType == Versatile.VersatileValue.nValueType_String:
                #value = str(data[nIdxStart:nIdxEnd]) # why not using fromPacket de string !!! - DONE
                value = Versatile.VersatileString.fromPacket(data[nIdxStart:nIdxEnd],nSize)[0]

            elif nType == Versatile.VersatileValue.nValueType_Image:
                value = Versatile.VersatileImage.fromPacket(data[nIdxStart:nIdxEnd]) # nSize from fromPacket isn't used  ?!?

            elif nType == Versatile.VersatileValue.nValueType_List:
                nNbrElement = struct.unpack_from('<I',data[nIdxStart:nIdxStart+4])[0]
                nIdxStart += 4
                value = []
                for i in range(nNbrElement):
                    v,size = Versatile.VersatileValue.fromPacket(data[nIdxStart:]) # we don't now size here
                    nIdxStart += size
                    value.append( v )
            else:
                print( "ERR: VersatileValue.fromPacket, unknown value type: %s" % str(nType) )
            #~ print( "DBG: VersatileValue.fromPacket, decoded value of type %s: %s" % (str(nType), str(value) ) )
            return value, nSize+5
    # class VersatileValue - end

    class VersatileImage:
        """
        store an image and its metadata
        """
        class WantedResolution:
            # client ask for a resolution that will be matched as nearest possible
            QQVGA=0
            QVGA=1
            VGA=2
            x960=3 # as: 1024x960
            x1024=4 # as: 1280x1024
            x1280=5 # as: 1920x1080

        class Format:
            GREY=1
            JPG=2
            PNG=3
            RAW=4

        def __init__( self, bVerbose = False ):
            self.data = None # image data ready for sending (one dimension, string encoded, ...)
            self.aCommand = [] # you can add a list of parameters # see addCommand for doc
            self.bVerbose = bVerbose

            self.bIsGrey = False
            self.timeStamp = 0,0
            self.data = ""

        def createFromCvImage( self, img, timeStamp = None, nFormat = Format.JPG ):
            """
            create a VersatileImage from a cv2.buffer
            """
            if img is None:
                return False

            # reducing image
            while img.shape[0]>1080:
                print("WRN: versatile.createFromCvImage: reducing image..." )
                img = cv2.resize( img, (img.shape[1]//2,img.shape[0]//2) )

            if timeStamp == None:
                timeStamp = time.time()
            self.timeStamp = int(timeStamp), int(timeStamp*1000)%1000 # on 2 int: seconds & ms
            self.bIsGrey = (nFormat == Versatile.VersatileImage.Format.GREY)
            #self.data = img.reshape( self.nW*self.nH*nChannel,1,1).tostring()
            # encoding:
            #~ print( "DBG: createFromCvImage: encoding format is %d" % nFormat )
            encodeParam = None
            if nFormat == Versatile.VersatileImage.Format.PNG:
                strFormat = ".png"
            if nFormat == Versatile.VersatileImage.Format.JPG:
                strFormat = ".jpg"
                encodeParam = [int(cv2.IMWRITE_JPEG_QUALITY),85]

            #~ print( "DBG: createFromCvImage: strFormat is %s" % strFormat )
            #~ print( "DBG: createFromCvImage: self.bIsGrey is %s" % self.bIsGrey )
            timeBegin = time.time()
            self.data = cv2.imencode( strFormat, img, encodeParam )[1].tostring()
            if 1:
                # encryption
                pass
            #~ print( "INF: VersatileImage.createFromCvImage: encoding image takes: %5.3fs" % (time.time()-timeBegin) ) # 0.01 on Pepper in VGA
            #print( "self.img in: %s" % str(img.size) )
            #print( "DBG: VersatileImage.createFromCvImage: compression out: %s in: %s" % ( len(self.data), img.size ) )

        def convertToCvImage( self ):
            """
            return a cv2.buffer from a VersatileImage
            """
            #return np.fromstring(self.data, np.uint8).reshape( self.nH,self.nW,nChannel)
            if self.bIsGrey:
                flag = 0
            else:
                try:
                    flag = cv2.CV_LOAD_IMAGE_COLOR # cv2 flag
                except:
                    flag = cv2.IMREAD_COLOR # cv3 flag
            #~ print( "DBG: convertToCvImage: flag is %s" % flag )
            #~ print( "DBG: convertToCvImage: type data: %s" % type(self.data) )
            #~ print( "DBG: convertToCvImage: len data: %s" % len(self.data) )

            if len(self.data) == 0:
                # empty image for some specific command
                print("WRN: convertToCvImage: received an empty image - possible for some command like clear_db...")
                img = None
            else:
                img = cv2.imdecode(np.fromstring(self.data, np.uint8), flag )
            #print( "DBG: VersatileImage.convertToCvImage: decoded size: %s" % ( img.size ) )
            return img

        def addCommand( self, listParameter ):
            """
            add a list of command and parameters to this image.
            The remote will send information related to this work
            eg:
                - ["facedetection"] => list face and confidence as an answer
                - ["facerecognition", "db_name", "learn", "some_name"] => True/False and confidence on learning (based on comparison on existing face)
                - ["facerecognition", "db_name", "recognise"] => recognition result
                - ["store"] => store this image on some disk
                - ["show"] => show this image on screen
                - ["broadcast"] => show this image to any viewer

            """
            for param in listParameter:
                if self.bVerbose: print( "INF: VersatileImage.addCommand: adding '%s'" % str(param) )
                self.aCommand.append(param)

        def toPacket( self ):
            """
            convert object value to string
            format: B: is grey, size of data_img, data_img
            """
            #~ print( "DBG: VersatileValue.toPacket: packeting a value of type %s, value: '%s'" % ( str(self.nType), str(self.v) ) )

            data = struct.pack('B', self.bIsGrey)  \
                        + struct.pack('<I', self.timeStamp[0]) \
                        + struct.pack('<I', self.timeStamp[1]) \
                        + struct.pack('<I', len(self.data)) \
                        + stringToBytes(self.data)
            data += Versatile.VersatileValue( self.aCommand ).toPacket()
            return data

        @staticmethod
        def fromPacket( data ):
            """
            decode a value from a packet
            return aVersatileImage, size used by the value in data
            """
            vi = Versatile.VersatileImage()
            offset = 0
            vi.timeStamp = [0,0]
            if sys.version_info[0] == 3:
                vi.bIsGrey = data[offset]; offset += 1
            else:
                vi.bIsGrey = struct.unpack_from('B',data[offset])[0]; offset += 1
            vi.timeStamp[0] = struct.unpack_from('<I',data[offset:offset+4])[0] ; offset += 4
            vi.timeStamp[1] = struct.unpack_from('<I',data[offset:offset+4])[0] ; offset += 4
            nSize = struct.unpack_from('<I',data[offset:offset+4])[0] ; offset += 4
            #vi.data = struct.unpack_from('B'*nSize,data[offset:offset+nSize]) # $$$$ found method to get a variable decoding!
            vi.data=data[offset:offset+nSize] ; offset += nSize
            #print( "datalen: %s" % (len(vi.data)) )

            param, sizeParams = Versatile.VersatileValue.fromPacket(data[offset:])
            print( "DBG: Versatile Image: decoded params: ")
            for i in range( len(param) ):
                print( "\t%s" % param[i])
            vi.aCommand = param
            offset += sizeParams
            print( "DBG: Versatile Image: decoded params - end")
            return vi, offset
    #class VersatileImage - end

    @staticmethod
    def VersatileImage_autoTest():
        print( "INF: VersatileImage_autoTest: starting..." )
        vi = Versatile.VersatileImage()
        w=640
        h = 480
        img = np.zeros((h,w,3), np.uint8)
        for j in range(h):
            for i in range(w):
                img[j,i,0] = i*(j+3)
        img = cv2.imread( "../camera_viewer_at_exit.png" )

        vi.createFromCvImage( img, nFormat = Versatile.VersatileImage.Format.PNG)
        imgOut = vi.convertToCvImage()
        assert( np.array_equal( img, imgOut) )

        vi.createFromCvImage( img, nFormat = Versatile.VersatileImage.Format.JPG)
        imgOut = vi.convertToCvImage()
        assert( not np.array_equal( img, imgOut) ) # different due to compression loss

        vi.createFromCvImage( img, nFormat = Versatile.VersatileImage.Format.PNG)
        data=vi.toPacket()
        vi, size = Versatile.VersatileImage.fromPacket(data)
        imgOut = vi.convertToCvImage()
        assert( np.array_equal( img, imgOut) )
        print( "INF: VersatileImage_autoTest: OK\n" )

    #########################################################
    # class Versatile method - start

    def __init__( self, nPort = 10001, bVerbose = False ):
        """
        """
        self.bVerbose = bVerbose
        self.nPort = nPort
        self.bMustStop = False
        self.threadSendImage=None
        self.aCameraSubs=[] # for each: client_socket,camera idx, reso, format, period, time last send
        self.dictClientID = dict() # a dict adress of client_socket => logical client ID (as returned by nCommandType_CreateClientID and set by nCommandType_SetClientID
        # NB: logical client ID is internally  a number!!!
        self.dictClientParams = dict() # a dict logical client ID => a dict of parameter (a dict of dict)
        self.dictBrodcastedImages = dict() # a dict logical client ID => a list of stored versatileimages

    def setVerbose( self, bVal ):
        self.bVerbose = bVal

    def registerToCamera( self, client_socket, nNumCamera, nResolution, nFormat, rPeriod = 0.2 ):
        print( "INF: client %s register for a camera (num camera: %d, res: %d, format: %d, period: %5.2fs)" % ( str(client_socket.getpeername()),nNumCamera, nResolution, nFormat, rPeriod) )
        self.aCameraSubs.append( [client_socket, nNumCamera, nResolution, nFormat, rPeriod, time.time()-100] )
        #self.startThreadSendImage() # can't prevent launching many => done at construction!

    def getNewImage( self, nNumCamera, nResolution, nFormat ):
        """
        return (time stamp, image)
        please inherits this method in your own program
        """
        img = np.zeros((480,640,3), np.uint8)
        inc = int(time.time()*100)
        x = (inc%160)
        y = (inc/160)%120
        for i in range(10):
            img[y,x+i,0] = 255 # make it move
            img[y,x+i,1] = 255 # make it move
            img[y,x+i,2] = 255 # make it move
        return (time.time(),img)

    def threadSendImageToClient( self ):
        import threading
        print( "DBG: threadSendImageToClient: start..." )
        while( 1 ):
            for i in range( len( self.aCameraSubs ) ):
                client_socket, nNumCamera, nResolution, nFormat, rPeriod, timeLastSend = self.aCameraSubs[i]
                if time.time() - timeLastSend >= rPeriod:
                    self.aCameraSubs[i][-1] = time.time()
                    ts, img = self.getNewImage( nNumCamera, nResolution, nFormat )
                    vi = Versatile.VersatileImage()
                    vi.createFromCvImage( img, ts )
                    vv = Versatile.VersatileValue( vi )
                    retVal = self._send( Versatile.nCommandType_Value, None, vv, optionnalClient = client_socket )
                    if retVal == False:
                        print( "INF: threadSendImageToClient: removing client %d" % i )
                        try:
                            del self.aCameraSubs[i]
                        except BaseException as err:
                            print( "WRN: error while deleting: %s" % str(err) )
                        break; # all indexs are offseted, let's restart iteration...
                        # HERE, we could finish this thread if no more subs
            time.sleep(0.01)
            #time.sleep(1.) # to debug
            #~ print( "DBG: threading.currentThread().getName(): %s" % threading.currentThread().getName())
            #~ print(".")

    def startThreadSendImage( self ):
        # todo: mutex
        print( "DBG: startThreadSendImage: start" )
        import multiprocessing
        import threading
        if self.threadSendImage != None:
            print( "DBG: startThreadSendImage: already started !!!" )
            return
        #~ print(dir(multiprocessing))
        # self.threadSendImageToClient() # pour tester sans multithreading (mais va locker apres le premier client)
        #self.threadSendImage = multiprocessing.Process( target=self.threadSendImageToClient ).start() # on windows, objects are pickled and socket is'nt picklable so, raise errors
        self.threadSendImage = True # to be sure
        self.threadSendImage = threading.Thread( target=self.threadSendImageToClient ).start() # under windows ne semble pas lancer dans un thread, mais bloque tout le thread
        print( "DBG: startThreadSendImage: end" )

    def handleCommand( self, command, command_parameters, clientSocket = None ):
        """
        please inherits this method in your own program.
        return None on error
        """

        if self.bVerbose: print( "DBG: Versatile.handleCommand: received command: %s,%s" % (command, command_parameters) )

        if not hasattr(self, "simulatedMemory"):
            self.simulatedMemory = dict() # to store command

        if command == Versatile.nCommandType_Ping:
            return 1

        if command == Versatile.nCommandType_Value:
            print( "WRN: Versatile.handleCommand: Abnormal case: Received this value: %s" % command_parameters )
            return None

        if command == Versatile.nCommandType_Get:
            #valueFromALMemory = "(a value taken from ALMemory data named: '%s')" % command_parameters[0]
            #valueFromALMemory = "toto"
            strDataName = command_parameters[0]
            try:
                valueFromALMemory = self.simulatedMemory[strDataName]
            except:
                if self.bVerbose: print( "DBG: data '%s' isn't in the Memory" % (strDataName) )
                valueFromALMemory = None # non existing value
            return valueFromALMemory

        if command == Versatile.nCommandType_Set:
            strDataName = command_parameters[0]
            value = command_parameters[1]
            if self.bVerbose: print( "DBG: data '%s' should be set to %s" % (strDataName, value) )
            self.simulatedMemory[strDataName] = value
            return 1

        if command == Versatile.nCommandType_CreateClientID:
            time.sleep(0.5)
            nNewClientID = int(time.time()*100)%0x80000000 # the goal is to ensure there won't be two client with same ID. (but that id is storable on a int32)
            print( "INF: Versatile.handleCommand: creating a new client ID: %s" % nNewClientID )
            return str(nNewClientID)

        if command == Versatile.nCommandType_SetClientID:
            value = str(command_parameters[0])
            if id(clientSocket) in self.dictClientID.keys():
                if self.dictClientID[id(clientSocket)] == value:
                    return 1 # don't do anything...
                self.handleClientRemoveIdentification( clientSocket )
            if self.bVerbose: print( "INF: client %s has idenfied himself as %s" % ( str(id(clientSocket)), value ) )
            self.dictClientID[id(clientSocket)] = value
            self.handleClientIdentified( clientSocket )
            return 1

        if command == Versatile.nCommandType_GetClientParam:
            #valueFromALMemory = "(a value taken from ALMemory data named: '%s')" % command_parameters[0]
            #valueFromALMemory = "toto"
            strDataName = command_parameters[0]

            try:
                clientID = self.dictClientID[id(clientSocket)]
                valueFromParams = self.dictClientParams[clientID][strDataName]
            except:
                if self.bVerbose:  print( "DBG: data '%s' isn't in the client params" % (strDataName) )
                valueFromParams = None # non existing value
            return valueFromParams

        if command == Versatile.nCommandType_SetClientParam:
            clientID = self.dictClientID[id(clientSocket)]
            strDataName = command_parameters[0]
            value = command_parameters[1]
            if clientID not in self.dictClientParams.keys():
                if self.bVerbose: print( "INF: Versatile.handleCommand: creating a client param id for client %s" % clientID )
                self.dictClientParams[clientID] = dict()
            self.dictClientParams[clientID][strDataName] = value
            return 1

        if command == Versatile.nCommandType_SubscribeCamera:
            if self.bVerbose: print( "DBG: SubscribeCamera: command_parameters: %s" % str(command_parameters) )
            self.registerToCamera( clientSocket, command_parameters[0], command_parameters[1], command_parameters[2], 1./command_parameters[3] )
            return 1

        if command == Versatile.nCommandType_GetBroadcasted:
            #valueFromALMemory = "(a value taken from ALMemory data named: '%s')" % command_parameters[0]
            #valueFromALMemory = "toto"
            strBroadcasterID = command_parameters[0]
            nBroadcasterID = int(strBroadcasterID)
            if not nBroadcasterID in self.dictBrodcastedImages.keys():
                print( "ERR: Versatile.handleCommand: broadcaster %d unknown" %  nBroadcasterID )
                return 0 # unknown client
            return self.dictBrodcastedImages[nBroadcasterID][-1]

        print( "ERR: Versatile.handleCommand: Abnormal case: unknown command type: %s" % command )
        return None

    # handleCommand - end

    def handleNewClientArrival( self, client ):
        """
        A new client arrived, do what you want in inherited classes...
        """
        pass

    def handleClientIdentified( self, client ):
        """
        A client has identified, do what you want in inherited classes...
        """
        pass

    def handleClientRemoveIdentification( self, client ):
        """
        this client want to stop using this identification, do what you want in inherited classes...
        """
        pass

    def handleClientLeft( self, client ):
        """
        Sad news: client left, do what you want in inherited classes...
        """
        pass

    def manageClientRequests( self, client ):
        self.handleNewClientArrival( client )
        try:
            while 1:
                command, command_parameters = Versatile._waitPacket(client, self.bVerbose)
                if command == Versatile.nCommandType_EOC:
                    client.close()
                    break
                if self.bVerbose: print( "DBG: versatile.manageClientRequests: before handling command..." )
                valueToReturn = self.handleCommand( command, command_parameters, client )
                self._send( Versatile.nCommandType_Value, None, valueToReturn, client )
        except socket.error as err:
            print( "ERR: when working with client, received error: %s" % err )
            client.close()
        self.handleClientLeft(client)

    def runServer( self ):
        """
        run an infinite server
        """
        import threading
        self.startThreadSendImage()
        self.socket_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        while True:
            try:
                self.socket_server.bind( ('', self.nPort ) )
                break
            except OSError as err:
                print( "ERR: connection error: %s" % str(err) )
                print( "ERR: retrying in 2 sec..." )
                time.sleep( 2 )

        print( "INF: versatile.runServer: server ready for client connection on port %s..." % self.nPort )

        self.bMustStop = False
        while not self.bMustStop:
            try:
                self.socket_server.listen(5)
                client, address = self.socket_server.accept()
                print( "Versatile: client connect from %s" % str(address) )
                #self.manageClientRequests(client) # only one at a time !
                threading.Thread( target=self.manageClientRequests, args=(client,) ).start() # the ',' after 'client' is important else it's not a tuple

            except socket.error as err:
                print( "ERR: when working with client, received error: %s" % err )
                client.close()

        self.socket_server.close()
        print( "INF: versatile.runServer: stopped." )

    def stopServer( self ):
        print( "INF: versatile.stopServer: stopping..." )
        self.bMustStop = True # known bug: we need to get out of the listen to be stopped !!! :(
        self.disconnect()
        try:
            socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect( ("localhost", self.nPort)) # so we generate a socket to ourself, weird, but works fine.
        except Exception as err:
            print( "DBG: stopServer: socket already closed? err: %s" % str(err) )

    def _reconnect( self, bFailSilently = False ):
        try:
            print( "INF: Versatile: connecting to %s:%d" % (self.ip, self.nPort) )
            self.socket_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.socket_server.connect((self.ip, self.nPort))
        except socket.error:
            if not bFailSilently:
                raise socket.error( "cannot connect to %s:%d" % (self.ip, self.nPort) )
            else:
                print( "WRN: can't (re)connect to %s:%d" % (self.ip, self.nPort) )

    def isConnect( self ):
        pass #TODO

    def connect( self, strIP, bPatientMode = True):
        """
        bPatientMode: if set, even if server not present at start, the module will wait
        """
        self.ip = strIP
        self._reconnect(bFailSilently=bPatientMode)

    def disconnect( self ):
        self.socket_server.shutdown(socket.SHUT_RDWR);
        self.socket_server.close()

    @staticmethod # why !?!
    def _waitPacket( socket_object, bVerbose = False ):
        """
        return a decoded packet as a tuple: (nCommand, command_parameters...)
        can return a nCommand == EOC: connection closed
        """
        p,size = _waitSizedData(socket_object,bVerbose)
        if p == None:
            return (Versatile.nCommandType_EOC,None)
        nCommand = struct.unpack_from("B",p)[0]
        offset = 1

        if bVerbose: print( "DBG: Versatile._waitPacket: receive command: %d (%s)" % (nCommand,Versatile.commandNumToLibelle(nCommand)) )

        commandParameters = []
        if Versatile.isCommandRequiringDataName( nCommand ):
            strDataName, size = Versatile.VersatileString.fromPacket(p[offset:])
            commandParameters.append( strDataName )
            offset+=size

        if Versatile.isCommandRequiringValue( nCommand ):
            value, size = Versatile.VersatileValue.fromPacket(p[offset:])
            commandParameters.append( value )
            offset+=size

        if nCommand == Versatile.nCommandType_SubscribeCamera:
            for i in range(4):
                value, size = Versatile.VersatileValue.fromPacket(p[offset:])
                commandParameters.append( value )
                offset+=size

        return (nCommand, commandParameters)

    @staticmethod
    def _waitValue( socket_object ):
        """
        return a decoded value as a python object (but encoded in a packet for more security)
        """
        #data,size = _waitSizedData(socket_object)
        #value, size = Versatile.VersatileValue.fromPacket(data)
        nCommand, commandParameters = _waitPacket( socket_object )
        assert( nCommand == Versatile.nCommandType_Value )
        return value

    def _send( self, nNumCommand, strDataName = None, value = None, optionnalClient = None ):
        """
        send packet to peer
        strDataName: in case of a stm mode
        value: a list of value to send
        optionnalClient: filed if the process is a server, permits to know the client (at least to print a debug msg)
        return:
            - False on error, or (nCommand, command_parameters...) in case of returned value from complex command
        """

        if self.bVerbose: print( "DBG: Versatile._send: sending command %s (%s), dataname: %s, value: %s" %(nNumCommand,Versatile.commandNumToLibelle(nNumCommand),strDataName, value) )

        data = struct.pack('B', nNumCommand)

        if nNumCommand == Versatile.nCommandType_SubscribeCamera:
            # value is a tuple
            for v in value:
                print( "DBG: _send: encoding command parameter: %s" % v )
                if self.bVerbose: print( "DBG: _send: encoding command parameter: %s" % v )
                vv = Versatile.VersatileValue(v)
                data += vv.toPacket()
        else:
            # ca mériterais un petit refactor ca (ce cas particulier est vraiment pas beau)
            if strDataName != None:
                vs = Versatile.VersatileString(strDataName)

            vv = Versatile.VersatileValue( value ) # in any case we can create a value, even if it's None

            if Versatile.isCommandRequiringDataName( nNumCommand ):
                # we encode the dataname
                data += vs.toPacket()

            if Versatile.isCommandRequiringValue( nNumCommand ):
                # we encode a value
                data += vv.toPacket()

        if self.bVerbose and 0:
            nPrintLen = 64
            print( "DBG: Versatile._send: data to send (without encoded len) (first %d bytes):"  % nPrintLen )
            for i in range(min(nPrintLen, len(data))):
                if isinstance(data[i], int):
                    print( "0x%X " % data[i] ),
                else:
                    print( "s0x%X " % ord(data[i]) ),
            print("" ) # \n

        data = struct.pack('<I', len(data)) + data

        if self.bVerbose: print( "DBG: Versatile._send: sending a packet of total size: %s" % str(len(data)) )

        socket_object = optionnalClient
        self.bCustomClientSocket = True
        if socket_object == None:
            self.bCustomClientSocket = False
            socket_object = self.socket_server

        try:
            socket_object.sendall( data )
        except socket.error as err:

            if self.bCustomClientSocket:
                print( "WRN: Versatile.send: got socket error, client disappear? err: %s" % str(err) )
                return False

            print( "WRN: Versatile.send: got socket error, serveur out, try to reconnect, err: %s" % str(err) )
            # ne fonctionne pas sous windows?
            self._reconnect(bFailSilently = True)
            socket_object = self.socket_server
            try:
                socket_object.sendall( data )
            except socket.error as err:
                print( "DBG: Versatile.send: socket disconnected, skipping this message, will try to reconnect next time..., err: %s" % str(err) )
                return False

        if self.bVerbose: print( "DBG: command sent..." )
        if nNumCommand == Versatile.nCommandType_Value:
            # a returned value never wait for ack
            return [1,[True]]

        if self.bVerbose: print( "DBG: waiting for an answer..." )
        retVal = Versatile._waitPacket(socket_object)
        if self.bVerbose: print( "DBG: waiting for an answer: received..." )
        return retVal

    def waitPacket( self ):
        """
        wait for some answer
        """
        retVal = Versatile._waitPacket(self.socket_server)
        if self.bVerbose: print( "DBG: waitPacket: returning: %s" % str(retVal ) )
        return retVal

    def isRunning( self ):
        try:
            retVal = self._send( Versatile.nCommandType_Ping )
            if self.bVerbose: print( "DBG: Versatile: isRunning: received: %s" % str(retVal) )
            return retVal != False and retVal[1][0] == 1
        except Exception as err:
            if self.bVerbose: print( "DBG: Versatile: isRunning: received this err: %s" % str(err) )
        return False

    ############### accessor to each command type - use them in your program

    def get( self, strDataName ):
        """
        return a pair: sucess,value
        """
        retVal = self._send( Versatile.nCommandType_Get, strDataName )
        if retVal == False:
            return [False,None]
        return [True, retVal[1][0] ]

    def set( self, strDataName, value ):
        retVal = self._send( Versatile.nCommandType_Set, strDataName, value )
        #~ print( "DBG: Versatile.set: retVal: %s" % str(retVal) )
        return retVal != False and retVal[1][0] == 1

    def sendValue( self, value ):
        retVal = self._send( Versatile.nCommandType_Value, None, value )
        return retVal != False and retVal[1][0] == 1

    def sendValueGeneratingResults( self, value ):
        """
        when sending a value generating a result, like an image and its analyse...
        """
        retVal = self._send( Versatile.nCommandType_Value, None, value )
        if self.bVerbose: print( "DBG: sendValueGeneratingResults: retVal: %s" % str(retVal) )
        if retVal == False or retVal[1][0] == 0:
            return False # error in sending values
        return self.waitPacket()

    def createClientID(self):
        retVal = self._send( Versatile.nCommandType_CreateClientID )
        if retVal == False:
            return [False,None]
        return [True, retVal[1][0] ]

    def setClientID(self, clientID):
        retVal = self._send( Versatile.nCommandType_SetClientID, None, str(clientID) )
        return retVal != False and retVal[1][0] == 1

    def setClientParam(self, strParamName, value ):
        retVal = self._send( Versatile.nCommandType_SetClientParam, str(strParamName), value )
        return retVal != False and retVal[1][0] == 1

    def getClientParam(self, strParamName, value ):
        retVal = self._send( Versatile.nCommandType_GetClientParam, str(strParamName) )
        if retVal == False:
            return [False,None]
        return [True, retVal[1][0] ]

    def subscribeCamera(self, nCameraIndex, nWantedResolution, nImageFormat, nFps ):
        retVal = self._send( Versatile.nCommandType_SubscribeCamera, None, (nCameraIndex,nWantedResolution,nImageFormat, nFps ) )
        return retVal != False and retVal[1][0] == 1

    def getImage( self ):
        """
        get an Image from a previously subscribed server
        return False on error
        """

        retVal = self.waitPacket()
        print("DBG: Versatile.getImage: retVal: %s" % str(retVal) )
        command, param = retVal
        if self.bVerbose: print("DBG: run_camera_subscriber: param: %s" % str(param))
        if param != None and len(param)> 0:
            data = param[0][0]
            if 1: # data.nType == Versatile.VersatileValue.nValueType_Image: # and command = Versatile.nCommandType_Value
                if self.bVerbose: print( "DBG: Versatile.getImage: image received" )
                im = data.convertToCvImage()
                if self.bVerbose: print( "DBG: Versatile.getImage: converted: shape: %s" % str(im.shape) )
                return im
        return False
    # getImage - end

    def getBrodcastedImage( self, strBroadcasterID ):
        """
        get a (redundant) pair [Versatile Image, opencv image] from a broadcaster
        return False on error
        """
        retVal = self._send( Versatile.nCommandType_GetBroadcasted, str(strBroadcasterID) )
        if self.bVerbose: print("DBG: getImage: retVal: %s" % str(retVal) )
        if retVal == None:
            return False
        command, param = retVal
        if self.bVerbose: print("DBG: run_camera_subscriber: param: %s" % str(param))
        if param != None and len(param)> 0:
            data = param[0][0]
            if 1: # data.nType == Versatile.VersatileValue.nValueType_Image: # and command = Versatile.nCommandType_Value
                if self.bVerbose: print( "DBG: image received" )
                im = data.convertToCvImage()
                if self.bVerbose: print( "DBG: converted: shape: %s" % str(im.shape) )
                return data,im
        return False
    # getBrodcastedImage - end

# class Versatile - end

def connectAndSend( strIP, strDataName, value = None ):
    v = Versatile()
    v.connect( strIP )
    if value == None:
        retVal = v.get( strDataName )
        #retVal = v.get( strDataName ) #test 2 calls in a raw
    else:
        retVal = v.set( strDataName, value )
    print( "connectAndSend: finished with value: %s" % str(retVal) )

def run_server(nPort=10001):
    print( "INF: versatile: starting server..." )
    v = Versatile(nPort=nPort)
    v.runServer()

def run_camera_subscriber( strServerIP, nPort=10001, bVerbose=False ):
    """
    duplicate from test_get_image
    """
    v = Versatile(nPort=nPort)
    v.connect( strServerIP )
    v.subscribeCamera( 0,Versatile.VersatileImage.WantedResolution.VGA,Versatile.VersatileImage.Format.PNG, 30 )

    nCptImage = 0
    nCptImageTotal = 0

    timeBegin = time.time()
    while 1:
        im = v.getImage()
        if not im is False:
                cv2.imshow("received", im)
                if bVerbose: print( "DBG: drawed" )
                nKey =  cv2.waitKey(1) & 0xFF
                bEnd = ( nKey == ord( 'q' ) or nKey == ord( 'Q' ) )
                if bEnd:
                    break

                nCptImage +=1
                nCptImageTotal += 1
                if nCptImage > 50:
                    print( "INF: fps: %5.1f (total image: %d)" % (nCptImage/(time.time()-timeBegin), nCptImageTotal) )
                    nCptImage = 0
                    timeBegin = time.time()

        else:
            print( "WRN: reception error...")

# run_camera_subscriber - end

def autoTest():
    """
    autotest as a client
    """
    Versatile.VersatileImage.autoTest()
    # system( "python %s start_server" % sys.argv )
    pass # a bit complicated

if( __name__ == "__main__" ):
    #Versatile.VersatileImage_autoTest()

    def print_syntax():
        print( "" )
        print( "syntax: python %s remote_ip variable_name [value]" % sys.argv[0])
        print( "syntax: to start a server: python %s start_server"  % sys.argv[0] )
        print( "syntax: to subscribe images from a server: python %s remote_ip subscribe_camera"  % sys.argv[0] )
        print( "" )
        print( "To start a local test:\nshell 1: python %s start_server\nshell 2: python %s localhost myTime 12h34 & python %s localhost myTime" % (sys.argv[0],sys.argv[0],sys.argv[0]))
        exit( 0 )

    nPort = 10001
    # eat options
    i = 1
    while i < len(sys.argv):
        if sys.argv[i][0] == '-':
            if sys.argv[i][1] == 'p':
                nPort = int(sys.argv[i+1])
                print( "INF: changing to port %d" % (nPort) )
            del sys.argv[i]
            del sys.argv[i] # was the i+1
        else:
            i += 1
    if len(sys.argv) < 2:
        print_syntax()
    strIP = sys.argv[1]
    if strIP == "start_server":
        run_server(nPort=nPort)
        exit(1)

    if len(sys.argv) < 3:
        print_syntax()

    strDataName = sys.argv[2]

    if strDataName == "subscribe_camera": # TODO: en cours de test: lancer un serveur d'un coté et subscribe_camera de l'autre
        run_camera_subscriber(strIP,nPort)
        exit(1)

    strValue = None
    if len(sys.argv)>3:
        strValue = sys.argv[3]

    retVal = connectAndSend( strIP, strDataName, strValue )
    print( "return:\n%s" % str(retVal) )


# -*- coding: utf-8 -*-
"""Concentration layer

This module contains code for calculating PSD concentration based on dilution and volume

"""
# Importing dependencies
import tensorflow as tf
from tensorflow.keras.layers import Input, Lambda
from tensorflow.keras.models import Model

class ConcentrationModel:
    """
    """
    @staticmethod
    def correction_in(concentration_input):
        # Split input
        y = concentration_input[0]
        correction_factor = concentration_input[1][:, 0]
        flowcell_volume = concentration_input[1][:, 1]
        # Return conversion factor
        return y/tf.reshape(correction_factor*flowcell_volume, (-1, 1))

    @staticmethod
    def correction_out(concentration_input):
        # Split input
        y = concentration_input[0]
        correction_factor = concentration_input[1][:, 0]
        flowcell_volume = concentration_input[1][:, 1]
        # Return conversion factor
        return y*tf.reshape(correction_factor*flowcell_volume, (-1, 1))

    def create_dilution_model(self, variable_dimension: int, correction_type: str, name: str) -> Model:
        # Get input vector
        variable = Input(shape=(variable_dimension,), name='Variable')
        dilution_factor = Input(shape=(2,), name='Dilution_factor')
        # Create concentration correction layer
        if correction_type == 'in':
            corrected_variable = Lambda(self.correction_in,
                                        output_shape=self.output_shape,
                                        name='Corrected_variable')([variable, dilution_factor])
        elif correction_type == 'out':
            corrected_variable = Lambda(self.correction_out,
                                        output_shape=self.output_shape,
                                        name='Corrected_variable')([variable, dilution_factor])
        else:
            raise ValueError("Dilution model correction type not properly set")
        # Generate dilution model
        dilution_model = Model(inputs=[variable, dilution_factor], outputs=[corrected_variable], name=name)
        dilution_model.units = {'input': '-', 'output': '-/mL'}
        return dilution_model

    @staticmethod
    def output_shape(input_shape):
        return tuple(input_shape)


from .values import *
from docx.shared import Pt
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT

def setParagraphType(paragraph):
    if paragraph[:7].upper() == CHAPTER_TEXT:
        type = CHAPTER_TEXT
    elif paragraph[:10].upper() == DEDICATION_TEXT:
        type = EXTRA_SECTION
    elif paragraph[:14].upper() == ACKNOWLEDGMENT_TEXT:
        type = EXTRA_SECTION
    elif paragraph[:9].upper() == BIOGRAPHY_TEXT:
        type = EXTRA_SECTION
    elif paragraph[:16].upper() == ABOUT_THE_AUTHOR_TEXT:
        type = EXTRA_SECTION
    elif paragraph[:4].upper() == PART_TEXT:
        type = PART_TEXT
    elif paragraph[:3] == SCENE_BREAK_1:
        type = SCENE_BREAK_1
    elif paragraph[:3] == SCENE_BREAK_2:
        type = SCENE_BREAK_2
    else:
        type = PARAGRAPH

    return type

def setHalfTitleFormat(document, dataFromHtml):
    paragraphWrite = document.add_paragraph(dataFromHtml.bookTitle)
    paragraphWrite.style = document.styles['Book Title']

    document.add_section()
    document.add_section()

def setTitleFormat(document, dataFromHtml):
    paragraphWrite = document.add_paragraph(dataFromHtml.bookTitle)
    paragraphWrite.style = document.styles['Book Title']

    paragraphWrite2 = document.add_paragraph(dataFromHtml.bookSubTitle)
    paragraphWrite2.style = document.styles['Book Subtitle']

    document.add_paragraph('')
    document.add_paragraph('')
    document.add_paragraph('')
    document.add_paragraph('')
    document.add_paragraph('')
    document.add_paragraph('')

    paragraphWrite3 = document.add_paragraph(dataFromHtml.author.upper())
    paragraphWrite3.style = document.styles['Author']

    document.add_section()

def setCopyrightFormat(document, dataFromHtml):
    i= 0
    while i <= 22:
        document.add_paragraph('')
        i = i + 1

    paragraphWrite = document.add_paragraph(dataFromHtml.bookTitle +' by ' + dataFromHtml.author)
    paragraphWrite.style = document.styles['Copyright']

    paragraphWrite = document.add_paragraph('Copyright© 2022 ' + dataFromHtml.author)
    paragraphWrite.style = document.styles['Copyright']

    paragraphWrite = document.add_paragraph(COPYRIGHT_TEXT)
    paragraphWrite.style = document.styles['Copyright']

    paragraphWrite = document.add_paragraph('ISBN: XXXX-XXXX')
    paragraphWrite.style = document.styles['Copyright']

    document.add_section()

def setPartFormat(document, paragraphRead):
    paragraphWrite = document.add_paragraph(paragraphRead)
    paragraphWrite.style = document.styles['Part']

def setChapterTitleFormat(document, paragraphRead, isPreviousAPart):
    paragraphWrite = document.add_paragraph(paragraphRead)
    paragraphWrite.style = document.styles['Chapter Title']

    paragraph_format = paragraphWrite.paragraph_format
    if isPreviousAPart:
        paragraph_format.space_before = Pt(45)
    else:
        paragraph_format.space_before = Pt(70)

    paragraph_format.space_after = Pt(20)
    paragraph_format.alignment = WD_PARAGRAPH_ALIGNMENT.CENTER
    paragraph_format.line_spacing = 1

def setChapterDescriptionFormat(document, paragraphRead):
    paragraphWrite = document.add_paragraph(paragraphRead)
    paragraphWrite.style = document.styles['Chapter Description']

    paragraphWrite.style.paragraph_format.line_spacing = 1

    paragraph_format = paragraphWrite.paragraph_format

    paragraph_format.space_before = Pt(10)
    paragraph_format.alignment = WD_PARAGRAPH_ALIGNMENT.CENTER

    paragraph_format.right_indent = Inches(0.7)
    paragraph_format.left_indent = Inches(0.7)

def setParagraphFormat(document, paragraphRead, indentFirstLine, paragrapghReadNext, chapterDescription, previousTitle, data):
    paragraphWrite = document.add_paragraph(paragraphRead)

    if (chapterDescription):
        paragraphWrite.style = document.styles['Chapter Description']
    else:
        if data.PublishOrEdit == 'toEdit':
            paragraphWrite.style = document.styles['Paragraph Editor']
            font = paragraphWrite.style.font
            font.size = Pt(12)
        else:
            paragraphWrite.style = document.styles['Paragraph']
            font = paragraphWrite.style.font
            font.size = Pt(11)

    paragraph_format = paragraphWrite.paragraph_format
    paragraph_format.keep_together = keep_together

    if indentFirstLine:
        paragraph_format.first_line_indent = Pt(18)

    typeNextParagraph = setParagraphType(paragrapghReadNext)

    if (typeNextParagraph == PART_TEXT or typeNextParagraph == CHAPTER_TEXT or typeNextParagraph == EXTRA_SECTION) :
        if previousTitle == EXTRA_SECTION:
            document.add_section()
            document.add_section()
        else:
            document.add_section()

def setSceneBreakFormat(document, paragraph):
    paragraphWrite = document.add_paragraph(paragraph)
    paragraphWrite.style = document.styles['Scene Break']

def setSectionFormat(document, dataFromHtml, doc):
    section = document.sections[0]

    #paragraphsCount = len(doc.paragraphs)
    #paragraphReadChapterCount = 0
    #countWords = 0
    #while paragraphReadChapterCount < paragraphsCount:
     #   paragraphRead = doc.paragraphs[paragraphReadChapterCount].text.strip()
      #  count = paragraphRead.split(" ")
       # countWords = countWords + len(count)
        #paragraphReadChapterCount = paragraphReadChapterCount + 1

    countWords = 100000
    if dataFromHtml.PageSize == '5" x 8" (12.7 x 20.32 cm)':
        section.page_width = Inches(5)
        section.page_height = Inches(8)
    elif dataFromHtml.PageSize == '5.25" x 8"(13.34 x 20.32 cm)':
        section.page_width = Inches(5.25)
        section.page_height = Inches(8)
    elif dataFromHtml.PageSize =='5.5" x 8.5"(13.97 x 21.59 cm)':
        section.page_width = Inches(5.5)
        section.page_height = Inches(8.5)
    elif dataFromHtml.PageSize == '6" x 9" (15.24 x 22.86 cm)':
        section.page_width = Inches(6)
        section.page_height = Inches(9)
    elif dataFromHtml.PageSize == '7" x 10" x (17.78 x 25.4 cm)':
        section.page_width = Inches(7)
        section.page_height = Inches(10)
    elif dataFromHtml.PageSize == '8" x 10" (20.32 x 25.4 cm)':
        section.page_width = Inches(8)
        section.page_height = Inches(10)
    elif dataFromHtml.PageSize == 'Letter 8.5" x 11" (21,59 x 27,94 cm)':
        section.page_width = Inches(8.5)
        section.page_height = Inches(11)
    elif dataFromHtml.PageSize == 'A4 8.27" x 11.69" (21 x 29,7 cm)':
        section.page_width = Inches(8.27)
        section.page_height = Inches(11.69)

    section.top_margin = Inches(0.8)
    section.bottom_margin = Inches(0.8)

    if dataFromHtml.PublishOrEdit == 'toEdit':
        section.left_margin = Inches(1)
        section.right_margin = Inches(1)
    else:
        section.right_margin = Inches(0.3)

        pageCount = countWords/250

        if pageCount < 150:
            marginCalc = 0.375
        elif pageCount < 300:
            marginCalc = 0.5
        elif pageCount < 500:
            marginCalc = 0.625
        elif pageCount < 700:
            marginCalc = 0.75
        elif pageCount < 828:
            marginCalc = 0.875
        else:
            marginCalc = 0.875

        margin = 0.375

        if margin > marginCalc:
            section.left_margin = Inches(margin)
        else:
            section.left_margin = Inches(marginCalc)

    section.different_first_page_header_footer = True
    document.settings.odd_and_even_pages_header_footer = True

    header = section.header
    #footer = section.footer
    evenHeader = section.even_page_header
    evenFooter = section.even_page_footer

    header.is_linked_to_previous = False

    textHeader = header.paragraphs[0]
    textHeader.text = dataFromHtml.bookTitle.upper()

    textEvenHeader = evenHeader.paragraphs[0]
    textEvenHeader.text = dataFromHtml.author.upper()


import re
import sqlite3
import os
import glob
import datetime as dt
import bokeh.models
import xarray
import numpy as np
from functools import lru_cache
from forest.exceptions import FileNotFound, IndexNotFound
from forest.old_state import old_state, unique
import forest.util
import forest.map_view
from forest import geo, locate

ENGINE = "h5netcdf"
MIN_DATETIME64 = np.datetime64("0001-01-01T00:00:00.000000")

def _natargmax(arr):
    """Find the arg max when an array contains NaT's"""
    no_nats = np.where(np.isnat(arr), MIN_DATETIME64, arr)
    return np.argmax(no_nats)

class Dataset:
    def __init__(self, pattern=None, database_path=None, minutes=15, **kwargs):
        self.pattern = pattern
        if database_path is None:
            database_path = ":memory:"
        self.database = Database(database_path)
        self.locator = Locator(self.pattern, self.database, minutes=minutes)

    def navigator(self):
        return Navigator(self.locator, self.database)

    def map_view(self, color_mapper):
        loader = Loader(self.locator)
        return forest.map_view.map_view(
            loader, color_mapper, use_hover_tool=False
        )

class Database:
    """Meta-data store for EIDA50 dataset"""

    def __init__(self, path=":memory:"):
        self.fmt = "%Y-%m-%d %H:%M:%S"
        self.path = path
        self.connection = sqlite3.connect(self.path)
        self.cursor = self.connection.cursor()

        # Schema
        query = """
            CREATE TABLE IF NOT EXISTS file (
                      id INTEGER PRIMARY KEY,
                    path TEXT,
                         UNIQUE(path));
        """
        self.cursor.execute(query)
        query = """
            CREATE TABLE IF NOT EXISTS time (
                      id INTEGER PRIMARY KEY,
                    time TEXT,
                 file_id INTEGER,
                 FOREIGN KEY(file_id) REFERENCES file(id));
        """
        self.cursor.execute(query)

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()

    def close(self):
        """Close connection gracefully"""
        self.connection.commit()
        self.connection.close()

    def insert_times(self, times, path):
        """Store times"""
        # Update file table
        query = """
            INSERT OR IGNORE INTO file (path) VALUES (:path);
        """
        self.cursor.execute(query, {"path": path})
        self.connection.commit()

        # Update time table
        query = """
            INSERT INTO time (time, file_id)
                 VALUES (:time, (SELECT id FROM file WHERE path = :path));
        """
        texts = [time.strftime(self.fmt) for time in times]
        args = [{"path": path, "time": text} for text in texts]
        self.cursor.executemany(query, args)
        self.connection.commit()

    def fetch_times(self, path=None):
        """Retrieve times"""
        if path is None:
            query = """
                SELECT time
                  FROM time;
            """
            rows = self.cursor.execute(query)
        else:
            query = """
                SELECT time.time
                  FROM time
                  JOIN file
                    ON file.id = time.file_id
                 WHERE file.path = :path;
            """
            rows = self.cursor.execute(query, {"path": path})
        rows = self.cursor.fetchall()
        texts = [text for text, in rows]
        times = [dt.datetime.strptime(text, self.fmt) for text in texts]
        return list(sorted(times))

    def fetch_paths(self):
        """Retrieve paths"""
        query = """
            SELECT path FROM file;
        """
        rows = self.cursor.execute(query).fetchall()
        texts = [text for text, in rows]
        return list(sorted(texts))

class Locator:
    """Locate EIDA50 satellite images"""

    def __init__(self, pattern, database, minutes=15):
        self.minutes = minutes
        self.pattern = pattern
        self._glob = forest.util.cached_glob(
            dt.timedelta(minutes=self.minutes)
        )
        self.database = database

    def all_times(self, paths):
        """All available times"""
        # Parse times from file names not in database
        unparsed_paths = []
        filename_times = []
        database_paths = set(self.database.fetch_paths())
        for path in paths:
            if path in database_paths:
                continue
            time = self.parse_date(path)
            if time is None:
                unparsed_paths.append(path)
                continue
            filename_times.append(time)

        # Store unparsed files in database
        for path in unparsed_paths:
            times = self.load_time_axis(path)  # datetime64[s]
            self.database.insert_times(times.astype(dt.datetime), path)

        # All recorded times
        database_times = self.database.fetch_times()

        # Combine database/timestamp information
        arrays = [
            np.array(database_times, dtype="datetime64[s]"),
            np.array(filename_times, dtype="datetime64[s]"),
        ]
        return np.unique(np.concatenate(arrays))

    def find(self, paths, date):
        """Find file and index related to date

        .. note:: Find should not write to disk
        """
        # Search file system
        path = self.find_file(paths, date)

        # Load times from database
        if path in self.database.fetch_paths():
            times = self.database.fetch_times(path)
            times = np.array(times, dtype="datetime64[s]")
        else:
            times = self.load_time_axis(path)  # datetime64[s]

        index = self.find_index(
            times, date, dt.timedelta(minutes=self.minutes)
        )
        return path, index

    def glob(self):
        return self._glob(self.pattern)

    @staticmethod
    @lru_cache()
    def load_time_axis(path):
        with xarray.open_dataset(path, engine=ENGINE) as nc:
            values = nc["time"]
        return np.array(values, dtype="datetime64[s]")

    def find_file(self, paths, user_date):
        """ "Find file likely to contain user supplied date

        .. note:: Search based on timestamp only

        .. warning:: Not suitable for searching unparsable file names
        """
        if isinstance(user_date, (dt.datetime, str)):
            user_date = np.datetime64(user_date, "s")
        dates = np.array(
            [self.parse_date(path) for path in paths], dtype="datetime64[s]"
        )
        mask = ~(dates <= user_date)
        if mask.all():
            msg = "No file for {}".format(user_date)
            raise FileNotFound(msg)
        before_dates = np.ma.array(dates, mask=mask, dtype="datetime64[s]")
        i = _natargmax(before_dates.filled())
        return paths[i]

    @staticmethod
    def find_index(times, time, length):
        """Search for index inside array of datetime64[s] values"""
        print(f"FIND_INDEX: {times} {time} {length}")
        dtype = "datetime64[s]"
        if isinstance(times, list):
            times = np.asarray(times, dtype=dtype)
        if isinstance(time, (dt.datetime, str)):
            time = np.datetime64(time, "s")
        bounds = locate.bounds(times, length)
        inside = locate.in_bounds(bounds, time)
        valid_times = np.ma.array(times, mask=~inside)
        if valid_times.mask.all():
            msg = "{}: not found".format(time)
            raise IndexNotFound(msg)
        return _natargmax(valid_times.filled())

    @staticmethod
    def parse_date(path):
        """Parse timestamp into datetime or None"""
        for regex, fmt in [
            (r"([0-9]{8})\.nc", "%Y%m%d"),
            (r"([0-9]{8}T[0-9]{4}Z)\.nc", "%Y%m%dT%H%MZ"),
        ]:
            groups = re.search(regex, path)
            if groups is None:
                continue
            else:
                return dt.datetime.strptime(groups[1], fmt)

class Loader:
    def __init__(self, locator):
        self.locator = locator
        self.empty_image = {"x": [], "y": [], "dw": [], "dh": [], "image": []}
        self.cache = {}
        paths = self.locator.glob()
        if len(paths) > 0:
            with xarray.open_dataset(paths[-1], engine=ENGINE) as nc:
                self.cache["longitude"] = nc["longitude"].values
                self.cache["latitude"] = nc["latitude"].values

    @property
    def longitudes(self):
        return self.cache["longitude"]

    @property
    def latitudes(self):
        return self.cache["latitude"]

    def image(self, state):
        if state.valid_time is None:
            data = self.empty_image
        else:
            try:
                data = self._image(forest.util.to_datetime(state.valid_time))
            except (FileNotFound, IndexNotFound) as e:
                print(f"EIDA50: {e}")
                data = self.empty_image
        return data

    def _image(self, valid_time):
        paths = self.locator.glob()
        path, itime = self.locator.find(paths, valid_time)
        return self.load_image(path, itime)

    def load_image(self, path, itime):
        lons = self.longitudes
        lats = self.latitudes
        with xarray.open_dataset(path, engine=ENGINE) as nc:
            values = nc["data"][itime].values

        # Use datashader to coarsify images from 4.4km to 8.8km grid
        scale = 2
        return geo.stretch_image(
            lons,
            lats,
            values,
            plot_width=int(values.shape[1] / scale),
            plot_height=int(values.shape[0] / scale),
        )

class Navigator:
    """Facade to map Navigator API to Locator"""

    def __init__(self, locator, database):
        self.locator = locator
        self.database = database

    def __call__(self, store, action):
        """Middleware interface"""
        import forest.db.control

        kind = action["kind"]
        if kind == forest.db.control.SET_VALUE:
            key, time = (action["payload"]["key"], action["payload"]["value"])
            if key == "valid_time":
                # Detect missing file
                paths = self.locator.glob()
                path = self.locator.find_file(paths, time)
                if path not in self.database.fetch_paths():
                    times = self.locator.load_time_axis(path)  # datetime64[s]
                    self.database.insert_times(times.astype(dt.datetime), path)

                # Update stale state
                store_times = store.state.get("valid_times", [])
                database_times = self._times()
                if len(store_times) != len(database_times):
                    yield forest.db.control.set_value(
                        "valid_times", database_times
                    )
        yield action

    def variables(self, pattern):
        return ["EIDA50"]

    def initial_times(self, pattern, variable):
        return [dt.datetime(1970, 1, 1)]

    def valid_times(self, pattern, variable, initial_time):
        """Get available times given application state"""
        return self._times()

    def _times(self):
        """Get available times given user selected valid_time

        :param valid_time: application state valid time
        """
        paths = self.locator.glob()
        datetimes = self.locator.all_times(paths)
        return np.array(datetimes, dtype="datetime64[s]")

    def pressures(self, pattern, variable, initial_time):
        return []


import agent.potil_lstm as agent
from expert_dataloader import ExpertLoader, ExpertLoader_each_step
import agent.utils as utils
import sys
sys.path.append('/catkin_workspace/src/ros_kortex/kortex_examples/src/move_it')
import torch
torch.backends.cudnn.benchmark = True
from mpl_toolkits import mplot3d
import numpy as np
import matplotlib.pyplot as plt
import queue

potil_agent = agent.POTILAgent(obs_shape=[4, 1], action_shape=[4, 1], device='cuda', lr=1e-3, feature_dim=512,
                               hidden_dim=128, critic_target_tau=0.01, num_expl_steps=0,
                               update_every_steps=2, stddev_schedule='linear(0.2,0.04,500000)', stddev_clip=0.1, use_tb=True, augment=True,
                               rewards='sinkhorn_cosine', sinkhorn_rew_scale=200, update_target_every=10000,
                               auto_rew_scale=True, auto_rew_scale_factor=10, obs_type='env', bc_weight_type='linear',
                               bc_weight_schedule='linear(0.15,0.03,20000)')

nstep = 32

# expert_loader = ExpertLoader(nstep, 50)
# expert_iter = iter(expert_loader)
# test_data = expert_loader.read_data(0,1)
# state, action = test_data
# seq_queue = queue.Queue(nstep)
# test_data_storage = []
# for i in range(nstep):
#     seq_queue.put(state[0])
# for i in range(state.shape[0]):
#     seq_queue.get()
#     seq_queue.put(state[i])
#     test_data_storage.append((np.stack(seq_queue.queue), action[i]))

# with step
expert_loader = ExpertLoader_each_step(nstep, 50)
expert_iter = iter(expert_loader)
test_data = expert_loader.read_data(0,1)
state, action = test_data
seq_queue = queue.Queue(nstep)
test_data_storage = []
for i in range(nstep):
    seq_queue.put(state[0])
for i in range(state.shape[0]):
    seq_queue.get()
    seq_queue.put(state[i])
    test_data_storage.append((np.stack(seq_queue.queue), action[i]))

cnt = 1
# def test():
#     obs = utils.to_torch(test_data, 'cuda')[0]
#     replay = []
#     for i in range(80):
#         with torch.no_grad():
#             action = potil_agent.act(obs, 0, True)
#         action[0] = (action[0] + 1.3) * 0.2 + 0.2
#         action[1] = (action[1] + 1) * 1.1 * -0.1
#         action[2] = (action[2] + 0.75) * 0.125 + 0.05
#         action[3] = (action[3] + 1) * 0.2
#         obs[:4] = torch.from_numpy(action).cuda()
#         replay.append(obs)

#     fig = plt.figure()
#     #创建3d绘图区域
#     ax = plt.axes(projection='3d')
#     #从三个维度构建
#     z = test_data[0][:,2]
#     x = test_data[0][:,0]
#     y = test_data[0][:,1]
#     #调用 ax.plot3D创建三维线图
#     ax.scatter3D(x, y, z,'gray', s=1)
#     replay = np.stack(replay)
#     z = replay[:,2]
#     x = replay[:,0]
#     y = replay[:,1]
#     #调用 ax.plot3D创建三维线图
#     ax.scatter3D(x, y, z,'gray', s=1)
#     plt.savefig(f'./figs/{cnt}.png')
#     cnt += 1

def scale(action_):
    action = action_.copy()
    action[0] = (action[0] + 1.31) * 0.2 + 0.2
    action[1] = (action[1] + 0.84) * 1.1 * -0.11
    action[2] = (action[2] + 0.83) * 0.15 + 0.05
    action[3] = (action[3] + 1) * 0.25
    return action

def test():
    loss = 0
    for data in test_data_storage:
        data = utils.to_torch(data, 'cuda')
        with torch.no_grad():
            action = potil_agent.act(data[0][:, :4], 0, True)
        loss += np.linalg.norm(scale(action) - scale(data[1].cpu().numpy()), ord=1)
    loss /= state.shape[0]
    print(loss)

# for i in range(100000):
#     batch = next(expert_iter)
#     obs_bc, action_bc = utils.to_torch(batch, 'cuda')
#     metrics = potil_agent.train_bc_norm(obs_bc, action_bc)
#     # if i % 10 == 1:
#         # print(metrics['bc_loss'])
#     if i % 20 == 0:
#         test()
#     if i % 100 == 0:
#         torch.save(potil_agent.actor, 'BC_actor_norm.pkl')

# for i in range(100000):
#     batch = next(expert_iter)
#     obs_bc, action_bc = utils.to_torch(batch, 'cuda')
#     metrics = potil_agent.train_bc_dist(obs_bc, action_bc)
#     # if i % 10 == 1:
#         # print(metrics['bc_loss'])
#     if i % 100 == 0:
#         test()
#     if i % 100 == 0:
#         torch.save(potil_agent.actor, 'BC_actor_dist.pkl')

# potil_agent.actor = torch.load('/catkin_workspace/src/ros_kortex/kortex_examples/src/move_it/BC_actor_dist.pkl').cuda()
potil_agent.actor.train()
potil_agent.actor_opt = torch.optim.Adam(potil_agent.actor.parameters(), lr=potil_agent.lr)
for i in range(100000):
    batch = next(expert_iter)
    obs_bc, action_bc = utils.to_torch(batch, 'cuda')
    metrics = potil_agent.train_bc_norm_step(obs_bc[:,:,:4], action_bc)
    # if i % 10 == 1:
        # print(metrics['bc_loss'])
    if i % 100 == 0:
        test()
    if i % 100 == 0:
        torch.save(potil_agent.actor, 'BC_actor_dist.pkl')


#!/usr/bin/env python
# coding: utf-8

# # Image_Segmentation

# ## 1. Contours

# ### Contours are continuous lines or curves that bound or cover the full-boundary of an object in an image

# In[1]:

import cv2                                       #importing necessary libraries
import numpy as np

# In[2]:

image=cv2.imread("2.jpg")                        #loading our image

# In[3]:

cv2.imshow("Input Image", image)                 #displaying our original image
cv2.waitKey(0)

# In[4]:

gray=cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)      #displaying the grayscale-image

# In[5]:

edged=cv2.Canny(gray,30,200)                     #finding canny edges
cv2.imshow("Canny Edges",edged)
cv2.waitKey(0)

# In[6]:

contours,hierarchy=cv2.findContours(edged,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_NONE)    #finding contours
cv2.imshow("Canny Edges after Contouring",edged)
cv2.waitKey(0)

# In[7]:

print("Number of contours found= " +str(len(contours)))

# In[8]:

cv2.drawContours(image,contours,-1,(0,255,0),3)    #drawing all contours

# In[9]:

cv2.imshow("Contours",image)                       #displaying all contours
cv2.waitKey(0)

# In[10]:

cv2.destroyAllWindows()

# In[ ]:


import numpy as np

import dezero.functions as F
import dezero.layers as L
from dezero import Variable

np.random.seed(0)
x = np.random.rand(100, 1)
y = np.sin(2 * np.pi * x) + np.random.rand(100, 1)
x = Variable(x)  # type:ignore
y = Variable(y)

I, H, O_ = 1, 10, 1
l1 = L.Linear(I, H)
l2 = L.Linear(H, O_)

def predict(x):
    y = l1(x)
    y = F.sigmoid(y)
    y = l2(y)
    return y

lr = 0.2
iters = 10000

for i in range(iters):
    y_pred = predict(x)
    loss = F.mean_squared_error(y, y_pred)

    l1.cleargrads()
    l2.cleargrads()
    loss.backward()

    for l_ in [l1, l2]:
        for p in l_.params():
            assert p.grad is not None
            p.data -= lr * p.grad.data
    if i % 1000 == 0:
        print(loss)


# Задание-1:
# Написать программу, выполняющую операции (сложение и вычитание) с простыми дробями.
# Дроби вводятся и выводятся в формате:
# n x/y ,где n - целая часть, x - числитель, у - знаменатель.
# Дроби могут быть отрицательные и не иметь целой части, или иметь только целую часть.
# Примеры:
# Ввод: 5/6 + 4/7 (всё выражение вводится целиком в виде строки)
# Вывод: 1 17/42  (результат обязательно упростить и выделить целую часть)
# Ввод: -2/3 - -2
# Вывод: 1 1/3

def math_magic(x):

    # Делим исходное выражение на целые и дробные части. От выражения 1 5/6 + 4/7 мы получим ('1', '5/6', '4/7', 0)
    def decompos(x):
            # Меняем '' на нули, если целых частей изначально нет.
            def space_check(x):
                if x == '':
                    x = 0
                return x
            x = x.split()
            if len(x) == 5:
                i = iter((0, 3))
            elif len(x) == 4:
                i = iter((0, 2, 3))
            else:
                i = iter((0, 2))
            a = a2 = c = b = b2 = ''
            for item in i:
                try:
                    if x[item].find('/') == (-1) and int(x[item]) != 0:
                        if a == '' and item == 0:
                            a = x[item]
                        else:
                            b = x[item]
                except ValueError:
                    pass
            for item in x:
                if item.find('/') != -1 and a2 == '' and item != a:
                    a2 = item
                elif item.find('/') != -1 and a2 > '' and item != b:
                    b2 = item
            return (a2, space_check(a), b2, space_check(b))

    # Получаем из дроби числитель и знаменатель в int
    def decompos_light(x):
        result = fract = '0'
        count = 0
        for item in x:
            if item.find('/') != -1:
                count = 1
            elif count == 0:
                result = result + item
            elif count == 1:
                fract = fract + item
        result = int(result)
        fract = int(fract)
        return result, fract

    # Приводим 2 дроби к одному знаменателю
    def generalization(a, x, b, y):
        if a == 0 or b == 0:
            return(a, b, x)
        a = a * y
        b = b * x
        x = x * y
        for i in range(len(str(x))):
            for i in range(x):
                i += 1
                if a / i == a // i and b / i == b // i and x / i == x // i:
                    a = a // i
                    b = b // i
                    x = x // i
        return a, b, x

    # Складываем
    def addition(x):
        a2, a, b2, b = decompos(x)
        integ = int(a) + int(b)
        a2, x = decompos_light(a2)
        b2, y = decompos_light(b2)
        a2, b2, fraction = generalization(a2, x, b2, y)
        float_part = a2 + b2
        while True:
            if float_part > fraction:
                float_part -= fraction
                integ += 1
            else:
                break
        # Еще раз упрощаем
        for i in range(fraction):
            i += 1
            if float_part / i == float_part // i and fraction / i == fraction // i:
                    float_part = float_part // i
                    fraction = fraction // i
        if integ != 0 and float_part != 0:
            result = str(integ)+" "+str(float_part)+"/"+str(fraction)
        elif float_part != 0:
            result = str(float_part)+"/"+str(fraction)
        else:
            result = str(integ)
        return result

    # Вычитаем
    def subtraction(x):
        a2, a, b2, b = decompos(x)
        integ = int(a) - int(b)
        a2, x = decompos_light(a2)
        b2, y = decompos_light(b2)
        a2, b2, fraction = generalization(a2, x, b2, y)
        float_part = a2 - b2
        while True:
            if float_part > fraction:
                float_part -= fraction
                integ += 1
            else:
                break
        # Еще раз упрощаем
        for i in range(fraction):
            i += 1
            if float_part / i == float_part // i and fraction / i == fraction // i:
                    float_part = float_part // i
                    fraction = fraction // i
        if integ != 0 and float_part != 0:
            result = str(integ)+" "+str(float_part)+"/"+str(fraction)
        elif float_part != 0:
            result = str(float_part)+"/"+str(fraction)
        else:
            result = str(integ)
        return result

    if x.find('+') == -1:
        result = subtraction(x)
    else:
        result = addition(x)
    print("\nИсходная задача: ", x)
    return result

print(math_magic('1 5/6 + 1 4/7'))
print(math_magic('5/6 + 1 4/7'))
print(math_magic('5/6 - 4/7'))
print(math_magic('1 5/6 + 1'))
print(math_magic('1 + 1'))
print(math_magic('1/3 + 1/3'))
print(math_magic('91/8322 + 155/34'))

# Задание-2:
# Дана ведомость расчета заработной платы (файл "data/workers").
# Рассчитайте зарплату всех работников, зная что они получат полный оклад,
# если отработают норму часов. Если же они отработали меньше нормы,
# то их ЗП уменьшается пропорционально, а за заждый час переработки
# они получают удвоенную ЗП, пропорциональную норме.
# Кол-во часов, которые были отработаны, указаны в файле "data/hours_of"

import os

def salary_counter(hours):
    addition = hours - hour_norm
    if addition > 0:
        addition *= 2
    result = (hour_norm + addition) * salary
    return result

# Tested
def line_to_tuple(x):
    iden = ''
    name = ''
    hours = ''
    counter = 0
    for item in x:
        if item != '/':
            if counter == 0:
                iden += item
            elif counter == 1:
                name += item
            else:
                hours += item
        else:
            counter += 1
    hours = int(hours)
    return iden, name, hours

def tuple_to_line(iden, name, salary):
    result = iden + " / " + name + " / " + str(salary)
    return result

def magic(x):
    iden, name, hours = line_to_tuple(x)
    salary = salary_counter(hours)
    result = tuple_to_line(iden, name, salary)
    return result

path_workers = os.path.join('data', 'workers.txt')
# В этом файле инфа хранится в формате id / ФИО /часы
# 001 / Виктор Иванов /160
# 002 / Сергей Агафонов /200
# 003 / Виктор Крищенко / 100
path_hours = os.path.join('data', 'hours_of.txt')
hour_norm = 160
salary = 100
line_out = []

with open(path_hours, "r") as input:
    for line_in in input:
        line_out.append(magic(line_in))
with open(path_workers, "w") as output:
    for line in line_out:
        output.write(line + "\n")

# Задание-3:
# Дан файл ("data/fruits") со списком фруктов.
# Записать в новые файлы все фрукты, начинающиеся с определенной буквы.
# Т.е. в одном файле будут все фрукты на букву “А”, во втором на “Б” и т.д.
# Файлы назвать соответственно.
# Пример имен файлов: fruits_А, fruits_Б, fruits_В ….
# Важно! Обратите внимание, что нет фруктов, начинающихся с некоторых букв.
# Напишите универсальный код, который будет работать с любым списком фруктов
# и распределять по файлам в зависимости от первых букв, имеющихся в списке фруктов.
# Подсказка:
# Чтобы получить список больших букв русского алфавита:
# print(list(map(chr, range(ord('А'), ord('Я')+1))))


from flask import render_template, url_for, flash, redirect, request, abort, jsonify, make_response
from app import app, db, bcrypt, mail
from forms import *
from models import *
from flask_login import login_user, current_user, logout_user, login_required
from routes.commonRoutes import commonTempRoute

@app.route("/supplier", methods=['GET', 'POST'])
@login_required
def suppliers():
    info = commonTempRoute(Supplier, AddSuppliers())
    return render_template('supplier/supplier.html',  mainArray=info["mainArray"], funName='listing', mainLabel='Supplier',
        formMain=info["formMain"], formLocation=info["formLocation"], formArea=info["formArea"], mainInd = 'listingInd')

@app.route("/supplier/new", methods=['POST'])
@login_required
def addSupplier():
    if (request.json!=None):
        if request.json.get('submit')=='supplier':
            checkName = Supplier.query.filter_by(name=request.json.get('name').strip()).all()
            locations = Location.query.filter_by(
                numberAndStreet=request.json.get('address').strip(),
                suburb=request.json.get('suburb').strip(),
                state_id = request.json.get('state')).first()
            if(len(checkName)>0):
                return jsonify({'status':'fail','reason':'Name already taken'})
            if(locations==None):
                return jsonify({'status':'fail','reason':'Location has not been registered'})
            if(len(request.json.get('name').strip())<1):
                return jsonify({'status':'fail','reason':'Invalid Name'})
            if(locations!=None and len(checkName)==0 and len(request.json.get('name').strip())>0):
                location_id  = locations.id
                supplier = Supplier(name=request.json.get('name').strip(), location_id=location_id, contact=request.json.get('contact'), website=request.json.get('website'))
                db.session.add(supplier)
                db.session.commit()
                return jsonify({'status':'success'})
                return redirect(url_for('storage'))
            else:
                return jsonify({'status':'fail', 'reason':'Please try again'})
    else:
        return jsonify({'status':'fail', 'reason':'No data'})

@app.route("/getSupplierInfo/<int:supplier_id>", methods=['GET'])
@login_required
def getSupplierInfo(supplier_id):
    info=Supplier.query.filter_by(id=supplier_id).first()
    infoJson={}
    infoJson["product_storage_listing"]=Product_Storage_Listing.query.filter_by(supplier_id=supplier_id).count()
    infoJson["verify_product_storage_listing"]=Verify_Product_Storage_Listing.query.filter_by(supplier_id=supplier_id).count()
    infoJson["id"]=info.id
    infoJson["name"]=info.name
    infoJson["contact"]=info.contact
    infoJson["website"]=info.website
    return jsonify(infoJson)

@app.route("/supplier/remove", methods=['POST'])
@login_required
def removeSupplier():
    if (request.json!=None):
        if request.json.get('submit')=='removeSupplier':
            deleteRecord = Supplier.query.filter_by(id=request.json.get('supplier_id')).first()
            if deleteRecord != None:
                db.session.delete(deleteRecord)
                db.session.commit()
                return jsonify({"status":"success"})
        return jsonify({"status":"fail"})
    else:
        return jsonify({'status':'fail', 'reason':'No data'})

@app.route("/supplier/edit", methods=['POST'])
@login_required
def editSupplier():
    if (request.json!=None):
        if request.json.get('submit')=='supplier edit':
            editRecord = Supplier.query.filter_by(id=request.json.get('supplier_id')).first()
            if editRecord != None:
                editRecord.name=request.json.get('newName')
                editRecord.contact=request.json.get('newContact')
                editRecord.website=request.json.get('newWebsite')
                db.session.commit()
                return jsonify({"status":"success"})
        return jsonify({"status":"fail"})
    else:
        return jsonify({'status':'fail', 'reason':'No data'})

@app.route("/supplier/area/<area_id>")
@login_required
def supplierFromArea(area_id):
    supplier = Supplier.query.filter(Supplier.location.has(Location.area_id==area_id)).all()
    supplierArray = []

    for sup in supplier:
        supObj = {}
        supObj['name']=sup.name+': '+ sup.location.numberAndStreet+' '+ sup.location.suburb+' '+ sup.location.area_loc.name+' '+sup.location.state_loc.name
        supObj['id']=sup.id
        supplierArray.append(supObj)
    return jsonify({'suppliers':supplierArray})

@app.route("/supplier/state/<state_id>")
@login_required
def supplierFromState(state_id):
    supplier = Supplier.query.filter(Supplier.location.has(Location.state_id==state_id)).all()
    supplierArray = []

    for sup in supplier:
        supObj = {}
        supObj['name']=sup.name+': '+ sup.location.numberAndStreet+' '+ sup.location.suburb+' '+ sup.location.area_loc.name+' '+sup.location.state_loc.name
        supObj['id']=sup.id
        supplierArray.append(supObj)
    return jsonify({'suppliers':supplierArray})


from movielog import api as movielog_api
from movielog.cli import confirm, radio_list

def prompt() -> None:
    options = [
        (None, "Go back"),
        (update_titles_and_people, "<cyan>Update titles and people</cyan>"),
        (
            update_watchlist_person_credits,
            "<cyan>Update watchlist person credits</cyan>",
        ),
    ]

    option_function = radio_list.prompt(
        title="IMDb:",
        options=options,
    )

    if option_function:
        option_function()
        prompt()

def update_titles_and_people() -> None:
    if confirm.prompt("<cyan>Download and update IMDb titles and people?</cyan>"):
        movielog_api.refresh_core_data()

def update_watchlist_person_credits() -> None:
    prompt_text = (
        "<cyan>This will update any non-frozen credits from the IMDb. Continue?</cyan>"
    )
    if confirm.prompt(prompt_text):
        movielog_api.refresh_credits()


#!/usr/bin/env python3
"""
https://adventofcode.com/2022/day/9
"""
from collections import namedtuple
from operator import add, sub

import aoc

PUZZLE = aoc.Puzzle(day=9, year=2022)

class Point(namedtuple('Point', ('x', 'y'))):
    """A point class with overridden operators"""

    def __abs__(self):
        return Point(*map(abs, self))

    def __add__(self, right):
        return Point(*map(add, self, right))

    def __sub__(self, right):
        return Point(*map(sub, self, right))

    def __repr__(self):
        return '(' + ', '.join(map(str, self)) + ')'

    def unit(self):
        """Return a Point() with each attribute set to -1, 0 or 1"""
        elements = []
        for element in self:
            sign = 0
            if element > 0:
                sign = 1
            elif element < 0:
                sign = -1
            elements.append(sign)
        return Point(*elements)

HEAD_DIRS = {
    'U': Point(1, 0),
    'D': Point(-1, 0),
    'L': Point(0, -1),
    'R': Point(0, 1),
}

def solve(part='a'):
    """Solve puzzle"""
    count = 2
    if part == 'b':
        count = 10
    knots = [Point(0, 0)] * count
    visited = set(knots[-1:])
    for line in PUZZLE.input.splitlines():
        direction, count = line.split()
        for _ in range(int(count)):
            knots[0] = knots[0] + HEAD_DIRS[direction]
            previous = knots[0]
            for offset, knot in enumerate(knots[1:], 1):
                diff = previous - knot
                if diff.x in (-1, 0, 1) and diff.y in (-1, 0, 1):
                    # tail is still "touching" head
                    previous = knot
                    continue
                knot = knot + diff.unit()
                knots[offset] = knot
                previous = knot
            visited.add(knots[-1])
    return len(visited)

if __name__ == "__main__":
    PUZZLE.report_a(solve('a'))
    PUZZLE.report_b(solve('b'))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Nov 26 2017

Description:
Input: symbols of chosen stocks
    Randomly assign weights to given stocks.
    Calculate portfolio return, volatility, and Sharp Ratio.
Output: portfolio with maximum SR or minimum volatility

@author: chenkai
"""
import numpy as np
import pandas as pd
import pandas_datareader.data as web
import matplotlib.pyplot as plt

#list of stocks in portfolio, i.e. stocks = ['AAPL','AMZN','MSFT','YHOO']
my_position = pd.read_csv('/Users/chenkai/Documents/Financial Analysis/cqa_challenge/OpenPosition_11_26_2017.csv')
stocks = my_position['Symbol']

#download daily price data for each of the stocks in the portfolio
data = web.DataReader(stocks,data_source='google',start='01/01/2010')['Close']

#convert daily stock prices into daily returns
returns = data.pct_change()

#calculate mean daily return and covariance of daily returns
mean_daily_returns = returns.mean()
cov_matrix = returns.cov()

#set number of runs of random portfolio weights
num_portfolios = 25000

#set up array to hold results
#We have increased the size of the array to hold the weight values for each stock
results = np.zeros((4+len(stocks)-1,num_portfolios))

for i in range(num_portfolios):
    #select random weights for portfolio holdings
    weights = np.array(np.random.random(len(stocks)))
    #rebalance weights to sum to 1
    weights /= np.sum(weights)

    #calculate portfolio return and volatility
    portfolio_return = np.sum(mean_daily_returns * weights) * 250
    portfolio_std_dev = np.sqrt(np.dot(weights.T,np.dot(cov_matrix, weights))) * np.sqrt(250)

    #store results in results array
    results[0,i] = portfolio_return
    results[1,i] = portfolio_std_dev
    #store Sharpe Ratio (return / volatility) - risk free rate element excluded for simplicity
    results[2,i] = results[0,i] / results[1,i]
    #iterate through the weight vector and add data to results array
    for j in range(len(weights)):
        results[j+3,i] = weights[j]

#convert results array to Pandas DataFrame
results_frame = pd.DataFrame(results.T)
result_list = pd.Series(['ret','stdev','sharpe'])
result_list = result_list.append(stocks)
result_list = result_list.reset_index(drop=True)
results_frame.rename(columns=result_list,inplace=True)

#locate position of portfolio with highest Sharpe Ratio
max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
#locate positon of portfolio with minimum standard deviation
min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]

#create scatter plot coloured by Sharpe Ratio
plt.scatter(results_frame.stdev,results_frame.ret,c=results_frame.sharpe,cmap='RdYlBu')
plt.xlabel('Volatility')
plt.ylabel('Returns')
plt.colorbar()
#plot red star to highlight position of portfolio with highest Sharpe Ratio
plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker=(5,1,0),color='r',s=1000)
#plot green star to highlight position of minimum variance portfolio
plt.scatter(min_vol_port[1],min_vol_port[0],marker=(5,1,0),color='g',s=1000)