from __future__ import absolute_import
from __future__ import division

import calendar
import errno
import logging
import os
import socket
import subprocess
import time
import xml.etree.ElementTree as etree

from vdsm.common import cmdutils
from vdsm.common import commands
from vdsm.network.netinfo import addresses

from . import exception as ge
from . import gluster_mgmt_api, gluster_api

_glusterCommandPath = cmdutils.CommandPath("gluster",
                                           "/usr/sbin/gluster",
                                           )
_TIME_ZONE = time.tzname[0]

_DEFAULT_TIMEOUT = 120  # secs

if hasattr(etree, 'ParseError'):
    _etreeExceptions = (etree.ParseError, AttributeError, ValueError)
else:
    _etreeExceptions = (SyntaxError, AttributeError, ValueError)

def _getGlusterVolCmd():
    return [_glusterCommandPath.cmd, "--mode=script", "volume"]

def _getGlusterPeerCmd():
    return [_glusterCommandPath.cmd, "--mode=script", "peer"]

def _getGlusterSystemCmd():
    return [_glusterCommandPath.cmd, "system::"]

def _getGlusterVolGeoRepCmd():
    return _getGlusterVolCmd() + ["geo-replication"]

def _getGlusterSnapshotCmd():
    return [_glusterCommandPath.cmd, "--mode=script", "snapshot"]

class BrickStatus:
    PAUSED = 'PAUSED'
    COMPLETED = 'COMPLETED'
    RUNNING = 'RUNNING'
    UNKNOWN = 'UNKNOWN'
    NA = 'NA'

class HostStatus:
    CONNECTED = 'CONNECTED'
    DISCONNECTED = 'DISCONNECTED'
    UNKNOWN = 'UNKNOWN'

class VolumeStatus:
    ONLINE = 'ONLINE'
    OFFLINE = 'OFFLINE'

class TransportType:
    TCP = 'TCP'
    RDMA = 'RDMA'

class TaskType:
    REBALANCE = 'REBALANCE'
    REPLACE_BRICK = 'REPLACE_BRICK'
    REMOVE_BRICK = 'REMOVE_BRICK'

class SnapshotStatus:
    ACTIVATED = 'ACTIVATED'
    DEACTIVATED = 'DEACTIVATED'

def _execGluster(cmd):
    try:
        return commands.run(cmd)
    except cmdutils.Error as e:
        raise ge.GlusterCmdFailedException(rc=e.rc, err=[e.msg])

def _getTree(out):
    try:
        tree = etree.fromstring(out)
        rv = int(tree.find('opRet').text)
        msg = tree.find('opErrstr').text
        errNo = int(tree.find('opErrno').text)
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=out)
    if rv == 0:
        return tree
    else:
        if errNo != 0:
            rv = errNo
        raise ge.GlusterCmdFailedException(rc=rv, err=[msg])

def _execGlusterXml(cmd):
    cmd.append('--xml')
    return _getTree(_execGluster(cmd))

def _execGlusterXmlWithTimeout(cmd, timeout=_DEFAULT_TIMEOUT):
    cmd.append('--xml')
    cmd = cmdutils.wrap_command(cmd)
    logging.debug(cmdutils.command_log_line(cmd))
    proc = subprocess.Popen(
        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    try:
        with commands.terminating(proc):
            out, err = proc.communicate(timeout=timeout)
    except subprocess.TimeoutExpired:
        raise ge.GlusterCommandTimeoutException(proc.returncode)

    logging.debug(cmdutils.retcode_log_line(proc.returncode, err))
    if proc.returncode != 0:
        raise ge.GlusterCmdExecFailedException(
            proc.returncode, out, err)

    return _getTree(out)

def _getLocalIpAddress():
    for ip in addresses.getIpAddresses():
        if not ip.startswith('127.'):
            return ip
    return ''

def _getGlusterHostName():
    try:
        return socket.getfqdn()
    except socket.herror:
        logging.exception('getfqdn')
        return ''

@gluster_mgmt_api
def hostUUIDGet():
    """
    Gluster command returns as
    $ gluster system:: uuid get
    UUID: d6b27c29-dce0-420e-9982-42f855bca9cd
    Strip the returned string starting from 6th char to fetch only uuid string
    """

    command = _getGlusterSystemCmd() + ["uuid", "get"]
    out = _execGluster(command)
    out = out.decode("utf-8")
    if not out.startswith('UUID: '):
        raise ge.GlusterHostUUIDNotFoundException()

    return out[6:].rstrip('\n')

def _parseVolumeStatus(tree):
    status = {'name': tree.find('volStatus/volumes/volume/volName').text,
              'bricks': [],
              'nfs': [],
              'shd': []}
    hostname = _getLocalIpAddress() or _getGlusterHostName()
    for el in tree.findall('volStatus/volumes/volume/node'):
        value = {}

        for ch in el:
            value[ch.tag] = ch.text or ''

        ports = {}
        for ch in el.find('ports'):
            ports[ch.tag] = ch.text or ''

        if value['path'] == 'localhost':
            value['path'] = hostname

        if value['status'] == '1':
            value['status'] = 'ONLINE'
        else:
            value['status'] = 'OFFLINE'

        if value['hostname'] == 'NFS Server':
            status['nfs'].append({'hostname': value['path'],
                                  'hostuuid': value['peerid'],
                                  'port': ports['tcp'],
                                  'rdma_port': ports['rdma'],
                                  'status': value['status'],
                                  'pid': value['pid']})
        elif value['hostname'] == 'Self-heal Daemon':
            status['shd'].append({'hostname': value['path'],
                                  'hostuuid': value['peerid'],
                                  'status': value['status'],
                                  'pid': value['pid']})
        else:
            status['bricks'].append({'brick': '%s:%s' % (value['hostname'],
                                                         value['path']),
                                     'hostuuid': value['peerid'],
                                     'port': ports['tcp'],
                                     'rdma_port': ports['rdma'],
                                     'status': value['status'],
                                     'pid': value['pid']})
    return status

def _parseVolumeStatusDetail(tree):
    status = {'name': tree.find('volStatus/volumes/volume/volName').text,
              'bricks': []}
    for el in tree.findall('volStatus/volumes/volume/node'):
        value = {}

        for ch in el:
            value[ch.tag] = ch.text or ''

        sizeTotal = int(value.get('sizeTotal', '0'))
        value['sizeTotal'] = sizeTotal / (1024.0 * 1024.0)
        sizeFree = int(value.get('sizeFree', '0'))
        value['sizeFree'] = sizeFree / (1024.0 * 1024.0)
        status['bricks'].append({'brick': '%s:%s' % (value['hostname'],
                                                     value['path']),
                                 'hostuuid': value['peerid'],
                                 'sizeTotal': '%.3f' % (value['sizeTotal'],),
                                 'sizeFree': '%.3f' % (value['sizeFree'],),
                                 'device': value.get('device', ''),
                                 'blockSize': value.get('blockSize', '0'),
                                 'mntOptions': value.get('mntOptions', ''),
                                 'fsName': value.get('fsName', '')})
    return status

def _parseVolumeStatusClients(tree):
    status = {'name': tree.find('volStatus/volumes/volume/volName').text,
              'bricks': []}
    for el in tree.findall('volStatus/volumes/volume/node'):
        hostname = el.find('hostname').text
        path = el.find('path').text
        hostuuid = el.find('peerid').text

        clientsStatus = []
        for c in el.findall('clientsStatus/client'):
            clientValue = {}
            for ch in c:
                clientValue[ch.tag] = ch.text or ''
            clientsStatus.append({'hostname': clientValue['hostname'],
                                  'bytesRead': clientValue['bytesRead'],
                                  'bytesWrite': clientValue['bytesWrite']})

        status['bricks'].append({'brick': '%s:%s' % (hostname, path),
                                 'hostuuid': hostuuid,
                                 'clientsStatus': clientsStatus})
    return status

def _parseVolumeStatusMem(tree):
    status = {'name': tree.find('volStatus/volumes/volume/volName').text,
              'bricks': []}
    for el in tree.findall('volStatus/volumes/volume/node'):
        brick = {'brick': '%s:%s' % (el.find('hostname').text,
                                     el.find('path').text),
                 'hostuuid': el.find('peerid').text,
                 'mallinfo': {},
                 'mempool': []}

        for ch in el.find('memStatus/mallinfo'):
            brick['mallinfo'][ch.tag] = ch.text or ''

        for c in el.findall('memStatus/mempool/pool'):
            mempool = {}
            for ch in c:
                mempool[ch.tag] = ch.text or ''
            brick['mempool'].append(mempool)

        status['bricks'].append(brick)
    return status

@gluster_mgmt_api
def volumeStatus(volumeName, brick=None, option=None):
    """
    Get volume status

    Arguments:
       * VolumeName
       * brick
       * option = 'detail' or 'clients' or 'mem' or None
    Returns:
       When option=None,
         {'name': NAME,
          'bricks': [{'brick': BRICK,
                      'hostuuid': UUID,
                      'port': PORT,
                      'rdma_port': RDMA_PORT,
                      'status': STATUS,
                      'pid': PID}, ...],
          'nfs': [{'hostname': HOST,
                   'hostuuid': UUID,
                   'port': PORT,
                   'rdma_port': RDMA_PORT,
                   'status': STATUS,
                   'pid': PID}, ...],
          'shd: [{'hostname': HOST,
                  'hostuuid': UUID,
                  'status': STATUS,
                  'pid': PID}, ...]}

      When option='detail',
         {'name': NAME,
          'bricks': [{'brick': BRICK,
                      'hostuuid': UUID,
                      'sizeTotal': SIZE,
                      'sizeFree': FREESIZE,
                      'device': DEVICE,
                      'blockSize': BLOCKSIZE,
                      'mntOptions': MOUNTOPTIONS,
                      'fsName': FSTYPE}, ...]}

       When option='clients':
         {'name': NAME,
          'bricks': [{'brick': BRICK,
                      'hostuuid': UUID,
                      'clientsStatus': [{'hostname': HOST,
                                         'bytesRead': BYTESREAD,
                                         'bytesWrite': BYTESWRITE}, ...]},
                    ...]}

       When option='mem':
         {'name': NAME,
          'bricks': [{'brick': BRICK,
                      'hostuuid': UUID,
                      'mallinfo': {'arena': int,
                                   'fordblks': int,
                                   'fsmblks': int,
                                   'hblkhd': int,
                                   'hblks': int,
                                   'keepcost': int,
                                   'ordblks': int,
                                   'smblks': int,
                                   'uordblks': int,
                                   'usmblks': int},
                      'mempool': [{'allocCount': int,
                                   'coldCount': int,
                                   'hotCount': int,
                                   'maxAlloc': int,
                                   'maxStdAlloc': int,
                                   'name': NAME,
                                   'padddedSizeOf': int,
                                   'poolMisses': int},...]}, ...]}
    """
    command = _getGlusterVolCmd() + ["status", volumeName]
    if brick:
        command.append(brick)
    if option:
        command.append(option)
    try:
        xmltree = _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeStatusFailedException(rc=e.rc, err=e.err)
    try:
        if option == 'detail':
            return _parseVolumeStatusDetail(xmltree)
        elif option == 'clients':
            return _parseVolumeStatusClients(xmltree)
        elif option == 'mem':
            return _parseVolumeStatusMem(xmltree)
        else:
            return _parseVolumeStatus(xmltree)
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

def _parseVolumeInfo(tree):
    """
        {VOLUMENAME: {'brickCount': BRICKCOUNT,
                      'bricks': [BRICK1, BRICK2, ...],
                      'options': {OPTION: VALUE, ...},
                      'transportType': [TCP,RDMA, ...],
                      'uuid': UUID,
                      'volumeName': NAME,
                      'volumeStatus': STATUS,
                      'volumeType': TYPE,
                      'disperseCount': DISPERSE_COUNT,
                      'redundancyCount': REDUNDANCY_COUNT,
                      'isArbiter': [True/False]}, ...}
    """
    volumes = {}
    for el in tree.findall('volInfo/volumes/volume'):
        value = {}
        value['volumeName'] = el.find('name').text
        value['uuid'] = el.find('id').text
        value['volumeType'] = el.find('typeStr').text.upper().replace('-', '_')
        status = el.find('statusStr').text.upper()
        if status == 'STARTED':
            value["volumeStatus"] = VolumeStatus.ONLINE
        else:
            value["volumeStatus"] = VolumeStatus.OFFLINE
        value['brickCount'] = el.find('brickCount').text
        value['distCount'] = el.find('distCount').text
        value['stripeCount'] = el.findtext('stripeCount', '1')
        value['replicaCount'] = el.find('replicaCount').text
        value['disperseCount'] = el.find('disperseCount').text
        value['redundancyCount'] = el.find('redundancyCount').text
        value['isArbiter'] = (el.find('arbiterCount').text == '1')
        transportType = el.find('transport').text
        if transportType == '0':
            value['transportType'] = [TransportType.TCP]
        elif transportType == '1':
            value['transportType'] = [TransportType.RDMA]
        else:
            value['transportType'] = [TransportType.TCP, TransportType.RDMA]
        value['bricks'] = []
        value['options'] = {}
        value['bricksInfo'] = []
        for b in el.findall('bricks/brick'):
            value['bricks'].append(b.text)
        for o in el.findall('options/option'):
            value['options'][o.find('name').text] = o.find('value').text
        for d in el.findall('bricks/brick'):
            brickDetail = {}
            # this try block is to maintain backward compatibility
            # it returns an empty list when gluster doesnot return uuid
            try:
                brickDetail['name'] = d.find('name').text
                brickDetail['hostUuid'] = d.find('hostUuid').text
                brickDetail['isArbiter'] = (d.find('isArbiter').text == '1')
                value['bricksInfo'].append(brickDetail)
            except AttributeError:
                break
        volumes[value['volumeName']] = value
    return volumes

def _parseVolumeProfileInfo(tree, nfs):
    bricks = []
    if nfs:
        brickKey = 'nfs'
        bricksKey = 'nfsServers'
    else:
        brickKey = 'brick'
        bricksKey = 'bricks'
    for brick in tree.findall('volProfile/brick'):
        fopCumulative = []
        blkCumulative = []
        fopInterval = []
        blkInterval = []
        brickName = brick.find('brickName').text
        if brickName == 'localhost':
            brickName = _getLocalIpAddress() or _getGlusterHostName()
        for block in brick.findall('cumulativeStats/blockStats/block'):
            blkCumulative.append({'size': block.find('size').text,
                                  'read': block.find('reads').text,
                                  'write': block.find('writes').text})
        for fop in brick.findall('cumulativeStats/fopStats/fop'):
            fopCumulative.append({'name': fop.find('name').text,
                                  'hits': fop.find('hits').text,
                                  'latencyAvg': fop.find('avgLatency').text,
                                  'latencyMin': fop.find('minLatency').text,
                                  'latencyMax': fop.find('maxLatency').text})
        for block in brick.findall('intervalStats/blockStats/block'):
            blkInterval.append({'size': block.find('size').text,
                                'read': block.find('reads').text,
                                'write': block.find('writes').text})
        for fop in brick.findall('intervalStats/fopStats/fop'):
            fopInterval.append({'name': fop.find('name').text,
                                'hits': fop.find('hits').text,
                                'latencyAvg': fop.find('avgLatency').text,
                                'latencyMin': fop.find('minLatency').text,
                                'latencyMax': fop.find('maxLatency').text})
        bricks.append(
            {brickKey: brickName,
             'cumulativeStats': {
                 'blockStats': blkCumulative,
                 'fopStats': fopCumulative,
                 'duration': brick.find('cumulativeStats/duration').text,
                 'totalRead': brick.find('cumulativeStats/totalRead').text,
                 'totalWrite': brick.find('cumulativeStats/totalWrite').text},
             'intervalStats': {
                 'blockStats': blkInterval,
                 'fopStats': fopInterval,
                 'duration': brick.find('intervalStats/duration').text,
                 'totalRead': brick.find('intervalStats/totalRead').text,
                 'totalWrite': brick.find('intervalStats/totalWrite').text}})
    status = {'volumeName': tree.find("volProfile/volname").text,
              bricksKey: bricks}
    return status

@gluster_api
@gluster_mgmt_api
def volumeInfo(volumeName=None, remoteServer=None):
    """
    Returns:
        {VOLUMENAME: {'brickCount': BRICKCOUNT,
                      'bricks': [BRICK1, BRICK2, ...],
                      'options': {OPTION: VALUE, ...},
                      'transportType': [TCP,RDMA, ...],
                      'uuid': UUID,
                      'volumeName': NAME,
                      'volumeStatus': STATUS,
                      'volumeType': TYPE}, ...}
    """
    command = _getGlusterVolCmd() + ["info"]
    if remoteServer:
        command += ['--remote-host=%s' % remoteServer]
        if not is_ipv4_fqdn(remoteServer):
            command += ['--inet6']
    if volumeName:
        command.append(volumeName)
    try:
        xmltree = _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumesListFailedException(rc=e.rc, err=e.err)
    try:
        return _parseVolumeInfo(xmltree)
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

def is_ipv4_fqdn(address):
    try:
        socket.getaddrinfo(address, None, family=socket.AF_INET)
    except OSError:  # cannot resolve to IPV4
        return False
    return True

@gluster_mgmt_api
def volumeCreate(volumeName, brickList, replicaCount=0, stripeCount=0,
                 transportList=[], force=False, arbiter=False):
    command = _getGlusterVolCmd() + ["create", volumeName]
    if stripeCount:
        command += ["stripe", "%s" % stripeCount]
    if replicaCount:
        command += ["replica", "%s" % replicaCount]
    if arbiter:
        command += ["arbiter", "1"]
    if transportList:
        command += ["transport", ','.join(transportList)]
    command += brickList

    if force:
        command.append('force')

    try:
        xmltree = _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeCreateFailedException(rc=e.rc, err=e.err)
    try:
        return {'uuid': xmltree.find('volCreate/volume/id').text}
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

@gluster_mgmt_api
def volumeStart(volumeName, force=False):
    command = _getGlusterVolCmd() + ["start", volumeName]
    if force:
        command.append('force')
    try:
        _execGluster(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeStartFailedException(rc=e.rc, err=e.err)
    return True

@gluster_mgmt_api
def volumeStop(volumeName, force=False):
    command = _getGlusterVolCmd() + ["stop", volumeName]
    if force:
        command.append('force')
    try:
        _execGlusterXml(command)
        return True
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeStopFailedException(rc=e.rc, err=e.err)

@gluster_mgmt_api
def volumeDelete(volumeName):
    command = _getGlusterVolCmd() + ["delete", volumeName]
    try:
        _execGlusterXml(command)
        return True
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeDeleteFailedException(rc=e.rc, err=e.err)

@gluster_mgmt_api
def volumeSet(volumeName, option, value):
    heal_is_set = option in ('cluster.granular-entry-heal',
                             'granular-entry-heal')
    volume_created = _checkIfVolumeCreated(volumeName)
    command = _getGlusterVolCmd()
    if heal_is_set and not volume_created:
        command += ['heal', volumeName, 'granular-entry-heal', value]
    else:
        command += ['set', volumeName, option, value]
    try:
        _execGlusterXml(command)
        return True
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeSetFailedException(rc=e.rc, err=e.err)

def _checkIfVolumeCreated(volumeName):
    vol_info_cmd = _getGlusterVolCmd() + ["info", volumeName]
    xmltree = _execGlusterXml(vol_info_cmd)
    vol_info = xmltree.find('volInfo/volumes/volume')
    status = vol_info.find('statusStr').text.upper()
    if status == "CREATED":
        return True
    return False

def _parseVolumeSetHelpXml(out):
    optionList = []
    tree = etree.fromstring(out)
    for el in tree.findall('option'):
        option = {}
        for ch in el:
            option[ch.tag] = ch.text or ''
        optionList.append(option)
    return optionList

@gluster_mgmt_api
def volumeSetHelpXml():
    command = _getGlusterVolCmd() + ["set", 'help-xml']
    try:
        out = _execGluster(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeSetHelpXmlFailedException(rc=e.rc, err=e.err)
    return _parseVolumeSetHelpXml(out)

@gluster_mgmt_api
def volumeReset(volumeName, option='', force=False):
    command = _getGlusterVolCmd() + ['reset', volumeName]
    if option:
        command.append(option)
    if force:
        command.append('force')
    try:
        _execGlusterXml(command)
        return True
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeResetFailedException(rc=e.rc, err=e.err)

@gluster_mgmt_api
def volumeAddBrick(volumeName, brickList,
                   replicaCount=0, stripeCount=0, force=False):
    command = _getGlusterVolCmd() + ["add-brick", volumeName]
    if stripeCount:
        command += ["stripe", "%s" % stripeCount]
    if replicaCount:
        command += ["replica", "%s" % replicaCount]
    command += brickList
    if force:
        command.append('force')
    try:
        _execGlusterXml(command)
        return True
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeBrickAddFailedException(rc=e.rc, err=e.err)

@gluster_mgmt_api
def volumeRebalanceStart(volumeName, rebalanceType="", force=False):
    command = _getGlusterVolCmd() + ["rebalance", volumeName]
    if rebalanceType:
        command.append(rebalanceType)
    command.append("start")
    if force:
        command.append("force")
    try:
        xmltree = _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeRebalanceStartFailedException(rc=e.rc,
                                                            err=e.err)
    try:
        return {'taskId': xmltree.find('volRebalance/task-id').text}
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

@gluster_mgmt_api
def volumeRebalanceStop(volumeName, force=False):
    command = _getGlusterVolCmd() + ["rebalance", volumeName, "stop"]
    if force:
        command.append('force')
    try:
        xmltree = _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeRebalanceStopFailedException(rc=e.rc,
                                                           err=e.err)

    try:
        return _parseVolumeRebalanceRemoveBrickStatus(xmltree, 'rebalance')
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

@gluster_mgmt_api
def _parseVolumeRebalanceRemoveBrickStatus(xmltree, mode):
    """
    returns {'hosts': [{'name': NAME,
                        'id': UUID_STRING,
                        'runtime': FLOAT_AS_STRING,
                        'filesScanned': INT AS STRING,
                        'filesMoved': INT AS STRING,
                        'filesFailed': INT AS STRING,
                        'filesSkipped': INT AS STRING,
                        'totalSizeMoved': INT AS STRING,
                        'status': STRING},...]
             'summary': {'runtime': FLOAT_AS_STRING,
                         'filesScanned': INT AS STRING,
                         'filesMoved': INT AS STRING,
                         'filesFailed': INT AS STRING,
                         'filesSkipped': INT AS STRING,
                         'totalSizeMoved': INT AS STRING,
                         'status': STRING}}
    """
    if mode == 'rebalance':
        tree = xmltree.find('volRebalance')
    elif mode == 'remove-brick':
        tree = xmltree.find('volRemoveBrick')
    else:
        return

    st = tree.find('aggregate/statusStr').text
    statusStr = st.replace(' ', '_').replace('-', '_')
    status = {
        'summary': {
            'runtime': tree.find('aggregate/runtime').text,
            'filesScanned': tree.find('aggregate/lookups').text,
            'filesMoved': tree.find('aggregate/files').text,
            'filesFailed': tree.find('aggregate/failures').text,
            'filesSkipped': tree.find('aggregate/skipped').text,
            'totalSizeMoved': tree.find('aggregate/size').text,
            'status': statusStr.upper()},
        'hosts': []}

    for el in tree.findall('node'):
        st = el.find('statusStr').text
        statusStr = st.replace(' ', '_').replace('-', '_')
        status['hosts'].append({'name': el.find('nodeName').text,
                                'id': el.find('id').text,
                                'runtime': el.find('runtime').text,
                                'filesScanned': el.find('lookups').text,
                                'filesMoved': el.find('files').text,
                                'filesFailed': el.find('failures').text,
                                'filesSkipped': el.find('skipped').text,
                                'totalSizeMoved': el.find('size').text,
                                'status': statusStr.upper()})

    return status

@gluster_mgmt_api
def volumeRebalanceStatus(volumeName):
    command = _getGlusterVolCmd() + ["rebalance", volumeName, "status"]
    try:
        xmltree = _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeRebalanceStatusFailedException(rc=e.rc,
                                                             err=e.err)
    try:
        return _parseVolumeRebalanceRemoveBrickStatus(xmltree, 'rebalance')
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

@gluster_mgmt_api
def volumeReplaceBrickCommitForce(volumeName, existingBrick, newBrick):
    command = _getGlusterVolCmd() + ["replace-brick", volumeName,
                                     existingBrick, newBrick, "commit",
                                     "force"]
    try:
        _execGlusterXml(command)
        return True
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeReplaceBrickCommitForceFailedException(rc=e.rc,
                                                                     err=e.err)

@gluster_mgmt_api
def volumeRemoveBrickStart(volumeName, brickList, replicaCount=0):
    command = _getGlusterVolCmd() + ["remove-brick", volumeName]
    if replicaCount:
        command += ["replica", "%s" % replicaCount]
    command += brickList + ["start"]
    try:
        xmltree = _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeRemoveBrickStartFailedException(rc=e.rc,
                                                              err=e.err)
    try:
        return {'taskId': xmltree.find('volRemoveBrick/task-id').text}
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

@gluster_mgmt_api
def volumeRemoveBrickStop(volumeName, brickList, replicaCount=0):
    command = _getGlusterVolCmd() + ["remove-brick", volumeName]
    if replicaCount:
        command += ["replica", "%s" % replicaCount]
    command += brickList + ["stop"]
    try:
        xmltree = _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeRemoveBrickStopFailedException(rc=e.rc,
                                                             err=e.err)

    try:
        return _parseVolumeRebalanceRemoveBrickStatus(xmltree, 'remove-brick')
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

@gluster_mgmt_api
def volumeRemoveBrickStatus(volumeName, brickList, replicaCount=0):
    command = _getGlusterVolCmd() + ["remove-brick", volumeName]
    if replicaCount:
        command += ["replica", "%s" % replicaCount]
    command += brickList + ["status"]
    try:
        xmltree = _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeRemoveBrickStatusFailedException(rc=e.rc,
                                                               err=e.err)
    try:
        return _parseVolumeRebalanceRemoveBrickStatus(xmltree, 'remove-brick')
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

@gluster_mgmt_api
def volumeRemoveBrickCommit(volumeName, brickList, replicaCount=0):
    command = _getGlusterVolCmd() + ["remove-brick", volumeName]
    if replicaCount:
        command += ["replica", "%s" % replicaCount]
    command += brickList + ["commit"]
    try:
        _execGlusterXml(command)
        return True
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeRemoveBrickCommitFailedException(rc=e.rc,
                                                               err=e.err)

@gluster_mgmt_api
def volumeRemoveBrickForce(volumeName, brickList, replicaCount=0):
    command = _getGlusterVolCmd() + ["remove-brick", volumeName]
    if replicaCount:
        command += ["replica", "%s" % replicaCount]
    command += brickList + ["force"]
    try:
        _execGlusterXml(command)
        return True
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeRemoveBrickForceFailedException(rc=e.rc,
                                                              err=e.err)

@gluster_mgmt_api
def peerProbe(hostName):
    command = _getGlusterPeerCmd() + ["probe", hostName]
    try:
        _execGlusterXml(command)
        return True
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterHostAddFailedException(rc=e.rc, err=e.err)

@gluster_mgmt_api
def peerDetach(hostName, force=False):
    command = _getGlusterPeerCmd() + ["detach", hostName]
    if force:
        command.append('force')
    try:
        _execGlusterXml(command)
        return True
    except ge.GlusterCmdFailedException as e:
        if e.rc == 2:
            raise ge.GlusterHostNotFoundException(rc=e.rc, err=e.err)
        else:
            raise ge.GlusterHostRemoveFailedException(rc=e.rc, err=e.err)

def _parsePeerStatus(tree, gHostName, gUuid, gStatus):
    hostList = [{'hostname': gHostName,
                 'uuid': gUuid,
                 'status': gStatus}]

    for el in tree.findall('peerStatus/peer'):
        if el.find('state').text != '3':
            status = HostStatus.UNKNOWN
        elif el.find('connected').text == '1':
            status = HostStatus.CONNECTED
        else:
            status = HostStatus.DISCONNECTED
        hostList.append({'hostname': el.find('hostname').text,
                         'uuid': el.find('uuid').text,
                         'status': status})

    return hostList

@gluster_mgmt_api
def peerStatus():
    """
    Returns:
        [{'hostname': HOSTNAME, 'uuid': UUID, 'status': STATE}, ...]
    """
    command = _getGlusterPeerCmd() + ["status"]
    try:
        xmltree = _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterHostsListFailedException(rc=e.rc, err=e.err)
    try:
        return _parsePeerStatus(xmltree,
                                _getLocalIpAddress() or _getGlusterHostName(),
                                hostUUIDGet(), HostStatus.CONNECTED)
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

@gluster_mgmt_api
def volumeProfileStart(volumeName):
    command = _getGlusterVolCmd() + ["profile", volumeName, "start"]
    try:
        _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeProfileStartFailedException(rc=e.rc, err=e.err)
    return True

@gluster_mgmt_api
def volumeProfileStop(volumeName):
    command = _getGlusterVolCmd() + ["profile", volumeName, "stop"]
    try:
        _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeProfileStopFailedException(rc=e.rc, err=e.err)
    return True

@gluster_mgmt_api
def volumeProfileInfo(volumeName, nfs=False):
    """
    Returns:
    When nfs=True:
    {'volumeName': VOLUME-NAME,
     'nfsServers': [
         {'nfs': SERVER-NAME,
          'cumulativeStats': {'blockStats': [{'size': int,
                                              'read': int,
                                              'write': int}, ...],
                              'fopStats': [{'name': FOP-NAME,
                                            'hits': int,
                                            'latencyAvg': float,
                                            'latencyMin': float,
                                            'latencyMax': float}, ...],
                              'duration': int,
                              'totalRead': int,
                              'totalWrite': int},
          'intervalStats': {'blockStats': [{'size': int,
                                            'read': int,
                                            'write': int}, ...],
                            'fopStats': [{'name': FOP-NAME,
                                          'hits': int,
                                          'latencyAvg': float,
                                          'latencyMin': float,
                                          'latencyMax': float}, ...],
                            'duration': int,
                            'totalRead': int,
                            'totalWrite': int}}, ...]}

    When nfs=False:
    {'volumeName': VOLUME-NAME,
     'bricks': [
         {'brick': BRICK-NAME,
          'cumulativeStats': {'blockStats': [{'size': int,
                                              'read': int,
                                              'write': int}, ...],
                              'fopStats': [{'name': FOP-NAME,
                                            'hits': int,
                                            'latencyAvg': float,
                                            'latencyMin': float,
                                            'latencyMax': float}, ...],
                              'duration': int,
                              'totalRead': int,
                              'totalWrite': int},
          'intervalStats': {'blockStats': [{'size': int,
                                            'read': int,
                                            'write': int}, ...],
                            'fopStats': [{'name': FOP-NAME,
                                          'hits': int,
                                          'latencyAvg': float,
                                          'latencyMin': float,
                                          'latencyMax': float}, ...],
                            'duration': int,
                            'totalRead': int,
                            'totalWrite': int}}, ...]}
    """
    command = _getGlusterVolCmd() + ["profile", volumeName, "info"]
    if nfs:
        command += ["nfs"]
    try:
        xmltree = _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeProfileInfoFailedException(rc=e.rc, err=e.err)
    try:
        return _parseVolumeProfileInfo(xmltree, nfs)
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

def _parseVolumeTasks(tree):
    """
    returns {TaskId: {'volumeName': VolumeName,
                      'taskType': TaskType,
                      'status': STATUS,
                      'bricks': BrickList}, ...}
    """
    tasks = {}
    for el in tree.findall('volStatus/volumes/volume'):
        volumeName = el.find('volName').text
        for c in el.findall('tasks/task'):
            taskType = c.find('type').text
            taskType = taskType.upper().replace('-', '_').replace(' ', '_')
            taskId = c.find('id').text
            bricks = []
            if taskType == TaskType.REPLACE_BRICK:
                bricks.append(c.find('params/srcBrick').text)
                bricks.append(c.find('params/dstBrick').text)
            elif taskType == TaskType.REMOVE_BRICK:
                for b in c.findall('params/brick'):
                    bricks.append(b.text)
            elif taskType == TaskType.REBALANCE:
                pass

            statusStr = c.find('statusStr').text.upper() \
                                                .replace('-', '_') \
                                                .replace(' ', '_')

            tasks[taskId] = {'volumeName': volumeName,
                             'taskType': taskType,
                             'status': statusStr,
                             'bricks': bricks}
    return tasks

@gluster_mgmt_api
def volumeTasks(volumeName="all"):
    command = _getGlusterVolCmd() + ["status", volumeName, "tasks"]
    try:
        xmltree = _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeTasksFailedException(rc=e.rc, err=e.err)
    try:
        return _parseVolumeTasks(xmltree)
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

@gluster_mgmt_api
def volumeGeoRepSessionStart(volumeName, remoteHost, remoteVolumeName,
                             remoteUserName=None, force=False):
    if remoteUserName:
        userAtHost = "%s@%s" % (remoteUserName, remoteHost)
    else:
        userAtHost = remoteHost
    command = _getGlusterVolGeoRepCmd() + [volumeName, "%s::%s" % (
        userAtHost, remoteVolumeName), "start"]
    if force:
        command.append('force')
    try:
        _execGlusterXml(command)
        return True
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeGeoRepSessionStartFailedException(rc=e.rc,
                                                                err=e.err)

@gluster_mgmt_api
def volumeGeoRepSessionStop(volumeName, remoteHost, remoteVolumeName,
                            remoteUserName=None, force=False):
    if remoteUserName:
        userAtHost = "%s@%s" % (remoteUserName, remoteHost)
    else:
        userAtHost = remoteHost
    command = _getGlusterVolGeoRepCmd() + [volumeName, "%s::%s" % (
        userAtHost, remoteVolumeName), "stop"]
    if force:
        command.append('force')
    try:
        _execGlusterXml(command)
        return True
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeGeoRepSessionStopFailedException(rc=e.rc,
                                                               err=e.err)

def _parseGeoRepStatus(tree):
    """
    Returns:
    {volume-name: [{sessionKey: 'key to identify the session',
                    remoteVolumeName: 'volume in remote gluster cluster'
                    bricks: [{host: 'local node',
                              hostUuid: 'uuid of brick host',
                              brickName: 'brick in the local volume',
                              remoteHost: 'secondary',
                              status: 'status'
                              remoteUserName: 'root'
                              timeZone: 'nodes time zone'
                              crawlStatus: 'crawlStatus'
                              lastSynced: 'last synced time'
                              entry: 'nos of entry operations pending'
                              data: 'nos of data operations pending'
                              meta: 'nos of meta operations pending'
                              failures: 'nos of failures'
                              checkpointTime: 'checkpoint set time'
                              checkpointCompletionTime: 'checkpoint completion
                                                         time'
                              checkpointCompleted: 'yes/no'}]...
               ]....
    }
    """
    gluster_old_format = tree.find(
        'geoRep/volume/sessions/session/session_secondary') is None
    status = {}
    for volume in tree.findall('geoRep/volume'):
        sessions = []
        volumeDetail = {}
        for session in volume.findall('sessions/session'):
            pairs = []
            sessionDetail = {}
            if gluster_old_format:
                sessionDetail['sessionKey'] = session.find(
                    'session_slave').text
            else:
                sessionDetail['sessionKey'] = session.find(
                    'session_secondary').text
            sessionDetail['remoteVolumeName'] = sessionDetail[
                'sessionKey'].split("::")[-1].split(":")[0]
            for pair in session.findall('pair'):
                pairDetail = {}
                if gluster_old_format:
                    pairDetail['host'] = pair.find('master_node').text
                    pairDetail['hostUuid'] = pair.find('master_node_uuid').text
                    pairDetail['brickName'] = pair.find('master_brick').text
                    pairDetail['remoteHost'] = pair.find('slave_node').text
                    if pairDetail['remoteHost'] is None:
                        pairDetail['remoteHost'] = pair.find(
                            'slave').text.split("::")[0]
                    pairDetail['remoteUserName'] = pair.find(
                        'slave_user').text
                else:
                    pairDetail['host'] = pair.find('primary_node').text
                    pairDetail['hostUuid'] = pair.find(
                        'primary_node_uuid').text
                    pairDetail['brickName'] = pair.find('primary_brick').text
                    pairDetail['remoteHost'] = pair.find(
                        'secondary').text.split("::")[0]
                    pairDetail['remoteUserName'] = pair.find(
                        'secondary_user').text
                pairDetail['status'] = pair.find('status').text
                pairDetail['crawlStatus'] = pair.find('crawl_status').text
                pairDetail['timeZone'] = _TIME_ZONE
                pairDetail['lastSynced'] = pair.find('last_synced').text
                if pairDetail['lastSynced'] != 'N/A':
                    pairDetail['lastSynced'] = calendar.timegm(
                        time.strptime(pairDetail['lastSynced'],
                                      "%Y-%m-%d %H:%M:%S"))

                pairDetail['checkpointTime'] = pair.find(
                    'checkpoint_time').text
                if pairDetail['checkpointTime'] != 'N/A':
                    pairDetail['checkpointTime'] = calendar.timegm(
                        time.strptime(pairDetail['checkpointTime'],
                                      "%Y-%m-%d %H:%M:%S"))

                pairDetail['checkpointCompletionTime'] = pair.find(
                    'checkpoint_completion_time').text
                if pairDetail['checkpointCompletionTime'] != 'N/A':
                    pairDetail['checkpointCompletionTime'] = calendar.timegm(
                        time.strptime(pairDetail['checkpointCompletionTime'],
                                      "%Y-%m-%d %H:%M:%S"))

                pairDetail['entry'] = pair.find('entry').text
                pairDetail['data'] = pair.find('data').text
                pairDetail['meta'] = pair.find('meta').text
                pairDetail['failures'] = pair.find('failures').text
                pairDetail['checkpointCompleted'] = pair.find(
                    'checkpoint_completed').text
                pairs.append(pairDetail)
            sessionDetail['bricks'] = pairs
            sessions.append(sessionDetail)
        volumeDetail['sessions'] = sessions
        status[volume.find('name').text] = volumeDetail
    return status

@gluster_mgmt_api
def volumeGeoRepStatus(volumeName=None, remoteHost=None,
                       remoteVolumeName=None, remoteUserName=None):
    if remoteUserName:
        userAtHost = "%s@%s" % (remoteUserName, remoteHost)
    else:
        userAtHost = remoteHost
    command = _getGlusterVolGeoRepCmd()
    if volumeName:
        command.append(volumeName)
    if remoteHost and remoteVolumeName:
        command.append("%s::%s" % (userAtHost, remoteVolumeName))
    command.append("status")

    try:
        xmltree = _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterGeoRepStatusFailedException(rc=e.rc, err=e.err)
    try:
        return _parseGeoRepStatus(xmltree)
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

@gluster_mgmt_api
def volumeGeoRepSessionPause(volumeName, remoteHost, remoteVolumeName,
                             remoteUserName=None, force=False):
    if remoteUserName:
        userAtHost = "%s@%s" % (remoteUserName, remoteHost)
    else:
        userAtHost = remoteHost
    command = _getGlusterVolGeoRepCmd() + [volumeName, "%s::%s" % (
        userAtHost, remoteVolumeName), "pause"]
    if force:
        command.append('force')
    try:
        _execGlusterXml(command)
        return True
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeGeoRepSessionPauseFailedException(rc=e.rc,
                                                                err=e.err)

@gluster_mgmt_api
def volumeGeoRepSessionResume(volumeName, remoteHost, remoteVolumeName,
                              remoteUserName=None, force=False):
    if remoteUserName:
        userAtHost = "%s@%s" % (remoteUserName, remoteHost)
    else:
        userAtHost = remoteHost
    command = _getGlusterVolGeoRepCmd() + [volumeName, "%s::%s" % (
        userAtHost, remoteVolumeName), "resume"]
    if force:
        command.append('force')
    try:
        _execGlusterXml(command)
        return True
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeGeoRepSessionResumeFailedException(rc=e.rc,
                                                                 err=e.err)

def _parseVolumeGeoRepConfig(tree):
    """
    Returns:
    {geoRepConfig:{'optionName': 'optionValue',...}
    }
    """
    conf = tree.find('geoRep/config')
    config = {}
    for child in conf:
        config[child.tag] = child.text
    return {'geoRepConfig': config}

@gluster_mgmt_api
def volumeGeoRepConfig(volumeName, remoteHost,
                       remoteVolumeName, optionName=None,
                       optionValue=None,
                       remoteUserName=None):
    if remoteUserName:
        userAtHost = "%s@%s" % (remoteUserName, remoteHost)
    else:
        userAtHost = remoteHost
    command = _getGlusterVolGeoRepCmd() + [volumeName, "%s::%s" % (
        userAtHost, remoteVolumeName), "config"]
    if optionName and optionValue:
        command += [optionName, optionValue]
    elif optionName:
        command += ["!%s" % optionName]

    try:
        xmltree = _execGlusterXml(command)
        if optionName:
            return True
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterGeoRepConfigFailedException(rc=e.rc, err=e.err)
    try:
        return _parseVolumeGeoRepConfig(xmltree)
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

@gluster_mgmt_api
def snapshotCreate(volumeName, snapName,
                   snapDescription=None,
                   force=False):
    command = _getGlusterSnapshotCmd() + ["create", snapName, volumeName]

    if snapDescription:
        command += ['description', snapDescription]
    if force:
        command.append('force')

    try:
        xmltree = _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterSnapshotCreateFailedException(rc=e.rc, err=e.err)
    try:
        return {'uuid': xmltree.find('snapCreate/snapshot/uuid').text,
                'name': xmltree.find('snapCreate/snapshot/name').text}
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

@gluster_mgmt_api
def snapshotDelete(volumeName=None, snapName=None):
    command = _getGlusterSnapshotCmd() + ["delete"]
    if snapName:
        command.append(snapName)
    elif volumeName:
        command += ["volume", volumeName]

    # xml output not used because of BZ:1161416 in gluster cli
    try:
        _execGluster(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterSnapshotDeleteFailedException(rc=e.rc, err=e.err)
    return True

@gluster_mgmt_api
def snapshotActivate(snapName, force=False):
    command = _getGlusterSnapshotCmd() + ["activate", snapName]

    if force:
        command.append('force')

    try:
        _execGlusterXml(command)
        return True
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterSnapshotActivateFailedException(rc=e.rc, err=e.err)

@gluster_mgmt_api
def snapshotDeactivate(snapName):
    command = _getGlusterSnapshotCmd() + ["deactivate", snapName]

    try:
        _execGlusterXml(command)
        return True
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterSnapshotDeactivateFailedException(rc=e.rc, err=e.err)

def _parseRestoredSnapshot(tree):
    """
    returns {'volumeName': 'vol1',
             'volumeUuid': 'uuid',
             'snapshotName': 'snap2',
             'snapshotUuid': 'uuid'
            }
    """
    snapshotRestore = {}
    snapshotRestore['volumeName'] = tree.find('snapRestore/volume/name').text
    snapshotRestore['volumeUuid'] = tree.find('snapRestore/volume/uuid').text
    snapshotRestore['snapshotName'] = tree.find(
        'snapRestore/snapshot/name').text
    snapshotRestore['snapshotUuid'] = tree.find(
        'snapRestore/snapshot/uuid').text

    return snapshotRestore

@gluster_mgmt_api
def snapshotRestore(snapName):
    command = _getGlusterSnapshotCmd() + ["restore", snapName]

    try:
        xmltree = _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterSnapshotRestoreFailedException(rc=e.rc, err=e.err)
    try:
        return _parseRestoredSnapshot(xmltree)
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

def _parseSnapshotConfigList(tree):
    """
    returns {'system':{'snap-max-hard-limit': 'hardlimit',
                       'snap-max-soft-limit': 'softLimit',
                       'auto-delete': 'enable/disable',
                       'activate-on-create': 'enable/disable'},
             'volume':{'name' :
                          {'snap-max-hard-limit: 'hardlimit'}
                      }
            }
    """
    systemConfig = {}
    systemConfig['snap-max-hard-limit'] = tree.find(
        'snapConfig/systemConfig/hardLimit').text
    systemConfig['snap-max-soft-limit'] = tree.find(
        'snapConfig/systemConfig/softLimit').text
    systemConfig['auto-delete'] = tree.find(
        'snapConfig/systemConfig/autoDelete').text
    systemConfig['activate-on-create'] = tree.find(
        'snapConfig/systemConfig/activateOnCreate').text

    volumeConfig = {}
    for el in tree.findall('snapConfig/volumeConfig/volume'):
        config = {}
        volumeName = el.find('name').text
        config['snap-max-hard-limit'] = el.find('effectiveHardLimit').text
        volumeConfig[volumeName] = config

    return {'system': systemConfig, 'volume': volumeConfig}

@gluster_mgmt_api
def snapshotConfig(volumeName=None, optionName=None, optionValue=None):
    command = _getGlusterSnapshotCmd() + ["config"]
    if volumeName:
        command.append(volumeName)
    if optionName and optionValue:
        command += [optionName, optionValue]

    try:
        xmltree = _execGlusterXml(command)
        if optionName and optionValue:
            return
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterSnapshotConfigFailedException(rc=e.rc, err=e.err)
    try:
        return _parseSnapshotConfigList(xmltree)
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

def _parseVolumeSnapshotList(tree):
    """
    {'v1': {'snapshots': [{'name': 'snap1_v1',
                           'description': description of the snapshot,
                           'id': '8add41ae-c60c-4023'
                                           '-a1a6-5093a5d35603',
                           'createTime': {'timeZone': 'IST',
                                          'epochTime': 1414427114}
                           'snapVolume': '5eeaf23def3f446d898e1de8461a6aa7'
                           'snapVolumeStatus': 'ACTIVATED'}, ...],
            'snapRemaining': 252}
    }
    """
    volume = {}
    volumeName = tree.find(
        'snapInfo/originVolume/name').text
    volume[volumeName] = {
        'snapRemaining': tree.find('snapInfo/originVolume/snapRemaining').text,
        'snapshots': []
    }
    if int(tree.find('snapInfo/count').text) == 0:
        return {}
    for el in tree.findall('snapInfo/snapshots/snapshot'):
        snapshot = {}
        snapshot['id'] = el.find('uuid').text
        snapshot['description'] = "" if el.find('description') is None \
                                  else el.find('description').text
        snapshot['createTime'] = {
            'epochTime': calendar.timegm(
                time.strptime(el.find('createTime').text,
                              "%Y-%m-%d %H:%M:%S")
            ),
            'timeZone': _TIME_ZONE
        }
        snapshot['snapVolume'] = el.find('snapVolume/name').text
        status = el.find('snapVolume/status').text
        if status.upper() == 'STARTED':
            snapshot['snapVolumeStatus'] = SnapshotStatus.ACTIVATED
        else:
            snapshot['snapVolumeStatus'] = SnapshotStatus.DEACTIVATED
        snapshot['name'] = el.find('name').text
        volume[volumeName]['snapshots'].append(snapshot)
    return volume

def _parseAllVolumeSnapshotList(tree):
    """
    {'v1': {'snapshots': [{'name': 'snap1_v1',
                           'description': description of the snapshot,
                           'id': '8add41ae-c60c-4023-'
                                           'a1a6-5093a5d35603',
                           'createTime': {'timeZone': 'IST',
                                          'epochTime': 141442711}
                           'snapVolume': '5eeaf23def3f446d898e1de8461a6aa7'
                           'snapVolumeStatus': 'ACTIVATED'}, ...],
            'snapRemaining': 252},
     'v2': {'snapshots': [{'name': 'snap1_v2',
                           'description': description of the snapshot,
                           'id': '8add41ae-c60c-4023'
                                           '-a1a6-1233a5d35603',
                           'createTime': {'timeZone': 'IST',
                                          'epochTime': 1414427114}
                           'snapVolume': '5eeaf23def3f446d898e1123461a6aa7'
                           'snapVolumeStatus': 'DEACTIVATED'}, ...],
            'snapRemaining': 252},...
    }
    """
    volumes = {}
    if int(tree.find('snapInfo/count').text) == 0:
        return {}
    for el in tree.findall('snapInfo/snapshots/snapshot'):
        snapshot = {}
        snapshot['id'] = el.find('uuid').text
        snapshot['description'] = "" if el.find('description') is None \
                                  else el.find('description').text
        snapshot['createTime'] = {
            'epochTime': calendar.timegm(
                time.strptime(el.find('createTime').text,
                              "%Y-%m-%d %H:%M:%S")
            ),
            'timeZone': _TIME_ZONE
        }
        snapshot['snapVolumeName'] = el.find('snapVolume/name').text
        status = el.find('snapVolume/status').text
        if status.upper() == 'STARTED':
            snapshot['snapVolumeStatus'] = SnapshotStatus.ACTIVATED
        else:
            snapshot['snapVolumeStatus'] = SnapshotStatus.DEACTIVATED
        snapshot['name'] = el.find('name').text
        volumeName = el.find('snapVolume/originVolume/name').text
        if volumeName not in volumes:
            volumes[volumeName] = {
                'snapRemaining': el.find(
                    'snapVolume/originVolume/snapRemaining').text,
                'snapshots': []
            }
        volumes[volumeName]['snapshots'].append(snapshot)
    return volumes

@gluster_mgmt_api
def snapshotInfo(volumeName=None):
    command = _getGlusterSnapshotCmd() + ["info"]
    if volumeName:
        command += ["volume", volumeName]
    try:
        xmltree = _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterSnapshotInfoFailedException(rc=e.rc, err=e.err)
    try:
        if volumeName:
            return _parseVolumeSnapshotList(xmltree)
        else:
            return _parseAllVolumeSnapshotList(xmltree)
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

@gluster_mgmt_api
def executeGsecCreate():
    command = _getGlusterSystemCmd() + ["execute", "gsec_create"]
    try:
        _execGluster(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterGeoRepPublicKeyFileCreateFailedException(
            rc=e.rc, err=e.err)
    return True

@gluster_mgmt_api
def executeMountBrokerUserAdd(remoteUserName, remoteVolumeName):
    command = _getGlusterSystemCmd() + ["execute", "mountbroker",
                                        "user", remoteUserName,
                                        remoteVolumeName]
    try:
        _execGluster(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterGeoRepExecuteMountBrokerUserAddFailedException(
            rc=e.rc, err=e.err)
    return True

@gluster_mgmt_api
def executeMountBrokerOpt(optionName, optionValue):
    command = _getGlusterSystemCmd() + ["execute", "mountbroker",
                                        "opt", optionName,
                                        optionValue]
    try:
        _execGluster(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterGeoRepExecuteMountBrokerOptFailedException(
            rc=e.rc, err=e.err)
    return True

@gluster_mgmt_api
def volumeGeoRepSessionCreate(volumeName, remoteHost,
                              remoteVolumeName,
                              remoteUserName=None, force=False):
    if remoteUserName:
        userAtHost = "%s@%s" % (remoteUserName, remoteHost)
    else:
        userAtHost = remoteHost
    command = _getGlusterVolCmd() + ["geo-replication", volumeName,
                                     "%s::%s" % (userAtHost, remoteVolumeName),
                                     "create", "no-verify"]
    if force:
        command.append('force')
    try:
        _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterGeoRepSessionCreateFailedException(rc=e.rc, err=e.err)
    return True

@gluster_mgmt_api
def volumeGeoRepSessionDelete(volumeName, remoteHost, remoteVolumeName,
                              remoteUserName=None):
    if remoteUserName:
        userAtHost = "%s@%s" % (remoteUserName, remoteHost)
    else:
        userAtHost = remoteHost
    command = _getGlusterVolCmd() + ["geo-replication", volumeName,
                                     "%s::%s" % (userAtHost, remoteVolumeName),
                                     "delete"]
    try:
        _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterGeoRepSessionDeleteFailedException(rc=e.rc, err=e.err)
    return True

@gluster_mgmt_api
def volumeHealInfo(volumeName=None):
    command = _getGlusterVolCmd() + ["heal", volumeName, 'info', 'summary']
    try:
        xmltree = _execGlusterXmlWithTimeout(command, timeout=_DEFAULT_TIMEOUT)
        return _parseVolumeHealInfo(xmltree)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeHealInfoFailedException(rc=e.rc, err=e.err)
    except _etreeExceptions:  # pylint: disable=catching-non-exception
        raise ge.GlusterXmlErrorException(err=[etree.tostring(xmltree)])

def _parseVolumeHealInfo(tree):
    """
    {'bricks': [{'name': 'Fully qualified brick path',
                           'status': 'Connected/Not Connected'
                           'numberOfEntries': int,
                           'hostUuid': 'UUID'},...]
    }
    """
    healInfo = {'bricks': []}
    for el in tree.findall('healInfo/bricks/brick'):
        brick = {}
        brick['name'] = el.find('name').text
        brick['status'] = el.find('status').text
        brick['hostUuid'] = el.get('hostUuid')
        if brick['status'] == 'Connected':
            brick['numberOfEntries'] = (
                el.find('totalNumberOfEntries').text)
        healInfo['bricks'].append(brick)
    return healInfo

@gluster_mgmt_api
def volumeResetBrickStart(volumeName, existingBrick):
    command = _getGlusterVolCmd() + ["reset-brick", volumeName,
                                     existingBrick, "start"]
    try:
        _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeResetBrickStartFailedException(rc=e.rc,
                                                             err=e.err)
    return True

@gluster_mgmt_api
def volumeResetBrickCommitForce(volumeName, existingBrick):
    command = _getGlusterVolCmd() + ["reset-brick", volumeName,
                                     existingBrick, existingBrick, "commit",
                                     "force"]
    try:
        _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeResetBrickCommitForceFailedException(rc=e.rc,
                                                                   err=e.err)
    return True

@gluster_mgmt_api
def globalVolumeOptions():
    command = _getGlusterVolCmd() + ["get", "all", "all"]

    try:
        xmltree = _execGlusterXml(command)
    except ge.GlusterCmdFailedException as e:
        raise ge.GlusterVolumeGetGlobalOptionsFailedException(rc=e.rc,
                                                              err=e.err)
    return _parseGlobalVolumeOptions(xmltree)

def _parseGlobalVolumeOptions(tree):
    """
    Structure as:

    {'globalOptionMap' : {'global_option_name' : 'global_option_value',...}}

    XML:
    <Opt>
      <Option>cluster.server-quorum-ratio</Option>
      <Value>51</Value>
    </Opt>
    parsed as

    {'globalOptionMap' :{'cluster.server-quorum-ratio' : '51'},...}
    """
    volumeOptions = {}
    for el in tree.findall('volGetopts/Opt'):
        volumeOptions[el.find('Option').text] = el.find('Value').text or ''
    return volumeOptions

def exists():
    try:
        return os.path.exists(_glusterCommandPath.cmd)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise
        return False


from django.conf.urls import patterns, url

from polls import views

urlpatterns = patterns('',
url(r'^add/(?P<x>\d+)/(?P<y>\d+)$', views.add, name="add"),
url(r'^about/$', views.hi, name="about"),
url(r'^contact/$', views.contact, name="contact"),
url(r'^polls/$', views.polls, name="polls"),

)


# -*- coding: utf-8 -*-
"""
    django_async_test.testcase
    ~~~~~~~~~~~~~~~~~~~~~~~~~~

    Internal module reference for ``django_async_test.testcase``.

"""
import asyncio
from asynctest import TestCase as AsyncTestCase
from django.test import TestCase as DjangoTestCase

class TestCase(DjangoTestCase, AsyncTestCase):
    """
    A testcase that wraps `django.test.TestCase`_ and `asynctest.TestCase`_.

    .. _django.test.TestCase: https://docs.djangoproject.com/en/1.9/topics/testing/tools/#django.test.TestCase
    .. _asynctest.TestCase: http://asynctest.readthedocs.org/en/latest/asynctest.case.html#asynctest.TestCase
    """

    def __init__(self, methodName='runTest', *args, **kwargs):
        self._origTestMethodName = getattr(self, methodName)
        AsyncTestCase.__init__(self, methodName='_run_test_method',  *args, **kwargs)
        DjangoTestCase.__init__(self, methodName='_run_test_method', *args, **kwargs)

    def setUp(self):
        """
        Override setup method.

        Note that ``asynctest`` supports ``setUp`` as a coroutine however
        :py:class:`django_async_test.TestCase` instead supports a
        ``setUpAsync`` method.
        """
        DjangoTestCase.setUp(self)

        self._init_loop()
        self.addCleanup(self._unset_loop)

        setUpAsync = getattr(self, 'setUpAsync', None)
        if setUpAsync is not None:
            if asyncio.iscoroutinefunction(setUpAsync):
                self.loop.run_until_complete(setUpAsync())
            else:
                setUpAsync()

        # don't take into account if the loop ran during setUp
        self.loop.__asynctest_ran = False

    def tearDown(self):
        """
        Override tearDown method.

        Note that ``asynctest`` supports ``tearDown`` as a coroutine however
        :py:class:`django_async_test.TestCase` instead supports a
        `tearDownAsync`` method.
        """
        tearDownAsync = getattr(self, 'tearDownAsync', None)
        if tearDownAsync is not None:
            if asyncio.iscoroutinefunction(tearDownAsync):
                self.loop.run_until_complete(tearDownAsync())
            else:
                tearDownAsync()

        DjangoTestCase.tearDown(self)

    def run(self, result=None):
        """
        Call `django.test.TestCase`_'s ``run`` method.

        .. _django.test.TestCase: https://docs.djangoproject.com/en/1.9/topics/testing/tools/#django.test.TestCase
        """
        return DjangoTestCase.run(self, result)

    def _run_test_method(self):
        return AsyncTestCase._run_test_method(self, self._origTestMethodName)


# https://projecteuler.net/problem=9
# created 1.11.2018 by CB Fay

from math import sqrt

def solution(target):
    for a in range(1, target//2):
        for b in range(a, target//2):
            c = sqrt(a**2 + b**2)
            if a**2 + b**2 == c**2 and a + b + c == 1000:
                print(a, b, c)
                print(a * b * c)
                exit()

solution(1000)

Euler 0009 - Special Pythagorean Theorem/euler_0009.py


import numpy as np
from astropy.io import fits
import os
from Project_Functions import MLAData,storing,StandardRebin

# Threshold for pixel rejection
reject_no = 500
# Spectral flux bin size
Bin_Size=10
slash =  os.path.normpath("/")
#Location of Data
Platedir = os.path.normpath(slash+"share"+slash+"data1"+slash+"boss_data"+slash+"sas"+slash+"dr12"+slash+"boss"+slash+"spectro"+slash+"redux"+slash+"v5_7_0")
#Location of where data will be stored
Bin_platedir =  os.path.normpath(slash+"share"+slash+"splinter"+slash+"hothi"+slash+"New_boss_data_rebinned_x10")

# Listing the plates and opening
plate_name = os.listdir(Platedir)
file="Full_Name.txt"
p = open(file, 'w')
for i in plate_name:
    p.write(i +'\n')

p.close()

with open(file) as f:
    Files = f.read().splitlines()

print(np.str(len(Files)))
Spectra_Files=[]
i=0
while i< len(Files):
    Spectra_Files.append(Files[i])
    i=i+1
PLATEIDs = []
BinInfos = []
Flux = []
MJDs = []
log_wavst=[]
ORMASK=[]
ANDMASK=[]
INVAR=[]

#Storing Plate Data
for f in Spectra_Files:
    file_list = os.listdir(Platedir+slash+f)
    for l in file_list:
        if 'spPlate' in l and ".fits"in l:
            c=Platedir+slash+f+slash+l
            plate_ = fits.open(c,memmap=True)
            Bin_info_ = plate_[5].data
            Flux_ = plate_[0].data
            primhdu_ = plate_[0]
            PLATEIDs.append(primhdu_.header['PLATEID'])
            ORMASK.append( plate_[3].data)
            ANDMASK.append( plate_[2].data)
            INVAR.append( plate_[1].data)
            log_wavst.append(primhdu_.header['COEFF0'])
            MJDs.append(primhdu_.header['MJD'])
            BinInfos.append(Bin_info_)
            Flux.append(Flux_)

 #Opening Superset data and restoring with only relevant data
list = fits.open('Superset_DR12Q.fits',memmap=True)#opening file
supers=list[1].data # storing  BINTABLE extension data
Full_Data = storing(PLATEIDs,supers)

#Matching the Superset and plates objects. Matched objects will be put
#in suitable form for the machine learning algorithm (MLA)
X,Y,Train_z, Train_mag,And, In, wavst, ID, MJ, MatchedPlates = MLAData(Full_Data,BinInfos,Flux, log_wavst,ANDMASK,INVAR,MJDs,PLATEIDs)
plate_no = 0
all_Testwav=[]
wav_ratio = 10**0.0001

#Creating the wavelength grid for each plate
while plate_no < len(wavst):
    append_count=0
    cent_wav = 10**wavst[plate_no]
    wavelengths = []
    wavelengths.append(cent_wav)
    current_wav = cent_wav
    while append_count < (4600-1):
        current_wav = current_wav*wav_ratio
        wavelengths.append(current_wav)
        append_count=append_count+1
    plate_no=plate_no+1
    all_Testwav.append(wavelengths)

plate_no = 0
Xbin_plate=[]
true_Y =[]
true_Train_z=[]
true_ID =[]
true_MJ = []
true_MatchedPlates=[]
w_plate=[]
l_plate=[]
minimum = []

#Binning the spectral flux of an object, in a given plate
while plate_no<len(X):
    plateX = X[plate_no]
    plateY = Y[plate_no]
    z0 = Train_z[plate_no]
    plateMask = And[plate_no]
    plateInv = In[plate_no]
    wave = all_Testwav[plate_no]
    pid  = ID[plate_no]
    r,w, r_lambda,m,rebin_class,rebin_z,rebin_id = StandardRebin(plateX,wave,plateMask,plateInv ,Bin_Size,reject_no,plateY,z0,pid)
    true_Y.append(rebin_class)
    true_Train_z.append(rebin_z)
    true_ID.append(rebin_id)
    minimum.append(m)
    Xbin_plate.append(r)
    w_plate.append(w)
    l_plate.append(r_lambda)
    plate_no=plate_no+1
minbin = 5000

#Finding the minimum number of bin fluxes
for f in minimum:
    if f< minbin:
        minbin=f

i=0
#Saving the plate data into new FITS files
First_Run=True
while i <len(MatchedPlates):
    C_Plate = MatchedPlates[i]
    a1 = Xbin_plate[i]
    a2 = np.array(true_Y[i])
    a3=l_plate[i]
    a4=w_plate[i]
    a5 = true_Train_z[i]
    a6= true_ID[i]
    a8 = MJ[i]
    a7 = wavst[i]
    col1 = fits.Column(name='Bin_Flux', format='PD()', array=np.array(a1,dtype=np.object))
    col2 = fits.Column(name='Class', format='I', array=np.array(a2))
    col3 = fits.Column(name='Bin_Cent', format='PD()', array=np.array(a3,dtype=np.object))
    col4 = fits.Column(name='INVAR', format='PD()', array=np.array(a4,dtype=np.object))
    col5 = fits.Column(name='Redshift', format='D', array=np.array(a5))
    col6 = fits.Column(name='Name', format='20A', array=np.array(a6))
    cols = fits.ColDefs([col1, col2,col3, col4,col5,col6])
    tbhdu = fits.BinTableHDU.from_columns(cols)
    prihdr = fits.Header()
    prihdr['Plate'] = C_Plate
    prihdr['LogWav'] = a7
    prihdr['MJDs'] = a8
    prihdr['Min'] = minbin
    prihdu = fits.PrimaryHDU(header=prihdr)
    file_dir = Bin_platedir+slash+np.str(C_Plate)+slash
    try:
        os.mkdir(file_dir)
        file_name = file_dir+np.str(C_Plate)+"_"+np.str(Bin_Size)+"_"+np.str(a8)+".fits"
        print(file_name)
        thdulist = fits.HDUList([prihdu, tbhdu])
        thdulist.writeto(file_name)
        i=i+1
    except FileExistsError as F:
        file_name = file_dir+np.str(C_Plate)+"_"+np.str(Bin_Size)+"_"+np.str(a8)+".fits"
        print(file_name)
        thdulist = fits.HDUList([prihdu, tbhdu])
        thdulist.writeto(file_name)
        i=i+1


from flask import jsonify, request, url_for, g
from app import db
from datetime import datetime
from app.api import api
from app.models import User, Deposit, Withdrawal, UserBankAccount, UserAddress
from app.auth.views import generate_account_no
from app.api.auth import token_auth
from .errors import bad_request
from dateutil.parser import parse

@api.get('/user_account_details/')
@token_auth.login_required
def get_user_acct():
    if g.current_user.account is not None:
        return jsonify(User.query.get_or_404(g.current_user.id).to_dict())
    else:
        return {}

@api.post('/deposit/')
@token_auth.login_required
def deposit_amt():
    user = User.query.get_or_404(g.current_user.id)
    data = request.get_json()['amt'] or {}

    deposit = Deposit(amount=int(data))
    user.deposits.append(deposit)
    if not user.account.initial_deposit_date:
        user.account.initial_deposit_date = datetime.utcnow()
    db.session.add(deposit)
    db.session.commit()

    user.account.balance += deposit.amount
    db.session.commit()
    return jsonify({'message': 'success'}), 201

@api.post('/withdraw/')
@token_auth.login_required
def withdraw_amt():
    user = User.query.get_or_404(g.current_user.id)
    data = request.get_json()['amt'] or {}

    if data > user.account.balance:
        return jsonify({'message': 'insufficient funds'})

    withdrawal = Withdrawal(amount=int(data))
    user.withdrawals.append(withdrawal)
    db.session.add(withdrawal)
    db.session.commit()

    user.account.balance -= withdrawal.amount
    db.session.commit()
    return jsonify({'message': 'success'}), 201

@api.post('/create_user')
def create_user():
    data = request.get_json() or {}
    if 'username' not in data or 'email' not in data or 'password' not in data:
        return bad_request('must include username, email and password fields')
    if User.query.filter_by(username=data['username']).first():
        return bad_request('please use a different username.')
    if User.query.filter_by(username=data['email']).first():
        return bad_request('please use a different email address.')
    user = User()
    user.from_dict(data, new_user=True)
    db.session.add(user)
    db.session.commit()

    accout_details = UserBankAccount(
        account_no=generate_account_no(),
        first_name=data['first_name'],
        last_name=data['last_name'],
        gender=data['gender'],
        phone_number=data['phone_number'],
        date_of_birth=parse(data['date_of_birth'])
    )

    address = UserAddress(
        country=data['country'],
        city=data['city'],
        street_address=data['street_address']
    )

    db.session.add_all([accout_details, address])

    user.account = accout_details
    user.address = address
    db.session.add(user)
    db.session.commit()

    g.current_user = user

    response = jsonify(user.to_dict())
    response.status_code = 201
    response.headers['Location'] = url_for('api.get_user_acct')
    return response


x = [int(i) for i in list(input().split(" "))]
# y = [int(i) for i in list(input().split(" "))]

dx = {}
# dy = {}

for i in x:
    if str(i) in dx.keys():
        dx[f"{i}"] += 1
    else:
        dx[f"{i}"] = 1

# for i in y:
#     if str(i) in dy.keys():
#         dy[f"{i}"] += 1
#     else:
#         dy[f"{i}"] = 1
sum = 0
print(dx)

for key in dx.keys():
    if int(key) == 0:
        sum += 1
    elif int(key) == dx.get(key)-1:
        sum += int(key)+1
    else:
        sum += (int(key))*2

print(sum)
print(sum)

# print("dx: ", dx)
# print("dy: ", dy)


from flask import request, redirect, url_for, render_template, session, jsonify, abort
from flask_babel import _
import os.path
from os import listdir
from os.path import isfile, join
from encodings.aliases import aliases

import pandas as pd

from ..models import Dataset
from ..models.modules import ModulePandasProfiling, ModuleBias
from .services.datasets import format_dataset, control_dataset, load_dataset_modules_in_background
from .services.datasets import load_pandas_prof_report, load_bias_module_without_df
from .services.modules import pandas_profiling
from .services.modules import bias
from .services.modules.pandas_profiling import get_save_path
from .services.commons import get_header_attributes
from .controller_class import Controller

from ..utils import add_in_db, exists_in_db

dataset_controller = Controller(component=Dataset,
                                format_fn=format_dataset,
                                control_fn=control_dataset,
                                module_fn=load_dataset_modules_in_background)

pandas_prof_controller = Controller(component=ModulePandasProfiling,
                                    format_fn=pandas_profiling.format_module,
                                    control_fn=pandas_profiling.control_module)

bias_controller = Controller(component=ModuleBias,
                             format_fn=bias.format_module,
                             control_fn=bias.control_module)

encodings = list(sorted(set([v for k, v in aliases.items()])))

def index():
    title = _('Datasets')
    header = get_header_attributes()
    datasets = dataset_controller.index()

    return render_template("datasets/index.html",
                           session=session,
                           datasets=datasets,
                           header=header)

def get_all_instances_json():
    datasets = dataset_controller.index()
    datasets = [elem.to_dict() for elem in datasets]
    return jsonify(datasets)

def new():
    title = _('Create a new dataset')
    header = get_header_attributes()
    previous = request.form

    if request.method == 'POST':
        dataset = dataset_controller.create()
        if dataset is not None:
            return redirect(url_for('datasets.get_instance',
                                    name=dataset.name))

    return render_template("datasets/new.html",
                           title=title,
                           session=session,
                           header=header,
                           previous=previous,
                           encodings=encodings)

def new_from_project(project_name):
    title = _('Create a new dataset')
    header = get_header_attributes()
    header['current_project'] = project_name
    previous = request.form

    if request.method == 'POST':
        dataset = dataset_controller.create()
        if dataset is not None:
            return redirect(url_for('datasets.get_instance',
                                    name=dataset.name))

    return render_template("projects/new-dataset.html",
                           title=title,
                           session=session,
                           header=header,
                           previous=previous,
                           project_name=project_name,
                           encodings=encodings)

def edit(name):
    title = _('Edit ') + name
    header = get_header_attributes()
    previous = None

    dataset = dataset_controller.get_instance(name)
    if dataset is not None:
        previous = dataset.to_dict()

    if request.method == 'POST':
        previous = request.form
        dataset = dataset_controller.update(name)
        if dataset is not None:
            return redirect(url_for('datasets.get_instance',
                                    name=dataset.name))

    return render_template("datasets/edit.html",
                           title=title,
                           session=session,
                           header=header,
                           previous=previous,
                           dataset=dataset,
                           encodings=encodings)

def get_instance(name):
    dataset = dataset_controller.get_instance(name)
    if dataset is None:
        return redirect(url_for('datasets.index'))

    title = name
    header = get_header_attributes()
    if dataset.project is not None:
        header['current_project'] = dataset.project.name

    return render_template("datasets/instance.html",
                           session=session,
                           dataset=dataset,
                           header=header,
                           title=title)

def get_instance_json(name):
    dataset = dataset_controller.get_instance(name)
    return jsonify(dataset.to_dict())

def create():
    dataset_controller.create()
    return redirect(url_for('datasets.index'))

def update(name):
    dataset_controller.update(name)
    return redirect(url_for('datasets.get_instance', name=name))

def delete(name):
    dataset = dataset_controller.get_instance(name)
    redirect_project = True if dataset.project is not None else False

    dataset_controller.delete(name)

    if redirect_project:
        return redirect(url_for('projects.get_instance', name=dataset.project.name))

    return redirect(url_for('index'))

def post_instance(name):
    form_data = request.form
    if '_method' not in form_data:
        return redirect(url_for('datasets.get_instance', name=name))

    method = form_data['_method']

    if method == 'POST':
        return create()
    elif method == 'PUT':
        return update(name)
    elif method == 'DELETE':
        return delete(name)

    return redirect(url_for('datasets.get_instance', name=name))

def analyse_dataset(name):
    title = _('Analyse Dataset: ') + name
    header = get_header_attributes()
    dataset = dataset_controller.get_instance(name)
    if dataset is None:
        return abort(404)

    module = dataset.module_pandas_profiling
    if module is None:
        return abort(404)

    previous = module.to_dict()
    previous['minimal'] = int(previous['minimal'])

    if dataset.project is not None:
        header['current_project'] = dataset.project.name

    if request.method == 'POST':
        previous = request.form
        module = pandas_prof_controller.update(module.id, id_col='id')

        if module is not None:
            load_pandas_prof_report(title=dataset.name,
                                    dataset=dataset,
                                    nrows=module.nrows,
                                    explorative=module.explorative,
                                    minimal=module.minimal)
            return redirect(url_for('datasets.analyse_dataset', name=name))

    return render_template("modules/analyse-dataset.html",
                           session=session,
                           dataset=dataset,
                           header=header,
                           title=title,
                           previous=previous)

def show_report():
    path = get_save_path()
    data = request.args
    files = [f for f in listdir(path) if isfile(join(path, f))]

    if 'name' in data:
        if data['name'] in files:
            return render_template("modules/pandas_profiling_reports/%s" %
                                   data['name'])

    return abort(404)

def analyse_performance(name):
    title = _('Analyse Performance: ') + name
    header = get_header_attributes()
    dataset = dataset_controller.get_instance(name)
    if dataset.project is not None:
        header['current_project'] = dataset.project.name

    return render_template("modules/analyse-performance.html",
                           session=session,
                           dataset=dataset,
                           header=header,
                           title=title)

def analyse_bias(name):
    title = _('Analyse Bias: ') + name
    header = get_header_attributes()
    dataset = dataset_controller.get_instance(name)
    if dataset is None:
        return abort(404)

    module = dataset.module_bias
    if module is None:
        return abort(404)

    previous = module.to_dict()

    if dataset.project is not None:
        header['current_project'] = dataset.project.name

    if request.method == 'POST':
        previous = request.form
        module = bias_controller.update(module.id, id_col='id')

        if module is not None:
            load_bias_module_without_df(dataset)
            return redirect(url_for('datasets.analyse_bias', name=name))

    return render_template("modules/analyse-bias.html",
                           session=session,
                           dataset=dataset,
                           header=header,
                           title=title,
                           previous=previous)

def bias_results(name):
    title = _('Bias report: ') + name
    header = get_header_attributes()
    dataset = dataset_controller.get_instance(name)
    if dataset is None:
        return abort(404)

    if dataset.project is not None:
        header['current_project'] = dataset.project.name

    module = dataset.module_bias
    if module is None:
        return abort(404)

    if module.results is None:
        return redirect(url_for('datasets.analyse_bias', name=name))

    return render_template("modules/bias-results.html",
                           session=session,
                           dataset=dataset,
                           header=header,
                           title=title)


import hashlib
import logging
import json
from sawtooth_sdk.processor.state import StateEntry
from sawtooth_sdk.processor.exceptions import InvalidTransaction
from sawtooth_sdk.processor.exceptions import InternalError
from sawtooth_sdk.protobuf.transaction_pb2 import TransactionHeader

LOGGER = logging.getLogger(__name__)

class PartTransactionHandler:
    def __init__(self, namespace_prefix):
        self._namespace_prefix = namespace_prefix

    @property
    def family_name(self):
        return 'pt'

    @property
    def family_versions(self):
        return ['1.0']

    @property
    def encodings(self):
        return ['csv-utf8']

    @property
    def namespaces(self):
        return [self._namespace_prefix]

    def apply(self, transaction, state_store):

        header = TransactionHeader()
        header.ParseFromString(transaction.header)

        try:
            # The payload is csv utf-8 encoded string
            pt_id,pt_name,checksum,version,alias,licensing,label,description,action,artifact_id,category_id,supplier_id = transaction.payload.decode().split(",")
        except ValueError:
            raise InvalidTransaction("Invalid payload serialization")

        validate_transaction( pt_id,action)

        data_address = make_part_address(self._namespace_prefix,pt_id)

        # Retrieve the data from state storage
        state_entries = state_store.get([data_address])

        if len(state_entries) != 0:
            try:

                    stored_pt_id, stored_pt_str = \
                    state_entries[0].data.decode().split(",",1)

                    stored_pt = json.loads(stored_pt_str)
            except ValueError:
                raise InternalError("Failed to deserialize data.")

        else:
            stored_pt_id = stored_pt = None

        if action == "create" and stored_pt_id is not None:
            raise InvalidTransaction("Invalid part already exists.")

        elif action == "AddArtifact" or action == "AddSupplier" or action == "AddCategory":
            if stored_pt_id is None:
                raise InvalidTransaction(
                    "Invalid the operation requires an existing part."
                )

        if action == "create":
            pt = create_part(pt_id,pt_name,checksum,version,alias,licensing,label,description)
            stored_pt_id = pt_id
            stored_pt = pt
            _display("Created a part.")

        if action == "AddArtifact":
            if artifact_id not in stored_pt_str:
                pt = add_artifact(artifact_id,stored_pt)
                stored_pt = pt

        if action == "AddSupplier":
            if supplier_id not in stored_pt_str:
                pt = add_supplier(supplier_id,stored_pt)
                stored_pt = pt

        if action == "AddCategory":
            if category_id not in stored_pt_str:
                pt = add_category(category_id,stored_pt)
                stored_pt = pt

        # 6. Put data back in state storage
        stored_pt_str = json.dumps(stored_pt)

        addresses = state_store.set([
            StateEntry(
                address=data_address,
                data=",".join([stored_pt_id, stored_pt_str]).encode()
            )
        ])
        return addresses

def add_artifact(uuid,parent_pt):

    pt_list = parent_pt['artifacts']
    pt_dic = {'artifact_id': uuid}
    pt_list.append(pt_dic)
    parent_pt['artifacts'] = pt_list
    return parent_pt

def add_supplier(uuid,parent_pt):

    pt_list = parent_pt['suppliers']
    pt_dic = {'supplier_id': uuid}
    pt_list.append(pt_dic)
    parent_pt['suppliers'] = pt_list
    return parent_pt

def add_category(uuid,parent_pt):

    pt_list = parent_pt['categories']
    pt_dic = {'category_id': uuid}
    pt_list.append(pt_dic)
    parent_pt['categories'] = pt_list
    return parent_pt

def create_part(pt_id,pt_name,checksum,version,alias,licensing,label,description):
    ptD = {'pt_id': pt_id,'pt_name': pt_name,'checksum': checksum,'version': version,'alias':alias,'licensing':licensing,'label':label,'description':description,'artifacts':[],'suppliers':[],'categories':[]}
    return ptD

def validate_transaction( pt_id,action):
    if not pt_id:
        raise InvalidTransaction('Part ID is required')

    if not action:
        raise InvalidTransaction('Action is required')

    if action not in ("AddArtifact", "create","AddCategory","AddSupplier","list-part","retrieve"):
        raise InvalidTransaction('Invalid action: {}'.format(action))

def make_part_address(namespace_prefix, part_id):
    return namespace_prefix + \
        hashlib.sha512(part_id.encode('utf-8')).hexdigest()[:64]

def _display(msg):
    n = msg.count("\n")

    if n > 0:
        msg = msg.split("\n")
        length = max(len(line) for line in msg)
    else:
        length = len(msg)
        msg = [msg]

    LOGGER.debug("+" + (length + 2) * "-" + "+")
    for line in msg:
        LOGGER.debug("+ " + line.center(length) + " +")
    LOGGER.debug("+" + (length + 2) * "-" + "+")


import os
import time
import shutil
import threading
import tkinter as tk
from tkinter import *
from tkinter.ttk import *
from tkinter import ttk
from tkinter import Toplevel
from tkinter import messagebox
from tkinter import filedialog
from tkinter.ttk import Progressbar
import datetime

os.getcwd()
print(os.getcwd())

os.chdir("/home/pi/TGView")
print(os.getcwd())

def CloseProgram():
    exit()

def SelectUSB():
    os.chdir("/media/pi")
    print(os.getcwd())
    global USB
    USB = filedialog.askdirectory()
    global destinations
    destinations = USB+"/TGView Backup (%s)"%datetime.datetime.now().strftime('%m-%d-%Y %H.%M.%S%p')
    print(USB)
    print(destinations)
    os.chdir(USB)
    print(os.getcwd())
    if os.getcwd() != "/media/pi":
        tk.Button(root, text="Create Backup", font=("century gothic",50,"bold"), fg="white", activeforeground="white", bg="#678277", activebackground="#678277", compound = "left", padx=30,command=CreateBackup).place(height=150,width=880,x=75,y=495)
    else:
        tk.Button(root, text="Create Backup", font=("century gothic",50,"bold"), fg="white", activeforeground="white", bg="grey35", activebackground="grey35", compound = "left", padx=30, state=tk.DISABLED).place(height=150,width=880,x=75,y=495)

def CreateBackup():

    tk.Button(root, text="Copying...", font=("century gothic",50,"bold"), fg="white", activeforeground="white", bg="#ff9500", activebackground="#ff9500", compound = "left", padx=30, command=None).place(height=150,width=630,x=75,y=255)
    tk.Label(root, text='This will automatically close once the copy is complete.', fg="White", bg="Grey25", font=("Century Gothic",40, "bold"), justify="center", wraplength=950).place(x=80,y=55)

    s1=ttk.Style()
    TROUGH_COLOR = 'grey35'
    BAR_COLOR = '#209736'
    s1.configure('red.Horizontal.TProgressbar', troughcolor=TROUGH_COLOR, bordercolor=TROUGH_COLOR,background=BAR_COLOR,lightcolor=BAR_COLOR,darkcolor=BAR_COLOR)
    global progress
    progress = Progressbar(root, style='red.Horizontal.TProgressbar', orient=tk.HORIZONTAL, length = 880, mode='determinate', maximum = 100, value=10)
    progress.place(x=75,y=495, height = 150)

    def bar():
        progress['value'] = 10
        root.update_idletasks()
        time.sleep(0.1)
        progress['value'] = 20
        root.update_idletasks()
        time.sleep(0.1)
        progress['value'] = 30
        root.update_idletasks()
        time.sleep(0.1)
        progress['value'] = 40
        root.update_idletasks()
        time.sleep(0.1)
        progress['value'] = 50
        root.update_idletasks()
        time.sleep(0.1)
        progress['value'] = 60
        root.update_idletasks()
        time.sleep(0.1)
        progress['value'] = 70
        root.update_idletasks()
        time.sleep(0.1)
        progress['value'] = 80
        root.update_idletasks()
        time.sleep(0.1)
        shutil.copytree("/home/pi/TGView", destinations)
        progress['value'] = 90
        root.update_idletasks()
        time.sleep(0.1)
        progress['value'] = 100
        root.update_idletasks()
        time.sleep(0.1)
    bar()

    tk.Label(root, text="Done!", fg="White", bg="#209736", font=("Century Gothic",50, "bold"), justify="center").place(x=395,y=530)
    print("DONE!!!")
    tk.Button(root, text="Closing...", font=("century gothic",50,"bold"), fg="white", activeforeground="white", bg="#ff9500", activebackground="#ff9500", compound = "left", padx=30, command=None).place(height=150,width=630,x=75,y=255)
    exit()

root = tk.Tk()
root.title('   Backup to USB   ')
p1=PhotoImage(file= '/home/pi/Pictures/USBcopy2.png')
root.iconphoto(False,p1)
root.resizable(False,False)
root.config(bg="Grey25")
# Width and height for the Tk root window
w=1030
h=720
# This gets the current screen width and height
ws = root.winfo_screenwidth()
hs = root.winfo_screenheight()
# Calculate the x and y coordinates based on the current screen size
sx = (ws/2) - (w/2)
sy = (hs/2) - (h/2)
# Open the root window in the middle of the screen
root.geometry('%dx%d+%d+%d' % (w, h, sx, sy))

print (os.getcwd())

tk.Button(root, text="Create Backup", font=("century gothic",50,"bold"), fg="white", activeforeground="white", bg="grey35", activebackground="grey35", compound = "left", padx=30, state=tk.DISABLED).place(height=150,width=880,x=75,y=495)
tk.Button(root, text="Select Drive", font=("century gothic",50,"bold"), fg="white", activeforeground="white", bg="#ff9500", activebackground="#ff9500", compound = "left", padx=30, command=SelectUSB).place(height=150,width=630,x=75,y=255)
tk.Button(root, text="X", font=("century gothic",50,"bold"), fg="white", activeforeground="white", bg="indianred", activebackground="indianred", compound = "left", padx=30, command=CloseProgram).place(height=150,width=150,x=800,y=255)

#tk.Button(root, text="Select Folder", font=("Century Gothic",31, "bold"), fg="white", activeforeground="white", bg="#678176", activebackground="#81a6a3", image=dashIcon, compound = "left", padx=25, command=root.destroy).place(height=125, width=350,x=75,y=75)
tk.Label(root, text='Double-tap the USB Drive folder then tap "OK"', fg="White", bg="Grey25", font=("Century Gothic",40, "bold"), justify="center", wraplength=850).place(x=120,y=55)

root.mainloop()