from bson import json_util
from copy import deepcopy
from datetime import datetime
from girderformindlogger.constants import AccessType, PREFERRED_NAMES, DEFINED_RELATIONS,       \
    HIERARCHY, KEYS_TO_DELANGUAGETAG, KEYS_TO_DEREFERENCE, KEYS_TO_EXPAND,     \
    MODELS, NONES, REPROLIB_CANONICAL, REPROLIB_PREFIXES
from girderformindlogger.exceptions import AccessException,                    \
    ResourcePathNotFound, ValidationException
from girderformindlogger.models.activity import Activity as ActivityModel
from girderformindlogger.models.applet import Applet as AppletModel
from girderformindlogger.models.collection import Collection as CollectionModel
from girderformindlogger.models.folder import Folder as FolderModel
from girderformindlogger.models.item import Item as ItemModel
from girderformindlogger.models.protocol import Protocol as ProtocolModel
from girderformindlogger.models.screen import Screen as ScreenModel
from girderformindlogger.models.user import User as UserModel
from girderformindlogger.utility import loadJSON
from girderformindlogger.utility.response import responseDateList
from girderformindlogger.models.cache import Cache as CacheModel
from bson.objectid import ObjectId
from pyld import jsonld

def getModelCollection(modelType):
    """
    Returns the Collection named for the given modelType, creating if not
    already extant.

    :param modelType: 'activity', 'screen', etc.
    :type modelType: str
    :returns: dict
    """
    from girderformindlogger.models import pluralize
    name = pluralize(modelType).title()
    collection = CollectionModel().findOne(
        {'name': name}
    )
    if not collection:
        collection = CollectionModel().createCollection(
            name=name,
            public=True,
            reuseExisting=True
        )
    return(collection)

def expandObj(contextSet, data):
    obj = deepcopy(data)
    context = {}

    if '@context' in data:
        for key in data['@context']:
            context.update(contextSet[key])
    obj['@context'] = context

    expanded = expand(obj)
    if '@context' in expanded:
        expanded.pop('@context')

    return expanded

def loadFromSingleFile(document, user):
    if 'protocol' not in document or 'data' not in document['protocol']:
        raise ValidationException(
            'should contain protocol field in the json file.',
        )
    if 'activities' not in document['protocol']:
        raise ValidationException(
            'should contain activities field in the json file.',
    )

    contexts = document.get('contexts', {})

    protocol = {
        'protocol': {},
        'activity': {},
        'screen': {}
    }

    expandedProtocol = expandObj(contexts, document['protocol']['data'])
    protocol['protocol'][expandedProtocol['@id']] = {
        'expanded': expandedProtocol
    }

    protocolId = None
    for activity in document['protocol']['activities'].values():
        expandedActivity = expandObj(contexts, activity['data'])
        protocol['activity'][expandedActivity['@id']] = {
            'parentKey': 'protocol',
            'parentId': expandedProtocol['@id'],
            'expanded': expandedActivity
        }

        if 'items' not in activity:
            raise ValidationException(
                'should contain at least one item in each activity.',
            )

        for item in activity['items'].values():
            expandedItem = expandObj(contexts, item)
            protocol['screen'][expandedItem['@id']] = {
                'parentKey': 'activity',
                'parentId': expandedActivity['@id'],
                'expanded': expandedItem
            }

    for modelType in ['protocol', 'activity', 'screen']:
        modelClass = MODELS()[modelType]()
        docCollection = getModelCollection(modelType)

        for model in protocol[modelType].values():
            prefName = modelClass.preferredName(model['expanded'])

            if modelClass.name in ['folder', 'item']:
                docFolder = FolderModel().createFolder(
                    name=prefName,
                    parent=docCollection,
                    parentType='collection',
                    public=True,
                    creator=user,
                    allowRename=True,
                    reuseExisting=(modelType!='applet')
                )

                metadata = {modelType: model['expanded']}

                tmp = model
                while tmp.get('parentId', None):
                    key = tmp['parentKey']
                    tmp = protocol[key][tmp['parentId']]
                    metadata['{}Id'.format(key)] = '{}/{}'.format(MODELS()[key]().name, tmp['_id'])

                if modelClass.name=='folder':
                    newModel = modelClass.setMetadata(
                        docFolder,
                        metadata
                    )
                elif modelClass.name=='item':
                    newModel = modelClass.setMetadata(
                        modelClass.createItem(
                            name=prefName if prefName else str(len(list(
                                FolderModel().childItems(
                                    FolderModel().load(
                                        docFolder,
                                        level=None,
                                        user=user,
                                        force=True
                                    )
                                )
                            )) + 1),
                            creator=user,
                            folder=docFolder,
                            reuseExisting=True
                        ),
                        metadata
                    )

                # we don't need this in the future because we will only using single-file-format
                modelClass.update(
                    {'_id': newModel['_id']},
                    {'$set': {
                        'loadedFromSingleFile': True
                    }})
                if modelType != 'protocol':
                    formatted = _fixUpFormat(formatLdObject(
                        newModel,
                        mesoPrefix = modelType,
                        user = user,
                        refreshCache = False
                    ))

                    createCache(newModel, formatted, modelType, user)
                model['_id'] = newModel['_id']

                if modelType == 'protocol':
                    protocolId = newModel['_id']

    return formatLdObject(
        ProtocolModel().load(protocolId, force=True),
        mesoPrefix = 'protocol',
        user = user,
        refreshCache = False
    )

def importAndCompareModelType(model, url, user, modelType):
    import threading
    from girderformindlogger.utility import firstLower

    if model is None:
        return(None, None)
    mt = model.get('@type', '')
    mt = mt[0] if isinstance(mt, list) else mt
    atType = mt.split('/')[-1].split(':')[-1]
    modelType = firstLower(atType) if len(atType) else modelType
    modelType = 'screen' if modelType.lower(
    )=='field' else 'protocol' if modelType.lower(
    )=='activityset' else modelType
    changedModel = (
        (atType != modelType and len(atType)) or (" " in modelType)
    )
    modelType = firstLower(atType) if changedModel else modelType
    modelType = 'screen' if modelType.lower(
    )=='field' else 'protocol' if modelType.lower(
    )=='activityset' else modelType
    modelClass = MODELS()[modelType]()
    prefName = modelClass.preferredName(model)
    cachedDocObj = {}
    model = expand(url)
    print("Loaded {}".format(": ".join([modelType, prefName])))
    docCollection=getModelCollection(modelType)
    if modelClass.name in ['folder', 'item']:
        docFolder = FolderModel().createFolder(
            name=prefName,
            parent=docCollection,
            parentType='collection',
            public=True,
            creator=user,
            allowRename=True,
            reuseExisting=(modelType!='applet')
        )
        if modelClass.name=='folder':
            newModel = modelClass.setMetadata(
                docFolder,
                {
                    modelType: {
                        **model,
                        'schema:url': url,
                        'url': url
                    }
                }
            )
        elif modelClass.name=='item':
            newModel = modelClass.setMetadata(
                modelClass.createItem(
                    name=prefName if prefName else str(len(list(
                        FolderModel().childItems(
                            FolderModel().load(
                                docFolder,
                                level=None,
                                user=user,
                                force=True
                            )
                        )
                    )) + 1),
                    creator=user,
                    folder=docFolder,
                    reuseExisting=True
                ),
                {
                    modelType: {
                        **model,
                        'schema:url': url,
                        'url': url
                    }
                }
            )

        modelClass.update(
            {'_id': newModel['_id']},
            {'$set': {
                'loadedFromSingleFile': False
            }}
        )
        newModel['loadedFromSingleFile'] = False

    formatted = _fixUpFormat(formatLdObject(
        newModel,
        mesoPrefix=modelType,
        user=user,
        refreshCache=True
    ))
    createCache(newModel, formatted, modelType, user)
    return(formatted, modelType)

def _createContextForStr(s):
    sp = s.split('/')
    k = '_'.join(
        sp[:-1] if '.' not in sp[-1] else sp
    ).replace('.','').replace(':','')
    return(
        (
            {k: '{}/'.format('/'.join(sp[:-1]))},
            "{}:{}".format(k, sp[-1])
        ) if '.' not in sp[-1] else (
            {k: s},
            k
        )
    )

def contextualize(ldObj):
    newObj = {}
    context = ldObj.get('@context', [])
    if isinstance(context, list):
        context.append(
            {
                "reprolib": REPROLIB_CANONICAL
            }
        )
    elif isinstance(context, dict):
        context["reprolib"] = REPROLIB_CANONICAL
    for k in ldObj.keys():
        if isinstance(ldObj[k], dict):
            context, newObj[k] = _deeperContextualize(
                ldObj[k],
                context
            )
        else:
            newObj[k] = ldObj[k]
    newObj['@context'] = reprolibCanonize(context)
    return(expand(newObj))

def _deeperContextualize(ldObj, context):
    newObj = {}
    for k in ldObj.keys():
        if isinstance(ldObj[k], dict) and '.' in k:
                (c, o) = _createContextForStr(k)
                newObj[o] = ldObj[k]
                if c not in context:
                    context.append(c)
        else:
            newObj[k] = reprolibPrefix(ldObj[k])
    return(context, newObj)

def childByParent(parent, applet, parentProfile=None):
    from girderformindlogger.models.profile import Profile

    parentProfile = Profile().getProfile(
        Profile().createProfile(
            applet['applet']['_id'].split('applet/')[-1],
            parent,
            "user"
        ).get('_id'),
        user=parent
    ) if parentProfile is None else parentProfile
    parentKnows = parentProfile.get('schema:knows', {})
    children = [Profile().displayProfileFields(
        Profile().load(
            p,
            force=True
        ),
        parent
    ) for p in list(
            set(parentKnows.get('rel:parentOf', {})).union(
                set(parentKnows.get('schema:children', {}))
            )
        )
    ]
    return([
        formatChildApplet(child, deepcopy(applet)) for child in children
    ])

def formatChildApplet(child, applet):
    applet['applet'] = _formatChildLabel(
        applet['applet'],
        child['displayName']
    )
    for act in applet['activities']:
        applet['activities'][act] = _formatChildLabel(
            applet['activities'][act],
            child['displayName']
        )
    return(_formatSubjectDocument(applet, child))

def _formatChildLabel(obj, label):
    for i, pl in enumerate(obj.get(
        "http://www.w3.org/2004/02/skos/core#prefLabel",
        []
    )):
        obj["http://www.w3.org/2004/02/skos/core#prefLabel"][i][
            "@value"
        ] = ": ".join([
            label,
            obj["http://www.w3.org/2004/02/skos/core#prefLabel"][
                i
            ].get("@value", "")
        ])
    return(obj)

def _formatSubjectDocument(obj, child):
    if isinstance(obj, str):
        return("?subjectId=".join([obj, str(child['_id'])]) if (
            obj.startswith("reprolib:") and not (
                obj.startswith("reprolib:terms") or obj.split(":")[-1].isupper()
            )
        ) else obj)
    elif isinstance(obj, list):
        return([_formatSubjectDocument(i, child) for i in obj])
    elif isinstance(obj, dict):
        n = {}
        for k in obj.keys():
            nk = "?subjectId=".join([k, str(child['_id'])]) if (
                k.startswith("reprolib:") and not (
                    k.startswith("reprolib:terms") or k.split(":")[-1].isupper()
                )
            ) else k
            n[nk] = (
                "?subjectId=".join([
                    obj[k],
                    str(child['_id'])
                ]) if isinstance(obj[k], str) else [
                    "?subjectId=".join([
                        o,
                        str(child['_id'])
                    ]) for o in obj[k]
                ] if isinstance(obj[k], list) else obj[k]
            ) if k in [
                "@index",
                "order"
            ] else {
                "?subjectId=".join([
                    adnK,
                    str(child['_id'])
                ]): ": ".join([
                    child['displayName'],
                    obj[k][adnK]
                ]) for adnK in obj[k]
            } if k=="activity_display_name" else {
                "?subjectId=".join([
                    vizK,
                    str(child['_id'])
                ]): obj[k][vizK] for vizK in obj[k]
            } if k=="visibility" else obj[
                k
            ] if k=="@type" else _formatSubjectDocument(obj[k], child)
        return(n)
    else:
        return(obj)

def inferRelationships(person):
    from girderformindlogger.models.invitation import Invitation
    from girderformindlogger.models.profile import Profile

    if "schema:knows" not in person:
        return(person)
    start = deepcopy(person)
    for rel in list(person['schema:knows'].keys()):
        if rel in DEFINED_RELATIONS.keys():
            if "owl:equivalentProperty" in DEFINED_RELATIONS[rel]:
                for ep in DEFINED_RELATIONS[rel]["owl:equivalentProperty"]:
                    if ep not in person['schema:knows']:
                        person['schema:knows'][ep] = []
                    for related in person['schema:knows'][rel]:
                        if related not in person['schema:knows'][ep]:
                            person['schema:knows'][ep].append(related)
    if any([
        bool(
            rp not in start.get('schema:knows', {}).get(rel, [])
        ) for rp in person['schema:knows'][rel] for rel in list(
            person['schema:knows'].keys()
        )
    ]):
        newPerson = Profile().load(person['_id'], force=True)
        if 'schema:knows' in newPerson:
            newPerson['schema:knows'].update(person['schema:knows'])
        else:
            newPerson['schema:knows'] = person['schema:knows']
        Profile().save(
            newPerson,
            validate=False
        ) if 'userId' in newPerson else Invitation().save(
            newPerson,
            validate=False
        )
    return(person)

def oidIffHex(s):
    """
    Function to return a list of the passed string and its ObjectId if the
    passed string is a valid hexidecimal string, or a list of just the passed
    string otherwise.

    :param s: string to check and potentially convert
    :type s: str
    :returns: list of strings, 1≤len≤2
    """
    from bson.objectid import ObjectId
    from bson.errors import InvalidId

    try:
        ObjectId(s)
        return([ObjectId(s), s])
    except InvalidId:
        return([s])

def reprolibPrefix(s):
    """
    Function to check if a string is a reprolib URL, and, if so, compact it to
    the prefix "reprolib:"

    :type s: str
    :returns: str
    """
    if isinstance(s, str):
        for prefix in REPROLIB_PREFIXES:
            if s.startswith(prefix) and s!=prefix:
                return(s.replace(prefix, 'reprolib:'))
    elif isinstance(s, dict):
        for k in s.keys():
            s[k] = reprolibPrefix(
                s[k]
            ) if k not in KEYS_TO_DEREFERENCE else dereference(s[k])
    elif isinstance(s, list):
        s = [reprolibPrefix(li) for li in s]
    return(s)

def schemaPrefix(s):
    """
    Function to toggle between "schema:" and "http://schema.org/" prefixes.

    :type s: str
    :returns: str
    """
    a = "schema:"
    b = "http://schema.org/"
    if isinstance(s, str):
        if s.startswith(a):
            return(s.replace(a, b))
        elif s.startswith(b):
            return(s.replace(b, a))
    return(s)

def reprolibCanonize(s):
    """
    Function to check if a string is a prfixed reprolib URL, and, if so,
    expand it to the current canonical prefix

    :type s: str
    :returns: str
    """
    if isinstance(s, str):
        s = reprolibPrefix(s).replace('reprolib:', REPROLIB_CANONICAL)
        return(s)
        ##
        ##Temporary disabled
        ##
        #if checkURL(s):
        #    return(s)
        #else:
        #    return(None)
    elif isinstance(s, list):
        return([reprolibCanonize(ls) for ls in s])
    elif isinstance(s, dict):
        return({
            reprolibCanonize(
                k
            ) if reprolibCanonize(
                k
            ) is not None else k: reprolibCanonize(v) for k, v in s.items()
        })
    return(s)

def delanguageTag(obj):
    """
    Function to take a language-tagged list of dicts and return an untagged
    string.

    :param obj: list of language-tagged dict
    :type obj: list
    :returns: string
    """
    if not isinstance(obj, list):
        return(obj)
    return((obj if len(obj) else [{}])[-1].get("@value", ""))

def expandOneLevel(obj):
    if obj is None:
        # We only want to catch `None`s here, not other falsy objects
        return(obj)
    try:
        newObj = jsonld.expand(obj)
    except jsonld.JsonLdError as e: # 👮 Catch illegal JSON-LD
        if e.type == "jsonld.InvalidUrl":
            try:
                newObj = jsonld.expand(reprolibCanonize(obj))
            except:
                print("Invalid URL: {}".format(e.details.get("url")))
                print(obj)
        elif e.cause.type == "jsonld.ContextUrlError":
            invalidContext = e.cause.details.get("url")
            print("Invalid context: {}".format(invalidContext))
            if isinstance(obj, str):
                obj = loadJSON(obj)
            if not isinstance(obj, dict):
                obj = {"@context": []}
            if invalidContext in obj.get("@context", []):
                obj["@context"] = obj["@context"].remove(invalidContext)
                obj["@context"].append(reprolibCanonize(invalidContext))
                if obj["@context"] is None:
                    obj["@context"] = []
            else:
                if isinstance(obj, dict):
                    for k in obj.keys():
                        if invalidContext in obj[k].get("@context", []):
                            obj[k]["@context"] = obj[k]["@context"].remove(
                                invalidContext
                            )
                            if obj[k]["@context"] is None:
                                obj[k]["@context"] = []
                            obj[k]["@context"].append(reprolibCanonize(
                                invalidContext
                            ))
            return(expandOneLevel(obj))
        return(obj)
    newObj = newObj[0] if (
        isinstance(newObj, list) and len(newObj)==1
    ) else newObj
    if isinstance(
        newObj,
        dict
    ):
        if not isinstance(obj, dict):
            obj={}
        for k, v in deepcopy(newObj).items():
            if not bool(v):
                newObj.pop(k)
            else:
                prefix_key = reprolibPrefix(k)
                if prefix_key != k:
                    newObj.pop(k)
                newObj[prefix_key] = reprolibPrefix(
                    v
                ) if prefix_key not in KEYS_TO_DEREFERENCE else dereference(v)
        for k in KEYS_TO_DELANGUAGETAG:
            if k in newObj.keys(
            ) and isinstance(newObj[k], list):
                newObj[k] = delanguageTag(newObj[k])
        newObj.update({
            k: reprolibPrefix(obj.get(k)) for k in obj.keys() if (
                bool(obj.get(k)) and k not in keyExpansion(
                    list(newObj.keys())
                )
            )
        })
        newObj.update({
            k: dereference(newObj[k]) for k in newObj.keys(
            ) if k in KEYS_TO_DEREFERENCE
        })
    return(newObj)

def dereference(prefixed):
    """
    Function to dereference values in given JSON.

    :param prefixed: JSON to dereference
    :type prefixed: dict, list, str, or None
    :returns: dereferenced same-type
    """
    if isinstance(prefixed, str):
        d = reprolibCanonize(prefixed)
        return(d if d is not None else prefixed)
    elif isinstance(prefixed, dict):
        return({
            k: dereference(v) for k, v in prefixed.items()
        })
    elif isinstance(prefixed, list):
        return([dereference(li) for li in prefixed])
    else: # bool, int, float, None
        return(prefixed)

def expand(obj, keepUndefined=False):
    """
    Function to take an unexpanded JSON-LD Object and return it expandeds.

    :param obj: unexpanded JSON-LD Object
    :type obj: dict
    :param keepUndefined: keep undefined-in-context terms?
    :param keepUndefined: bool
    :returns: list, expanded JSON-LD Array or Object
    """
    if obj is None:
        return(obj)
    newObj = expandOneLevel(obj)
    if isinstance(newObj, dict):
        for k in KEYS_TO_EXPAND:
            if k in newObj.keys():
                if isinstance(newObj.get(k), list):
                    v = [
                        expand(lv.get('@id')) for lv in newObj.get(k)
                    ]
                    v = v if v!=[None] else None
                else:
                    v = expand(newObj[k])
                if bool(v):
                    newObj[k] = delanguageTag(
                        v
                    ) if k in KEYS_TO_DELANGUAGETAG else dereference(
                        v
                    ) if k in KEYS_TO_DEREFERENCE else reprolibPrefix(v)
        if k in KEYS_TO_DELANGUAGETAG:
            if k in newObj:
                newObj[k] = reprolibCanonize(
                    delanguageTag(newObj[k])
                )
        if k in KEYS_TO_DEREFERENCE:
            if k in newObj:
                newObj[k] = dereference(newObj[k])
        return(_fixUpFormat(newObj) if bool(newObj) else None)
    else:
        expanded = [expand(n, keepUndefined) for n in newObj]
        return(_fixUpFormat(expanded) if bool(expanded) else None)

def fileObjectToStr(obj):
    """
    Function to load a linked file in a JSON-LD object and return a string.

    :param obj: Object
    :type obj: dict
    :returns: String from loaded file
    """
    import requests
    from requests.exceptions import ConnectionError, MissingSchema
    try:
        r = requests.get(obj.get('@id'))
    except (AttributeError, ConnectionError, MissingSchema):
        r = obj.get("@id") if isinstance(obj, dict) else ""
        raise ResourcePathNotFound("Could not load {}".format(r))
    return(r.text)

def checkURL(s):
    """
    Function to check if a URL is dereferenceable

    :param s: URL
    :type s: string
    :returns: bool
    """
    import requests
    try:
        if (requests.get(s).status_code==404):
            return(False)
        else:
            return(True)
    except:
        return(False)

def compactKeys(obj):
    context = obj.get('@context', [])
    if not isinstance(context, list):
        context = [context]
    newObj = {}
    for k in list(obj.keys()):
        if "." in k:
            c, nk = _createContext(k)
            if c not in context:
                context.append(c)
            nc, newObj[nk] = _deepCompactKeys(obj[k])
            for c in nc:
                if c not in context:
                    context.append(c)
        else:
            nc, newObj[k] = _deepCompactKeys(obj[k])
            for c in nc:
                if c not in context:
                    context.append(c)
    newObj['@context'] = context
    return(newObj)

def _deepCompactKeys(obj):
    context = []
    if not isinstance(obj, dict):
        return(context, obj)
    newObj = {}
    for k in list(obj.keys()):
        if "." in k:
            c, nk = _createContext(k)
            if c not in context:
                context.append(c)
            nc, newObj[nk] = _deepCompactKeys(obj[k])
            for c in nc:
                if c not in context:
                    context.append(c)
        else:
            nc, newObj[k] = _deepCompactKeys(obj[k])
            for c in nc:
                if c not in context:
                    context.append(c)
    return(context, newObj)

def _createContext(key):
    s = key.split('/')
    k = s[-1]
    key = '{}/'.format('/'.join(s[0:-1]))
    return({key.split('://')[-1].replace('.', '_dot_'): key}, k)

def createCache(obj, formatted, modelType, user = None):
    obj = MODELS()[modelType]().load(obj['_id'], force=True)
    if modelType in NONES:
        print("No modelType!")
        print(obj)
    if formatted is None:
        print("formatting failed!")
        print(obj)

    if obj.get('cached'):
        cache_id = obj['cached']
        CacheModel().updateCache(cache_id, MODELS()[modelType]().name, obj['_id'], modelType, formatted)
    else:
        saved = CacheModel().insertCache(MODELS()[modelType]().name, obj['_id'], modelType, formatted)
        obj['cached'] = saved['_id']
        MODELS()[modelType]().update({'_id': ObjectId(obj['_id'])}, {'$set': {'cached': obj['cached']}}, False)
    return obj

def loadCache(id):
    cache = CacheModel().getCacheData(id)
    return cache

def _fixUpFormat(obj):
    if isinstance(obj, dict):
        newObj = {}
        for k in obj.keys():
            rk = reprolibPrefix(k)
            if k in KEYS_TO_DELANGUAGETAG:
                newObj[rk] = reprolibCanonize(
                    delanguageTag(obj[k])
                )
            elif k in KEYS_TO_DEREFERENCE:
                newObj[rk] = dereference(obj[k])
            elif isinstance(obj[k], list):
                newObj[rk] = [_fixUpFormat(li) for li in obj[k]]
            elif isinstance(obj[k], dict):
                newObj[rk] = _fixUpFormat(obj[k])
            else: # bool, int, float
                newObj[rk] = obj[k]
            if isinstance(obj[k], str) and k not in KEYS_TO_DEREFERENCE:
                c = reprolibPrefix(obj[k])
                newObj[rk] = c if c is not None else obj[k]
            s2k = schemaPrefix(rk)
            if s2k!=rk:
                newObj[s2k] = deepcopy(newObj[rk])
        if "@context" in newObj:
            newObj["@context"] = reprolibCanonize(newObj["@context"])
        for k in ["schema:url", "http://schema.org/url"]:
            if k in newObj and newObj[k] is not None:
                newObj["url"] = newObj["schema:url"] = newObj[k]
        return(newObj)
    elif isinstance(obj, str):
        return(reprolibPrefix(obj))
    else:
        return(obj)

def formatLdObject(
    obj,
    mesoPrefix='folder',
    user=None,
    keepUndefined=False,
    dropErrors=False,
    refreshCache=False,
    responseDates=False
):
    """
    Function to take a compacted JSON-LD Object within a Girder for Mindlogger
    database and return an exapanded JSON-LD Object including an _id.

    :param obj: Compacted JSON-LD Object
    :type obj: dict or list
    :param mesoPrefix: Girder for Mindlogger entity type, defaults to 'folder'
                       if not provided
    :type mesoPrefix: str
    :param user: User making the call
    :type user: User
    :param keepUndefined: Keep undefined properties
    :type keepUndefined: bool
    :param dropErrors: Return `None` instead of raising an error for illegal
        JSON-LD definitions.
    :type dropErrors: bool
    :param refreshCache: Refresh from Dereferencing URLs?
    :type refreshCache: bool
    :param responseDates: Include list of ISO date strings of responses
    :type responseDates: bool
    :returns: Expanded JSON-LD Object (dict or list)
    """
    from girderformindlogger.models import pluralize

    refreshCache = False if refreshCache is None else refreshCache

    try:
        if obj is None:
            return(None)
        if isinstance(obj, dict):
            oc = obj.get("cached")
            if all([
                not refreshCache,
                oc is not None
            ]):
                return(loadCache(oc))
            if 'meta' not in obj.keys():
                return(_fixUpFormat(obj))
        mesoPrefix = camelCase(mesoPrefix)
        if type(obj)==list:
            return(_fixUpFormat([
                formatLdObject(
                    o,
                    mesoPrefix,
                    refreshCache=refreshCache,
                    user=user
                ) for o in obj if o is not None
            ]))
        if not type(obj)==dict and not dropErrors:
            raise TypeError("JSON-LD must be an Object or Array.")
        newObj = obj.get('meta', obj)
        newObj = newObj.get(mesoPrefix, newObj)

        if not obj.get('loadedFromSingleFile', False):
            newObj = expand(newObj, keepUndefined=keepUndefined)

        if type(newObj)==list and len(newObj)==1:
            try:
                newObj = newObj[0]
            except:
                raise ValidationException(str(newObj))
        if type(newObj)!=dict:
            newObj = {}
        objID = str(obj.get('_id', 'undefined'))
        if objID=='undefined':
            raise ResourcePathNotFound('unable to load object')
        newObj['_id'] = "/".join([snake_case(mesoPrefix), objID])
        if mesoPrefix=='applet':
            protocolUrl = obj.get('meta', {}).get('protocol', obj).get(
                'http://schema.org/url',
                obj.get('meta', {}).get('protocol', obj).get('url')
            )

            # get protocol data from id
            protocol = None
            protocolId = obj.get('meta', {}).get('protocol', {}).get('_id' ,'').split('/')[-1]
            if protocolId:
                cache = ProtocolModel().getCache(protocolId)
                if cache:
                    protocol = loadCache(cache)

            if protocolUrl is not None and not protocol:
                # get protocol from url
                protocol = ProtocolModel().getFromUrl(
                            protocolUrl,
                            'protocol',
                            user,
                            thread=False,
                            refreshCache=refreshCache
                        )[0]

            # format protocol data
            protocol = formatLdObject(
                protocol,
                'protocol',
                user,
                refreshCache=refreshCache
            )

            applet = {}
            applet['activities'] = protocol.pop('activities', {})
            applet['items'] = protocol.pop('items', {})
            applet['protocol'] = {
                key: protocol.get(
                    'protocol',
                    protocol.get(
                        'activitySet',
                        {}
                    )
                ).pop(
                    key
                ) for key in [
                    '@type',
                    '_id',
                    'http://schema.org/url',
                    'schema:url',
                    'url'
                ] if key in list(protocol.get('protocol', {}).keys())
            }
            applet['applet'] = {
                **protocol.pop('protocol', {}),
                **obj.get('meta', {}).get(mesoPrefix, {}),
                '_id': "/".join([snake_case(mesoPrefix), objID]),
                'url': "#".join([
                    obj.get('meta', {}).get('protocol', {}).get("url", "")
                ])
            }

            if 'appletName' in obj and obj['appletName']:
                suffix = obj['appletName'].split('/')[-1]
                inserted = False

                candidates = ['prefLabel', 'altLabel']
                for candidate in candidates:
                    for key in applet['applet']:
                        if not inserted and str(key).endswith(candidate) and len(applet['applet'][key]) and len(applet['applet'][key][0].get('@value', '')):
                            if len(suffix):
                                applet['applet'][key][0]['@value'] += (' ' + suffix)

                            if not len(applet['applet']['url']): # for development
                                applet['applet'][key][0]['@value'] += ' ( single-file )'

                            AppletModel().update({'_id': obj['_id']}, {'$set': {'displayName': applet['applet'][key][0]['@value']}})

                            inserted = True
            createCache(obj, applet, 'applet', user)
            if responseDates:
                try:
                    applet["applet"]["responseDates"] = responseDateList(
                        obj.get('_id'),
                        user.get('_id'),
                        user
                    )
                except:
                    applet["applet"]["responseDates"] = []
            return(applet)
        elif mesoPrefix=='protocol':
            protocol = {
                'protocol': newObj,
                'activities': {},
                "items": {}
            }

            if obj.get('loadedFromSingleFile', False):
                activities = list(ActivityModel().find({'meta.protocolId': '{}/{}'.format(MODELS()['protocol']().name, obj['_id'])}))
                items = list(ScreenModel().find({'meta.protocolId': '{}/{}'.format(MODELS()['protocol']().name, obj['_id'])}))

                for activity in activities:
                    formatted = formatLdObject(activity, 'activity', user)
                    protocol['activities'][formatted['@id']] = formatted
                for item in items:
                    formatted = formatLdObject(item, 'screen', user)
                    protocol['items'][formatted['@id']] = formatted
            else:
                try:
                    protocol = componentImport(
                        newObj,
                        deepcopy(protocol),
                        user,
                        refreshCache=refreshCache
                    )
                except:
                    print("636")
                    protocol = componentImport(
                        newObj,
                        deepcopy(protocol),
                        user,
                        refreshCache=True
                    )

                newActivities = protocol.get('activities', {}).keys()
                newItems = protocol.get('items', {}).keys()

                while(any([len(newActivities), len(newItems)])):
                    activitiesNow = set(
                        protocol.get('activities', {}).keys()
                    )
                    itemsNow = set(protocol.get('items', {}).keys())
                    for activityURL in newActivities:
                        activity = protocol['activities'][activityURL]
                        activity = activity.get(
                            'meta',
                            {}
                        ).get('activity', activity)
                        try:
                            protocol = componentImport(
                                deepcopy(activity),
                                deepcopy(protocol),
                                user,
                                refreshCache=refreshCache
                            )
                        except:
                            print("670")
                            protocol = componentImport(
                                deepcopy(activity),
                                deepcopy(protocol),
                                user,
                                refreshCache=True
                            )
                    for itemURL in newItems:
                        activity = protocol['items'][itemURL]
                        activity = activity.get(
                            'meta',
                            {}
                        ).get('screen', activity)
                        try:
                            protocol = componentImport(
                                deepcopy(activity),
                                deepcopy(protocol),
                                user,
                                refreshCache=refreshCache
                            )
                        except:
                            print("691")
                            protocol = componentImport(
                                deepcopy(activity),
                                deepcopy(protocol),
                                user,
                                refreshCache=True
                            )
                    newActivities = list(
                        set(
                            protocol.get('activities', {}).keys()
                        ) - activitiesNow
                    )
                    newItems = list(
                        set(
                            protocol.get('items', {}).keys()
                        ) - itemsNow
                    )

            formatted = _fixUpFormat(protocol)

            createCache(obj, formatted, 'protocol')
            return formatted
        else:
            return(_fixUpFormat(newObj))
    except:
        if refreshCache==False:
            return(_fixUpFormat(formatLdObject(
                obj,
                mesoPrefix,
                user,
                keepUndefined,
                dropErrors,
                refreshCache=True,
                responseDates=responseDates
            )))
        import sys, traceback
        print(sys.exc_info())
        print(traceback.print_tb(sys.exc_info()[2]))

def componentImport(
    obj,
    protocol,
    user=None,
    refreshCache=False,
    modelType=['activity', 'item']
):
    """
    :param modelType: model or models to search
    :type modelType: str or iterable
    :returns: protocol (updated)
    """
    import itertools
    from girderformindlogger.models import pluralize, smartImport
    from girderformindlogger.utility import firstLower

    updatedProtocol = deepcopy(protocol)
    obj2 = {k: v for k, v in expand(deepcopy(obj)).items() if v is not None}
    try:
        for order in obj2.get(
            "reprolib:terms/order",
            {}
        ):
            for activity in order.get("@list", []):
                IRI = activity.get(
                    'url',
                    activity.get('@id')
                )
                if reprolibPrefix(IRI) not in list(
                    itertools.chain.from_iterable([
                        protocol.get(mt, {}).keys() for mt in [
                            "activities",
                            "items"
                        ]
                    ])
                ):
                    result =         \
                        smartImport(
                            IRI,
                            user=user,
                            refreshCache=refreshCache
                        ) if (IRI is not None and not IRI.startswith(
                            "Document not found"
                        )) else (None, None, None)
                    activityComponent, activityContent, canonicalIRI = result

                    activity["url"] = activity["schema:url"] = canonicalIRI if(
                        canonicalIRI is not None
                    ) else IRI
                    activityComponent = pluralize(firstLower(
                        activityContent.get(
                            '@type',
                            ['']
                        )[0].split('/')[-1].split(':')[-1]
                    )) if (activityComponent is None and isinstance(
                        activityContent,
                        dict
                    )) else activityComponent
                    if activityComponent is not None:
                        activityComponents = (
                            pluralize(
                                activityComponent
                            ) if activityComponent != 'screen' else 'items'
                        )
                        updatedProtocol[activityComponents][
                            canonicalIRI
                        ] = deepcopy(formatLdObject(
                            activityContent,
                            activityComponent,
                            user,
                            refreshCache=refreshCache
                        ))
        return(updatedProtocol.get(
            'meta',
            updatedProtocol
        ).get(modelType if isinstance(
            modelType,
            str
        ) else modelType[0], updatedProtocol))
    except:
        import sys, traceback
        print("error!")
        print(sys.exc_info())
        print(traceback.print_tb(sys.exc_info()[2]))

def getByLanguage(object, tag=None):
    """
    Function to get a value or IRI by a language tag following
    https://tools.ietf.org/html/bcp47.

    :param object: The JSON-LD Object to language-parse
    :type object: dict or list
    :param tag: The language tag to use.
    :type tag: str
    :returns: str, either a literal or an IRI.
    """
    if not tag:
        from girderformindlogger.api.v1.context import Context
        tag = FolderModel().findOne({
            'name': 'JSON-LD',
            'parentCollection': 'collection',
            'parentId': CollectionModel().findOne({
                'name': 'Context'
            }).get('_id')
        })
        tag = tag.get('meta', {}).get('@context', {}).get(
            '@language'
        ) if tag else None
    if isinstance(tag, str):
        tags = getMoreGeneric(tag)
        tags = tags + ["@{}".format(t) for t in tags]
        tags.sort(key=len, reverse=True)
        if isinstance(object, dict):
            return(
                getFromLongestMatchingKey(object, tags, caseInsensitive=True)
            )
        if isinstance(object, list):
            return([getFromLongestMatchingValue(
                objectList=object,
                listOfValues=tags,
                keyToMatch='@language',
                caseInsensitive=True
            )])
    if isinstance(object, str):
        return(object)

def getFromLongestMatchingKey(object, listOfKeys, caseInsensitive=True):
    """
    Function to take an object and a list of keys and return the value of the
    longest matching key or None if no key matches.

    :param object: The object with the keys.
    :type object: dict
    :param listOfKeys: A list of keys to try to match
    :type listOfKeys: list of string keys
    :param caseInsensitive: Case insensitive key matching?
    :type caseInsensitive: boolean
    :returns: value of longest matching key in object
    """
    listOfKeys = listOfKeys.copy()
    if caseInsensitive:
        object = {k.lower():v for k,v in object.items()}
        listOfKeys = [k.lower() for k in listOfKeys]
    key = max(
        [str(k) for k in listOfKeys],
        key=len
    ) if len(listOfKeys) else None
    if key and key in listOfKeys:
        listOfKeys.remove(key)
    return(
        object.get(
            key,
            getFromLongestMatchingKey(object, listOfKeys)
        ) if key else None
    )

def getFromLongestMatchingValue(
    objectList,
    listOfValues,
    keyToMatch,
    caseInsensitive=True
):
    """
    Function to take a list of objects, a list of values and a key to match and
    return the object with the longest matching value for that key or None if
    no value matches for that that key.

    :param objectList: The list of objects.
    :type objectList: list of dicts
    :param listOfValues: A list of values to try to match
    :type listOfValues: list of string values
    :param keyToMatch: key in which to match the value
    :type keyToMatch: str
    :param caseInsensitive: Case insensitive value matching?
    :type caseInsensitive: boolean
    :returns: dict with longest matching value for specified key in object
    """
    objectList = objectList.copy()
    if caseInsensitive:
        listOfValues = [k.lower() for k in listOfValues]
    value = max(
        [str(k) for k in listOfValues],
        key=len
    ) if len(listOfValues) else None
    if value and value in listOfValues:
        listOfValues.remove(value)
    for object in sorted(
        objectList,
        key=lambda i: len(i.get(keyToMatch, "")),
        reverse=True
    ):
        if (
            object.get(keyToMatch, '').lower(
            ) if caseInsensitive else object.get(keyToMatch, '')
        )==value:
            return(object)
    if len(listOfValues)>=1:
        return(getFromLongestMatchingValue(
            objectList,
            listOfValues,
            keyToMatch,
            caseInsensitive
        ))
    for object in sorted(
        objectList,
        key=lambda i: len(i.get(keyToMatch, "")),
        reverse=False
    ):
        generic = object.get(keyToMatch, '').lower(
        ) if caseInsensitive else object.get(keyToMatch, '')
        generic = generic.split('-')[0] if '-' in generic else generic
        if generic==value:
            return(object)
    return({})

def getMoreGeneric(langTag):
    """
    Function to return a list of decreasingly specific language tags, given a
    language tag.

    :param langTag: a language tag following https://tools.ietf.org/html/bcp47
    :type langTag: str
    :returns: list
    """
    langTags = [langTag]
    while '-' in langTag:
        langTag = langTag[::-1].split('-', 1)[1][::-1]
        langTags.append(langTag)
    return(langTags)

def keyExpansion(keys):
    return(list(set([
        k.split(delimiter)[-1] for k in keys for delimiter in [
            ':',
            '/'
        ] if delimiter in k
    ] + [
        k for k in keys if (':' not in k and '/' not in k)
    ])))

def camelCase(snake_case):
    """
    Function to convert a snake_case_string to a camelCaseString

    :param snake_case: snake_case_string
    :type snake_case: str
    :returns: camelCaseString
    """
    words = snake_case.split('_')
    return('{}{}'.format(
        words[0],
        ''.join([
            word.title() for word in words[1:]
        ])
    ))

def snake_case(camelCase):
    """
    Function to convert a camelCaseString to a snake_case_string

    :param camelCase: camelCaseString
    :type camelCase: str
    :returns: snake_case_string
    """
    import re
    first_cap_re = re.compile('(.)([A-Z][a-z]+)')
    all_cap_re = re.compile('([a-z0-9])([A-Z])')
    return(
        all_cap_re.sub(
            r'\1_\2',
            first_cap_re.sub(
                r'\1_\2',
                camelCase
            )
        ).lower()
    )

def xsdNow():
    """
    Function to return an XSD formatted datetime string for the current
    datetime.now()
    """
    return(datetime.now(datetime.utcnow().astimezone().tzinfo).isoformat())


import os
import xml.etree.ElementTree as parse

src_path = os.path.dirname(os.path.realpath(__file__))
print(src_path)
xml = os.path.join(src_path, "zookeys.pensoft.net.xml")
print(xml)

xml_tree = parse.parse(xml)

root = xml_tree.getroot()

for child in root:
    for elem in child:
        if elem.tag == "article-meta":
            for sub_elem in elem:
                if sub_elem.tag == "abstract":
                    for ss in sub_elem:
                        # print(ss.tag)
                        for sss in ss:
                            # print(sss.tag)
                            if sss.tag == "{http://www.plazi.org/taxpub}taxon-name" or "italic":
                                for tn in sss:
                                    if tn.tag == "{http://www.plazi.org/taxpub}taxon-name":
                                        for tnp in tn:
                                            print(tnp.attrib)
                                    else:
                                        print(tn.attrib)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from __future__ import print_function
from codecs import open

import os
# _en_set = os.environ.get('HAS_SYS_ENCODING_SET', 0)
# if _en_set == 0:
#     import sys
#     reload(sys)
#     sys.setdefaultencoding('utf-8')
import json
from helper import SUID, PathHelper
import time
import shutil

def _fill_contents(tpl_sch_dir, sch_name, ct_dict):
    sch_path = os.path.join(tpl_sch_dir, sch_name + '.json')
    with open(sch_path, 'r', encoding='utf-8') as _f:
        tpl_sch = json.load(_f)
    var_bi, var_ctt = {}, {}
    if 'extends' in tpl_sch:
        var_bi, var_ctt = _fill_contents(tpl_sch_dir, tpl_sch['extends'], ct_dict)
    if 'build_in' in tpl_sch:
        for _k, _v in tpl_sch['build_in'].items():
            var_bi[_k] = ''  # TODO: Do Something
    if 'contents' in tpl_sch:
        for _k, _v in tpl_sch['contents'].items():
            optional = _v.get('optional', False)
            if _k in ct_dict:
                var_ctt[_k] = ct_dict[_k]
            else:
                if not optional:
                    raise KeyError('Content var: %s is required from template: %s !' % (_k, sch_name))
            # TODO: Validate !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    return var_bi, var_ctt

def _update_nav(root_fs, sp_path):
    # if repo_tree is None:
    #     repo_tree = {'': [0, 'root', 'P', {'cn': {'n': '首页'}, 'en': {'n': 'index'}}, {}]}
    #     cur_dir = repo_tree
    #     _p = root_fs
    #     for _d in sp_path[: -1]:
    #         cur_dir = cur_dir[_d][4]
    #         _p = os.path.join(_p, _d)
    #         with open(os.path.join(_p, 'repo.json'), 'r') as _f:
    #             repo = json.load(_f)
    #         for idx, item in enumerate(repo['items']):
    #             name, _type = item['n'], item['t']
    #             if _type == 'P':
    #                 cur_dir[name] = [idx, name, _type, item['inf'], {}]
    # cur_dir = repo_tree
    # for _d in sp_path:
    #     cur_dir = cur_dir[_d][4]
    # _d = sp_path[-1]
    _p = os.path.join(root_fs, *sp_path)
    with open(os.path.join(_p, 'repo.json'), 'r', encoding='utf-8') as _f:
        repo = json.load(_f)
    # for idx, item in enumerate(repo['items']):
    #     name, _type = item['n'], item['t']
    #     if _type == 'P':
    #         cur_dir[name] = [idx, name, _type, item['inf'], {}]

    for item in repo['items']:
        name, _type = item['n'], item['t']
        if _type == 'P':
            sub_sp_path = sp_path + [name]
            _update_nav(root_fs, sub_sp_path)

    with open(os.path.join(_p, 'nav.json'), 'r', encoding='utf-8') as _f:
        nav = json.load(_f)

    # if nav['t'] == 'A':
    #     _path = []
    #     cur_dir = repo_tree
    #     for _d in nav['ns'][: -1]:
    #         _path.append(_d)
    #         cur_dir = cur_dir[_d][4]
    #     for _d in sp_path[len(nav['ns']) - 1: ]:
    #         _path.append(_d)
    #         cur_dir = cur_dir[_d][4]
    if nav['t'] == 'A':
        _path = []
        for _d in nav['ns']:
            _path.append(_d)
        _pp = os.path.join(root_fs, *_path)
        with open(os.path.join(_pp, 'prop.json'), 'r', encoding='utf-8') as _f:
            prop = json.load(_f)
        info = {"en": {"n": prop['inf']['en']['nm']}, "cn": {"n": prop['inf']['cn']['nm']}}
        _u_p = '/'.join(_path)
        nav_tree = {'nm': prop['pn'], 't': 'P', 'inf': info, 'ch': [], 'p': _u_p}
        next_b = nav_tree
        for _d in sp_path[len(nav['ns']) + 1:] + [None]:
            cur_b = next_b
            with open(os.path.join(_pp, 'repo.json'), 'r', encoding='utf-8') as _f:
                repo = json.load(_f)
            for item in repo['items']:
                name, _type = item['n'], item['t']
                if _type == 'P':
                    _b = {'nm': name, 't': _type, 'inf': item['inf'], 'ch': [], 'p': _u_p + '/' + name}
                    cur_b['ch'].append(_b)
                    if name == _d:
                        next_b = _b
            if _d is None:
                break
            _path.append(_d)
            _u_p += '/' + _d
            _pp = os.path.join(root_fs, *_path)
        nav['nt'] = [nav_tree]
        # print json.dumps(nav_tree)
        with open(os.path.join(_p, 'nav.json'), 'w', encoding='utf-8') as _f:
            json.dump(nav, _f, ensure_ascii=False)

# _update_nav('./data', [''])

# with open(os.path.join(fs_path, 'repo.json'), 'r') as _f:
    #     repo = json.load(_f)
    # cur_dir = {}
    # for idx, item in enumerate(repo['items']):
    #     name, _type = item['n'], item['t']
    #     if _type == 'P'
    #         cur_dir[name] = [idx, name, _type]

# _fill_contents('./templates/template_schema', 'general_view', {'body_content': '23333', 'title': 'hhhhhh', 'useless': '213213'})

def get_auth():
    # TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    return True, 'admin', 'admins'

def check_perm(need_perms):
    def perm_dec(func):
        def dec_func(cls, path, *args, **kw):
            if isinstance(path, tuple):
                path, sp_path, fs_path, path_type = path
            else:
                path, sp_path, fs_path, path_type = cls.path_h.do_all(path)
            is_login, user, group = get_auth()
            print('Method: %s for user: %s, group: %s need permission: %s:' % (str(func), user, group, need_perms))  # TODO: debug
            # TODO: permission check !!!!!!!!!!
            kw['user'], kw['group'] = user, group
            kw['inh_perm'] = 508  # TODO !!!!!!!!!!!!! inherent permission
            func(cls, (path, sp_path, fs_path, path_type), *args, **kw)
        return dec_func
    return perm_dec

class DirSite(object):
    def __init__(self, root_dir, template_dir='./templates'):
        self._root_dir = root_dir
        self.path_h = PathHelper(root_dir)
        self.tpl_sch_dir = os.path.join(template_dir, 'template_schema')

    @check_perm('w')
    def add_page(self, path_inf, contents, page_name=None, permission=None, extra_perm=None,
                 name_en='', name_cn=None, tags=None, **kw):

        par_path, par_sp_path, par_fs_path, par_path_type = path_inf
        uuid = SUID.timeUID()
        # TODO: validate page_name is plain text
        if page_name is None:
            page_name = uuid

        sp_path = par_sp_path + [page_name]
        path = '/'.join(sp_path)
        path, sp_path, fs_path, path_type = self.path_h.do_all(path)

        if path_type is not None:
            raise IOError('Path or File: %s already exists !' % fs_path)
        user = kw['user']
        group = kw['group']
        if permission is None:
            permission = kw['inh_perm']

        if extra_perm:
            # TODO:  Check extra permission !
            print("Not Implement: extra permission !")

        if tags:
            assert isinstance(tags, list)
            # TODO: Check tags
        else:
            tags = []

        page_prop = {
            "id": uuid,
            "pn": page_name,
            "pms": {"o": user, "g": group, "p": permission, "ep": []},
            "cr": user,
            "crt": int(time.time()),
            "lmt": int(time.time()),
            "tgs": tags,
            "inf": {
                "cn": {
                    "nm": name_cn if name_cn else name_en
                },
                "en": {
                    "nm": name_en
                }
            }
        }

        page_repo = {
            "id": uuid,
            "items": [],
        }

        if len(sp_path) == 1:
            nav_start = []
            nav_type = 'N'
        else:
            nav_start = sp_path[0: 2]
            nav_type = 'A'
        page_nav = {
            "id": uuid,
            "t": nav_type,   # type: A: automatic, N: None
            "ns": nav_start,  # Nav Start splited Path
            "nt": [],  # Nav Tree
        }

        os.makedirs(fs_path)
        path_type = 'D'

        with open(os.path.join(fs_path, 'prop.json'), 'w', encoding='utf-8') as _f:
            json.dump(page_prop, _f, ensure_ascii=False)
        with open(os.path.join(fs_path, 'repo.json'), 'w', encoding='utf-8') as _f:
            json.dump(page_repo, _f, ensure_ascii=False)
        with open(os.path.join(fs_path, 'nav.json'), 'w', encoding='utf-8') as _f:
            json.dump(page_nav, _f, ensure_ascii=False)

        self.edit_page((path, sp_path, fs_path, path_type), contents)

        self._add_child(path_inf, page_name, 'P', {'cn': {'n': name_cn}, 'en': {'n': name_en}})
        self.update_nav(path_inf)

    @check_perm('w')
    def edit_page(self, path_inf, contents, **kw):
        path, sp_path, fs_path, path_type = path_inf
        if path_type == 'D':
            for lang, ctt_upd in contents['contents'].items():
                page_fs_p = os.path.join(fs_path, 'page.%s.json' % lang)
                if os.path.exists(page_fs_p):
                    with open(page_fs_p, 'r', encoding='utf-8') as _f:
                        page = json.load(_f)
                    tpl_name = page['tpl']  # TODO: Can't change template
                else:
                    tpl_name = contents['template']
                    page = {
                        "tpl": tpl_name,  # template
                        'bi': {},  # build in
                        "ctt": {},  # contents
                    }
                ctt_fill = page['ctt']
                ctt_fill.update(ctt_upd)
                page['bi'], page['ctt'] = _fill_contents(self.tpl_sch_dir, tpl_name, ctt_fill)
                with open(page_fs_p, 'w', encoding='utf-8') as _f:
                    json.dump(page, _f, ensure_ascii=False)
        else:
            raise ValueError

    def _add_child(self, path_inf, name, ch_type, info, **kw):
        assert isinstance(path_inf, tuple)
        path, sp_path, fs_path, path_type = path_inf
        repo_path = os.path.join(fs_path, 'repo.json')
        with open(repo_path, 'r', encoding='utf-8') as _f:
            repo = json.load(_f)
        repo['items'].append({
            'n': name, 't': ch_type, 'inf': info,
        })
        with open(repo_path, 'w', encoding='utf-8') as _f:
            json.dump(repo, _f, ensure_ascii=False)

    @check_perm('w')
    def delete_page(self, path_inf, **kw):
        path, sp_path, fs_path, path_type = path_inf
        assert len(sp_path) > 1, 'Can not remove root directory !'
        par_sp_path = sp_path[: -1]
        par_path = '/'.join(par_sp_path)
        par_fs_path = self.path_h.fs_path(par_path)
        par_repo_fp = os.path.join(par_fs_path, 'repo.json')
        with open(par_repo_fp, 'r', encoding='utf-8') as _f:
            par_repo = json.load(_f)
        resource_name = sp_path[-1]
        for item in par_repo['items']:
            if item['n'] == resource_name:
                par_repo['items'].remove(item)
                break
        with open(par_repo_fp, 'w', encoding='utf-8') as _f:
            json.dump(par_repo, _f, ensure_ascii=False)
        if path_type == 'D':
            shutil.rmtree(fs_path)
        else:
            # TODO
            pass
        self.update_nav((par_path, par_sp_path, par_fs_path, 'D'))

    @check_perm('')
    def update_nav(self, path_inf, **kw):
        path, sp_path, fs_path, path_type = path_inf
        _update_nav(self._root_dir, sp_path)

# ss = DirSite('./data')
#
# ctt = {
#     'template': 'general_view',
#     'contents': {
#         'cn': {
#             'title': '中心简介222',
#             'body_content': '哈哈哈哈222',
#         },
#         'en': {
#             'title': 'Introduction222',
#             'body_content': 'hhhhh222',
#         }
#     }
# }
# ss.add_page('/', ctt, page_name='introduction', name_cn='中心简介', name_en='introduction')
# ss.add_page('/introduction', ctt, page_name='111', name_cn='111', name_en='111')
# ss.add_page('/introduction/111', ctt, page_name='222', name_cn='222', name_en='222')
# ss.add_page('/introduction/111/222', ctt, page_name='333', name_cn='333', name_en='333')
# ss.add_page('/introduction', ctt, page_name='com', name_cn='com', name_en='com')
#
# ss.edit_page('/introduction', ctt)


import matplotlib.pyplot as plt
import tensorflow
from sklearn.cluster import KMeans, AgglomerativeClustering, Birch, FeatureAgglomeration
from sklearn.cross_decomposition import PLSCanonical, CCA, PLSSVD, PLSRegression
from sklearn.decomposition import PCA, FastICA
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.manifold import LocallyLinearEmbedding, TSNE, MDS, Isomap
from sklearn.preprocessing import StandardScaler

from helpers.visualize import plot_true_labels, subplot_components

def visualize(x, y):
    """
    https://scikit-learn.org/stable/modules/clustering.html

    :param x:
    :param y:
    :return:
    """

    # make sure data is scaled
    scaler = StandardScaler()
    x = scaler.fit_transform(x, y)

    # cross decomposition
    cross_decomposers = {
        "PLSCanonical": PLSCanonical(n_components=2),  # 2 because the 3rd component is just zero
        "CCA": CCA(n_components=2),  # 2 because the 3rd component is just zero
        "PLSSVD": PLSSVD(n_components=3),
        "PLSRegression": PLSRegression(n_components=3),
    }

    for extractor_name, extractor in cross_decomposers.items():
        x_components, y_component = extractor.fit_transform(x, tensorflow.keras.utils.to_categorical(y, 3))
        plot_true_labels(x, y, x_components, extractor_name)

    feature_extractors = {
        # Discriminant Analysis
        # https://scikit-learn.org/stable/modules/classes.html#module-sklearn.discriminant_analysis
        "LDA": LinearDiscriminantAnalysis(n_components=2),
        # decomposition
        # https://scikit-learn.org/stable/modules/classes.html#module-sklearn.decomposition
        "PCA": PCA(n_components=3),
        "FastICA": FastICA(n_components=3, random_state=41),
        # Manifold Learning
        # https://scikit-learn.org/stable/modules/classes.html#module-sklearn.manifold
        "Isomap": Isomap(n_components=3, n_jobs=-1),
        "LocallyLinearEmbedding": LocallyLinearEmbedding(n_components=3, n_jobs=-1, random_state=41),
        "MDS": MDS(n_components=3, n_jobs=-1, random_state=41),
        "TSNE": TSNE(n_components=3, n_jobs=-1, random_state=41),
        # Clustering
        # https://scikit-learn.org/stable/modules/classes.html#module-sklearn.cluster
        "FeatureAgglomeration": FeatureAgglomeration(n_clusters=3, linkage='ward')
    }

    cluster_dict = {
        "KMeans": KMeans(n_clusters=3, random_state=41),
        "Birch": Birch(threshold=0.8, n_clusters=3),
        # linkage : {'ward', 'complete', 'average', 'single'}
        "AgglomerativeClustering": AgglomerativeClustering(n_clusters=3, linkage='ward'),
    }
    for extractor_name, extractor in feature_extractors.items():
        components = extractor.fit_transform(x, y)
        plot_true_labels(x, y, components, extractor_name)

    plt.show()

    """
    for name, clusterer in cluster_dict.items():
        plot_predicted_clusters(x, y, components, clusterer, extractor_name + " - " + name)
    """

def plot_predicted_clusters(x, y, components, clusterer, title="", interactive=True):
    if interactive:
        # pip install PyQt5
        import matplotlib as mpl
        mpl.use("Qt5Agg")

    fig = plt.figure(figsize=(12, 12))
    fig.suptitle(title, fontsize=20)
    fig.set_dpi(150)

    # plot ture values
    y_pred = y.values.T
    subplot_components(components, 221, "True labels", y_pred)

    # plot cluster prediction based on x
    y_pred = clusterer.fit_predict(x)

    subplot_components(components, 222, "Predicted on whole input data", y_pred)

    # plot cluster prediction based only on the supplied components
    y_pred = clusterer.fit_predict(components)
    subplot_components(components, 212, "Predicted on only the supplied components", y_pred)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2013-08-24 16:23:58
# @Author  : vfasky (vfasky@gmail.com)
# @Link    : http://vfasky.com
# @Version : $Id$

import os
import glob

path = os.path.dirname(__file__)
handlers = []

for name in os.listdir(path):
    if os.path.isdir(os.path.join(path, name)):
        handlers.append(name)

__all__ = handlers


## Devin Higgins, 903187870

## I have neither given nor recieved any unacknowledged aid
## on this assessment

def mixtapeFire(timesPlayed, rating):

## write a function that determines how fire your mixtape is
## user enters number of times played and rating
## 10000 or greater and rating is 6 or greater mixtape is fire
## if either is not met you should quit the rap game
## if rating is greater than 10 invalid input

    if timesPlayed >= 10000 and rating >= 6 and rating <= 10:

        return "Your mixtape is fire!"

    elif timesPlayed <10000 or rating < 6:

        return "You should quit the rap game."

    elif rating > 10:

        return "Invalid input. Try again."

def myFavSong(song,artist):

## write a function with a song as a parameter and ask the user
    ##to guess the song
## second parameter should be the artist who wrote the song
    ##artist will be used if they need a hint
## they get 5 guesses, after the 5th try tell the user they exceeded
## the number of tries
## user can ask for a hint using hint, HINT, or Hint
## Hint is the artist of the song is " " use print
## after user guesses right say great job and how many guesses
## caps dont matter and thank the user after the game
    ## must use a while loop

    tries = 0

    songGuess = input("Guess my favorite song: ").lower()

    hints = 0

    while( tries < 5):

        tries += 1

        if songGuess == song.lower():
            print("Great Job! It took you {} tries and {} hints to guess my favorite song".format(tries,hints))
            break

        elif songGuess == 'hint':
            print('the artist of the song is {}'.format(artist))
            hints += 1
            tries -= 1

        if tries < 5:
            songGuess = input("Try again. Guess my favorite song: ").lower()

        if tries == 5:
            print('You have exceeded your number of tries')

    print('Thanks for playing!')

def illuminatiConfirmed(secretMsg):

    ## takes a secret message and determines if it is illuminati
    ## must use a  for loop
    ## if 3 @ signs replace @s with As
    ## print new string and illuminati confirmed
    ## if not 3 @s print old and probably not illuminati

    counter = 0
    string = ""

    ## goes through string and counts the @s

    for i in (secretMsg):

        if i == '@':

            counter += 1

    ## if there are 3 @s
    if counter == 3:

        for x in (secretMsg):

            if x == '@':
                string = string + "a"

            else:
                string = string + x

        print(string)

        return "Illuminati Confirmed"

    elif counter != 3:

        print(secretMsg)

        return "Probably not the Illuminati"

def decNum(aString,aNum):

    ## parameters- a string and a single digit integer
    ## if the string has the number it will decrement it by one
    ## if it doesnt have the number, try a different number

    newString = ""

    if str(aNum) in aString:

        for i in aString:

            if i == str(aNum):

                newString += str(aNum-1)

            else:
                newString += i

        return newString

    else:
        return "Try a different number."

def numberTie(num):

    ##take half the width of the tie as a parameter
    ##draw a tie using three number pyramids using print function
    ## one upside down, one right side up, and another upside down
    ## must use a for loop

    blank = " "

    width = num*2
    counter1 = 0
    counter2 = num-1
    counter3 = 0

    #pyramid 1
    for i in range(num,0,-1):

        ## adds blanks in front and back according to what i is
        x = blank * counter1

        print(x + (str(i)*(i*2)) + x)

        counter1 += 1

    #pyramid 2
    for i in range(1,num+1):

        y = blank * counter2

        print(y + (str(i)*(i*2)) + y)

        counter2 -= 1

    #pyramid 3
    for i in range(num,0,-1):

        z = blank * counter3

        print(z + (str(i)*(i*2)) + z)

        counter3 += 1

def countDown(start,limit,decrement):

    # takes three parameters start limit and decrement
    ## printed same line seperated by commas

    string = ""

    for i in range(start,limit,(-1)*decrement):

        if i - decrement <= limit:
            string = string + str(i)
        else:
            string = string  + str(i) + ','

    print(string)


#!/usr/bin/env python3

# Copyright 2014 Brett Slatkin, Pearson Education Inc.
#
# Udostępniono na licencji Apache w wersji 2.0 ("Licencja").
# Tego pliku można używać jedynie zgodnie z warunkami Licencji.
# Treść Licencji znajdziesz na stronie:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# O ile obowiązujące prawo nie stanowi inaczej lub czegoś innego nie
# uzgodniono w formie pisemnej, oprogramowanie objęte Licencją jest
# dostarczane w stanie, w jakim jest (wersja "AS IS"), BEZ JAKIEJKOLWIEK
# GWARANCJI, ani wyrażonej otwarcie, ani domyślnej. Dokładne zasady
# i warunki Licencji znajdziesz w jej treści.

# Przygotowania mające na celu odtworzenie środowiska użytego w książce.
import logging
from pprint import pprint
from sys import stdout as STDOUT

# Przykład 1.
from datetime import datetime, timedelta

class Bucket(object):
    def __init__(self, period):
        self.period_delta = timedelta(seconds=period)
        self.reset_time = datetime.now()
        self.quota = 0

    def __repr__(self):
        return 'Wiadro(quota=%d)' % self.quota

bucket = Bucket(60)
print(bucket)

# Przykład 2.
def fill(bucket, amount):
    now = datetime.now()
    if now - bucket.reset_time > bucket.period_delta:
        bucket.quota = 0
        bucket.reset_time = now
    bucket.quota += amount

# Przykład 3.
def deduct(bucket, amount):
    now = datetime.now()
    if now - bucket.reset_time > bucket.period_delta:
        return False
    if bucket.quota - amount < 0:
        return False
    bucket.quota -= amount
    return True

# Przykład 4.
bucket = Bucket(60)
fill(bucket, 100)
print(bucket)

# Przykład 5.
if deduct(bucket, 99):
    print('Potrzebne 99 jednostek danych')
else:
    print('Nie ma 99 jednostek danych')
print(bucket)

# Przykład 6.
if deduct(bucket, 3):
    print('Potrzebne 3 jednostki danych')
else:
    print('Nie ma 3 jednostek danych')
print(bucket)

# Przykład 7.
class Bucket(object):
    def __init__(self, period):
        self.period_delta = timedelta(seconds=period)
        self.reset_time = datetime.now()
        self.max_quota = 0
        self.quota_consumed = 0

    def __repr__(self):
        return ('Bucket(max_quota=%d, quota_consumed=%d)' %
                (self.max_quota, self.quota_consumed))

# Przykład 8.
    @property
    def quota(self):
        return self.max_quota - self.quota_consumed

# Przykład 9.
    @quota.setter
    def quota(self, amount):
        delta = self.max_quota - amount
        if amount == 0:
            # Liczba jednostek danych jest wyzerowana dla nowego przedziału czasu.
            self.quota_consumed = 0
            self.max_quota = 0
        elif delta < 0:
            # Liczba jednostek danych przygotowana dla nowego przedziału czasu.
            assert self.quota_consumed == 0
            self.max_quota = amount
        else:
            # Liczba jednostek danych wykorzystana w przedziale czasu.
            assert self.max_quota >= self.quota_consumed
            self.quota_consumed += delta

# Przykład 10.
bucket = Bucket(60)
print('Początkowo', bucket)
fill(bucket, 100)
print('Wypełniono', bucket)

if deduct(bucket, 99):
    print('Potrzebne 99 jednostek danych')
else:
    print('Nie ma 99 jednostek danych')

print('Teraz', bucket)

if deduct(bucket, 3):
    print('Potrzebne 3 jednostki danych')
else:
    print('Nie ma 3 jednostek danych')

print('Pozostało', bucket)

Python/Python - efektywny Python/item_30.py


#!/usr/bin/python
# -*- coding: utf-8 -*-

import datetime

PATH_TO_INTERPRETER = "/cm/shared/apps/intel/composer_xe/python3.5/intelpython3/bin/python3"  # plafrim

env = 'HalfCheetah-v2'
trial_id = list(range(120, 144))
a_optimizer = 'kfra'
c_optimizer = 'kfra'
a_lrs = [1e-2, 1e-1]
a_dampings = [1e-1]
c_lrs = [1.0, 1e-1, 5e-1]
c_dampings = [100]

script_path = '../'
filename = script_path + 'run_' + a_optimizer + '_' + c_optimizer + '_trials' + str(trial_id[0]) + '_' + str(trial_id[-1]) + '.sh'

with open(filename, 'w') as f:
    f.write('#!/bin/sh\n')
    f.write('#SBATCH --mincpus 24\n')
    f.write('#SBATCH -p long_sirocco\n')
    f.write('#SBATCH -t 8:00:00\n')
    f.write('#SBATCH -e ' + filename[1:] + '.err\n')
    f.write('#SBATCH -o ' + filename[1:] + '.out\n')
    f.write('rm log.txt; \n')
    f.write("export EXP_INTERP='%s' ;\n" % PATH_TO_INTERPRETER)
    t_id = trial_id[0]
    for c_lr in c_lrs:
        for a_lr in a_lrs:
            for a_dmp in a_dampings:
                for c_dmp in c_dampings:
                    for seed in range(4):
                        name = (a_optimizer+"/"+c_optimizer+" : trial %s, %s" % (str(t_id), str(datetime.datetime.now()))).title()
                        f.write("echo '=================> %s';\n" % name)
                        f.write("echo '=================> %s' >> log.txt;\n" % name)
                        f.write("$EXP_INTERP gep.py --trial %s --env_id %s --actor-optimizer %s --critic-optimizer %s --critic-lr %s --actor-lr %s --actor-damping %s --critic-damping %s  & \n" % (
                            str(t_id), env, a_optimizer, c_optimizer, str(c_lr), str(a_lr), str(a_dmp), str(c_dmp) ))

                        t_id += 1
    f.write('wait')


###与Spark相似，tensorflow也是构建一个有向无环的工作流，通过创建常量、变量、占位量，并定义三者之间的关系，最后放入工作流执行

import numpy as np
import tensorflow as t
#唯有将常量、变量、占位量放入session执行，才会有结果，类似于spark中的transform和show，可存放超参数
s = t.Session()

##创建常量，常量一旦创建，别无法改变

t.zeros([2,2])
t.ones([2,2])
t.fill(1,[2,2])
t.range(1.0,2,3)
t.constant(43,shape=[10,10])
c = t.constant([1,2,3,4,5,6],shape=[2,3])
s.run(c)
#random_shuffle可将矩阵的行随即调换位置
t.random_shuffle(c)

#均匀分布
t.random_uniform([3,3],1,10)
#高斯分布
t.random_normal([3,3,3])
#截断高斯，即与中心相距不超过2个标准差的区域
t.truncated_normal([2, 3])

#将np.array转为tensor.array
a = np.random.rand(3,3)
at = t.convert_to_tensor(a)

##创建变量,变量在每次迭代中都会被改变，需要由初始值和初始化，可存放训练参数和每层训练后的临时结果
#设初始值
t.Variable(c)
v1 = t.Variable(at,name="at")
#初始化
v2 = t.Variable(v1)
s.run(v2.initializer)
s.run(v2)
#全局初始化，所有变量都在初始化池中，可单个初始化，也可全部初始化
init_op = t.global_variables_initializer()
s.run(init_op)

#占位量，只需指定类型和维度，无需初始化，用于存放输出值
x = t.placeholder(t.float32,[2,2])
y = t.identity(x-1)
x_vals = np.random.rand(2,2)
s.run(y,feed_dict={x:x_vals})

##矩阵操作
#定义对角矩阵
identity_matrix = t.diag([1.0, 1.0, 1.0])

A = t.truncated_normal([2, 3])
B = t.fill([2,3], 5.0)
C = t.random_uniform([3,2])
D = t.convert_to_tensor(np.array([[1., 2., 3.],[-3., -7.,
-1.],[0., 5., -2.]]))
print(s.run(identity_matrix))

#加
s.run(A+B)
#点乘
s.run(t.matmul(B,identity_matrix*2))
#转置
s.run(t.transpose(D))
#行列式
s.run(t.matrix_determinant(D))
#求逆
s.run(t.matrix_inverse(D))
#用cholesky求特征值和特征向量
s.run(t.self_adjoint_eig(D))

#除
s.run(t.div(3,4))

sr = s.run
#将int转为float后除
sr(t.truediv(3,4))
#反转类型
sr(t.floordiv(3.0,4.0))
#求余
sr(t.mod(22.0,3))
#叉乘
sr(t.cross([1.0,0.,0.],[0.,1.,0.]))
#自定义操作
def custom_polynomial(value):
    return(t.subtract(3 * t.square(value), value) + 10)
print(s.run(custom_polynomial(11)))

#激活函数
s.run(t.nn.relu([-3,0,3]))
s.run(t.nn.relu6([-3,0,3]))
s.run(t.nn.sigmoid([1.0,2.0,3]))
s.run(t.nn.tanh([1.0,2.0,3]))
s.run(t.nn.softsign([1.0,2.0,3]))
s.run(t.nn.softplus([1.0,2.0,3]))
s.run(t.nn.elu(np.array([-1,0,1],dtype=float)))

#读入iris
from sklearn import datasets
iris = datasets.load_iris()
print(len(iris.data),iris.data[1:20])

#读入mnist
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
print(len(mnist.train.images),len(mnist.test.images),len(mnist.validation.images))
mnist.train.labels[1,:]


#######################################################################################################################
# ITP 2017 Homework A1-1 (Version 1.0.0)
#
# Runyu Hong
#
# 09/26/2017
#
# A simple DNA sequence converter
#
#######################################################################################################################

# Define a codon converter method
def convert(a):
    b = ''
    if a == 'A' or a == 'a':
        b = 'T'
    elif a == 'T' or a == 't':
        b = 'A'
    elif a == 'G' or a == 'g':
        b = 'C'
    elif a =='C' or a == 'c':
        b = 'G'
    return b

# Define a codon count method
def charcount(str):
    A = 0
    T = 0
    G = 0
    C = 0
    for a in str:
        if a == 'A' or a == 'a':
            A += 1
        elif a == 'T' or a == 't':
            T += 1
        elif a == 'G' or a == 'g':
            G += 1
        elif a =='C' or a == 'c':
            C += 1
    return A, T, G, C

# Ask for how many sequences expected
n = int(input("How many sequences would you like to convert?"))

# Set up loop count, two empty lists for input and output
count = 0
listold = []
newlist = []

# Set up valid input character list
valid = set('AaTtGgCc')

# Set up total length count and single codon count
length = 0
A = 0
T = 0
G = 0
C = 0

# Main loop
while count in range (0, n):
    # Read in user input sequence
    seq = input("Please enter your sequence here:")
    # Check if sequence is valid
    if set(seq) <= valid:
        # Check duplicate sequence
        if seq in listold:
           print(seq," is already in the list.")
           count = count
        else:
            # Length count
            length = length + len(seq)
            # Save input to list
            listold.append(seq)
            # Convert sequence
            com = ''
            for m in seq:
                new = convert(m)
                com += new
            # 5' to 3' reformat
            rev = com[::-1]
            # Count single codon
            a,t,g,c = charcount(rev)
            A += a
            T += t
            G += g
            C += c
            # Add output sequence to list
            newlist.append(rev)
            print(rev," has been added to the list.")
            count += 1
    else:
        print("Input invalid. Not a DNA sequence.")
        count = count
# Average sequence length
AveLen = length/n

# Summary and Statistics
print("\nSequences:\n", newlist)
print("\n--* Statistics *-- \n frequencies:")
print("A/a: ", A, "/", length)
print("T/t: ", T, "/", length)
print("G/g: ", G, "/", length)
print("C/c: ", C, "/", length)
print("number of sequences:\n", n)
print("Average sequence length:\n", AveLen)


#! coding: utf8

#chemins à modfier

#***************************************************

import rake
import glob
import operator
import os
from pprint import pprint

rake_object = rake.Rake("C:\\Users\\marcel\\Downloads\\RAKE-tutorial-master\\FrenchStoplist.txt", 5, 3, 4)

compteur = 0
for d in os.listdir("C:\\Users\\marcel\\Desktop\\programmation\\TIM-master\\M2\\2017\\Corpus\\types\\"):
	compteur +=1
	corpus = ""
	for filename in glob.glob("C:\\Users\\marcel\\Desktop\\programmation\\TIM-master\\M2\\2017\\Corpus\\types\\"+d+"\\*"):
		try:
			fic = open(filename, 'r',encoding="utf8")
			text = fic.read()
			corpus +=text
		# keywords += rake_object.run(text)
		except UnicodeDecodeError:
			pass
	print("FINI CONCAT")
	outc = open("C:\\Users\\marcel\\Desktop\\programmation\\TIM-master\\M2\\2017\\Corpus\\corpus\\corpus"+str(compteur)+".txt", 'w')
	outc.write(corpus)
	outc.close()
	keywords=rake_object.run(corpus)
	print("FINI EXTRACT")
	out = open("C:\\Users\\marcel\\Desktop\\programmation\\TIM-master\\M2\\2017\\Corpus\\resultats\\resultat"+str(compteur)+".txt", 'w', encoding="utf8")
	out.write(str(keywords[:300]))
	out.close()


import argparse
import time
from pathlib import Path

import numpy as np
from PIL import Image

import seam_carving

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('src', type=str)
    parser.add_argument('-o', dest='dst', type=str, default='a.jpg')
    parser.add_argument('--keep', type=str, default=None)
    parser.add_argument('--drop', type=str, default=None)
    parser.add_argument('--dw', type=int, default=0)
    parser.add_argument('--dh', type=int, default=0)
    parser.add_argument('--energy', type=str, default='backward',
                        choices=['backward', 'forward'])
    parser.add_argument('--order', type=str, default='width-first',
                        choices=['width-first', 'height-first', 'optimal'])
    args = parser.parse_args()

    try:
        print('Loading source image from {}'.format(args.src))
        src = np.array(Image.open(args.src))

        drop_mask = None
        if args.drop is not None:
            print('Loading drop_mask from {}'.format(args.drop))
            drop_mask = np.array(Image.open(args.drop).convert('L'))

        keep_mask = None
        if args.keep is not None:
            print('Loading keep_mask from {}'.format(args.keep))
            keep_mask = np.array(Image.open(args.keep).convert('L'))

        print('Performing seam carving...')
        start = time.time()
        if drop_mask is not None:
            dst = seam_carving.remove_object(src, drop_mask, keep_mask)
        else:
            src_h, src_w, _ = src.shape
            dst = seam_carving.resize(src, (src_w + args.dw, src_h + args.dh),
                                      args.energy, args.order, keep_mask)
        print('Done at {:.4f} second(s)'.format(time.time() - start))

        print('Saving output image to {}'.format(args.dst))
        Path(args.dst).parent.mkdir(parents=True, exist_ok=True)
        Image.fromarray(dst).save(args.dst)
    except Exception as e:
        print(e)
        exit(1)

if __name__ == "__main__":
    main()


import os

from paramiko import SSHClient

#file = open("/Volumes/Liberty.PolicyAdmin Debugging/")

def rating_trap(rpath,rfile):
  rmpath = rpath

  list_of_files = os.listdir(rmpath)
  # print(list_of_files)
  each_file = list_of_files

  for file in each_file:

    if file.startswith(rfile):
      print("found!!!")
      rreqf = os.path.join(rmpath, file)
      print(rreqf)
      rreqf = f"'{rreqf}'"
      print(rreqf)
      dreqf = dirn
      dreqf = f"'{dreqf}'"

      cmder = 'cp -av ' + rreqf + ' ' + dreqf
      print(cmder)

      os.popen(cmder)

def direct_log(quote_n):
  global dirn
  try:
    BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    print(BASE_DIR)
    LOG_DIR = os.path.join(BASE_DIR, "testapi/Log_data/")
    dirn = LOG_DIR + quote_n
    os.mkdir(dirn)

  except FileExistsError:
    print("Directory already exist!!!")
    pass

  # os.mkdir('', 0755);....................check : run trace log

    # cpfile = os.path.join(dirpath, each_file)
    # os.popen('cp ' + cpfile + 'Rating_engine_req.log')

"""lines = file.readlines()
for line in lines:
    print(line)

file.close()
"""

if __name__ == '__main__':
  quoteno = 'SHL000012569'

  direct_log(quoteno)

  get_me = ['rengreq','xml']

  req_rmpath = '/Volumes/Liberty.PolicyAdmin Debugging/'
  xmlio_rmpath = '/Volumes/RatingEngine/CalculationOutput/'

  for gtm in get_me:
    if gtm == 'xml':
      prefile = quoteno
      rmpath = xmlio_rmpath
    elif gtm == 'rengreq':
      prefile = 'RatingEngineRequest_'+quoteno
      rmpath = req_rmpath

    rating_trap(rmpath,prefile)


import copy

from shapes.Shape import Shape

class Triangle(Shape):
    """
    A Polygon with 3 Sides
    """

    def __init__(self, a=1, b=1, c=1):
        self.name = "Triangle"

        self._side_a = a
        self._side_b = b
        self._side_c = c

    @property
    def side_a(self):
        return self._side_a

    @side_a.setter
    def side_a(self, s):
        self._side_a = s

    @property
    def side_b(self):
        return self._side_b

    @side_b.setter
    def side_b(self, s):
        self._side_b = s

    @property
    def side_c(self):
        return self._side_c

    @side_c.setter
    def side_c(self, s):
        self._side_c = s

    def area(self):
        """
        Compute the area using Heron's Formula.
        Use @f$ s = \frac{1}{2}Perimeter @f$ and
        @f$ Area = \sqrt{ s(s-a)(s-b)(s-c) } @f$

        :return: area
        """

        s = self.perimeter() / 2.0

        return (
            s * (s - self.side_a)
              * (s - self.side_b)
              * (s - self.side_c)
        ) ** 0.5

    def perimeter(self):
        """
        Compute the perimeter

        :return: perimeter
        """

        return (self.side_a + self.side_b + self.side_c)

    def __deepcopy__(self, memo):
        """
        Return a new duplicate Shape
        """

        return Triangle(
            copy.deepcopy(self.side_a),
            copy.deepcopy(self.side_b),
            copy.deepcopy(self.side_c)
        )

    def __str__(self):
        """
        Print the Triangle
        """

        formatStr = (
            "{:<" +
            str(Shape.WIDTH_LABEL) +
            "}:{:>" +
            str(Shape.WIDTH_VALUE) +
            ".4f}\n"
        )

        return (
            super(Triangle, self).__str__() +
            formatStr.format("Side A", self.side_a) +
            formatStr.format("Side B", self.side_b) +
            formatStr.format("Side C", self.side_c) +
            formatStr.format("Perimeter", self.perimeter()) +
            formatStr.format("Area", self.area())
        )

if __name__ == "__main__":
    s = Triangle(2, 2, 2)

    print(s)
    print(s.side_a)


import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
import seaborn as sns
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.linear_model import LinearRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve, auc
from sklearn.metrics import accuracy_score
from sklearn.model_selection import permutation_test_score
from sklearn.model_selection import StratifiedKFold

def bootstrap_resample(X, n=None):
    if n == None:
        n = len(X)

    resample_i = np.floor(np.random.rand(n)*len(X)).astype(int)
    X_resample = np.array(X[resample_i])
    return X_resample

if __name__ == '__main__':
    sns.set_style('whitegrid')

    for extreme_case in ['pyro', 'chanting', 'throwing', 'banner', 'others']:
        features_df = pd.DataFrame.from_csv(
            '/media/vasiliy/01D266BF62940460/Linux/football_research/output_data/max_features.csv', sep=';')

        data_df = features_df[[extreme_case, 'posts_number', 'stands_filling', 'posts_before_game_number', 'mean_text_length',
                               'mean_likes', 'mutual_friends_graph_density', 'mutual_communities_graph_density',
                               'friends_among_visitors_graph_density']]

        pyro_df = pd.DataFrame()

        pyro_positive_df = data_df[data_df[extreme_case] == 1]
        pyro_positive_df.reset_index(drop=True, inplace=True)

        pyro_negative_df = data_df[data_df[extreme_case] == 0]
        pyro_negative_df.reset_index(drop=True, inplace=True)

        for _ in range(5):
            data_df_resampled = pd.DataFrame()
            for col in data_df.columns:
                data_df_resampled[col] = bootstrap_resample(pyro_positive_df[col], 100)
            pyro_df = pyro_df.append(data_df_resampled)

            data_df_resampled = pd.DataFrame()
            for col in data_df.columns:
                data_df_resampled[col] = bootstrap_resample(pyro_negative_df[col], 100)
            pyro_df = pyro_df.append(data_df_resampled)

        X = pyro_df.drop([extreme_case], axis=1)
        Y = pyro_df[extreme_case].get_values()

        X = X.as_matrix()

        test = SelectKBest(chi2, k=4)
        fit = test.fit(X, Y)
        X_new = fit.transform(X)

        rocs = []

        for i in range(100):
            clf = LinearRegression()

            # Y = np.random.permutation(Y)

            X_train, X_test, Y_train, Y_test = train_test_split(X, Y)

            # Y_test = np.random.permutation(Y_test)

            clf.fit(X_train, Y_train)

            y_score = clf.predict(X_test)

            # kfold = StratifiedKFold(5)
            # score, permutation_scores, p_value = permutation_test_score(clf, X, Y, scoring="roc_auc", cv=kfold, n_permutations=100, n_jobs=1)
            # print('score = {0}, permutation_scores = {1}, p_value = {2}'.format(score, permutation_scores, p_value))
            # print(accuracy_score(Y_test, y_score))
            # compute ROC

            fpr = dict()
            tpr = dict()
            roc_auc = dict()

            fpr[0], tpr[0], _ = roc_curve(Y_test, y_score)
            roc_auc[0] = auc(fpr[0], tpr[0])

            rocs.append(roc_auc[0])

            fpr["micro"], tpr["micro"], _ = roc_curve(Y_test.ravel(), y_score.ravel())
            roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])

            lw = 2

        mean_roc = np.mean(rocs)
        median_roc = np.median(rocs)
        std_roc = np.std(rocs)
        # print('mean roc = {0}'.format(mean_roc))
        # print('std roc = {0}'.format(std_roc))
        plt.figure()
        sns.distplot(rocs)
        plt.title('LinearRegression - ' + extreme_case + ' mean roc = {0:.2f}, median = {1:.2f}, std = {2:.2f}'.
                  format(round(mean_roc, 2), round(median_roc, 2), round(std_roc, 2)))
        rocs.clear()
    plt.show()
        # plt.savefig('/media/vasiliy/01D266BF62940460/Linux/football_research/output_data/img/rocs/artificial_data_rocs/' + 'LinearRegression' + extreme_case + '.png')


class Stack():
    def __init__(self, sample):
        print("New Stack object")
        self.stack = []
        sample = "sample string"

    def expect_string(i):
        if not isinstance(i, type(sample)):
            raise ValueError('Expected string but got {}'.format(type(i)))
        else:
            print('Got a string: {}'.format(i))
            return true

    def pop(self):
        print("Popping...")
        self.stack.pop()

    def push(self, value):
        print("Pushing...")
        if value == expect_string():
            self.stack.append()
        else:
            expect_string()