#! python3
# mcb.pyw - Saves and loads pieces of text to the clipboard.
# usage: py.exe mcb.pyw save <keyword> - Saves clipbord to keyword.
# 		py.exe mcb.pw <keyword> - Loads keyword to clipboard.
# 		py.exe mcb.pyw list - Loads all keywords to clipboard.
#		py.exe mcb.pyw delete <keyword> - delete's keyword from record
#		py.exe mcb.pyw delete - deletes all keywords from record

import shelve, pyperclip, sys

mcbShelf = shelve.open('mcb')

	#Save clipboard content
if len(sys.argv) == 3 and sys.argv[1].lower() == 'save':
	mcbShelf[sys.argv[2]] = pyperclip.paste()
	#delete a keyword
elif len(sys.argv) == 3 and sys.argv[1].lower() == 'delete':
	del mcbShelf[sys.argv[2]]

elif len(sys.argv) == 2:
	#List keywords and load content
	if sys.argv[1].lower() == 'list':
		pyperclip.copy(str(list(mcbShelf.keys())))
	#delete all keywords
	elif sys.argv[1].lower() == 'delete':
		for x in mcbShelf:
			del mcbShelf[x]
	#List keywords and load content
	elif sys.argv[1] in mcbShelf:
		pyperclip.copy(mcbShelf[sys.argv[1]])

	mcbShelf.close()


# -*- coding: utf-8 -*-
from os.path import join
from tensorflow.python.keras.models import load_model, save_model

#%%

ROOT_DIR = '.'
MODEL_DIR = join(ROOT_DIR, '..', 'data_out', 'model')
#%%

def saveModel(model, tag='1'):
    save_model(model, join(MODEL_DIR, '%s.h5' % tag), include_optimizer = True, overwrite = True)
    return tag

def loadModel(tag = '1', trainable = False):
    model = load_model(filepath = join(MODEL_DIR, '%s.h5' % tag), compile = False)

    print("Loaded model from disk")

    if not trainable:
        for layer in model.layers:
            layer.trainable = False

    return model


import os
import re

from MesaHandler.support import *
from MesaHandler.MesaFileHandler.MesaFileInterface import IMesaInterface

class MesaEnvironmentHandler(IMesaInterface):
    def __init__(self):
        IMesaInterface.__init__(self)
        self.mesaDir, self.defaultsDir = self.readMesaDirs(mesa_env)
        for section, file in defaultsFileDict.items():
            fileContent = self.readFile(self.defaultsDir + file)
            self.dataDict[section] = self.getParameters(fileContent)

    def readMesaDirs(self, envVar):
        try:
            mesaDir = os.environ[envVar]
        except KeyError:
            raise EnvironmentError("MESA_DIR is not set in your enviroment. \
                                   Be sure to set it properly!!")
        defaultsDir = mesaDir + defaultsPath

        if not os.path.exists(mesaDir):
            raise FileNotFoundError("Mesa dir " + mesaDir + " does not exist. \
                                    Be sure that it exists on your machine")

        if not os.path.exists(defaultsDir):
            raise FileNotFoundError("Defaults directory " + defaultsDir +
                                    " does not exist.")

        return mesaDir, defaultsDir

    def checkParameter(self, parameter, value=None):
        regex = r'(\w*) (\( [0-9]+ \))'
        x_ctrls = ['x_ctrl', 'x_integer_ctrl', 'x_logical_ctrl']
        for section, paramDict in self.dataDict.items():
            if parameter in paramDict.keys():
                if (value is None or
                        type(value) == type(paramDict[parameter])):
                    return section, paramDict[parameter]
                elif (isinstance(value, int) and
                        isinstance(paramDict[parameter], float)):
                    return section, paramDict[parameter]
                else:
                        raise TypeError('Type ' + str(type(value)) +
                                        ' for parameter ' + parameter +
                                        ' is wrong, expected type ' +
                                        str(type(paramDict[parameter])))
            else:
                match = re.search(regex, parameter, re.VERBOSE)
                if(match):
                    prefix = match.group(1)
                    if(prefix in x_ctrls):
                        suffix = '(1:num_x_ctrls)'
                    else:
                        suffix = '(:)'
                    p = prefix + suffix

                    if(p in paramDict.keys()):
                        if (value is None or
                                type(value) == type(paramDict[p])):
                            return section, paramDict[p]
                        elif (isinstance(value, int) and
                                isinstance(paramDict[p], float)):
                            return section, paramDict[p]
                        else:
                            raise TypeError('Type ' + str(type(value)) +
                                            ' for parameter ' + p +
                                            ' is wrong, expected type ' +
                                            str(type(paramDict[parameter])))
            #  Perhaps re.sub is a better solution?

        return "", value


# -*- coding: utf-8 -*-
"""
:mod:`re.Filing`
~~~~~~~~~~~~~~~~~~~
Edgar(tm) Renderer was created by staff of the U.S. Securities and Exchange Commission.
Data and content created by government employees within the scope of their employment
are not subject to domestic copyright protection. 17 U.S.C. 105.
"""

from gettext import gettext as _
from collections import defaultdict
import os, re, math, datetime, dateutil.relativedelta, lxml
import arelle.ModelValue, arelle.XbrlConst
import Cube, Embedding, Report, PresentationGroup, Summary, ErrorMgr, Utils, Xlout

def mainFun(controller, modelXbrl, outputFolderName):
    filing = Filing(controller, modelXbrl, outputFolderName)
    filing.populateAndLinkClasses()

    sortedCubeList = sorted(filing.cubeDict.values(), key=lambda cube : cube.definitionText)

    for cube in sortedCubeList:
        filing.mainFunDriverBeforeFlowThroughSuppression(cube)

    if not filing.hasEmbeddings:
        filing.filterOutColumnsWhereAllElementsAreInOtherReports(sortedCubeList)
        #filing.handleFlowThroughSuppression(sortedCubeList)

    # WARNING -- right now, the only place that excludes from numbering is called before flow through, so we can do here
    # might need to do later in the future though.
    # i think flow through can also exclude from numbering?  so do this after flow through for now.
    # number the cubes
    nextFile = controller.nextFile
    for cube in sortedCubeList:
        if not cube.excludeFromNumbering and len(cube.factMemberships) > 0:
            # even though there is embedding, and cubes might have more than one embedding and thus more than one report,
            # we still keep the fileNumber attribute on the cube and not the report, because if there are multiple embeddings
            # they all print in one file.
            cube.fileNumber = nextFile
            nextFile += 1
            filing.addToLog(_('File R{!s} is {} {}')
                              .format(cube.fileNumber, cube.linkroleUri, cube.definitionText)
                              , messageCode='info')

    xlWriter = None
    if filing.hasEmbeddings:
        # sort is a stable sort, so since this is already sorted by definition text, it will now be sorted first by
        # whether it is embedded or not and then by definition text.  Cubes that are embedded will be at the beginning
        # so that their reports will be there when it's time to embed them into cells of reports that are not embedded.
        sortedCubeList = sorted(sortedCubeList, key=lambda cube : cube.isEmbedded, reverse = True)
        if controller.excelXslt:
            filing.addToLog(_("Excel XSLT is not applied to instance {} having embedded commands."
                              ).format(filing.fileNameBase),messageCode='info')
    elif controller.excelXslt:
        xlWriter = controller.xlWriter
        if not xlWriter:
            controller.xlWriter = xlWriter = Xlout.XlWriter(controller, outputFolderName)

    #import win32process
    #print('memory '  + str(int(win32process.GetProcessMemoryInfo(win32process.GetCurrentProcess())['WorkingSetSize'] / (1024*1024))))

    # hang onto a list of skipped facts before cube properties all get dereferenced.
    skippedFactsList = []
    for cube in sortedCubeList:
        skippedFactsList += list(cube.skippedFactMembershipSet)

    # handle the steps after flow through and then emit all of the XML and write the files
    filing.addToLog(_('Generating rendered reports in {}').format(outputFolderName), messageCode='info')
    for cube in sortedCubeList:
        if cube.noFactsOrAllFactsSuppressed:
            for embedding in cube.embeddingList:
                Utils.releaseUnneededMemory(embedding)
        else:
            for embedding in cube.embeddingList:
                if embedding.isEmbeddingOrReportBroken:
                    Utils.releaseUnneededMemory(embedding)
                else:
                    filing.mainFunDriverAfterFlowThroughSuppressionAndReleaseMemory(cube, embedding, xlWriter)

    controller.nextFile = filing.nextFile()
    filing.unusedFactSet = set(modelXbrl.facts) - filing.usedOrBrokenFactSet

    for args in skippedFactsList:
        (fact,role,cube,ignore,shortName) = args
        if fact in filing.unusedFactSet:
            controller.logWarn(cube.strExplainSkippedFact(fact,role,shortName))

    if len(filing.unusedFactSet) > 0:
        filing.handleUncategorizedCube(xlWriter)
        controller.nextUncategorizedFile -= 1

    controller.instanceSummaryList += [Summary.InstanceSummary(filing, modelXbrl)]
    return True

class Filing(object):
    def __init__(self, controller, modelXbrl, outputFolderName):
        self.modelXbrl = modelXbrl

        self.cubeDict = {}
        self.axisDict = {}
        self.memberDict = {}
        self.elementDict = {}
        self.factToQlabelDict = {}
        self.symbolLookupDict = {}
        self.presentationUnitToConceptDict = {}

        self.usedOrBrokenFactSet = set()
        self.unusedFactSet = set()

        self.isRR = False
        self.hasEmbeddings = False
        self.isInvestTaxonomyInDTS = False
        for namespace in self.modelXbrl.namespaceDocs:
            if re.compile('http://xbrl.(sec.gov|us)/rr/20.*').match(namespace):
                self.isRR = True
            elif re.compile('http://xbrl.sec.gov/invest/*').match(namespace):
                self.isInvestTaxonomyInDTS = True
        # These namespaces contain elements treated specially in some layouts.
        self.stmNamespace = next((n for n in self.modelXbrl.namespaceDocs.keys() if re.search('/us-gaap/20',n) is not None),None)
        self.deiNamespace = next((n for n in self.modelXbrl.namespaceDocs.keys() if re.search('/dei/20',n) is not None), None)
        self.builtinEquityColAxes = [('dei',self.deiNamespace,'LegalEntityAxis'),
                                     ('us-gaap',self.stmNamespace,'StatementEquityComponentsAxis'),
                                     ('us-gaap',self.stmNamespace,'PartnerCapitalComponentsAxis'),
                                     ('us-gaap',self.stmNamespace,'StatementClassOfStockAxis')]
        self.builtinEquityRowAxes = [('us-gaap',self.stmNamespace,'CreationDateAxis'),
                                     ('us-gaap',self.stmNamespace,'StatementScenarioAxis'),
                                     ('us-gaap',self.stmNamespace,'AdjustmentsForNewAccountingPronouncementsAxis'),
                                     ('us-gaap',self.stmNamespace,'AdjustmentsForChangeInAccountingPrincipleAxis'),
                                     ('us-gaap',self.stmNamespace,'ErrorCorrectionsAndPriorPeriodAdjustmentsRestatementByRestatementPeriodAndAmountAxis')                                     ]
        self.builtinAxisOrders = [(arelle.ModelValue.QName('us-gaap',self.stmNamespace,'StatementScenarioAxis'),
                                   ['ScenarioPreviouslyReportedMember',
                                    'RestatementAdjustmentMember',
                                    'ChangeInAccountingPrincipleMember'],
                                   ['ScenarioUnspecifiedDomain'])]
        self.builtinLineItems = [arelle.ModelValue.QName('us-gaap',self.stmNamespace,'StatementLineItems')]
        self.segmentHeadingStopList = [arelle.ModelValue.QName(x,y,z) for x,y,z in self.builtinEquityRowAxes]
        # TODO: change flags like isRR, isInvest to contain the actual namespace or None.
        self.factToEmbeddingDict = {}
        self.factFootnoteDict = defaultdict(list)
        self.startEndContextDict = {}

        self.numReports = 0
        self.numBarcharts = 0

        self.controller = controller
        self.reportXmlFormat = 'xml' in controller.reportFormat.casefold()
        self.reportHtmlFormat = 'html' in controller.reportFormat.casefold()
        self.fileNamePrefix = 'R'
        self.fileNameBase = os.path.normpath(os.path.join(os.path.dirname(controller.webCache.normalizeUrl(modelXbrl.fileSource.url)) ,outputFolderName))
        if not os.path.exists(self.fileNameBase):  # This is usually the Reports subfolder.
            os.mkdir(self.fileNameBase)

        if controller.reportXslt:
            self.transform = lxml.etree.XSLT(lxml.etree.parse(controller.reportXslt))
        if controller.summaryXslt:
            self.summary_transform = lxml.etree.XSLT(lxml.etree.parse(controller.summaryXslt))
        self.reportSummaryList = []

        self.rowSeparatorStr = ' | '
        self.titleSeparatorStr = ' - '
        self.verboseHeadingsForDebugging = False
        self.ignoredPreferredLabels = [] # locations where the preferred label role was incompatible with the concept type.
        self.entrypoint = modelXbrl.modelDocument.basename

    def __str__(self):
        return "[Filing "+str(self.entrypoint)+"]"

    def populateAndLinkClasses(self, uncategorizedCube = None):
        duplicateFacts = set()

        if uncategorizedCube is not None:
            for fact in self.unusedFactSet:
                # we know these facts aren't broken, because broken facts weren't added to self.unusedFactSet.
                try:
                    element = self.elementDict[fact.qname]
                except KeyError:
                    element = Element(fact.concept)
                    self.elementDict[fact.qname] = element
                    element.linkCube(uncategorizedCube)
            uncategorizedCube.presentationGroup = PresentationGroup.PresentationGroup(self, uncategorizedCube)
            facts = self.unusedFactSet

        else:
            # build cubes
            for linkroleUri in self.modelXbrl.relationshipSet(arelle.XbrlConst.parentChild).linkRoleUris:
                cube = Cube.Cube(self, linkroleUri)
                self.cubeDict[linkroleUri] = cube
                cube.presentationGroup = PresentationGroup.PresentationGroup(self, cube)

            # initialize elements
            for qname, factSet in self.modelXbrl.factsByQname.items():

                # we are looking to see if we have "duplicate" facts.  a duplicate fact is one with the same qname, context and unit
                # as another fact.  in this case, we keep the first fact with an 'en-US' language, or if there is none, keep the first fact.
                # the others need to be proactively added to the set of unused facts.
                if len(factSet) > 1:
                    sortedFactList = sorted(factSet, key = lambda thing : (thing.contextID, thing.unitID, thing.sourceline))
                    while len(sortedFactList) > 0:

                        firstFact = sortedFactList.pop(0)
                        if firstFact.xmlLang == 'en-US':
                            firstEnUsLangFact = firstFact
                        else:
                            firstEnUsLangFact = None

                        discardedLineNumberList = []
                        counter = 0

                        # finds facts with same qname, context and unit as firstFact
                        while (len(sortedFactList) > 0 and
                               sortedFactList[0].qname == firstFact.qname and
                               sortedFactList[0].context == firstFact.context and
                               sortedFactList[0].unitID == firstFact.unitID):
                            counter += 1
                            fact = sortedFactList.pop(0)
                            if firstEnUsLangFact is None and fact.xmlLang == 'en-US':
                                firstEnUsLangFact = fact # keeping this fact
                            else:
                                duplicateFacts.add(fact) # not keeping this fact
                                discardedLineNumberList += [str(fact.sourceline)] # these are added in sorted order by sourceline

                        if counter > 0:
                            if firstEnUsLangFact is None:
                                lineNumOfFactWeAreKeeping = firstFact.sourceline # there is no en-US fact, keep the first fact we found
                            else:
                                lineNumOfFactWeAreKeeping = firstEnUsLangFact.sourceline # we did find an en-US fact, keep it.
                                if firstFact != firstEnUsLangFact:
                                    duplicateFacts.add(firstFact)
                                    discardedLineNumberList = [str(firstFact.sourceline)] + discardedLineNumberList # prepend to maintain sorted order

                            # start it off because we can assume that these facts have a qname and a context
                            # note that even if we are keeping firstEnUsLangFact, we are only printing qname, contextID and unitID,
                            # which are the same for both firstFact and firstEnUsLangFact.
                            qnameContextIDUnitStr = 'qname {!s}, context {}'.format(firstFact.qname, firstFact.contextID)
                            if firstFact.unit is not None:
                                qnameContextIDUnitStr += ', unit ' + firstFact.unitID
                            message = ErrorMgr.getError('DUPLICATE_FACT_SUPPRESSION').format(qnameContextIDUnitStr, lineNumOfFactWeAreKeeping, ', '.join(discardedLineNumberList))
                            self.addToLog(message, messageCode='warn')

                for fact in factSet: # we only want one thing, but we don't want to pop from the set so we "loop" and then break right away

                    elementBroken = False

                    if fact.concept is None:
                        conceptErrStr = ErrorMgr.getError('FACT_DECLARATION_BROKEN').format(qname)
                        self.addToLog(conceptErrStr, messageCode='warn')
                        elementBroken = True

                    elif fact.concept.type is None:
                        typeErrStr = ErrorMgr.getError('TYPE_DECLARATION_BROKEN').format(qname)
                        self.addToLog(typeErrStr, messageCode='warn')
                        elementBroken = True

                    if fact.context is None or elementBroken: # we will print the error if firstContext is broken later
                        continue # see if there are other facts for this concept with good contexts before we break from the loop, we still might make the Element yet

                    self.elementDict[qname] = Element(fact.concept)
                    break # we don't need to look at more facts from the fact set, we're just trying to make elements.

            # build presentation groups
            for concept in self.modelXbrl.qnameConcepts.values():
                for relationship in self.modelXbrl.relationshipSet(arelle.XbrlConst.parentChild).toModelObject(concept):
                    cube = self.cubeDict[relationship.linkrole]
                    cube.presentationGroup.traverseToRootOrRoots(concept, None, None, None, set())
                    try:
                        element = self.elementDict[concept.qname] # retrieve active Element
                        element.linkCube(cube) # link element to this cube.
                    except KeyError:
                        pass

            # footnotes
            for relationship in self.modelXbrl.relationshipSet('XBRL-footnotes').modelRelationships:
                # relationship.fromModelObject is a fact
                # relationship.toModelObject is a resource
                # make sure Element is active and that no Error is caught by relationshipErrorThrower()
                # if relationship.fromModelObject.qname in self.elementDict and not self.relationshipErrorThrower(relationship, 'Footnote'):
                #if relationship.fromModelObject.qname in self.elementDict:
                self.factFootnoteDict[relationship.fromModelObject].append((relationship.toModelObject, relationship.toModelObject.viewText()))

            facts = self.modelXbrl.facts

        for fact in facts:
            if fact.isTuple:
                tupleErrStr = ErrorMgr.getError('UNSUPPORTED_TUPLE_FOUND').format(fact.qname)
                self.addToLog(tupleErrStr, messageCode='warn')
                self.usedOrBrokenFactSet.add(fact) #now bad fact won't come back to bite us when processing isUncategorizedFacts
                continue

            if fact.context is None:
                contextErrStr1 = ErrorMgr.getError('CONTEXT_BROKEN').format(fact.qname, fact.value)
                self.addToLog(contextErrStr1, messageCode='warn')
                self.usedOrBrokenFactSet.add(fact) #now bad fact won't come back to bite us when processing isUncategorizedFacts
                continue

            if fact.context.scenario is not None:
                scenarioErrStr = ErrorMgr.getError('IMPROPER_CONTEXT_FOUND').format(fact.context.id)
                self.addToLog(scenarioErrStr, messageCode='warn')
            try:
                element = self.elementDict[fact.qname]
            except KeyError:
                self.usedOrBrokenFactSet.add(fact) #now bad fact won't come back to bite us when processing isUncategorizedFacts
                continue # fact was rejected in first loop of this function because of problem with the Element

            # this is after we check for the bad stuff so that we make sure not to put those into usedOrBrokenFactSet
            # so that they don't break when processing isUncategorizedFacts
            if fact in duplicateFacts:
                continue

            # first see if fact's value is an embedded command, then check if it's a qlabel fact.
            if not fact.isNumeric:
                if fact.concept.isTextBlock:
                    isEmbeddedCommand = self.checkForEmbeddedCommandAndProcessIt(fact)
                else:
                    isEmbeddedCommand = False

                if not isEmbeddedCommand and re.compile('[a-zA-Z-]+:[a-zA-Z]+').match(fact.value):
                    try:
                        prefix, ignore, localName = fact.value.partition(':')
                        namespaceURI = self.modelXbrl.prefixedNamespaces[prefix]
                        qname = arelle.ModelValue.QName(prefix, namespaceURI, localName)
                        if qname in self.modelXbrl.qnameConcepts:
                            self.factToQlabelDict[fact] = qname
                    except KeyError:
                        pass

            axisMemberLookupDict = {}

            # add period and unit to axisMemberLookupDict
            startEndContext = None
            con = fact.context
            if fact.context is not None:
                if con.instantDatetime is not None: # is an instant
                    startEndTuple = (None, con.instantDatetime)
                else: # is a startEndContext
                    startEndTuple = (con.startDatetime, con.endDatetime)
                try:
                    startEndContext = self.startEndContextDict[startEndTuple]
                except KeyError:
                    startEndContext = StartEndContext(con, startEndTuple)
                    self.startEndContextDict[startEndTuple] = startEndContext
                axisMemberLookupDict['period'] = startEndContext

            if fact.unit is not None:
                axisMemberLookupDict['unit'] = fact.unit.id

            # add each axis to axisMemberLookupDict
            for arelleDimension in fact.context.qnameDims.values():
                dimensionConcept = arelleDimension.dimension
                memberConcept = arelleDimension.member
                if dimensionConcept is None:
                    errStr1 = ErrorMgr.getError('XBRL_DIMENSIONS_INVALID_AXIS_BROKEN').format(fact.context.id, fact.qname)
                    self.addToLog(errStr1, messageCode='warn')

                elif memberConcept is None:
                    errStr2 = ErrorMgr.getError('XBRL_DIMENSIONS_INVALID_AXIS_MEMBER_BROKEN').format(dimensionConcept.qname, fact.qname, fact.context.id)
                    self.addToLog(errStr2, messageCode='warn')

                else:
                    try:
                        axis = self.axisDict[dimensionConcept.qname]
                    except KeyError:
                        axis = Axis(dimensionConcept)
                        for relationship in self.modelXbrl.relationshipSet(arelle.XbrlConst.dimensionDefault).fromModelObject(dimensionConcept):
                            axis.defaultArelleConcept = relationship.toModelObject
                            break
                        self.axisDict[dimensionConcept.qname] = axis
                    if arelleDimension.isExplicit: # if true, Member exists, else None. there's also isTyped, for typed dims.
                        try:
                            member = self.memberDict[memberConcept.qname]
                        except KeyError:
                            member = Member(memberConcept)
                            self.memberDict[memberConcept.qname] = member
                        member.linkAxis(axis)
                        axis.linkMember(member)
                    axisMemberLookupDict[axis.arelleConcept.qname] = member.arelleConcept.qname

                    # while we're at it, do some other stuff
                    for cube in element.inCubes.values():
                        cube.hasAxes[axis.arelleConcept.qname] = axis
                        cube.hasMembers[member.arelleConcept.qname] = member
                        axis.linkCube(cube)

            for cube in element.inCubes.values():
                # the None in the tuple is only to handle periodStartLabels and periodEndLabels later on
                cube.factMemberships += [(fact, axisMemberLookupDict, None)]
                cube.hasElements.add(fact.concept)
                if fact.unit is not None:
                    cube.unitAxis[fact.unit.id] = fact.unit
                if startEndContext is not None:
                    cube.timeAxis.add(startEndContext)

    def checkForEmbeddedCommandAndProcessIt(self, fact):
        # partition('~') on a string breaks up a string into a tuple with before the first ~, the ~, and then after the ~.
        ignore, tilde, rightOfTilde = fact.value.partition('~')
        if tilde == '':
            return False
        leftOfTilde, tilde, ignore = rightOfTilde.partition('~')
        if tilde == '' or leftOfTilde == '':
            return False
        commandText = leftOfTilde

        # we take out the URI first, because it might have double quotes in it and we want to cleanse the rest of the
        # command of double quotes since the separator command wraps the separator character in double quotes.
        commandTextList = commandText.split(maxsplit=1) # this is a list of length 1 or 2.
        linkroleUri = commandTextList.pop(0) # now commandTextList is a list of length 0 or 1
        try:
            cube = self.cubeDict[linkroleUri]
        except KeyError:
            return False # not a valid linkroleUri
        if len(commandTextList) > 0:
            commandTextList = commandTextList[0].replace('"', ' ')
            commandTextList = commandTextList.split()

        outputList = []
        tokenCounter = 1
        while len(commandTextList) > 0:
            listToAddToOutput = []

            token0 = commandTextList.pop(0)
            tokenCounter += 1
            token0Lower = token0.casefold()
            if token0Lower in {'row', 'column'}:
                listToAddToOutput += [token0Lower]
            else:
                errorStr = Utils.printErrorStringToDisambiguateEmbeddedOrNot(fact)
                message = ErrorMgr.getError('EMBEDDED_COMMAND_TOKEN_NOT_ROW_OR_COLUMN_ERROR').format(token0, tokenCounter, errorStr)
                self.addToLog(message, messageCode='error')
                return False

            token1 = commandTextList.pop(0)
            tokenCounter += 1
            token1Lower = token1.casefold()
            if token1Lower in {'period', 'unit', 'primary'}:
                listToAddToOutput += [token1Lower]
            elif '_' in token1:
                listToAddToOutput += [arelle.ModelValue.qname(fact, token1.replace('_',':',1))] # only replace first _, because qnames can have _

            # separator is not supported, we just pop it off and ignore it
            elif token1Lower == 'separator':
                if commandTextList.pop(0).casefold() == 'segment':
                    commandTextList.pop(0)
                    tokenCounter += 1
                tokenCounter += 1
                errorStr = Utils.printErrorStringToDisambiguateEmbeddedOrNot(fact)
                message = ErrorMgr.getError('EMBEDDED_COMMAND_SEPARATOR_USED_WARNING').format(token1, tokenCounter, errorStr)
                self.addToLog(message, messageCode='warn')
                continue

            else:
                errorStr = Utils.printErrorStringToDisambiguateEmbeddedOrNot(fact)
                message = ErrorMgr.getError('EMBEDDED_COMMAND_INVALID_FIRST_TOKEN_ERROR').format(token1, tokenCounter, errorStr)
                self.addToLog(message, messageCode='error')
                return False

            token2 = commandTextList.pop(0)
            tokenCounter += 1
            token2Lower = token2.casefold()
            if token2Lower in {'compact', 'nodisplay'}:
                listToAddToOutput += [token2Lower]
            elif token2Lower == 'grouped':
                listToAddToOutput += ['compact']
                errorStr = Utils.printErrorStringToDisambiguateEmbeddedOrNot(fact)
                message = ErrorMgr.getError('EMBEDDED_COMMAND_GROUPED_USED_WARNING').format(token2, tokenCounter, errorStr)
                self.addToLog(message, messageCode='warn')
            elif token2Lower == 'unitcell':
                listToAddToOutput += ['compact']
                errorStr = Utils.printErrorStringToDisambiguateEmbeddedOrNot(fact)
                message = ErrorMgr.getError('EMBEDDED_COMMAND_UNITCELL_USED_WARNING').format(token2, tokenCounter, errorStr)
                self.addToLog(message, messageCode='warn')
            else:
                errorStr = Utils.printErrorStringToDisambiguateEmbeddedOrNot(fact)
                message = ErrorMgr.getError('EMBEDDED_COMMAND_INVALID_SECOND_TOKEN_ERROR').format(token2, tokenCounter, errorStr)
                self.addToLog(message, messageCode='error')
                return False

            # there could be multiple members, so grab them all here
            tempList = []
            while len(commandTextList) > 0 and commandTextList[0].casefold() not in {'row', 'column'}:
                tempList += [commandTextList.pop(0)]

            for tokenMember in tempList:
                tokenCounter += 1
                if '_' in tokenMember:
                    listToAddToOutput += [arelle.ModelValue.qname(fact, tokenMember.replace('_',':',1))]
                elif tokenMember == '*' and len(tempList) == 1:
                    listToAddToOutput += [tokenMember]
                else:
                    errorStr = Utils.printErrorStringToDisambiguateEmbeddedOrNot(fact)
                    message = ErrorMgr.getError('EMBEDDED_COMMAND_INVALID_MEMBER_NAME_ERROR').format(tokenMember, tokenCounter, errorStr)
                    self.addToLog(message, messageCode='error')
                    return False

            outputList += [listToAddToOutput]

        cube.isEmbedded = True
        self.hasEmbeddings = True

        embedding = Embedding.Embedding(self, cube, outputList, factThatContainsEmbeddedCommand = fact)
        cube.embeddingList += [embedding]
        self.factToEmbeddingDict[fact] = embedding
        return True

    def handleUncategorizedCube(self, xlWriter):
        # get a fresh start, kill all the old data structures that get built in populateAndLinkClasses()
        for element in self.elementDict.values():
            element.__dict__.clear()
            del element
        self.elementDict = {}

        for cube in self.cubeDict.values():
            cube.__dict__.clear()
            del cube
        self.cubeDict = {}

        for member in self.memberDict.values():
            member.__dict__.clear()
            del member
        self.memberDict = {}

        for axis in self.axisDict.values():
            axis.__dict__.clear()
            del axis
        self.axisDict = {}

        for startEndContext in self.startEndContextDict.values():
            startEndContext.__dict__.clear()
            del startEndContext
        self.startEndContextDict = {}

        uncategorizedCube = Cube.Cube(self, 'http://xbrl.sec.gov/role/uncategorizedFacts')
        uncategorizedCube.fileNumber = self.controller.nextUncategorizedFile
        uncategorizedCube.shortName = uncategorizedCube.definitionText = 'Uncategorized Items - '+self.entrypoint
        uncategorizedCube.isElements = True

        # now run populateAndLinkClasses() again and let it re-populate and re-link everything from scratch but let it do so
        # only with filing.unusedFactSet as it's fact set and with only the uncategorizedCube, and no other cubes.
        self.cubeDict[uncategorizedCube.linkroleUri] = uncategorizedCube
        self.populateAndLinkClasses(uncategorizedCube = uncategorizedCube)

        self.mainFunDriverBeforeFlowThroughSuppression(uncategorizedCube)

        embedding = uncategorizedCube.embeddingList[0]

        self.mainFunDriverAfterFlowThroughSuppressionAndReleaseMemory(uncategorizedCube, embedding, xlWriter)

    def mainFunDriverBeforeFlowThroughSuppression(self, cube):
        if cube.isUncategorizedFacts:
            cube.presentationGroup.generateUncategorizedFactsPresentationGroup()
        else:
            if len(cube.hasElements) == 0 or len(cube.factMemberships) == 0:
                cube.noFactsOrAllFactsSuppressed = True
                return
            cube.presentationGroup.startPreorderTraversal()
            if cube.noFactsOrAllFactsSuppressed:
                return
            cube.checkForTransposedUnlabeledAndElements()
            if len(cube.periodStartEndLabelDict) > 0:
                cube.handlePeriodStartEndLabel() # needs preferred labels from the presentationGroup
            cube.DetermineAxesWhereDefaultFilteredOut()

        cube.populateUnitPseudoaxis()
        cube.populatePeriodPseudoaxis()

        if self.controller.debugMode:
            cube.printCube()

        if len(cube.embeddingList) == 0: # add the non-embedded cube to the end, the rest will plug into this one.
            cube.embeddingList = [Embedding.Embedding(self, cube, [])]

        for embedding in cube.embeddingList:
            embedding.generateStandardEmbeddedCommandsFromPresentationGroup()
            if cube.isTransposed:
                embedding.handleTransposedByModifyingCommandText()
            embedding.buildAndProcessCommands()
            if embedding.isEmbeddingOrReportBroken:
                continue

            embedding.processOrFilterFacts()
            if embedding.isEmbeddingOrReportBroken:
                continue

            embedding.possiblyReorderUnitsAfterTheFactAccordingToPresentationGroup()

            if self.controller.debugMode:
                embedding.printEmbedding()

            report = embedding.report = Report.Report(self, cube, embedding)
            report.generateRowsOrCols('col', sorted(embedding.factAxisMemberGroupList, key=lambda thing: thing.axisMemberPositionTupleColList))

            # this is because if the {Elements} view is used, then you might have lots of facts right next to each other with the same qname
            # this is fine, but each time you render, they might appear in a different order.  so this will sort the facts by value
            # so that each run the same facts don't appear in different orders.
            if cube.isElements:
                sortedFAMGL = sorted(embedding.factAxisMemberGroupList, key=lambda thing: (thing.axisMemberPositionTupleRowList, thing.fact.value))
            else:
                sortedFAMGL = sorted(embedding.factAxisMemberGroupList, key=lambda thing: thing.axisMemberPositionTupleRowList)
            report.generateRowsOrCols('row', sortedFAMGL)

            if not cube.isElements:
                report.promoteAxes()

                if report.isUnitsAxisOnRowsOrColumns == 'row':
                    report.mergeRowsOrColsIfUnitsCompatible('row', report.rowList)
                elif report.isUnitsAxisOnRowsOrColumns == 'col':
                    report.mergeRowsOrColsIfUnitsCompatible('col', report.colList)
                if report.isPeriodAxisOnRowsOrColumns == 'row':
                    report.mergeRowsOrColsInstantsIntoDurationsIfUnitsCompatible('row', report.rowList)
                elif report.isPeriodAxisOnRowsOrColumns == 'col':
                    report.mergeRowsOrColsInstantsIntoDurationsIfUnitsCompatible('col', report.colList)
                report.hideRedundantColumns()

    def mainFunDriverAfterFlowThroughSuppressionAndReleaseMemory(self, cube, embedding, xlWriter):
        report = embedding.report

        report.scaleUnitGlobally()

        if      (len(self.factFootnoteDict) > 0 and
                 not {factAxisMemberGroup.fact for factAxisMemberGroup in embedding.factAxisMemberGroupList}.isdisjoint(self.factFootnoteDict)):
            report.handleFootnotes()
            report.setAndMergeFootnoteRowsAndColumns('row', report.rowList)
            report.setAndMergeFootnoteRowsAndColumns('col', report.colList)

        report.removeVerticalInteriorSymbols()

        #if len(embedding.groupedAxisQnameSet) > 0:
        #    report.handleGrouped()

        if cube.isElements or not cube.isEmbedded:
            if      (report.isPeriodAxisOnRowsOrColumns == 'col' or
                     {command.pseudoAxis for command in embedding.rowCommands}.isdisjoint(self.segmentHeadingStopList)):
                report.makeSegmentTitleRows()

            # if report.rowPrimaryPosition != -1, then primary elements aren't on the rows, so no abstracts to add
            if report.rowPrimaryPosition != -1 and not cube.isUnlabeled:
                report.addAbstracts()

        if report.isPeriodAxisOnRowsOrColumns=='row':
            report.HideAdjacentInstantRows()
        elif cube.isStatementOfCashFlows:
            self.RemoveStuntedCashFlowColumns(report,cube)

        report.generateRowAndOrColHeadings()

        report.emitRFile()

        if xlWriter:
            # we pass the cube's shortname since it doesn't have units and stuff tacked onto the end.
            xlWriter.createWorkSheet(cube.fileNumber, cube.shortName)
            xlWriter.buildWorkSheet(report)

        if not cube.isEmbedded:
            reportSummary = ReportSummary()
            report.createReportSummary(reportSummary)
            report.writeHtmlAndOrXmlFiles(reportSummary)
            self.reportSummaryList += [reportSummary]

            # if the cube is embedded, we need to hold on to it, since we might not have embedded it yet.
            # but once we embed it, we garbage collect it in the cell function insertEmbeddingOrBarChartEmbedding()
            Utils.releaseUnneededMemory(embedding)

    def RemoveStuntedCashFlowColumns(self,report,cube):
        visibleColumns = [col for col in report.colList if not col.isHidden]
        didWeHideAnyCols = False
        if len(visibleColumns)>0:
            remainingVisibleColumns = visibleColumns.copy()
            maxMonths = max(col.startEndContext.numMonths for col in visibleColumns)
            minFacts = min(len(col.factList) for col in visibleColumns if col.startEndContext.numMonths==maxMonths)
            minToKeep = math.floor(.25*minFacts)
            for col in visibleColumns:
                if (col.startEndContext.numMonths < maxMonths
                    and len(col.factList) < minToKeep):
                    self.controller.logInfo(("Columns in cash flow ''{}'' have maximum duration {} months and at least {} values. "+
                                  "Shorter duration columns must have at least one fourth ({}) as many values. "+
                                  "Column '{}' is shorter ({} months) and has only {} values, so it is being removed.")
                                  .format(cube.shortName,maxMonths,minFacts,minToKeep,col.startEndContext,col.startEndContext.numMonths,len(col.factList))
                                  )
                    col.isHidden = True
                    didWeHideAnyCols = True
                    remainingVisibleColumns.remove(col)
                    for fact in col.factList:
                        appearsInOtherColumn = False
                        for otherCol in remainingVisibleColumns:
                            if fact in otherCol.factList:
                                appearsInOtherColumn = True
                                break
                        if not appearsInOtherColumn:
                            cube.factMemberships = [fm for fm in cube.factMemberships if not fact is fm[0]] # local removal
                            # Go look whether the fact is now completely uncategorized
                            if fact in self.usedOrBrokenFactSet:
                                element = self.elementDict[fact.elementQname]
                                appearsInOtherCube = False
                                for c in element.inCubes.values():
                                    if hasattr(c,'factMemberships'): # Some Cube objects seem uninitialized, not sure why.
                                        for fm in c.factMemberships:
                                            if fm[0] is fact: # Assumes that factMemberships is an accurate list of facts presented.
                                                appearsInOtherCube = True
                                                break # Only need to find one other place the fact appears
                                        if appearsInOtherCube is True:
                                            break
                                if appearsInOtherCube is False:
                                    # This was the only place the fact was presented, and now it's hidden.
                                    self.usedOrBrokenFactSet.remove(fact)

        if didWeHideAnyCols:
            Utils.hideEmptyRows(report.rowList)

    # this is only for financial reports, so we know that there are no embedded commands.  this means that for each cube,
    # there is only one embedding object, but we have to get that this way: cube.embeddingList[0].  also, always, in general
    # each embedding only ever has one and only one report.  so for financial reports, report is cube.embeddingList[0].report.
    def filterOutColumnsWhereAllElementsAreInOtherReports(self, sortedCubeList):
        # we separate nonStatementElementsAndElementMemberPairs from all of the cubes because we can do this just once and not run through them each time
        statementCubesList = []
        nonStatementElementsAndElementMemberPairs = set()
        for cube in sortedCubeList:
            if not cube.noFactsOrAllFactsSuppressed and len(cube.embeddingList) == 1 and not cube.embeddingList[0].isEmbeddingOrReportBroken:
                if cube.cubeType == 'statement' and not cube.isStatementOfEquity:
                    statementCubesList += [cube]
                else:
                    nonStatementElementsAndElementMemberPairs.update(cube.embeddingList[0].hasElementsAndElementMemberPairs)

        for i, cube in enumerate(statementCubesList):
            # initialize to all the non statement elements and non statement element member pairs
            elementQnamesInOtherReportsAndElementQnameMemberPairs = nonStatementElementsAndElementMemberPairs.copy()
            # now add other statements
            for j, otherStatement in enumerate(statementCubesList):
                if i != j:
                    elementQnamesInOtherReportsAndElementQnameMemberPairs.update(otherStatement.embeddingList[0].hasElementsAndElementMemberPairs)
            # now elementQnamesInOtherReportsAndElementQnameMemberPairs has all the qnames used in every other report except for the statement we're using

            report = cube.embeddingList[0].report # we know there's no embeddings, so the report is on the first and only embedding
            columnsToKill = []
            nonHiddenColCount = 0

            elementQnamesThatWillBeKeptProvidingThatWeHideTheseCols = set() # if we kill a few cols, we will want to update embedding.hasElements
            for col in report.colList:

                if not col.isHidden:
                    nonHiddenColCount += 1
                    setOfElementQnamesInCol = {fact.qname for fact in col.factList}
                    setOfElementQnamesAndQnameMemberPairsForCol = setOfElementQnamesInCol.union(col.elementQnameMemberForColHidingSet)
                    # the operator <= means subset, not a proper subset.  so if all the facts in the column are elsewhere, then hide column.
                    if setOfElementQnamesAndQnameMemberPairsForCol <= elementQnamesInOtherReportsAndElementQnameMemberPairs:
                        columnsToKill += [col]
                    else:
                        elementQnamesThatWillBeKeptProvidingThatWeHideTheseCols.update(setOfElementQnamesInCol)

            if 0 < len(columnsToKill) < nonHiddenColCount:
                cube.embeddingList[0].hasElements = elementQnamesThatWillBeKeptProvidingThatWeHideTheseCols # update hasElements, might have less now
                for col in columnsToKill:
                    col.isHidden = True
                    self.addToLog('{} column {!s} was just killed'.format(cube.linkroleUri, col.index))

                Utils.hideEmptyRows(report.rowList)

    def addToLog(self,message,messageCode='debug',messageArgs=(),file='Filing.py'):
        self.controller.addToLog(message, messageCode=messageCode, messageArgs=messageArgs, file=file)

    def nextFile(self):
        if len(self.reportSummaryList)==0: return 1
        return self.reportSummaryList[-1].fileNumber + 1

class ReportSummary(object):
    def __init__(self):
        self.order = None
        self.isDefault = None
        self.hasEmbeddedReports = None
        self.longName = None
        self.shortName = None
        self.role = None
        self.logList = None
        self.xmlFileName = None
        self.htmlFileName = None
        self.isUncategorized = False

class StartEndContext(object):
    def __init__(self, context, startEndTuple):
        # be really careful when using this context.  many contexts from the instance can share this object,
        # probably not the right one.
        self.context = context
        self.startTime = startEndTuple[0]
        self.endTime = startEndTuple[1]
        self.label = (self.endTime - datetime.timedelta(days=1)).strftime('%b. %d, %Y')

        if self.startTime == None:
            self.periodTypeStr = 'instant'
            self.startTimePretty = None
            self.numMonths = 0
        else:
            self.periodTypeStr = 'duration'
            self.startTimePretty = (self.startTime).strftime('%Y-%m-%dT%H:%M:%S')
            self.numMonths = self.startEndContextInMonths()
        self.endTimePretty = (self.endTime - datetime.timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%S')

    def startEndContextInMonths(self):
        modifiedEndTime = self.endTime + datetime.timedelta(days=15) # we add to it because it rounds down
        delta = dateutil.relativedelta.relativedelta(modifiedEndTime, self.startTime)
        return delta.years * 12 + delta.months

    def startOrInstantTime(self):
        if self.startTime is None:
            return self.endTime
        return self.startTime

    def __str__(self):
        if self.periodTypeStr=='instant':
            return "[{}]".format(self.endTimePretty[:10])
        else:
            return "[{} {}m {}]".format(self.startTimePretty[:10],self.numMonths,self.endTimePretty[:10])

class Axis(object):
    def __init__(self, arelleConcept):
        self.inCubes = {}
        self.hasMembers = {}
        self.arelleConcept = arelleConcept
        self.defaultArelleConcept = None
    def linkCube(self, cubeObj):
        self.inCubes[cubeObj.linkroleUri] = cubeObj
    def linkMember(self, memObj):
        self.hasMembers[memObj.arelleConcept.qname] = memObj
    def __repr__(self):
        return "axis(arelleConcept={}, default={})".format(self.arelleConcept, self.defaultArelleConcept)

class Member(object):
    def __init__(self, arelleConcept):
        self.hasMembers = {}
        self.arelleConcept = arelleConcept
        self.axis = None
        self.parent = None
    def linkAxis(self, axisObj):
        self.axis = axisObj
    def linkParent(self, parentObj):
        self.parent = parentObj

class Element(object):
    def __init__(self, arelleConcept):
        self.inCubes = {}
        self.arelleConcept = arelleConcept
    def linkCube(self, cube):
        self.inCubes[cube.linkroleUri] = cube

#===============================================================================
#     # to do, take out linkrole uri's and prints.  only for testing.
#     def handleFlowThroughSuppression(self, sortedCubeList):
#         reportList = []
#         for cube in sortedCubeList:
#             if not cube.noFactsOrAllFactsSuppressed and not cube.isElements:
#                 for embedding in cube.embeddingList:
#                     if not embedding.isEmbeddingOrReportBroken:
#                         report = embedding.report
#                         reportList += [(report, cube.linkroleUri)]
#
#                         # this is just for speed.  better to convert lists to sets only once, rather than each time.
#                         for row in report.rowList:
#                             row.factSet = set(row.factList)
#                         for col in report.colList:
#                             col.factSet = set(col.factList)
#
#         reversedReportList = reportList.copy()
#         reversedReportList.reverse()
#         for reportToTest, reportToTestLinkroleUri in reportList:
#             if not reportToTest.cube.isStatementOfEquity:
#                 self.testEachRowAndCol(reversedReportList, reportToTest, reportToTestLinkroleUri, reportToTest.rowList, 'row')
#             self.testEachRowAndCol(reversedReportList, reportToTest, reportToTestLinkroleUri, reportToTest.colList, 'column')
#
#             if len(reportToTest.factSetForFlowThroughSuppression) == 0:
#                 message = ErrorMgr.getError('FLOWTHROUGH_SUPPRESSION').format(reportToTest.cube.linkroleUri)
#                 self.addToLog(message, messageCode='Info')
#                 reportToTest.embedding.isEmbeddingOrReportBroken = True
#
#
#     def testEachRowAndCol(self, reversedReportList, reportToTest, reportToTestLinkroleUri, rowOrColList, rowOrColStr):
#         for rowOrColToTest in rowOrColList:
#             if not rowOrColToTest.isHidden:
#                 for reportToTestAgainst, reportToTestAgainstLinkroleUri in reversedReportList:
#
#                     # < tests proper subset
#                     # here we are checking a row or col against an entire report.  If the row or col is not in the entire report, stop
#                     if (reportToTest != reportToTestAgainst and
#                         rowOrColToTest.factSet < reportToTestAgainst.factSetForFlowThroughSuppression):
#
#                         # since rowOrColToTest is contained in the report, we check to see if it is a proper subset of
#                         # any of the rows or cols of the report.  it might not be.  if it is, we stop.
#                         returnValueRows = self.findContainingRowOrCol(rowOrColToTest, reportToTest, reportToTestLinkroleUri, rowOrColStr,
#                                                                       reportToTestAgainstLinkroleUri, 'row', reportToTestAgainst.rowList)
#                         if returnValueRows:
#                             break
#
#                         returnValueCols = self.findContainingRowOrCol(rowOrColToTest, reportToTest, reportToTestLinkroleUri, rowOrColStr,
#                                                                       reportToTestAgainstLinkroleUri, 'col', reportToTestAgainst.colList)
#                         if returnValueCols:
#                             break
#
#
#     def findContainingRowOrCol(self, rowOrColToTest, reportToTest, reportToTestLinkroleUri, rowOrColToTestStr,
#                                reportToTestAgainstLinkroleUri, rowOrColToTestAgainstStr, reportToTestAgainstRowOrColList):
#         for rowOrColToTestAgainst in reportToTestAgainstRowOrColList:
#             if not rowOrColToTestAgainst.isHidden and rowOrColToTest.factSet < rowOrColToTestAgainst.factSet:
#
#                 self.addToLog('{} {} {}'.format(rowOrColToTestStr, rowOrColToTest.index, reportToTestLinkroleUri))
#                 for fact in sorted(rowOrColToTest.factList,        key=lambda fact : fact.value):
#                     if fact.value:
#                         self.addToLog(fact.value)
#
#                 self.addToLog('{} {} {}'.format(rowOrColToTestAgainstStr, rowOrColToTestAgainst.index, reportToTestAgainstLinkroleUri))
#                 for fact in sorted(rowOrColToTestAgainst.factList, key=lambda fact : fact.value):
#                     if fact.value:
#                         self.addToLog(fact.value)
#
#                 self.addToLog('\n\n')
#
#                 rowOrColToTest.isHidden = True
#                 reportToTest.factSetForFlowThroughSuppression -= rowOrColToTest.factSet
#                 reportToTest.logList += ['Process Flow-Through: Removing ' + rowOrColToTestStr + ' ' + ' | '.join(rowOrColToTest.headingList)]
#                 return True
#===============================================================================


import pygame, colour as col, dimensions as dim, buttons as but

screen = pygame.display.set_mode((dim.width, dim.height))
game_start = 0

def text(text, colour, x, y, size):
    font = pygame.font.Font('freesansbold.ttf', size)
    text = font.render(text, True, colour)
    textRect = text.get_rect()
    textRect.center = (x, y)
    screen.blit(text, textRect)

def text_1(text, colour, colour_1, x, y, size, rotate):
    font = pygame.font.Font('freesansbold.ttf', size)
    text = font.render(text, True, colour, colour_1)
    textRect = text.get_rect()
    textRect.center = (x, y)
    text = pygame.transform.rotate(text, rotate)
    screen.blit(text, textRect)

def line(colour, from_x, from_y, to_x, to_y, thick):
    pygame.draw.line(screen, colour, [from_x, from_y], [to_x, to_y], thick)

def rect(colour, x, y, width, height):
    pygame.draw.rect(screen, colour, [x, y, width, height])

def pic(location, w, h, x, y):
    image = pygame.image.load(location)
    image = pygame.transform.scale(image, (w, h))
    rect = image.get_rect()
    rect = rect.move((x, y))
    screen.blit(image, rect)

def shop_map():
    #Map
    screen.fill(col.BROWN)
    pic('risk.jpg', dim.width - dim.shop_width - 20, dim.height - 20, 10, 10)
    rect(col.BLACK, dim.width - dim.shop_width, 0, dim.shop_width, dim.height)
    rect(col.DARK_BROWN, dim.width - 240, 10, 230, dim.height - 20)

    #Totals
    rect(col.BLACK, dim.shop_column_1 - 27, 18, 204, 54)
    rect(col.LIGHT_GRAY, dim.shop_column_1 - 25, 20, dim.shop_box_w * 4, dim.shop_box_h)

    #Food Per Turn
    rect(col.BLACK, dim.shop_column_1 - 17, 78, 84, 54)
    rect(col.YELLOW, dim.shop_column_1 - 15, 80, dim.per_turn_w, dim.per_turn_h)

    #Wood Per Turn
    rect(col.BLACK, dim.shop_column_2 - 17, 78, 84, 54)
    rect(col.BROWN, dim.shop_column_2 - 15, 80, dim.per_turn_w, dim.per_turn_h)

    #Steel Per Turn
    rect(col.BLACK, dim.shop_column_1 - 17, 138, 84, 54)
    rect(col.STEEL, dim.shop_column_1 - 15, 140, dim.per_turn_w, dim.per_turn_h)

    #Food Per Turn
    rect(col.BLACK, dim.shop_column_2 - 17, 138, 84, 54)
    rect(col.OIL, dim.shop_column_2 - 15, 140, dim.per_turn_w, dim.per_turn_h)

    #Turn Marker
    rect(col.BLACK, dim.shop_column_1 - 27, 218, 104, 54)
    rect(col.LIGHT_GRAY, dim.shop_column_1 - 25, 220, dim.shop_box_w * 2, dim.shop_box_h)
    text('Turn:', col.BLACK, dim.shop_column_1 + 25, 245, 16)
    rect(col.BLACK, dim.shop_column_2 - 2, 218, 54, 54)
    rect(col.WHITE, dim.shop_column_2, 220, dim.shop_box_w, dim.shop_box_h)

    #On Deck Marker
    rect(col.BLACK, dim.shop_column_1 - 27, 288, 104, 54)
    rect(col.LIGHT_GRAY, dim.shop_column_1 - 25, 290, dim.shop_box_w * 2, dim.shop_box_h)
    text('Place Unit:', col.BLACK, dim.shop_column_1 + 25, 315, 16)
    rect(col.BLACK, dim.shop_column_2 - 2, 288, 54, 54)
    rect(col.BROWN, dim.shop_column_2, 290, dim.shop_box_w, dim.shop_box_h)

    line(col.BLACK, dim.width - 240, 365, dim.width, 365, 10)

    #Shop Markers
    rect(col.BLACK, dim.shop_column_1 - 2, dim.shop_row_5 - 2, 54, 54)
    rect(col.BLACK, dim.shop_column_1 - 2, dim.shop_row_4 - 2, 54, 54)
    rect(col.BLACK, dim.shop_column_1 - 2, dim.shop_row_3 - 2, 54, 54)
    rect(col.BLACK, dim.shop_column_1 - 2, dim.shop_row_2 - 2, 54, 54)
    rect(col.BLACK, dim.shop_column_1 - 2, dim.shop_row_1 - 2, 54, 54)
    rect(col.BLACK, dim.shop_column_2 - 2, dim.shop_row_5 - 2, 54, 54)
    rect(col.BLACK, dim.shop_column_2 - 2, dim.shop_row_4 - 2, 54, 54)
    rect(col.BLACK, dim.shop_column_2 - 2, dim.shop_row_3 - 2, 54, 54)
    rect(col.BLACK, dim.shop_column_2 - 2, dim.shop_row_2 - 2, 54, 54)
    rect(col.BLACK, dim.shop_column_2 - 2, dim.shop_row_1 - 2, 54, 54)
    rect(col.WHITE, dim.shop_column_1, dim.shop_row_5, dim.shop_box_w, dim.shop_box_h)
    rect(col.WHITE, dim.shop_column_1, dim.shop_row_4, dim.shop_box_w, dim.shop_box_h)
    rect(col.WHITE, dim.shop_column_1, dim.shop_row_3, dim.shop_box_w, dim.shop_box_h)
    rect(col.WHITE, dim.shop_column_1, dim.shop_row_2, dim.shop_box_w, dim.shop_box_h)
    rect(col.WHITE, dim.shop_column_1, dim.shop_row_1, dim.shop_box_w, dim.shop_box_h)
    rect(col.WHITE, dim.shop_column_2, dim.shop_row_5, dim.shop_box_w, dim.shop_box_h)
    rect(col.WHITE, dim.shop_column_2, dim.shop_row_4, dim.shop_box_w, dim.shop_box_h)
    rect(col.WHITE, dim.shop_column_2, dim.shop_row_3, dim.shop_box_w, dim.shop_box_h)
    rect(col.WHITE, dim.shop_column_2, dim.shop_row_2, dim.shop_box_w, dim.shop_box_h)
    rect(col.WHITE, dim.shop_column_2, dim.shop_row_1, dim.shop_box_w, dim.shop_box_h)

    #Name Countries + Stats
    text_1('Madagascar', col.BLACK, None, 803, 631, 15, 60)
    text_1('South', col.BLACK, None, 670, 652, 15, 0)
    text_1('Africa', col.BLACK, None, 670, 667, 15, 0)
    text_1('Congo', col.BLACK, None, 662, 564, 15, 0)
    text_1('East', col.BLACK, None, 715, 517, 15, 0)
    text_1('Africa', col.BLACK, None, 740, 532, 15, 0)
    text_1('Africa', col.BLACK, None, 605, 480, 16, 0)
    text_1('North', col.BLACK, None, 547, 480, 16, 0)
    text_1('Egypt', col.BLACK, None, 665, 440, 15, 0)
    text_1('Middle East', col.BLACK, None, 773, 376, 15, 0)
    text_1('India', col.BLACK, None, 904, 389, 15, 0)
    text_1('Siam', col.BLACK, None, 1002, 424, 15, 0)
    text_1('China', col.BLACK, None, 979, 333, 15, 0)
    text_1('Mongolia', col.BLACK, None, 1015, 260, 15, 0)
    text_1('Japan', col.BLACK, None, 1145, 270, 15, 0)
    text_1('Afghanistan', col.BLACK, None, 843, 276, 15, 0)
    text_1('Ural', col.BLACK, None, 855, 181, 15, 0)
    text_1('Siberia', col.BLACK, None, 925, 121, 15, 0)
    text_1('Yakutsk', col.BLACK, None, 1025, 92, 15, 0)
    text_1('Irkutsk', col.BLACK, None, 1013, 185, 15, 0)
    text_1('Kamchatka', col.BLACK, None, 1125, 93, 15, 0)
    text_1('Ukraine', col.BLACK, None, 734, 188, 16, 0)
    text_1('Scandinavia', col.BLACK, None, 632, 112, 15, 45)
    text_1('Northern', col.BLACK, None, 617, 250, 15, 0)
    text_1('Europe', col.BLACK, None, 618, 263, 15, 0)
    text_1('Southern', col.BLACK, None, 620, 310, 15, 0)
    text_1('Europe', col.BLACK, None, 640, 324, 15, 0)
    text_1('Iceland', col.BLACK, None, 527, 150, 15, 0)
    text_1('Great Britian', col.BLACK, None, 493, 239, 15, 0)
    text_1('Western Europe', col.BLACK, None, 548, 292, 15, 55)
    text_1('Greenland', col.BLACK, None, 425, 76, 15, 0)
    text_1('Quebec', col.BLACK, None, 352, 194, 15, 0)
    text_1('Ontario', col.BLACK, None, 262, 195, 15, 0)
    text_1('Alberta', col.BLACK, None, 184, 180, 15, 0)
    text_1('Northwest Territoy', col.BLACK, None, 200, 117, 15, 0)
    text_1('Alaska', col.BLACK, None, 80, 120, 15, 0)
    text_1('Western', col.BLACK, None, 190, 245, 15, 0)
    text_1('United States', col.BLACK, None, 194, 262, 15, 0)
    text_1('Eastern', col.BLACK, None, 270, 286, 15, 0)
    text_1('United States', col.BLACK, None, 270, 298, 15, 0)
    text_1('Argentina', col.BLACK, None, 325, 605, 15, 0)
    text_1('Peru', col.BLACK, None, 313, 533, 15, 0)
    text_1('Brazil', col.BLACK, None, 371, 492, 16, 0)
    text_1('Venezuela', col.BLACK, None, 286, 426, 15, 0)
    text_1('Eastern', col.BLACK, None, 1159, 625, 15, 0)
    text_1('Australia', col.BLACK, None, 1158, 637, 15, 0)
    text_1('Western', col.BLACK, None, 1063, 655, 15, 0)
    text_1('Australia', col.BLACK, None, 1088, 669, 15, 0)
    text_1('New', col.BLACK, None, 1130, 512, 15, 0)
    text_1('Guinea', col.BLACK, None, 1138, 525, 15, 0)
    text_1('Ind', col.BLACK, None, 998, 548, 16, 15)
    text_1('one', col.BLACK, None, 1024, 548, 16, 0)
    text_1('sia', col.BLACK, None, 1049, 548, 16, -15)
    text_1('Central', col.BLACK, None, 194, 314, 15, -72)
    text_1('America', col.BLACK, None, 220, 368, 15, -18)

    pic('warrior.png', 50, 50, dim.shop_column_1, dim.shop_row_1)
    pic('horse.png', 50, 50, dim.shop_column_2, dim.shop_row_1)
    pic('big_boat.png', 50, 50, dim.shop_column_2, dim.shop_row_3)
    pic('plane.png', 50, 50, dim.shop_column_1, dim.shop_row_4)
    pic('nuke.png', 50, 50, dim.shop_column_2, dim.shop_row_5)
    pic('barrier.png', 50, 50, dim.shop_column_1, dim.shop_row_5)
    pic('tank.png', 50, 50, dim.shop_column_2, dim.shop_row_4)
    pic('int boat.png', 50, 50, dim.shop_column_1, dim.shop_row_3)
    pic('small_boat.png', 50, 50, dim.shop_column_2, dim.shop_row_2)
    pic('bus.png', 50, 50, dim.shop_column_1, dim.shop_row_2)

def start_screen():
    rect(col.DARK_BROWN, dim.map_center - 300, 250, 600, 250)
    rect(col.RED, dim.map_center - 280, 270, 560, 210)
    rect(col.DARK_BROWN, dim.map_center - 275, 275, 550, 200)
    text('Fishing For Bass', col.BLACK, dim.map_center, dim.height / 2, 50)

def start_menu():
    rect(col.BLACK, dim.map_center + 175, 435, 80, 30)
    rect(col.RED, dim.map_center + 178, 438, 74, 24)
    text('Click to Start', col.BLACK, dim.map_center + 215, 450, 10)

def end_menu():
    rect(col.BLACK, 18, 698, 84, 34)
    rect(col.RED, 20, 700, 80, 30)
    text('End Turn', col.BLACK, 60, 715, 15)

def start_game(x, y, left):
    global game_start
    if game_start > 0:
        end_menu()
    elif but.start_button(x, y, left) > 0:
        game_start = 1
    else:
        start_screen()
        start_menu()

def turn_stats(x, y, left, turn):
    text('{}'.format(turn), col.BLACK, dim.shop_column_2 + 25, 245, 30)
    but.end_button(x, y, left)


# Train a tensorflow model on the training data

import os

import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
from tensorflow import keras
import cv2

CHARLIST = [
    '00', '01', '02', '03', '04', '05', '06', '07', '08', '09',
    '10', '11', '12', '13', '14', '15', '16', '17', '18', '19',
    '20', '21', '22', '23', '24', '25', '26', '27', '28', '29'
]

training_images = []
training_answer = []
with open("Training_sample/data.txt") as f:
    for line in f:
        path, value = line.strip().split(" ")

        # Get the number in the image
        value = int(value)
        value = "{:02d}".format(value)

        # Get the image data (20x20)
        image_raw = tf.io.read_file(path)
        image_tf  = tf.image.decode_image(image_raw, dtype=tf.dtypes.uint8)

        training_images.append(image_tf)
        training_answer.append(CHARLIST.index(value))

training_answer = np.asarray(training_answer)
training_images = np.asarray(training_images)

# Scale the images from 0 -> 255 to 0 -> 1
training_images = training_images / 255.0

# The above reads images as (X, Y, COLORS). Even when B/W, the COLORS are a
# list of one. Flatten that out
training_images = np.squeeze(training_images)

# Split the training set into two, for training and testing
n_images = len(training_answer)
sl = int(n_images/2)

train_data = training_images[sl:, ...]
train_keys = training_answer[sl:, ...]

test_data = training_images[:sl, ...]
test_keys = training_answer[:sl, ...]

# Plot a sample of the images
plt.figure(figsize=(4,4))
for i in range(10):
    plt.subplot(2,5,i+1)
    plt.xticks([])
    plt.yticks([])
    plt.grid(False)
    plt.imshow(train_data[i], cmap=plt.cm.binary)
    plt.xlabel(CHARLIST[train_keys[i]])
plt.show()

# Build a model
input_shape = train_data[0].shape
middle_layer = 128
output_shape = len(CHARLIST)
print("Model will have the layers:")
print("{} --> {} --> {}".format(input_shape, middle_layer, output_shape))

model = keras.Sequential([
    keras.layers.Flatten(input_shape=input_shape),
    keras.layers.Dense(middle_layer, activation=tf.nn.relu),
    keras.layers.Dense(output_shape, activation=tf.nn.softmax)
])

model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

# Train the model on the data
model.fit(
    train_data,
    train_keys,
    epochs=100,
    verbose=0
)

print("\n\nTESTING ON SECOND DATASET:")
# Let's test it.
test_loss, test_acc = model.evaluate(test_data, test_keys)
print('Test accuracy:', test_acc)
if test_acc != 1.0:
    print("Get more training data!")
    exit()

# The URL to this camera
# http://www.insecam.org/en/view/805350/
URL = "http://210.148.5.180:8084/?action=stream"

# Open a connection to the weird camera
cap = cv2.VideoCapture(URL)

# I need a permanent dir to hold training data for OCR
if not os.path.isdir("Training_sample"):
    os.mkdir("Training_sample")

margin = 180

for _ in range(50):
    ret, image = cap.read()

    # Massage into something the NN can parse
    image = image[margin:-margin, margin:-margin]
    frame = cv2.resize(image, (20,20))
    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    _, frame = cv2.threshold(frame, 127, 255, cv2.THRESH_BINARY)

    img_tf = np.array([frame])
    val = model.predict(img_tf)
    index = np.argmax(val)

    # Get the answer
    num = CHARLIST[index]
    print("I think this image is {}".format(num))

    # show to the user
    cv2.imshow('Grabbed this', image)
    cv2.waitKey(1000)


import requests
from queue import Queue
import random

class LongPoll:
    def __init__(self,basicToken,groupId,debug=False,verbouse=False,network="requests"):
        self.basicAddress = 'https://api.vk.com/method/'
        self.longpollApi = 'https://api.vk.com/method/groups.getLongPollServer'
        self.messageApi = 'https://api.vk.com/method/messages.send'
        self.participiantApi = 'messages.getConversationMembers'
        self.usersApi = 'users.get'
        self.groupId = groupId
        self.basicToken = basicToken
        self.LongpollServer = None
        self.longPollToken = None
        self.debug = debug
        self.verbouse = verbouse
        self.ts = None
        self.waitDefalt = 25
        self.initialized = False
        self.longpoolRes = Queue()
        self.longpoolResMax = 10
    def updateInner(self):
        for _ in range (0,4):
            if (self.initialized != True) or (self.needToReconnect == True):
                self.getLongPoll()
            else:
                return
        raise Exception("Attempt to init failed!")
    def getLongPoll(self):
        try:
            print(self.basicToken)
            answer = requests.post(\
                self.longpollApi,\
                data = {'group_id':self.groupId ,\
                        'access_token':self.basicToken,\
                        'v':'5.110'})
        except requests.ConnectionError:
            if (self.debug == True):
                print("Connection Error" + "\n")
            return
        if (answer.status_code == 200):
            if (self.debug == True):
                print("Longpoll: 200, answer:" + answer.text + "\n")
            ansJson = answer.json()
            self.LongpollServer = ansJson["response"]["server"]
            self.longPollToken = ansJson["response"]["key"]
            self.logPollToken = ansJson["response"]["ts"]
            self.initialized = True
            self.needToReconnect = False
        else:
            if (self.debug == True):
                print("Longpoll status: " + str(anwer.status_code) + "\n" )
            raise Exception("Status code is incorrect ")
    def longPoll(self):
        if (self.longpoolRes.qsize() < self.longpoolResMax):
            res = requests.post(self.LongpollServer, data = {\
                    'act':'a_check','key':self.longPollToken,\
                    'ts':self.ts,'wait':self.waitDefalt})
            if (res.status_code == 200):
                failedSafe = False
                json = res.json()
                try:
                    self.ts = json['ts']
                except:
                    if self.debug:
                        print(json)
                    failedSafe = True
                try:
                    if (json['failed'] == 1):
                        failedSafe = True
                    if (json['failed'] == 2):
                        self.needToReconnect = True
                except:
                    pass
                if (failedSafe == False):
                    self.longpoolRes.put(json['updates'])
            else:
                raise Exception("Status code is incorrect")
    def sendMessage(self,dialog,text):
        resMsgRes = requests.post(self.messageApi, data = {\
            'peer_id':dialog, 'group_id':self.groupId, 'message':text,\
            'random_id':random.randint(1, 1000), 'access_token':self.basicToken, 'v':'5.123'})
        if self.debug == True:
            print(resMsgRes.text)
    def getMembers(self,dialog):
        resMembers = requests.post(self.basicAddress + self.participiantApi, data = {\
            'peer_id':dialog, 'group_id':self.groupId, 'access_token':self.basicToken, 'v':'5.123'})
        if self.debug == True:
            print(resMembers.text)
        json = resMembers.json()
        return json
    def getUserName(self,userId):
        resUser = requests.post(self.basicAddress + self.usersApi, data = {\
            'user_ids':userId, 'name_case':"Nom", 'access_token':self.basicToken, 'v':'5.120'})
        if self.debug == True:
            print(resUser.text)
        body = resUser.json()["response"][0]
        return body["first_name"] , body["last_name"]


load("//aspects:utils/utils.bzl", "create_struct", "file_location")

def extract_python_info(target, ctx):
    if PyInfo not in target:
        return None

    if PyRuntimeInfo in target:
        provider = target[PyRuntimeInfo]
    else:
        provider = None

    return create_struct(
        interpreter = file_location(getattr(provider, "interpreter", None)),
        version = getattr(provider, "python_version", None),
    )


from copy import copy

from legoc.parser.types.pexp.pexpression import PExpression
from legoc.parser.types.poperation.pbinoperation import PBinOperation
from legoc.parser.types.ptype.ptempltype import PTemplType

class PTBrackets(PExpression):
    def __init__(self, child=None):
        super(PTBrackets, self).__init__(child)
        self.tstack.append(PTBrackets.__name__)

    def get(self):
        if not isinstance(self.child, list):
            return self.child
        print('Множественный дочерний элеемент\n'
              'Возможно нужен get_list')
        return self

    def get_list(self):
        if isinstance(self.child, list):
            return self.child
        else:
            return [self.child]

    def left_reduce(self, tkn):
        if 'PSimpleType' in tkn.tstack:
            pttype = PTemplType(tkn)
            pttype.args = self.child if isinstance(self.child, list) \
                else [self.child]
            return pttype

        elif 'POperation' in tkn.get().tstack and 'PSimpleType' in tkn.get().right.tstack:
            pttype = PTemplType(copy(tkn.get().right))
            pttype.args = self.child if isinstance(self.child, list) \
                else [self.child]
            tkn.get().right = pttype
            return tkn

        return None


#!/usr/bin/python3
# -*- coding:utf-8 -*-

"""
@author:sunzhihua
@filename:demo2.py
@time:2021/4/18 16:36
@descirpt:async_demo2
"""

import asyncio

async def other():
    print("start")
    await asyncio.sleep(2)
    print("end")
    return "返回值"

async def func():
    print("执行协程函数异步代码")
    response1 = await other()
    print("IO请求结束,结果为:", response1)
    response2 = await other()
    print("IO请求结束,结果为:", response2)

asyncio.run(func())


import logging
from contextlib import contextmanager
from typing import ContextManager

from requests import Session
from torpy.http.requests import tor_requests_session

from twitscrape.settings import settings

logger = logging.getLogger(__name__)
logger.setLevel(settings.log_level)
authorization = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0",
    "content-type": "application/json",
    "authorization": authorization,
}

@contextmanager
def twitter_session() -> ContextManager[Session]:
    """Context manager providing an authenticated twitter session over a tor circuit"""
    while True:
        with tor_requests_session(headers=headers) as s:
            r = s.post("https://api.twitter.com/1.1/guest/activate.json")
            if r.status_code == 429:
                logger.debug("Client rate limited, re-circuiting...")
            elif r.status_code != 200:
                raise ValueError(f"Failed to get guest token: {r!r}")
            token = r.json()["guest_token"]
            s.headers.update({"x-guest-token": token})
            logger.debug("Obtained token")
            yield s
            return


import sqlite3
from threading import Lock

class Database:
    """Simple database wrapper to retrieve packages and their corresponding categories"""
    def __init__(self, file):
        self.con = sqlite3.connect(file, check_same_thread=False)
        self.c = self.con.cursor()
        self.lock = Lock()

    #region Getting

    def get_package_id(self, package):
        """Returns the package ID if package is found, None otherwise"""
        result = self.c.execute('select id from package where package=?', (package,)).fetchone()
        return result[0] if result else None

    def get_categories(self, package):
        """Returns a list containing strings representing the categories to which this package belongs to"""
        package_id = self.get_package_id(package)
        if package_id is not None:
            self.c.execute('select category_id from package_category where package_id=?',
                           (package_id,))

            return [             # Return a list made up from the categories text
                self.c.execute("""
                select category -- Select the category text
                from category   -- From the category table
                where id=?      -- Where the ID matches the returned many-to-many IDs
                """, (i[0],))    # With the first ID from the resulting ID tuple
                .fetchone()[0]   # (Fetch the first element, the category name, taking it out its tuple)
                for i in self.c.fetchall()  # Which IDs are returned in a list of tuples
            ]
        else:
            # Return an empty list if there is no package
            return []

    #endregion

    #region Others

    def close(self):
        """Closes the database"""
        self.con.close()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    #endregion


import os
import re
import math
import time
import numpy as np
import snowballstemmer as stem
from scipy.sparse import csr_matrix, linalg
import sklearn.preprocessing

def gen_words(text, stemming=stem.stemmer('english')):
    """Create generator.

    :param text: some string
    :param stemming: variant of stemming algorithm
    :return: generator giving stemmed words from text
    """
    for word in stemming.stemWords(re.findall(r"[\w']+", text.lower())):
        yield word

def parse_file(file_path, bow, stemming):
    """Create bow set by parsing file.

    :param file_path: path to file
    :param bow: bag_of_words set
    :param stemming: variant of stemming algorithm
    :return: bag_of_words set for parsed file
    """
    terms = {}
    with open(file_path, 'r', encoding='utf-8') as file:
        for line in file.readlines():
            for word in gen_words(line.lower(), stemming):
                if word not in bow:
                    bow[word] = len(bow)

                if bow[word] in terms:
                    terms[bow[word]] += 1
                else:
                    terms[bow[word]] = 1

    return terms

def idf_matrix(matrix, bow):
    """Use inverse document frequency to reduce meaning of frequent words.

    :param bow: bag_of_words set
    :param matrix: term_by_document matrix
    :return: none
    """
    for word in bow:
        docs = sum(1 for doc in matrix if bow[word] in matrix[doc])
        idf = math.log10(len(matrix) / docs)
        for doc in matrix:
            if bow[word] in matrix[doc]:
                matrix[doc][bow[word]] *= idf
                if not matrix[doc][bow[word]]:
                    del matrix[doc][bow[word]]

def create_matrix(dir_path, svd=False, k=30, idf=True):
    """Create bag_of_words and term_by_document matrix.

    :param idf: whether to apply idf for each word
    :param k: parameter for svd
    :param svd: whether to apply low rank approximation
    :param dir_path: path to directory with documents
    :returns: bag_of_words, documents names, term_by_document matrix
    """
    bow = {}
    stemming = stem.stemmer('english')
    docs_terms = {name: parse_file(dir_path + "/" + name, bow, stemming)
                  for name in os.listdir(dir_path)}

    if idf:
        idf_matrix(docs_terms, bow)

    rows = []
    cols = []
    data = []
    docs = list(docs_terms.keys())
    for i, doc in enumerate(docs):
        col = docs_terms[doc].keys()
        data.extend([docs_terms[doc][x] for x in col])
        cols.extend(col)
        rows.extend([i] * len(col))

    matrix = csr_matrix((data, (rows, cols)),
                        shape=(len(docs_terms), len(bow)), dtype=float)

    if svd:
        u, s, v = linalg.svds(matrix, k)
        matrix = csr_matrix(u.dot(np.diag(s)).dot(v))

    sklearn.preprocessing.normalize(matrix, norm='l1', axis=1, copy=False)

    return bow, docs, matrix

def create_vec(text, bow):
    """Change user query into bag_of_words vector.

    :param text: user query
    :param bow: bag_of_words set
    :return: normalised bag_of_words vector
    """
    vec = np.zeros(shape=len(bow))
    for word in gen_words(text.lower()):
        if word in bow:
            vec[bow[word]] += 1
    return vec / np.linalg.norm(vec)

def fix_arr(max_values, indexes, n):
    """Sort in reverse order max_values.

    :param max_values: arr of max_values
    :param indexes: arr of indexes
    :param n: len of indexes/max_values
    :return: none
    """
    i = n - 1
    while i > 0 and max_values[i] > max_values[i - 1]:
        max_values[i], max_values[i - 1] = max_values[i - 1], max_values[i]
        indexes[i], indexes[i - 1] = indexes[i - 1], indexes[i]
        i -= 1

def query(matrix, vec, n=1):
    """Find n documents with highest correlation to query.

    :param matrix: term_by_document matrix
    :param vec: query from user
    :param n: number of documents to find
    :return: n max files indexes with highest correlation
    """
    arr = matrix.dot(vec)
    max_values = np.zeros(shape=n)
    indexes = np.zeros(shape=n, dtype=int)

    for i, elem in enumerate(arr):
        if elem > max_values[-1]:
            max_values[-1] = elem
            indexes[-1] = i
            fix_arr(max_values, indexes, n)
    return indexes

def display_results(docs, indexes):
    """Display results of query.

    :param docs: arr of names of files
    :param indexes: indexes of docs with highest correlation
    :return: none
    """
    print("\nTop %d results are:" % len(indexes))
    for index in indexes:
        print(docs[index])

def main(n=10, k=800, svd=True, idf=True, path='wiki'):

    bow, docs, matrix = create_matrix(path, svd, k, idf)

    text = input('\nEnter sentence: ')
    while text != 'quit':
        start_time = time.time()
        vec = create_vec(text, bow)
        display_results(docs, query(matrix, vec, n))
        end_time = time.time()
        print("time: " + str(end_time - start_time))
        text = input('\nEnter sentence: ')

if __name__ == "__main__":
    main()


# importing the module
import os
import time
#importing pyfiglet module for fonts
import pyfiglet

os.system("clear")

# sets the text colour to green
os.system("tput setaf 2")
print("\t\t\t\t\t\tLaunching Terminal User Interface")

# sets the text color to white
os.system("tput setaf 7")

print("\t\t\t\t\t-------------------------------------------------")
print("\t\t\t\t\t\t\tHOLD ON!!!!!!!!!\n\t\t\t\t\t\tYou are entering to the linux world")
print("\t\t\t\t\t-------------------------------------------------")
print("\n")
while True:
          os.system("tput setaf 3")
          result = pyfiglet.figlet_format("\t\t\t\tWelcome to ", font = "bulbhead" )
          result1 = pyfiglet.figlet_format("\t\t\t\t  the World of", font = "bulbhead" )
          os.system("tput setaf 1")
          result2 = pyfiglet.figlet_format("\t\t\t\t   REDHAT LINUX", font = "bulbhead" )
          print(result)
          print(result1)
          print(result2)
          os.system("tput setaf 4")
          print('\t\t\t1.Print date {:<25} 11.Create a folder'.format(''))
          print('\t\t\t2.Print cal {:<25}  12.To see all the folders '.format(''))
          print('\t\t\t3.Configure yum {:<21}  13.To see yum is configured or not '.format(''))
          print('\t\t\t4.To check the status of webserver {:<2}  14.Print Working Directory '.format(''))
          print('\t\t\t5.Install & start docker {:<12}  15.Reference manual pages for commands/programs '.format(''))
          print('\t\t\t6.To check the status of docker {:<5}  16.display the disk space used in the file system '.format(''))
          print('\t\t\t7.To stop docker {:<20}  17.Show previously used commands '.format(''))
          print('\t\t\t8.Add user {:<26}  18.Who you are logged in as '.format(''))
          print('\t\t\t9.Delete user {:<23}  19.Display free and used memory '.format(''))
          print('\t\t\t10.Create a file {:<20}  20.Display CPU information '.format(''))
          print('\t\t\t21.To create a webserver {:<13} 22.To Create Partition'.format(''))
          print('\t\t\t23.To mount a file {:<19} 24.To Create physical volume(pv)'.format(''))
          print('\t\t\t25.To create volume group {:<12} 26.To Create logical volume'.format(''))
          print('\t\t\t27.Exit ')

          os.system("tput setaf 5")
          ch = int(input("Enter your choice:  \t"))
          if (ch == 1):
              os.system("tput setaf 6")
              print("today's date is \n")
              os.system("date")
          elif ch == 2:
              os.system("tput setaf 6")
              print("Here is your calender")
              os.system("cal")
          elif ch == 3:
              os.system("yum install httpd -y")
              print(os.system("systemctl start httpd"))
          elif ch == 4:
              os.system("systemctl status httpd")
          elif ch == 5:
              print("installing docker............")
              print("starting docker...................")
             # os.system("yum install docker-ce -y")
              os.system("systemctl start docker")
          elif ch == 6:
              print(os.system("systemctl status docker"))
          elif ch == 7:
              print("stopping docker.................")
              os.system("systemctl stop docker")
              print(os.system("systemctl status docker"))
          elif ch == 8:
              os.system("tput setaf 6")
              new_user=input("Enter the name of new user: ")
              os.system("useradd {}".format(new_user))
              os.system("id -u {}".format(new_user))
          elif ch == 9:
              del_user=input("Enter the name of the user to delete: ")
              os.system("userdel {}".format(del_user))
          elif ch == 10:
              filename=input("Enter the filename: ")
              f=os.system("touch {}".format(filename))
              if f!=0:
                  print("Some error occurred")
              else:
                  print("File created successfully")
          elif ch == 11:
              os.system("tput setaf 6")
              foldername=input("Enter the foldername: ")
              f=os.system("mkdir {}".format(foldername))
              if f!=0:
                  print("tput setaf 1")
                  print("Some error occurred")
              else:
                  print("Folder created successfully")
          elif ch == 12:
              os.system("tput setaf 6")
              os.system("ls -a")
          elif ch == 13:
              os.system("tput setaf 6")
              os.system("yum repolist")
          elif ch == 14:
              os.system("tput setaf 6")
              os.system("pwd")
          elif ch == 15:
              os.system("tput setaf 6")
              command_name = input("Enter the command whose manual you want to see:")
              os.system("man {}".format(command_name))
          elif ch == 16:
              print("disk space...........")
              os.system("tput setaf 6")
              os.system("df")
          elif ch == 17:
              print("history...........")
              os.system("tput setaf 6")
              print(os.system("history"))
          elif ch == 18:
              os.system("tput setaf 6")
              os.system("whoami")
          elif ch == 19:
              os.system("tput setaf 6")
              os.system("free -h")
          elif ch == 20:
              os.system("tput setaf 6")
              os.system("cat /proc/cpuinfo")
          elif ch == 21:
              os.system("tput setaf 6")
              print("Stoping firewall............")
              os.system("systemctl stop firewalld")
              print("Starting the apache services...........")
              os.system("systemctl start httpd")
              print("STATUS OF THE APACHE WEBSERVER")
              s= os.system("systemctl status httpd")
              print(s)
              os.system("tput setaf 2")
              print("\nCopy your index.html files in /var/www/html/ folder ")
              f = input("Enter your webfile name:")
              os.system("cp {} /var/www/html".format(f) )
              time.sleep(2)
              print("file copied......................")
              print("*********IP ADDRESS*********")
              os.system("ifconfig enp0s3")
              print("Enter the ip in webbrowser for visiting the webpage")
              time.sleep(4)
          elif ch == 22:
              os.system("tput setaf 6")
              disk=input("enter your disk name ")
              os.system("sudo fdisk /dev/{}".format(disk))
          elif ch == 23:
              os.system("tput setaf 6")
              initdisk=input("Enter the destination where to mount ")
              finaldisk=input("Enter the disk path ")
              os.system("sudo mount -t ntfs {}  {}".format(initdisk, finaldisk))
              time.sleep(4)
          elif ch == 24:
              os.system("tput setaf 6")
              initdisk=input("Enter the destination where to mount ")
              finaldisk=input("Enter the disk path ")
              os.system("sudo pvcreate {}  {}".format(initdisk, finaldisk))
              time.sleep(4)
          elif ch == 25:
              os.system("tput setaf 6")
              grname=input("Enter your group name ")
              initdisk=input("Enter the destination where to mount ")
              finaldisk=input("Enter the disk path ")
              os.system("sudo vgcreate {}  {} {}".format(grname,initdisk, finaldisk))
              time.sleep(4)
          elif ch == 26:
              os.system("tput setaf 6")
              l= os.system("sudo lvdisplay")
              print(l)
              time.sleep(4)
          elif ch == 27:
              os.system("tput setaf 6")
              print("\n\t\t\t\t\t\tSEE YOU SOON!!\n")
              os.system("tput setaf 7")
              exit()
          else:
              print("Invalid entry")
          os.system("tput setaf 9")
          input("\n\nPress enter to continue")
          os.system("clear")