from __future__ import absolute_import
from builtins import object
from builtins import range
from builtins import object

import json
import logging
import lxml.etree as et
import re
import sys
from gettext import gettext as _

import splunk.util
from splunk.models.view_escaping import forminput
from splunk.models.view_escaping.base import POST_SEARCH_MODE, SAVED_SEARCH_MODE, TEMPLATE_SEARCH_MODE
from splunk.models.view_escaping.base import DEFAULT_SEARCH_ID
from splunk.models.view_escaping.dashboard import SimpleDashboard
from splunk.models.view_escaping.drilldown import parseDrilldown
from splunk.models.view_escaping.drilldown import parseDrilldownAction
from splunk.models.view_escaping.panel import Panel
# I am leaving the * because refactoring may have un foreseen issues. the code uses `globals().items()` :(
from splunk.models.view_escaping.panelElement import *
from splunk.models.view_escaping.panelElement import normalizeBoolean, Event, BasePanel, Chart
from splunk.models.view_escaping.row import Row
from splunk.models.view_escaping.search import Search, createSearchFromSearchXml
from splunk.models.view_escaping.tokendeps import parseTokenDeps
from splunk.models.view_escaping.validation_helper import WarningsCollector

logger = logging.getLogger('splunk.models.view_escaping')

ID_PATTERN = re.compile(r'^[a-z]\w*$', re.IGNORECASE)

def getValidationMessages(dashboardNode, viewName=None, digest=None, sourceApp=None):
    logger.info('Validating dashboard XML for view=%s', viewName)
    collector = WarningsCollector(logger=logger)
    try:
        with collector:
            createDashboardFromXml(dashboardNode, viewName, digest, sourceApp)
    except:
        pass
    return collector.getMessages()

def createDashboardFromXml(dashboardNode, viewName=None, digest=None, sourceApp=None):
    """
    @param dashboardNode: lxml node representing a dashboard or form
    @param viewName: the name of the xml file
    @param digest: boolean
    @return:
    """

    logger.debug("Parsing view: simplexml=%s" % viewName)
    dashboard = SimpleDashboard(viewName=viewName, digest=digest, sourceApp=sourceApp)

    # Parse global options
    dashboard.label = dashboardNode.findtext('./label')
    dashboard.description = dashboardNode.findtext('./description')

    dashboard.customScript = dashboardNode.get('script', None)
    dashboard.customStylesheet = dashboardNode.get('stylesheet', None)
    dashboard.onUnloadCancelJobs = normalizeBoolean(
        dashboardNode.get('onunloadCancelJobs', dashboard.onUnloadCancelJobs))

    dashboard.matchTagName = dashboardNode.tag
    if dashboard.matchTagName not in ['form', 'dashboard']:
        raise Exception(_('Invalid root element'))

    if dashboardNode.tag == 'form':
        # update the fieldset
        fieldsetNode = dashboardNode.find('./fieldset')
        if fieldsetNode is not None:
            dashboard.autoRun = splunk.util.normalizeBoolean(
                fieldsetNode.get('autoRun', False))
            dashboard.submitButton = splunk.util.normalizeBoolean(
                fieldsetNode.get('submitButton', True))

            for item in fieldsetNode:
                logger.debug("Found new element in fieldset: form=%s item=%s" % (viewName, item.tag))
                if item.tag == 'html':
                    panelInstance = createPanelElementFromXml(item)
                    dashboard.fieldset.append(panelInstance)
                elif item.tag == 'input':
                    inputDefaults = dict()
                    if not dashboard.submitButton:
                        inputDefaults['searchWhenChanged'] = True
                    inputInstance = forminput.createInput(item.get('type'), inputDefaults)
                    inputInstance.fromXml(item, sourceApp)

                    dashboard.fieldset.append(inputInstance)

    # set core view attributes
    for k in dashboard.standardAttributeMap:
        v = dashboardNode.get(k)
        if v is not None:
            if k in dashboard.booleanAttributeKeys:
                v = splunk.util.normalizeBoolean(v)
            elif k in dashboard.integerAttributeKeys:
                try:
                    v = int(v)
                except:
                    msg = "Dashboard attribute %s should have an integer value. Value found: %s" % (k, v)
                    logger.error(msg)
                    raise Exception(msg)
            setattr(dashboard, k, v)

    # Legacy searchTemplate extraction
    search = createSearchFromXml(dashboardNode, id=DEFAULT_SEARCH_ID)
    if search:
        dashboard.searches.append(search)
    for searchNode in dashboardNode.findall('search'):
        search = createSearchFromSearchXml(searchNode)
        if search:
            logger.debug("Appending new search to view=%s" % viewName)
            dashboard.searches.append(search)

    for rowNode in dashboardNode.findall('row'):
        logger.debug("Appending new row to view=%s" % viewName)
        dashboard.rows.append(createRowFromXml(rowNode, sourceApp))

    normalizeIdentifiers(dashboard)

    def prepareEventSearch(searchObj):
        if searchObj is not None:
            if searchObj.searchMode == POST_SEARCH_MODE:
                prepareEventSearch(dashboard.get_search(searchObj.baseSearchId))
            else:
                searchObj.statusBuckets = 300

    for row in dashboard.rows:
        for panel in row.panels:
            for panelElement in panel.panelElements:
                if isinstance(panelElement, Event):
                    prepareEventSearch(panelElement.search)

    return dashboard

def verify_id(id, seen, type):
    if id is not None:
        if id in seen:
            raise Exception(_('Duplicate ID "%s" of %s' % (id, type)))
        if not ID_PATTERN.match(id):
            raise Exception(_(
                'ID "%s" for %s does not match pattern %s ' +
                '(ID has to begin with a letter and must not '
                'container characters other than letters, numbers and "_")') % (id, type, ID_PATTERN.pattern))
        seen.add(id)

class IdSeq(object):
    def __init__(self):
        self.val = 1

    def __next__(self):
        while True:
            yield self.val
            self.val += 1

def auto_id(pattern, seen, seq):
    id = None
    for seq_no in next(seq):
        id = pattern % seq_no
        if id not in seen:
            break
    seen.add(id)
    return id

def normalizeIdentifiers(dashboard):
    """
    Ensure all dashboard elements have an unique id

    @param dashboard: Dashboard object
    @return: no return value
    """
    seen = set()
    for row in dashboard.rows:
        verify_id(row.id, seen, "row")
    for panel in dashboard.all_panels():
        verify_id(panel.id, seen, "panel")
    for el in dashboard.all_elements():
        verify_id(el.id, seen, "panel element")
    for el in dashboard.all_fields():
        verify_id(el.id, seen, "input field")
    for (search, obj) in dashboard.all_searches():
        verify_id(search.id, seen, "search")

    globalSearch = None

    search_seq = IdSeq()
    row_seq = IdSeq()
    panel_seq = IdSeq()
    element_seq = IdSeq()
    input_seq = IdSeq()

    for (search, obj) in dashboard.all_searches():
        if search.id is None:
            search.id = globalSearch = auto_id("search%d", seen, search_seq)

    for row in dashboard.rows:
        if row.id is None:
            row.id = auto_id("row%d", seen, row_seq)
            row.idGenerated = True
    for panel in dashboard.all_panels():
        if panel.id is None:
            panel.id = auto_id("panel%d", seen, panel_seq)
            panel.idGenerated = True
    for el in dashboard.all_elements():
        if el.id is None:
            el.id = auto_id("element%d", seen, element_seq)
        if el.context is None:
            if getattr(el, 'searchMode', None):
                el.context = auto_id("search%d", seen, search_seq)
            elif globalSearch is not None:
                el.context = globalSearch
    for field in dashboard.all_fields():
        if field.id is None:
            field.id = auto_id("input%d", seen, input_seq)
        if getattr(field, 'context', None) is None:
            if getattr(field, 'search', None) or getattr(field, 'savedSearch', None):
                field.context = auto_id("search%d", seen, search_seq)

def createSearchFromXml(searchNode, id=None):
    """
    Parses a search from search, dashboard, panel element xml nodes
    @param searchNode: Lxml representing a form or dashboard element
    @param id: and optional id to force id to
    @return:
    """

    # define search mode XML node -> object property mappings
    searchModeNodeMap = [
        ('searchString', TEMPLATE_SEARCH_MODE),
        ('searchName', SAVED_SEARCH_MODE),
        ('searchTemplate', TEMPLATE_SEARCH_MODE),
        ('searchPostProcess', POST_SEARCH_MODE)
    ]

    opt = dict()
    for pair in searchModeNodeMap:
        if searchNode.find(pair[0]) is not None:
            opt['searchMode'] = pair[1]
            opt['searchCommand'] = (
                searchNode.findtext(pair[0])).replace("\n", " ").replace("\t", " ")
            break
    else:
        return False
    for node in ['earliestTime', 'latestTime']:
        nodeVal = searchNode.findtext(node)
        if nodeVal:
            opt[node] = nodeVal
    tokenDeps = parseTokenDeps(searchNode)
    if id:
        opt['id'] = id
    if tokenDeps:
        opt['tokenDeps'] = tokenDeps
    return Search(**opt)

def createRowFromXml(rowNode, sourceApp=None):
    """
    Parses a row xml node
    @param rowNode: Lxml representing a form or dashboard element
    @return:
    """
    logger.debug("parsing dashboard row node")
    if rowNode.get('grouping'):
        rowGroupings = list(map(
            int,
            rowNode.get('grouping').replace(' ', '').strip(',').split(',')))
        logger.debug("Found row grouping=%s" % rowGroupings)
    else:
        rowGroupings = None

    row = Row(rowGroupings)
    row.tokenDeps = parseTokenDeps(rowNode)
    row.id = rowNode.get('id')

    if len(rowNode) is 0:
        logger.warn('Dashboard row is empty (line %d)', rowNode.sourceline)

    else:
        hasPanels = False
        hasVisualizations = False
        for panelElementNode in rowNode:
            if panelElementNode.tag == "panel":
                if hasVisualizations:
                    raise Exception(_('Row, on line=%s, should not combine visualizations and panels. Panel, on line=%s') % (rowNode.sourceline, panelElementNode.sourceline))
                hasPanels = True
                if rowGroupings is not None:
                    raise Exception(_('Row on line=%s specifies row grouping but has <panel> children, which is not allowed') % rowNode.sourceline)
                row.panels.append(createPanelFromXML(panelElementNode, sourceApp))
            elif panelElementNode.tag == et.Comment:
                continue
            else:
                if hasPanels:
                    raise Exception(_('Row, on line=%s, should not combine visualizations and panels. Visualization, on line=%s') % (rowNode.sourceline, panelElementNode.sourceline))
                hasVisualizations = True
                try:
                    panelElement = createPanelElementFromXml(panelElementNode)
                    if panelElement:
                        row.appendPanelElement(panelElement)
                except NotImplementedError:
                    raise Exception(_('Row, on line=%s, contains unknown node=%s on line=%s.') % (rowNode.sourceline, panelElementNode.tag, panelElementNode.sourceline))
    return row

def createPanelFromXML(panelNode, sourceApp=None):
    panel = Panel(None)
    panel.tokenDeps = parseTokenDeps(panelNode)
    panel.id = panelNode.get('id', None)

    if len(panelNode) is 0:
        logger.warn('Dashboard panel is empty (line %d)', panelNode.sourceline)

    ref = panelNode.get('ref', None)
    if ref:
        panel.ref = ref
        panel.app = panelNode.get('app', sourceApp)
    else:
        panel.title = panelNode.findtext('title')
        for panelElementNode in panelNode:
            if panelElementNode.tag == 'input':
                inputInstance = forminput.createInput(
                    panelElementNode.get('type'), dict(searchWhenChanged=True))
                inputInstance.fromXml(panelElementNode, sourceApp)
                panel.fieldset.append(inputInstance)
            elif panelElementNode.tag == 'search':
                search = createSearchFromSearchXml(panelElementNode)
                if search:
                    panel.searches.append(search)
            elif panelElementNode.tag not in ('title', 'description'):
                panelElement = createPanelElementFromXml(panelElementNode)
                if panelElement:
                    panel.appendPanelElement(panelElement)
    return panel

def extractHtmlContent(node):
    if normalizeBoolean(node.attrib.get("encoded", False)):
        return node.text
    else:
        if len(node) == 0 and node.text is None:
            return ''
        else:
            parser = et.XMLParser(resolve_entities=False)
            clone = et.XML(et.tostring(node).strip(), parser)
            # Strip attributes on outer <html> node, so we can safely strip it once serialized
            clone.attrib.clear()
            html = et.tostring(clone)
            if sys.version_info > (3, 0) and isinstance(html, bytes):
                html = html.decode()
            if html.startswith('<html>') and html.endswith('</html>'):
                return html[6:-7].strip()
            else:
                return html

def _isPrimarySearchNode(node):
    type = node.get('type')
    return type == 'primary' or type is None

def _findPrimarySearchNode(panelElementNode):
    searchNodes = panelElementNode.findall('search')
    return next((sn for sn in searchNodes if _isPrimarySearchNode(sn)), None)

def createPanelElementFromXml(panelElementNode):
    def createHtmlPanelElementFromXml(panelElementNode, panelInstance):
        """
        Override default parser and just get the text
        """
        logger.debug("parsing html panel element")
        panelInstance.options['useTokens'] = normalizeBoolean(panelElementNode.attrib.get("tokens", True))

        src = panelElementNode.get('src')
        if src:
            panelInstance.options['serverSideInclude'] = src
            return

        panelInstance.options['rawcontent'] = extractHtmlContent(panelElementNode)
        panelInstance.tokenDeps = parseTokenDeps(panelElementNode)

    def createChartPanelElementFromXml(node, element):
        createDefaultPanelElementFromXml(node, element)
        selectionNode = node.find('selection')
        if selectionNode:
            element.selection = []
            for actionNode in [node for node in selectionNode if et.iselement(node) and isinstance(node.tag, str)]:
                action = parseDrilldownAction(actionNode)
                if action:
                    element.selection.append(action)

    def createTablePanelElementFromXml(panelElementNode, panelInstance):
        """
        Add a format tag to provide cell formatting
        (primarily for sparklines at the current time)
        Each format tag may have some option sub-tags
        Each option may contain list or option tags generating
        lists or dictionaries respectively

        Each format tag may specify a field to filter on
        (defaults to all fields) and a type to apply (eg 'sparkline')
        """
        logger.debug("parsing table panel element")

        def parseOption(node):
            """Extract nested options/lists"""
            listNodes = node.findall('list')
            optionNodes = node.findall('option')

            if listNodes and optionNodes:
                raise ValueError("Tag cannot contain both list and option subtags")
            elif listNodes:
                result = []
                for listNode in listNodes:
                    result.append(parseOption(listNode))
                return result
            elif optionNodes:
                result = {}
                for optionNode in optionNodes:
                    result[optionNode.get('name')] = parseOption(optionNode)
                return result
            else:
                return node.text

        createDefaultPanelElementFromXml(panelElementNode, panelInstance)
        fieldFormats = {}
        for formatNode in panelElementNode.findall('format'):
            logger.debug("Parsing format view node")
            field = formatNode.get('field', '*')
            formatType = formatNode.get('type', 'text')
            options = {}
            for optionNode in formatNode.findall('option'):
                options[optionNode.get('name')] = parseOption(optionNode)
            fieldFormats.setdefault(field, []).append({
                'type': formatType,
                'options': options
            })
        panelInstance.fieldFormats = fieldFormats

    def createVizPanelElementFromXml(panelElementNode, panelInstance):
        createDefaultPanelElementFromXml(panelElementNode, panelInstance)
        panelInstance.type = panelElementNode.get('type', None)

    def createDefaultPanelElementFromXml(panelElementNode, panelInstance):
        logger.debug("Parsing default panel element options")
        title = panelElementNode.findtext('title')
        if title is not None:
            setattr(panelInstance, 'title', title)

        optionTypeMap = {}
        if hasattr(panelInstance.__class__, 'optionTypeMap'):
            optionTypeMap = getattr(panelInstance.__class__, 'optionTypeMap')

        # option params get their own container
        for node in panelElementNode.findall('option'):
            optionName = node.get('name')
            optionValue = node.text
            if optionName in optionTypeMap:
                optionValue = optionTypeMap[optionName](optionValue)
            if isinstance(optionValue, str) and optionValue != None:
                optionValue = optionValue.strip()
            if optionName not in ['id', 'el', 'managerid', 'tokenDependencies', 'resizable']:
                panelInstance.options[optionName] = optionValue

        # handle different search modes
        if getattr(panelInstance.__class__, 'hasSearch'):
            # SPL-143552: make sure the primary search is found. Other types of search nodes will be ignored for now.
            primarySearchNode = _findPrimarySearchNode(panelElementNode)
            search = createSearchFromSearchXml(primarySearchNode) if primarySearchNode is not None else None
            if search:
                panelInstance.search = search
            else:
                search = createSearchFromXml(panelElementNode)
                if search:
                    panelInstance.search = search
                else:
                    # create a default search if there is not one
                    panelInstance.search = Search(searchMode=POST_SEARCH_MODE)

        # handle field lists
        if panelElementNode.find('fields') is not None:
            fields = panelElementNode.findtext('fields').strip()
            if len(fields) and fields[0] == '[' and fields[-1] == ']':
                panelInstance.searchFieldList = json.loads(fields)
            else:
                panelInstance.searchFieldList = splunk.util.stringToFieldList(fields)

        # extract simple XML drilldown params
        panelInstance.simpleDrilldown = parseDrilldown(panelElementNode.find('drilldown'))

        panelInstance.tokenDeps = parseTokenDeps(panelElementNode)

        # extract the contents of all top-level comment nodes
        for node in panelElementNode.xpath('./comment()'):
            panelInstance.comments.append(node.text)

        # extract the comments from inside the drilldown tag
        for node in panelElementNode.xpath('./drilldown/comment()'):
            panelInstance.drilldownComments.append(node.text)

    def createPanel(name):
        """
        Factory method for creating an appropriate panel object based upon the
        name.  Returns an instance of a BasePanel subclass, or throws a
        NotImplementedError if no suitable mapper is found.

        This method works by inspecting all objects that subclass BasePanel and
        attempting to match their matchTagName class attribute.
        """

        if not name:
            raise ValueError('Cannot create panel from nothing')

        for obj in globals().values():
            try:
                if issubclass(obj, BasePanel) and name == obj.matchTagName:
                    #  only Chart objects need to be instantiated
                    if obj is Chart:
                        return Chart()
                    else:
                        return obj()
            except:
                pass
        raise NotImplementedError(
            _('Cannot find object mapper for panel type: %s') % name)

    if not isinstance(panelElementNode.tag, splunk.util.string_type):
        return False
    panelType = panelElementNode.tag
    panelInstance = createPanel(panelType)

    id = panelElementNode.attrib.get('id')
    if id is not None:
        panelInstance.id = id

    logger.debug("found panel element type=%s" % panelType)
    if panelType == 'table':
        createTablePanelElementFromXml(panelElementNode, panelInstance)
    elif panelType == 'chart':
        createChartPanelElementFromXml(panelElementNode, panelInstance)
    elif panelType == 'html':
        createHtmlPanelElementFromXml(
            panelElementNode, panelInstance)
    elif panelType == 'viz':
        createVizPanelElementFromXml(panelElementNode, panelInstance)
    else:
        createDefaultPanelElementFromXml(panelElementNode, panelInstance)
    return panelInstance

if __name__ == '__main__':
    import unittest

    def getRowXml(args='', panels=1):
        nodes = ['<row %(args)s>']
        for i in range(0, panels):
            nodes.append('<single></single>')
        nodes.append('</row>')
        xml = ''.join(nodes)
        return xml % {'args': args}

    def getPanelElementXml(type="foo", options=None, args=None):
        options = options or (
            '<searchString> | metadata type="sources" | '
            'stats count</searchString>')
        args = args or ''
        xml = '''
            <%(type)s %(args)s>
                %(options)s
            </%(type)s>
        '''
        return xml % {'type': type, 'options': options, 'args': args}

    class CreatePanelElementTests(unittest.TestCase):

        def createPanel(self, type="foo", options=None, args=None):
            xml = getPanelElementXml(type, options, args)
            root = et.fromstring(xml)
            return createPanelElementFromXml(root)

        def testCreateUnknownPanel(self):
            with self.assertRaises(NotImplementedError):
                d = self.createPanel('foo')

        def testCreateAllowedPanels(self):
            for panelType in ['single', 'chart', 'table',
                              'html', 'map', 'event', 'list', 'viz']:
                d = self.createPanel(panelType)
                self.assertTrue(d.matchTagName == panelType)

        def testCreateHTMLPanel(self):
            d = self.createPanel(
                'html', args='src="/foo/bar"')
            self.assertTrue(d.options.get('serverSideInclude') == '/foo/bar')
            self.assertFalse(d.options.get('rawcontent'))
            d = self.createPanel(
                'html', options='<div>Test</div>', args='src="/foo/bar"')
            self.assertTrue(d.options.get('serverSideInclude') == '/foo/bar')
            self.assertFalse(d.options.get('rawcontent'))
            d = self.createPanel(
                'html', options='<div>Test</div>')
            self.assertFalse(d.options.get('serverSideInclude'))
            self.assertEqual(d.options.get('rawcontent'), '<div>Test</div>')

        def testCreateTablePanel(self):
            """tables only have special format fields"""
            d = self.createPanel('table', options='''
                <format>
                    <option name="fff">
                        <list>foo</list>
                        <list>bop</list>
                    </option>
                    <option name="bippity">
                        <option name="bippity">bop</option>
                    </option>
                </format>
                <format field="foobar" type="num">
                    <option name="fff">
                        <list>foo</list>
                        <list>bop</list>
                    </option>
                    <option>bar</option>
                </format>
                ''')
            self.assertTrue(d.fieldFormats == { # pylint: disable=E1103
                '*': [{'type': 'text',
                       'options': {'bippity': {'bippity': 'bop'},
                                   'fff': ['foo', 'bop']
                       }
                      }],
                'foobar': [{'type': 'num',
                            'options': {None: 'bar',
                                        'fff': ['foo', 'bop']
                            }
                           }]
            })
            #  formats can't mix options and lists.
            #  lists can contain options and options can contain list but
            #     neither can contain both.
            with self.assertRaises(ValueError):
                d = self.createPanel(
                    'table',
                    options='''
                        <format>
                            <option name="fff">
                                <list>foo</list>
                                <list>bop</list>
                                <option name="should">not work</option>
                            </option>
                            <option>bar</option>
                        </format>
                    ''')

        def testCreateChartPanelWithSelection(self):
            chart = self.createPanel('chart', options='''
                <selection>
                    <set token="foo">$start$</set>
                    <unset token="bar" />
                </selection>
                ''')

            self.assertIsNotNone(chart.selection)
            self.assertEqual(2, len(chart.selection))
            self.assertEqual("settoken", chart.selection[0].type)
            self.assertEqual("unsettoken", chart.selection[1].type)

        def testCreateCustomVizPanel(self):
            customViz = self.createPanel('viz', args='type="testapp.testviz"')
            self.assertTrue(customViz.type == 'testapp.testviz')

        def testCreateDefaultPanel(self):
            """All panels have some things in common"""
            #s test common nodes
            d = self.createPanel('single', options='''
                <title>Title1</title>
                <searchString>search string</searchString>
                <earliestTime>0</earliestTime>
                <latestTime>50</latestTime>
                <fields>foo bar baz</fields>
                ''')
            self.assertEqual(d.search.earliestTime, '0')
            self.assertEqual(d.search.latestTime, '50')
            self.assertEqual(d.title, 'Title1')
            self.assertEqual(d.searchFieldList, 'foo bar baz'.split())

            # should be able to accept one of the search modes
            d = self.createPanel('single', options='''
                <searchString>search string</searchString>
                ''')
            self.assertTrue(hasattr(d, 'search'),  'panel elements should have a search object')
            self.assertEqual(getattr(d.search, 'searchMode'), 'template')
            self.assertEqual(getattr(d.search, 'searchCommand'), 'search string')
            d = self.createPanel('single', options='''
                <searchName>search saved</searchName>
                <searchTemplate>search template</searchTemplate>
                <searchPostProcess>search postsearch</searchPostProcess>
                ''')
            self.assertEqual(getattr(d.search, 'searchMode'), 'saved')
            self.assertEqual(getattr(d.search, 'searchCommand'), 'search saved')
            d = self.createPanel('single', options='''
                <searchTemplate>search template</searchTemplate>
                ''')
            self.assertEqual(getattr(d.search, 'searchMode'), 'template')
            self.assertEqual(getattr(d.search, 'searchCommand'), 'search template')
            d = self.createPanel('single', options='''
                <searchPostProcess>search postsearch</searchPostProcess>
                ''')
            self.assertEqual(getattr(d.search, 'searchMode'), 'postsearch')
            self.assertEqual(getattr(d.search, 'searchCommand'), 'search postsearch')
            self.assertEqual(getattr(d.search, 'baseSearchId'), 'global')

            # Options should not override internal settings
            d = self.createPanel('chart', options='''
                <option name="id">xyz</option>
                <option name="managerid">abc</option>
                <option name="el">abc</option>
                <option name="tokenDependencies">abc</option>
                <option name="resizable">False</option>
                ''')

            for option in d.options:
                if option in ['id', 'el', 'managerid', 'tokenDependencies', 'resizable']:
                    self.assertTrue(False, 'Options should not override internal settings')

        def testCreateSavedSearch(self):
            """All panels have some things in common"""
            #s test common nodes
            d = self.createPanel('single', options='''
                <title>Title1</title>
                <search ref="search"/>
                ''')
            self.assertEqual(d.search.searchCommand, 'search')

        def testTokenDependencies(self):
            for panelType in ('table', 'chart', 'single', 'map', 'list', 'html'):
                panel = createPanelElementFromXml(et.fromstring('''
                    <%(type)s depends="$foo$">
                    </%(type)s>
                ''' % dict(type=panelType)))
                self.assertIsNotNone(panel)
                self.assertIsNotNone(panel.tokenDeps)
                self.assertEquals(panel.tokenDeps.depends, '$foo$')
                self.assertEquals(panel.tokenDeps.rejects, '')

                panel = createPanelElementFromXml(et.fromstring('''
                    <%(type)s rejects="$foo$">
                    </%(type)s>
                ''' % dict(type=panelType)))
                self.assertIsNotNone(panel)
                self.assertIsNotNone(panel.tokenDeps)
                self.assertEquals(panel.tokenDeps.rejects, '$foo$')
                self.assertEquals(panel.tokenDeps.depends, '')

                panel = createPanelElementFromXml(et.fromstring('''
                    <%(type)s id="foo" depends="$foo$" rejects="$bar$">
                    </%(type)s>
                ''' % dict(type=panelType)))
                self.assertIsNotNone(panel)
                self.assertIsNotNone(panel.tokenDeps)
                self.assertEquals(panel.tokenDeps.depends, '$foo$')
                self.assertEquals(panel.tokenDeps.rejects, '$bar$')

                panel = createPanelElementFromXml(et.fromstring('''
                    <%(type)s>
                    </%(type)s>
                ''' % dict(type=panelType)))
                self.assertIsNotNone(panel)
                self.assertIsNone(panel.tokenDeps)

                panel = createPanelElementFromXml(et.fromstring('''
                    <%(type)s depends="" rejects="">
                    </%(type)s>
                ''' % dict(type=panelType)))
                self.assertIsNotNone(panel)
                self.assertIsNone(panel.tokenDeps)

                dashboard = createDashboardFromXml(et.fromstring('''
                    <form>

                        <fieldset>
                            <input token="foobar" rejects="$foobar$" />
                        </fieldset>

                        <row depends="$foobar$">
                            <panel rejects="$foobar$">
                                <chart depends="$x$" rejects="$y$">

                                </chart>
                            </panel>
                        </row>
                    </form>
                '''))

                self.assertIsNotNone(dashboard.rows[0].tokenDeps)
                self.assertEquals(dashboard.rows[0].tokenDeps.depends, "$foobar$")
                self.assertEquals(dashboard.rows[0].tokenDeps.rejects, "")
                self.assertIsNotNone(dashboard.rows[0].panels[0].tokenDeps)
                self.assertEquals(dashboard.rows[0].panels[0].tokenDeps.depends, "")
                self.assertEquals(dashboard.rows[0].panels[0].tokenDeps.rejects, "$foobar$")
                self.assertIsNotNone(dashboard.rows[0].panels[0].tokenDeps)
                self.assertEquals(dashboard.rows[0].panels[0].tokenDeps.depends, "")
                self.assertEquals(dashboard.rows[0].panels[0].tokenDeps.rejects, "$foobar$")
                self.assertIsNotNone(dashboard.rows[0].panels[0].panelElements[0].tokenDeps)
                self.assertEquals(dashboard.rows[0].panels[0].panelElements[0].tokenDeps.depends, "$x$")
                self.assertEquals(dashboard.rows[0].panels[0].panelElements[0].tokenDeps.rejects, "$y$")
                self.assertIsNotNone(dashboard.fieldset[0].tokenDeps)
                self.assertEquals(dashboard.fieldset[0].tokenDeps.depends, "")
                self.assertEquals(dashboard.fieldset[0].tokenDeps.rejects, "$foobar$")
                dashboard = createDashboardFromXml(et.fromstring('''
                    <form>

                        <fieldset>
                            <input token="multiselect" type="multiselect" rejects="$foobar$">
                                <search ref="foo"/>
                            </input>
                        </fieldset>

                        <row depends="$foobar$">
                            <panel rejects="$foobar$">
                                <chart depends="$x$" rejects="$y$">

                                </chart>
                            </panel>
                        </row>
                    </form>
                '''))
                self.assertEquals(dashboard.fieldset[0].search.searchCommand, "foo")

        def testSimpleDrilldownPopulated(self):
            for panelType in ('table', 'chart', 'single', 'map', 'list'):
                xmlNode = et.fromstring('''
                    <%(type)s id="panel1">
                        <title>Panel 1</title>
                        <drilldown>
                            <set token="foobar">($click.value$)</set>
                        </drilldown>
                    </%(type)s>
                ''' % dict(type=panelType))
                panel = createPanelElementFromXml(xmlNode)
                self.assertIsNotNone(panel)
                self.assertEquals(len(panel.simpleDrilldown), 1)

    class _findPrimarySearchNodeTests(unittest.TestCase):
        def testAnnotationBeforePrimary(self):
            chart = et.fromstring('''
              <chart>
                <title>test_chart_annotation_pdf_printing</title>
                <search type="annotation">
                  <query>index=_internal (log_level="WARN" OR log_level="ERROR" OR log_level="INFO") | eval annotation_label = "Category is" | eval annotation_category = log_level | table _time annotation_label annotation_category</query>
                  <earliest>-1m@m</earliest>
                  <latest>now</latest>
                </search>
                <search>
                  <query>index=_internal | timechart count</query>
                  <earliest>-1m@m</earliest>
                  <latest>now</latest>
                </search>
                <option name="charting.annotation.categoryColors">{"ERROR":"0xFF0000","WARN":"0x0000FF", "INFO": "0x008000"}</option>
                <option name="charting.chart">area</option>
              </chart>
            ''')
            primary = _findPrimarySearchNode(chart)
            self.assertIsNone(primary.get('type'), 'primary search type')
            self.assertEqual(primary.findtext('query'), 'index=_internal | timechart count', 'primary search type')

        def test_isPrimarySearchNode(self):
            primaryNode = et.fromstring('''
                <search type="primary">
                  <query>index=_internal | timechart count</query>
                  <earliest>-1m@m</earliest>
                  <latest>now</latest>
                </search>
            ''')
            self.assertTrue(_isPrimarySearchNode(primaryNode))

            defaultNode = et.fromstring('''
                <search>
                  <query>index=_internal | timechart count</query>
                  <earliest>-1m@m</earliest>
                  <latest>now</latest>
                </search>
            ''')
            self.assertTrue(_isPrimarySearchNode(defaultNode))

            annotationNode = et.fromstring('''
                <search type="annotation">
                  <query>index=_internal (log_level="WARN" OR log_level="ERROR" OR log_level="INFO") | eval annotation_label = "Category is" | eval annotation_category = log_level | table _time annotation_label annotation_category</query>
                  <earliest>-1m@m</earliest>
                  <latest>now</latest>
                </search>
            ''')
            self.assertFalse(_isPrimarySearchNode(annotationNode))

    class CreatePanelTests(unittest.TestCase):
        def getPanel(self, xml='<panel></panel>'):
            root = et.fromstring(xml)
            return createPanelFromXML(root)

        def testEmptyPanel(self):
            self.assertTrue(isinstance(self.getPanel(), Panel), '')

        def testPanelId(self):
            panel = self.getPanel(xml='<panel id="foo"></panel>')
            self.assertTrue(isinstance(panel, Panel), 'Panel should exist')
            self.assertEqual(panel.id, "foo", 'Panel should have id=foo')

        def testPanelMultipleSearches(self):
            """Extract new search format and multiple searches"""
            #s test common nodes
            d = self.getPanel(xml='''
            <panel>
                <search id="panel_search">
                    <query>search string also</query>
                    <earliest>-15m</earliest>
                    <latest>now</latest>
                </search>
                <search base="panel_search">
                    <query>search postsearch</query>
                </search>
            </panel>
            ''')
            self.assertEqual(len(d.searches), 2)
            self.assertEqual(getattr(d.searches[0], 'searchMode'), 'template')
            self.assertEqual(getattr(d.searches[0], 'searchCommand'), 'search string also')
            self.assertEqual(getattr(d.searches[0], 'id'), 'panel_search')
            self.assertEqual(getattr(d.searches[0], 'latestTime'), 'now')
            self.assertEqual(getattr(d.searches[0], 'earliestTime'), '-15m')
            self.assertEqual(getattr(d.searches[1], 'searchMode'), 'postsearch')
            self.assertEqual(getattr(d.searches[1], 'searchCommand'), 'search postsearch')
            self.assertEqual(getattr(d.searches[1], 'baseSearchId'), 'panel_search')

        def testPanelSavedSearch(self):
            """Extract new search format saved search"""
            d = self.getPanel(xml='''
            <panel>
                <search ref="panel_search"/>
            </panel>
            ''')
            self.assertEqual(len(d.searches), 1)
            self.assertEqual(getattr(d.searches[0], 'searchMode'), 'saved')
            self.assertEqual(getattr(d.searches[0], 'searchCommand'), 'panel_search')

    class CreateRowTests(unittest.TestCase):
        def getRowLxml(self, args='', panels=1):
            xml = getRowXml(args, panels)
            root = et.fromstring(xml)
            return createRowFromXml(root)

        def testCreateRow(self):
            d = self.getRowLxml()
            self.assertTrue(d)
            self.assertEqual(len(d.panels), 1)
            self.assertEqual(
                d.panels[0].panelElements[0].matchTagName, 'single')

        def testRowGrouping(self):
            d = self.getRowLxml(args='grouping="2,1"', panels=3)
            self.assertTrue(d)
            self.assertEqual(len(d.panels), 2)
            self.assertEqual(len(d.panels[0].panelElements), 2)
            self.assertEqual(len(d.panels[1].panelElements), 1)

        def testRowId(self):
            d = self.getRowLxml(args='id="foo"')
            self.assertTrue(d)
            self.assertEqual(d.id, "foo", "Row Id should be foo")

        def testCreateRowWith3Panels(self):
            d = self.getRowLxml(args='', panels=3)
            self.assertTrue(d)
            self.assertEqual(len(d.panels), 3)

        def testCreateRowWithPanels(self):
            xml = '''
                    <row>
                        <panel>
                            <single/>
                        </panel>
                    </row>
                '''
            root = et.fromstring(xml)
            d =  createRowFromXml(root)
            self.assertTrue(d)
            self.assertEqual(len(d.panels), 1)
            self.assertEqual(
                d.panels[0].panelElements[0].matchTagName, 'single')

        def testRowGroupingWithPanels(self):
            xml = '''
                    <row>
                        <panel>
                            <single/>
                            <single/>
                        </panel>
                        <panel>
                            <single/>
                        </panel>
                    </row>
                '''
            root = et.fromstring(xml)
            d =  createRowFromXml(root)
            self.assertTrue(d)
            self.assertEqual(len(d.panels), 2)
            self.assertEqual(len(d.panels[0].panelElements), 2)
            self.assertEqual(len(d.panels[1].panelElements), 1)

        def testCreateRowWithComments(self):
            xml = '''
                    <row>
                        <!-- this better work -->
                        <panel/>
                    </row>
                '''
            root = et.fromstring(xml)
            row =  createRowFromXml(root)
            xml = '''
                    <row>
                        <!-- this better work -->
                        <chart/>
                    </row>
                '''
            root = et.fromstring(xml)
            row =  createRowFromXml(root)

        def testCreateRowWithTitleException(self):
            with self.assertRaises(Exception):
                xml = '''
                        <row>
                            <title/>
                            <panel/>
                        </row>
                    '''
                root = et.fromstring(xml)
                row =  createRowFromXml(root)
            with self.assertRaises(Exception):
                xml = '''
                        <row>
                            <title/>
                            <chart/>
                        </row>
                    '''
                root = et.fromstring(xml)
                row =  createRowFromXml(root)

    class CreateDashboardTests(unittest.TestCase):
        def getSimpleLxml(self, root='dashboard', rows=1, fieldset=''):
            nodes = []
            nodes.append('<%(root)s>')
            nodes.append(fieldset)
            for i in range(0, rows):
                nodes.append(getRowXml())
            nodes.append('</%(root)s>')
            xml = ''.join(nodes)
            xml = xml % {'root': root}
            root = et.fromstring(xml)
            return createDashboardFromXml(root)

        def testCreateDashboard(self):
            d = self.getSimpleLxml()
            self.assertTrue(d)
            self.assertEqual(d.matchTagName, 'dashboard')

        def testCreateForm(self):
            d = self.getSimpleLxml(root='form')
            self.assertTrue(d)
            self.assertEqual(d.matchTagName, 'form')

        def testCreateFormWithFieldset(self):
            fieldset = '''
            <fieldset>
                <input token="foo" searchWhenChanged="True"></input>
                <html></html>
                <shouldntShow></shouldntShow>
            </fieldset>
            '''
            d = self.getSimpleLxml(root='form', fieldset=fieldset)
            self.assertTrue(d)
            self.assertTrue(d.fieldset)
            self.assertEqual(len(d.fieldset), 2)
            self.assertEqual(d.fieldset[0].__class__.__name__, 'TextInput')
            self.assertEqual(d.fieldset[0].searchWhenChanged, True)
            self.assertEqual(d.fieldset[1].matchTagName, 'html')
            self.assertEqual(d.matchTagName, 'form')

        def testCreateUnsupportedRoot(self):
            with self.assertRaises(Exception):
                self.getSimpleLxml(root='notFormOrDashboard')

        def testValidationMessages(self):
            msgs = getValidationMessages(et.fromstring('<dashboard></dashboard>'))
            self.assertIsNotNone(msgs)
            self.assertEquals(len(msgs), 0)

            msgs = getValidationMessages(et.fromstring('''
                <dashboard>
                    <row>
                        <table>
                            <drilldown>
                                <set token="foo">...</set>
                                <condition field="bar"></condition>
                            </drilldown>
                        </table>
                    </row>
                </dashboard>
            '''))
            self.assertIsNotNone(msgs)
            self.assertGreater(len(msgs), 0)

            msgs = getValidationMessages(et.fromstring('''
                <dashboard>
                    <row>
                        <table>
                            <drilldown>
                                <set field="bar" token="foo">...</set>
                            </drilldown>
                        </table>
                    </row>
                </dashboard>
            '''))
            self.assertIsNotNone(msgs)
            self.assertGreater(len(msgs), 0)

            msgs = getValidationMessages(et.fromstring('''
                <dashboard>
                    <row>
                        <table>
                            <drilldown>
                                <link field="bar">...</link>
                                <link field="bar">...</link>
                            </drilldown>
                        </table>
                    </row>
                </dashboard>
            '''))
            self.assertIsNotNone(msgs)
            self.assertGreater(len(msgs), 0)

            msgs = getValidationMessages(et.fromstring('''
                <dashboard>
                    <row>
                    </row>
                </dashboard>
            '''))
            self.assertIsNotNone(msgs)
            self.assertGreater(len(msgs), 0)

            msgs = getValidationMessages(et.fromstring('<foobar></foobar>'))
            self.assertIsNotNone(msgs)
            self.assertGreater(len(msgs), 0)

    class ExtractHTMLTests(unittest.TestCase):
        def testExtractEncodedHTML(self):
            self.assertEqual(" foo ", extractHtmlContent(et.fromstring('<html encoded="1"><![CDATA[ foo ]]></html>')))

        def testExtractEmptyHTML(self):
            self.assertEqual('', extractHtmlContent(et.fromstring('<html></html>')))
            self.assertEqual('', extractHtmlContent(et.fromstring('<html/>')))
            self.assertEqual('', extractHtmlContent(et.fromstring('<html id="foobar" />')))

        def testExtractInlineHTML(self):
            self.assertEqual('<h1>test</h1><p>foo</p>', extractHtmlContent(et.fromstring('<html id="foobar"><h1>test</h1><p>foo</p></html>')))

    loader = unittest.TestLoader()
    unittest.TextTestRunner(verbosity=2).run(unittest.TestSuite([
        loader.loadTestsFromTestCase(CreatePanelElementTests),
        loader.loadTestsFromTestCase(_findPrimarySearchNodeTests),
        loader.loadTestsFromTestCase(CreateRowTests),
        loader.loadTestsFromTestCase(CreateDashboardTests),
        loader.loadTestsFromTestCase(CreatePanelTests),
        loader.loadTestsFromTestCase(ExtractHTMLTests)
    ]))


from mcpi.minecraft import Minecraft
from mcpi import block
from   time import sleep
import random
import math
pi = math.pi
sin = math.sin
cos = math.cos
def init():
 ipString = "127.0.0.1"
 #ipString = "192.168.7.226"
 #mc = Minecraft.create("127.0.0.1", 4711)
 mc = Minecraft.create(ipString, 4711)
 mc.setting("world_immutable",False)
 #x, y, z = mc.player.getPos()
 return mc

def createSphere(mc,x,y,z,r):
	N=200
	lst = []
	thetas = [(2*pi*i)/N for i in range(N)]
	print("thetas", thetas)
	phis = [(pi*i)/N for i in range(N)]
	x1,y1,z1=mc.player.getPos()
	for theta in thetas:
		for phi in phis:
			x = (r * sin(phi) * cos(theta)) + x1
			y = r * sin(phi) * sin(theta) +y1
			z = r * cos(phi) + z1
			mc.setBlock(x,y,z,57)

def main():
 mc = init()
 #x,y,z = mc.player.getPos()
 #mc.setBlocks(x-5,y-1,z-5,x+5,y-1,z+5,4)
 createSphere(10,mc)
 #worldWalker(mc)
 #worldEater(mc)
 #mc.postToChat("Hondo21")

if __name__ == "__main__":
 main()

"""
AIR                   0
STONE                 1
GRASS                 2
DIRT                  3
COBBLESTONE           4
WOOD_PLANKS           5
SAPLING               6
BEDROCK               7
WATER_FLOWING         8
WATER                 8
WATER_STATIONARY      9
LAVA_FLOWING         10
LAVA                 10
LAVA_STATIONARY      11
SAND                 12
GRAVEL               13
GOLD_ORE             14
IRON_ORE             15
COAL_ORE             16
WOOD                 17
LEAVES               18
GLASS                20
LAPIS_LAZULI_ORE     21
LAPIS_LAZULI_BLOCK   22
SANDSTONE            24
BED                  26
COBWEB               30
GRASS_TALL           31
WOOL                 35
FLOWER_YELLOW        37
FLOWER_CYAN          38
MUSHROOM_BROWN       39
MUSHROOM_RED         40
GOLD_BLOCK           41
IRON_BLOCK           42
STONE_SLAB_DOUBLE    43
STONE_SLAB           44
BRICK_BLOCK          45
TNT                  46
BOOKSHELF            47
MOSS_STONE           48
OBSIDIAN             49
TORCH                50
FIRE                 51
STAIRS_WOOD          53
CHEST                54
DIAMOND_ORE          56
DIAMOND_BLOCK        57
CRAFTING_TABLE       58
FARMLAND             60
FURNACE_INACTIVE     61
FURNACE_ACTIVE       62
DOOR_WOOD            64
LADDER               65
STAIRS_COBBLESTONE   67
DOOR_IRON            71
REDSTONE_ORE         73
SNOW                 78
ICE                  79
SNOW_BLOCK           80
CACTUS               81
CLAY                 82
SUGAR_CANE           83
FENCE                85
GLOWSTONE_BLOCK      89
BEDROCK_INVISIBLE    95
STONE_BRICK          98
GLASS_PANE          102
MELON               103
FENCE_GATE          107
GLOWING_OBSIDIAN    246
NETHER_REACTOR_CORE 247
"""


from flask import current_app
from . import db
from .models import Build, BuildStatus
import os

def run_builds():
    build = Build.query.filter_by(status=0).first()
    if build != None:
        build.status = BuildStatus.BUILD
        db.session.commit()
        uploads = os.path.join(current_app.config['APP_ROOT'], current_app.config['UPLOAD_FOLDER'])
        file = open(uploads + "/" + build.build_hash + ".txt", "w")
        print(uploads + "/" + build.build_hash + ".txt", "w")
        for i in range(0, 30):
            file.write("build for: " + build.build_hash + "\n")
        file.close()
        build.status = BuildStatus.SUCCESS
        db.session.commit()
    print(build)


import socket

host = '127.0.0.1'  # Endereco IP do Servidor
porta = 5000           # Porta que o Servidor esta
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
orig = (host, porta)

tcp.bind(orig)  #associando servidor ao endereço
tcp.listen(1)   #ouvindo pedidos de conexão

while True:
    conexao, cliente = tcp.accept() #aceita conexão do cliente
    print ('Conectado por', cliente)

    while True:
        msg = conexao.recv(1024)    #recebe mensagem do cliente
        msg = msg.split()

        if not msg:
            break

        idade = float(msg[0])

        if idade < 5:
            categoria = 'Nenhuma'

        elif idade >= 5 and idade <= 7:
            categoria = 'Infantil A'

        elif idade >= 8 and idade <= 10:
            categoria = 'Infantil B'

        elif idade >= 11 and idade <= 13:
            categoria = 'Juvenil A'

        elif idade >= 14 and idade <= 17:
            categoria = 'Juvenil B'

        elif idade >= 18:
            categoria = 'Adulto'

        conexao.send(categoria.encode()) #envia resultado ao cliente conectado

    print ('Finalizando conexao do cliente', cliente)
    conexao.close()


import logging
from settings import API_ID, API_HASH
from telethon import TelegramClient, client, events
from forwarder import _
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
                    level=logging.INFO)

SENT_VIA = f'\n__Sent via__ `{str(__file__)}`'

client = TelegramClient('forwarder', API_ID, API_HASH)

@client.on(events.NewMessage(outgoing=True, pattern=r'\.id'))
async def chat_id_handler(event):
    chat_id = event.chat_id
    await event.edit(str(chat_id))

@client.on(events.NewMessage(outgoing=True, pattern=r'\.info'))
async def chat_info_handler(event):
    chat_info = await event.get_chat()
    await event.edit(str(chat_info))

async def main():
    await client.send_message('me', f'''Hi!
        \n**Telegram Chat Forward** is made by @AahnikDaw.
        \nPlease star 🌟 on [GitHub](https://github.com/aahnik/telegram-chat-forward).
        \nYou can send `.id` to any chat/group/channel to get its chat id.
        \nTo get full info, send `.info`.
        \nYou may first try it here 😃
        {SENT_VIA}''', link_preview=False)

if __name__ == "__main__":
    client.start()
    client.loop.run_until_complete(main())
    print('''If this script is succesfully running,
    you will see a new message in your Saved Messages.
    Open your Telegram App and send .id to any chat, to get the chat id.
    To get all info send .info
    \nPress Ctrl + C to stop the process.''')
    client.run_until_disconnected()


"""This is the main program file for the auto grader program.
The auto grader assumes that the following files are in the same directory as the autograder:
  - autotraining.txt  --> file of tagged words used to train the HMM tagger
  - autotest.txt      --> file of untagged words to be tagged by the HMM
  - autosolution.txt  --> file with correct tags for autotest words
This auto grader generates a file called results.txt that records the test results.
"""
import os
import time

if __name__ == '__main__':
    start = time.time()
    # Invoke the shell command to train and test the HMM tagger
    print("Training on autotraining.txt, running tests on autotest.txt. "
          "Output --> autooutput.txt")
    os.system("python ../starter/tagger.py -d  ../training-test/training3.txt -t ../training-test/test3.txt -o autooutput.txt")

    # Compare the contents of the HMM tagger output with the reference solution.
    # Store the missed cases and overall stats in results.txt
    with open("autooutput.txt", "r") as output_file, \
         open("../training-test/training3.txt", "r") as solution_file, \
         open("results.txt", "w") as results_file:
        # Each word is on a separate line in each file.
        output = output_file.readlines()
        solution = solution_file.readlines()
        total_matches = 0

        # generate the report
        for index in range(len(output)):
            if output[index] != solution[index]:
                results_file.write(f"Line {index + 1}: "
                                   f"expected <{solution[index].strip()}> "
                                   f"but got <{output[index].strip()}>\n")
            else:
                total_matches = total_matches + 1

        # Add stats at the end of the results file.
        results_file.write(f"Total words seen: {len(output)}.\n")
        results_file.write(f"Total matches: {total_matches}.\n")

        time_use = round(time.time() - start, 2)
        print("Total time used: {}".format(time_use))
        acc = round(total_matches / len(output), 4)
        results_file.write("Accuracy: {}%".format(acc * 100))
        print("Accuracy: {}%".format(acc * 100))


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('events', '0011_auto_20150812_1138'),
    ]

    operations = [
        migrations.RenameField(
            model_name='event',
            old_name='q_num',
            new_name='refQ_num',
        ),
        migrations.AddField(
            model_name='event',
            name='sort_num',
            field=models.IntegerField(default=0, verbose_name=b'Sort Number'),
            preserve_default=True,
        ),
    ]


#!/usr/bin/env python

from itertools import takewhile
import logging
import os
from random import randint
import re
import sys
from urllib.parse import urlencode
from urllib.request import urlopen

from bs4 import BeautifulSoup
import yaml

import typos

# http://www.foia.gov/foiareport.js
# Excludes "ALL" (All agencies, though it's not, really).
# Excludes " ", which in `agenciesAb` is "SIGIR", the Special Inspector
# General for Iraq Reconstruction, which no longer accepts FOIA requests.
AGENCIES = [
    'USDA', 'DOC', 'DoD', 'ED', 'DOE', 'HHS', 'DHS', 'HUD', 'DOI', 'DOJ',
    'U.S. DOL', 'State', 'DOT', 'Treasury', 'VA', 'ACUS', 'USAID', 'ABMC',
    'NRPC', 'AFRH', 'BBG', 'CIA', 'CSB', 'USCCR', 'CPPBSD', 'CFTC', 'CFPB',
    'U.S. CPSC', 'CNCS', 'CIGIE', 'CSOSA', 'DNFSB', 'EPA', 'EEOC', 'CEQ',
    'OMB', 'ONDCP', 'OSTP', 'USTR', 'Ex-Im Bank', 'FCA', 'FCSIC', 'FCC',
    'FDIC', 'FEC', 'FERC', 'FFIEC', 'FHFA', 'FLRA', 'FMC', 'FMCS', 'FMSHRC',
    'FOMC', 'FRB', 'FRTIB', 'FTC', 'GSA', 'IMLS', 'IAF', 'LSC', 'MSPB', 'MCC',
    'NASA', 'NARA', 'NCPC', 'NCUA', 'NEA', 'NEH', 'NIGC', 'NLRB', 'NMB',
    'NSF', 'NTSB', 'USNRC', 'OSHRC', 'OGE', 'ONHIR', 'OPM', 'OSC', 'ODNI',
    'OPIC', 'PC', 'PCLOB', 'PBGC', 'PRC', 'RATB', 'US RRB', 'SEC', 'SSS',
    'SBA', 'SSA', 'SIGAR', 'STB', 'TVA', 'US ADF', 'CO', 'USIBWC', 'USITC',
    'USPS', 'USTDA']

PHONE_RE = re.compile(
    r"""(?P<prefix>\+?[\d\s\(\)\-]*)"""
    r"""(?P<area_code>\(?\d{3}\)?[\s\-\(\)]*)"""
    r"""(?P<first_three>\d{3}[\-\s\(\)]*)"""
    r"""(?P<last_four>\d{4}[\-\s]*)"""
    r"""(?P<extension>[\s\(,]*?ext[ .]*?\d{3,5})?"""
    r"""(?P<tty>\s*\(tty)?""", re.IGNORECASE)

ADDY_RE = re.compile(
    r"""(?P<city>.*)"""
    r"""[\s]*?,[\s]*?(?P<state>[A-Z\.]{2,4})"""
    r"""\s*(?P<zip>[0-9-]+)""")

def agency_description(doc):
    """Account for BRs and such while finding the description."""
    description = ""
    after_h2 = doc("h2")[-1].next_elements
    next(after_h2)      # skip the text *within* the h2
    while_text = takewhile(lambda el: el.name is None or el.name == 'br',
                           after_h2)    # stop when we a different type
    for el in while_text:
        if el.name is None:
            description += el.string.strip()
        # Only want one new line when there are two BRs
        elif el.name == "br" and not description.endswith("\n"):
            description += "\n"
    return description.strip()

def clean_paragraphs(doc):
    """Find all paragraphs with content. Return paragraph els + content
    strings. Beautiful Soup doesn't handle unclosed tags very graciously, so
    account for paragraphs within paragraphs."""
    lines, ps = [], []
    for p in doc("p"):
        text = ""
        for child in p.contents:
            if child.name != 'p' and child.string and child.string.strip():
                text += child.string.strip()
        if text:
            lines.append(text)
            ps.append(p)
    return lines, ps

def clean_phone_number(line):
    """
    Given "(123) 456-7890 (Telephone)", extract the number and format
    """
    match = PHONE_RE.search(line)
    if match:
        # kill all non-numbers
        prefix = "".join(ch for ch in match.group("prefix") if ch.isdigit())
        area_code = "".join(ch for ch in match.group("area_code")
                            if ch.isdigit())
        first_three = "".join(ch for ch in match.group("first_three")
                              if ch.isdigit())
        last_four = "".join(ch for ch in match.group("last_four")
                            if ch.isdigit())
        number = "-".join([area_code, first_three, last_four])
        if prefix:
            number = "+" + prefix + " " + number
        extension = match.group("extension")
        if extension:
            extension = re.sub("\D", "", extension)
            number = number + " x" + extension
        if match.group("tty"):
            number += " (TTY)"
        return number

    else:
        raise Exception("Error extracting phone number",
                        "phone line: " + line)

def address_list_to_dict(address_list):
    """
    Converts a list containing address elements into a dictionary

    Address List
    ["Martha R. Sell", "FOIA Assistant", "2300 Clarendon Boulevard",
        "Arlington, VA 22201"]

    Address Dict
    {'state': 'VA', 'address_lines': ['Martha R. Sell', 'FOIA Assistant'],
        'city': 'Arlington', 'street': '2300 Clarendon Boulevard',
        'zip': '22201'}

    Only returns an address dict if it contains a valid street, zip,
    state, and city.
    """
    address_dict = {}

    if len(address_list) > 1:
        address_dict['street'] = address_list[-2]
    if len(address_list) > 2:
        address_dict['address_lines'] = address_list[0:-2]

    match = ADDY_RE.match(address_list[-1])
    if match:
        address_dict['zip'] = match.group('zip').strip()
        address_dict['state'] = re.sub("\W", "", match.group('state'))
        address_dict['city'] = match.group('city').strip()

    if re.search('\d', address_dict['street']) \
            and address_dict.get('zip') \
            and address_dict.get('state') \
            and address_dict.get('city'):
        return address_dict

def split_address_from(lines):
    """ Extracts address from lines into a list like the one below:
    Address list
    ["Martha R. Sell", "FOIA Assistant", "2300 Clarendon Boulevard",
        "Arlington, VA 22201"]"""

    address_list, remaining = [], []
    cues = ("phone", "fax", "service center")
    for line in lines:
        if remaining:   # already switched over
            remaining.append(line)
        elif PHONE_RE.search(line) and any(q in line.lower() for q in cues):
            remaining.append(line)
        else:
            # Separate line breaks
            address_list.extend(re.split(r"[\n\r]+", line))
    if not remaining:
        raise Exception("error finding address end", lines)
    else:
        return address_list, remaining

def find_emails(lines, ps):
    """Find email address, then associated mailto"""
    email_re = re.compile(r"\be\-?mail", re.IGNORECASE)
    emails = []
    for idx, line in enumerate(lines):
        if email_re.search(line):
            a = ps[idx].a
            if a:
                emails_str = a["href"].replace("mailto:", "").strip()
                if "http://" not in emails_str:
                    emails.extend(re.split(r";\s*", emails_str))
            else:
                raise Exception("Error extracting email", line, idx,
                                ps[idx].prettify())
    return emails

def extract_numbers(phones):
    """
    Extracts all phone numbers from a line and adds them to a list
    """

    clean_numbers = []
    phones = re.sub('[,\s(]+ext', ' ext', phones)
    phones = phones.split(",")
    for phone in phones:
        phone = phone.strip()
        if PHONE_RE.match(phone):
            clean_numbers.append(clean_phone_number(phone))
    return clean_numbers

def organize_contact(contact_info):
    """
    Organize contact info into a dictionary to facilitate extraction
    """
    contact_info = contact_info.split("Phone: ")
    name_str = contact_info[0].strip(" ,'")
    phone_str = contact_info[-1]
    clean_numbers = extract_numbers(phone_str)

    cleaned_contact_info = {}
    if name_str:
        cleaned_contact_info['name'] = name_str
    if clean_numbers:
        cleaned_contact_info['phone'] = clean_numbers

    if cleaned_contact_info:
        return cleaned_contact_info

def find_bold_fields(ps):
    """Remaining fields: website, request form, anything else"""
    simple_search = ["service center", "public liaison", "notes",
                     "foia officer"]
    link_search = ["website", "request form"]
    for p in ps:
        strong = p.strong
        text = strong.string.replace(":", "").strip() if strong else ""
        lower = text.lower()
        if strong and text != "FOIA Contact":
            try:
                value = strong.next_sibling.string
            except:
                value = p.next_sibling.string
            if value:
                value = value.strip()
            yielded = False
            for term in simple_search:
                if term in lower:
                    yield term.replace(" ", "_"), value
                    yielded = True

            for term in link_search:
                if term in lower:
                    if p.a:
                        yield term.replace(" ", "_"), p.a["href"].strip()
                        yielded = True
                    else:
                        raise Exception("error extracting " + term,
                                        p.prettify())
            if not yielded:
                # FTC: FOIA Hotline
                # GSA: Program Manager
                yield "misc", (text, value)

def parse_department(elem, name):
    """Get data from a 'div' (elem) associated with a single department"""
    data = {"name": name, "top_level": False}
    lines, ps = clean_paragraphs(elem)
    # remove first el (which introduces the section)
    lines, ps = lines[1:], ps[1:]

    address_list, lines = split_address_from(lines)
    address_dict = address_list_to_dict(address_list=address_list)
    if address_dict:
        data['address'] = address_dict
    else:
        logging.warning("Valid address not found for %s", name)

    ps = ps[-len(lines):]   # Also throw away associated paragraphs
    for line in lines:
        lower = line.lower()
        if ('phone' in lower and 'public liaison' not in lower
                and 'service center' not in lower and 'phone' not in data):
            data['phone'] = clean_phone_number(line)
        elif 'fax' in lower and 'fax' not in data and PHONE_RE.match(line):
            data['fax'] = clean_phone_number(line)
    emails = find_emails(lines, ps)
    if emails:
        data['emails'] = emails
    for key, value in find_bold_fields(ps):
        if key == 'misc':
            misc_key, misc_value = value
            if 'misc' not in data:
                data['misc'] = {}
            misc_value = organize_contact(misc_value)
            if misc_value:
                data['misc'][misc_key] = misc_value
        else:
            if key in ['service_center', 'public_liaison', 'foia_officer']:
                value = organize_contact(value)

            if value:
                data[key] = value
    return data

def parse_agency(abb, doc):
    """Make sense of a block of HTML from FOIA.gov"""
    agency_name = doc.h1.text.strip()
    description = agency_description(doc)

    # get each dept id and name, parse department from its div. Skip the first
    # as it is always a 'please select'
    departments = []
    for option in doc("option")[1:]:
        opt_id = option['value']
        elem = doc(id=opt_id)[0]
        # Needed to replace the ? with - in order to
        # accomate Carlsbad Field Office
        dept_name = option.string.strip().replace('?', '–')
        departments.append(parse_department(elem, dept_name))

    agency = {"abbreviation": abb,
              "name": agency_name,
              "description": description,
              "departments": departments}
    return agency

def fix_known_typos(text):
    """Account for faulty data"""
    for error, fix in typos.REPLACEMENTS.items():
        text = text.replace(error, fix)
    return text

def agency_yaml_filename(data_directory, agency_abbr):
    return os.path.join(data_directory, '%s.yaml' % agency_abbr)

def read_manual_data(agency_abbr, manual_data_dir='manual_data'):
    if os.path.isdir(manual_data_dir):
        filename = agency_yaml_filename(manual_data_dir, agency_abbr)
        if os.path.exists(filename):
            with open(filename, 'r') as yaml_file:
                manual_data = yaml.load(yaml_file)
            return manual_data

def update_list_in_dict(data, field, new_values_list):
    original_values = set(data.get(field, []))
    data[field] = sorted(list(original_values | set(new_values_list)))

def update_non_departments(agency_data, manual_data):
    """ Apply all the non-department changes from manual_data to agency_data.
    """

    agency_data = dict(agency_data)

    list_fields = ['common_requests', 'keywords']

    for field in manual_data.keys():
        if field not in list_fields + ['departments']:
            agency_data[field] = manual_data[field]

    for field in list_fields:
        if field in manual_data and field not in ['departments']:
            update_list_in_dict(agency_data, field, manual_data[field])
    return agency_data

def actual_apply(agency_data, manual_data):
    """ Actually apply the changes in manual_data to agency_data. This handles
    the departments. """

    agency_data = update_non_departments(agency_data, manual_data)
    manual_depts = {}
    if 'departments' in manual_data:
        for dept in manual_data['departments']:
            manual_depts[dept['name']] = dept

        departments = []
        if 'departments' in agency_data:
            for dept in agency_data['departments']:
                if dept['name'] in manual_depts:
                    new_department = update_non_departments(
                        dept, manual_depts[dept['name']])
                else:
                    new_department = dict(dept)
                departments.append(new_department)
            agency_data['departments'] = departments
    return agency_data

def apply_manual_data(agency_abbr, agency_data):
    """ In the manual data directory, we have all the manual over-rides for
    various contact fields. Apply those here. """
    manual_data = read_manual_data(agency_abbr)
    if manual_data:
        return actual_apply(agency_data, manual_data)
    else:
        return agency_data

def get_unknown_office_details(agency_data):
    for dept in agency_data['departments']:
        if dept['name'] == "I don't know which office":
            return dict(dept)

def all_but_unknown(agency_data):
    departments = [d for d in agency_data['departments']
                   if d['name'] != "I don't know which office"]
    return departments

def populate_parent(agency_data):
    unknown_office = get_unknown_office_details(agency_data)
    if unknown_office:
        departments = all_but_unknown(agency_data)
        agency_data = dict(agency_data, departments=departments)
        for field, value in unknown_office.items():
            if field not in ['name']:
                agency_data[field] = value
    return agency_data

def save_agency(abb):
    """For a given agency, download (if not already present) their HTML,
    process it, and save the resulting YAML"""
    os.makedirs('html', exist_ok=True)
    html_path = "html" + os.sep + "%s.html" % abb
    if not os.path.isfile(html_path):
        body = ""
        body = download_agency(abb)
        if body:
            with open(html_path, 'w') as f:
                f.write(body)
            logging.info("[%s] Downloaded.", abb)
        else:
            logging.warning("[%s] DID NOT DOWNLOAD, NO.", abb)
            return
    else:
        logging.info("[%s] Already downloaded.", abb)

    with open(html_path, 'r') as f:
        text = f.read()
    text = fix_known_typos(text)
    data = parse_agency(abb, BeautifulSoup(text))
    data = populate_parent(data)
    data = apply_manual_data(abb, data)
    save_agency_data(abb, data)

def save_agency_data(agency_abbr, data, data_directory='data'):
    """ Actually do the save. """
    os.makedirs(data_directory, exist_ok=True)

    if data:
        with open(data_directory + os.sep + "%s.yaml" % agency_abbr, 'w') as f:
            f.write(yaml.dump(data, default_flow_style=False,
                    allow_unicode=True))
            logging.info("[%s] Parsed.", agency_abbr)
    else:
        logging.warning("[%s] DID NOT PARSE, NO.", agency_abbr)

def save_agencies():
    """Save all agencies"""
    for agency in AGENCIES:
        save_agency(agency)

def agency_url(abb):
    """Construct download url, add cache busting -- the site does this too"""
    params = {"agency": abb, "Random": randint(1, 1000)}
    return "http://www.foia.gov/foia/FoiaMakeRequest?" + urlencode(params)

def download_agency(abb):
    """Agency HTML files"""
    url = agency_url(abb)
    return urlopen(url).read().decode("utf-8")

if __name__ == "__main__":
    """
        python scraper.py <<agency_abbreviation>>
        will only scrape and save the data for the provided agency.

        python scraper.py will scrape and save data for all the agencies.
    """
    logging.basicConfig(level=logging.INFO)

    agency_abbr = None
    if len(sys.argv) > 1:
        agency_abbr = sys.argv[1]

    if agency_abbr:
        save_agency(agency_abbr)
    else:
        save_agencies()


import sys

import MapReduce

mr = MapReduce.MapReduce()

def mapper(record):
    # record: (document name, document contents)
    key = record[0]
    value = record[1]
    words = value.split()
    for w in words:
        mr.emit_intermediate(w, key)

def reducer(key, list_of_values):
    # key: word
    # list_of_values: list of documents it appears in
    mr.emit((key, list(set(list_of_values))))

if __name__ == '__main__':
    with open(sys.argv[1], 'r') as input_data:
        mr.execute(input_data, mapper, reducer)

Coursera - Data Manipulation at Scale - Systems and Algorithms/week 3/invertedindex.py


#!/usr/bin/env python

# pip install tensorflow
# pip install keras

import cv2   as cv
import numpy as np
from threading import Thread
import time
from umucv.util import putText
from umucv.stream import autoStream

import keras
import keras.backend as K
from keras.applications.inception_v3 import InceptionV3, preprocess_input, decode_predictions
from keras.preprocessing import image

model = InceptionV3(weights='imagenet')

def classify(img):
    h,w,_ = img.shape
    dw = (w-299)//2
    dh = (h-299)//2
    win = img[dh:299+dh,dw:299+dw]
    arr = preprocess_input(np.expand_dims(win.astype(np.float32), axis=0))
    preds = model.predict(arr)
    _,lab,p = decode_predictions(preds, top=3)[0][0]
    if p < 0.5: lab = ''
    return win, lab

# la primera llamada debe hacerse fuera del hilo
classify(np.zeros((480,640,3), np.uint8))

frame = None
goon = True
win  = None

def work():
    global win,lab
    while goon:
        if frame is not None:
            t0 = time.time()
            win,lab = classify(frame)
            t1 = time.time()
            putText(win, '{:.0f}ms  {}'.format(1000*(t1-t0), lab))

t = Thread(target=work,args=())
t.start()

for key, frame in autoStream():
    cv.imshow('cam',frame)
    if win is not None:
            cv.imshow('inception', win)
            win = None

goon = False


# coding:utf-8
import re
import urllib.request
stock_CodeUrl = 'http://quote.eastmoney.com/stocklist.html'

def urlTolist(url):
    allCodeList = []
    html = urllib.request.urlopen(url).read()
    html = html.decode('gbk')
    s = r'
'
    pat = re.compile(s)
    code = pat.findall(html)
    for item in code:
        if item[0]=='6' or item[0]=='3' or item[0]=='0':
            allCodeList.append(item)
    return allCodeList

allCodelist = urlTolist(stock_CodeUrl)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# file: postprocess_prediction.py
# description:
# code for postprocessing model outputs for SQuAD.

import json
import math
import collections
import numpy as np

from transformers.tokenization_bert import BasicTokenizer

def _numpy_sigmoid(x):
    return 1/(1 + np.exp(-x))

def _get_best_indexes(logits, n_best_size):
    """Get the n-best logits from a list."""
    index_and_score = sorted(enumerate(logits), key=lambda x: x[1], reverse=True)
    best_indexes = []
    for i in range(len(index_and_score)):
        if i >= n_best_size:
            break
        best_indexes.append(index_and_score[i][0])
    return best_indexes

def _compute_softmax(scores):
    """Compute softmax probability over raw logits."""
    if not scores:
        return []

    max_score = None
    for score in scores:
        if max_score is None or score > max_score:
            max_score = score

    exp_scores = []
    total_sum = 0.0
    for score in scores:
        x = math.exp(score - max_score)
        exp_scores.append(x)
        total_sum += x

    probs = []
    for score in exp_scores:
        probs.append(score / total_sum)
    return probs

def compute_predictions_logits(
    all_examples,
    all_features,
    all_results,
    n_best_size,
    max_answer_length,
    do_lower_case,
    output_prediction_file,
    output_nbest_file,
    output_null_log_odds_file,
    verbose_logging,
    version_2_with_negative,
    null_score_diff_threshold,
    tokenizer,
    sigmoid=True
):
    """Write final predictions to the json file and log-odds of null if needed."""

    example_index_to_features = collections.defaultdict(list)
    for feature in all_features:
        example_index_to_features[feature.example_index].append(feature)

    unique_id_to_result = {}
    for result in all_results:
        unique_id_to_result[result['unique_id']] = result

    feature_lst = [tmp.unique_id for tmp in all_features]

    _PrelimPrediction = collections.namedtuple(  # pylint: disable=invalid-name
        "PrelimPrediction", ["feature_index", "start_index", "end_index", "start_logit", "end_logit"]
    )

    all_predictions = collections.OrderedDict()
    all_nbest_json = collections.OrderedDict()
    scores_diff_json = collections.OrderedDict()

    for (example_index, example) in enumerate(all_examples):
        features = example_index_to_features[example_index]

        prelim_predictions = []
        # keep track of the minimum score of null start+end of position 0
        score_null = 1000000  # large and positive
        min_null_feature_index = 0  # the paragraph slice with min null score
        null_start_logit = 0  # the start logit at the slice with min null score
        null_end_logit = 0  # the end logit at the slice with min null score
        for (feature_index, feature) in enumerate(features):
            if feature.unique_id in unique_id_to_result.keys():
                result = unique_id_to_result[feature.unique_id]
                if sigmoid:
                    result["start_logits"] = _numpy_sigmoid(result["start_logits"].to("cpu").numpy()).tolist()[0]
                    result["end_logits"] = _numpy_sigmoid(result["end_logits"].to("cpu").numpy()).tolist()[0]
                else:
                    result["start_logits"] = _compute_softmax(result["start_logits"].to("cpu").numpy().tolist()[0])
                    result["end_logits"] = _compute_softmax(result["end_logits"].to("cpu").numpy().tolist()[0])

                start_indexes = _get_best_indexes(result["start_logits"], n_best_size)
                end_indexes = _get_best_indexes(result["end_logits"], n_best_size)

                # if we could have irrelevant answers, get the min score of irrelevant
                if version_2_with_negative:
                    feature_null_score = result["start_logits"][0] + result["end_logits"][0]
                    if feature_null_score < score_null:
                        score_null = feature_null_score
                        min_null_feature_index = feature_index
                        null_start_logit = result["start_logits"][0]
                        null_end_logit = result["end_logits"][0]
                for start_index in start_indexes:
                    for end_index in end_indexes:
                        # We could hypothetically create invalid predictions, e.g., predict
                        # that the start of the span is in the question. We throw out all
                        # invalid predictions.
                        if start_index >= len(feature.tokens):
                            continue
                        if end_index >= len(feature.tokens):
                            continue
                        if start_index not in feature.token_to_orig_map:
                            continue
                        if end_index not in feature.token_to_orig_map:
                            continue
                        if not feature.token_is_max_context.get(start_index, False):
                            continue
                        if end_index < start_index:
                            continue
                        length = end_index - start_index + 1
                        if length > max_answer_length:
                            continue
                        prelim_predictions.append(
                            _PrelimPrediction(
                                feature_index=feature_index,
                                start_index=start_index,
                                end_index=end_index,
                                start_logit=result["start_logits"][start_index],
                                end_logit=result["end_logits"][end_index],
                            )
                        )
        if version_2_with_negative:
            prelim_predictions.append(
                _PrelimPrediction(
                    feature_index=min_null_feature_index,
                    start_index=0,
                    end_index=0,
                    start_logit=null_start_logit,
                    end_logit=null_end_logit,
                )
            )
        prelim_predictions = sorted(prelim_predictions, key=lambda x: (x.start_logit + x.end_logit), reverse=True)

        _NbestPrediction = collections.namedtuple(  # pylint: disable=invalid-name
            "NbestPrediction", ["text", "start_logit", "end_logit"]
        )

        seen_predictions = {}
        nbest = []
        for pred in prelim_predictions:
            if len(nbest) >= n_best_size:
                break
            feature = features[pred.feature_index]
            if pred.start_index > 0:  # this is a non-null prediction
                tok_tokens = feature.tokens[pred.start_index : (pred.end_index + 1)]
                orig_doc_start = feature.token_to_orig_map[pred.start_index]
                orig_doc_end = feature.token_to_orig_map[pred.end_index]
                orig_tokens = example.doc_tokens[orig_doc_start : (orig_doc_end + 1)]

                tok_text = tokenizer.convert_tokens_to_string(tok_tokens)

                # tok_text = " ".join(tok_tokens)
                #
                # # De-tokenize WordPieces that have been split off.
                # tok_text = tok_text.replace(" ##", "")
                # tok_text = tok_text.replace("##", "")

                # Clean whitespace
                tok_text = tok_text.strip()
                tok_text = " ".join(tok_text.split())
                orig_text = " ".join(orig_tokens)

                final_text = get_final_text(tok_text, orig_text, do_lower_case, verbose_logging)
                if final_text in seen_predictions:
                    continue

                seen_predictions[final_text] = True
            else:
                final_text = ""
                seen_predictions[final_text] = True

            nbest.append(_NbestPrediction(text=final_text, start_logit=pred.start_logit, end_logit=pred.end_logit))
        # if we didn't include the empty option in the n-best, include it
        if version_2_with_negative:
            if "" not in seen_predictions:
                nbest.append(_NbestPrediction(text="", start_logit=null_start_logit, end_logit=null_end_logit))

            # In very rare edge cases we could only have single null prediction.
            # So we just create a nonce prediction in this case to avoid failure.
            if len(nbest) == 1:
                nbest.insert(0, _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0))

        # In very rare edge cases we could have no valid predictions. So we
        # just create a nonce prediction in this case to avoid failure.
        if not nbest:
            nbest.append(_NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0))

        assert len(nbest) >= 1, "No valid predictions"

        total_scores = []
        best_non_null_entry = None
        for entry in nbest:
            total_scores.append(entry.start_logit + entry.end_logit)
            if not best_non_null_entry:
                if entry.text:
                    best_non_null_entry = entry

        probs = _compute_softmax(total_scores)

        nbest_json = []
        for (i, entry) in enumerate(nbest):
            output = collections.OrderedDict()
            output["text"] = entry.text
            output["probability"] = probs[i]
            output["start_logit"] = entry.start_logit
            output["end_logit"] = entry.end_logit
            nbest_json.append(output)

        assert len(nbest_json) >= 1, "No valid predictions"

        if not version_2_with_negative:
            all_predictions[example.qas_id] = nbest_json[0]["text"]
        else:
            # predict "" iff the null score - the score of best non-null > threshold
            score_diff = score_null - best_non_null_entry.start_logit - (best_non_null_entry.end_logit)
            scores_diff_json[example.qas_id] = score_diff
            if score_diff > null_score_diff_threshold:
                all_predictions[example.qas_id] = ""
            else:
                all_predictions[example.qas_id] = best_non_null_entry.text
        all_nbest_json[example.qas_id] = nbest_json

    if output_prediction_file:
        with open(output_prediction_file, "w") as writer:
            writer.write(json.dumps(all_predictions, indent=4) + "\n")

    if output_nbest_file:
        with open(output_nbest_file, "w") as writer:
            writer.write(json.dumps(all_nbest_json, indent=4) + "\n")

    if output_null_log_odds_file and version_2_with_negative:
        with open(output_null_log_odds_file, "w") as writer:
            writer.write(json.dumps(scores_diff_json, indent=4) + "\n")

    return all_predictions

def get_final_text(pred_text, orig_text, do_lower_case, verbose_logging=False):
    """Project the tokenized prediction back to the original text."""

    # When we created the data, we kept track of the alignment between original
    # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So
    # now `orig_text` contains the span of our original text corresponding to the
    # span that we predicted.
    #
    # However, `orig_text` may contain extra characters that we don't want in
    # our prediction.
    #
    # For example, let's say:
    #   pred_text = steve smith
    #   orig_text = Steve Smith's
    #
    # We don't want to return `orig_text` because it contains the extra "'s".
    #
    # We don't want to return `pred_text` because it's already been normalized
    # (the SQuAD eval script also does punctuation stripping/lower casing but
    # our tokenizer does additional normalization like stripping accent
    # characters).
    #
    # What we really want to return is "Steve Smith".
    #
    # Therefore, we have to apply a semi-complicated alignment heuristic between
    # `pred_text` and `orig_text` to get a character-to-character alignment. This
    # can fail in certain cases in which case we just return `orig_text`.

    def _strip_spaces(text):
        ns_chars = []
        ns_to_s_map = collections.OrderedDict()
        for (i, c) in enumerate(text):
            if c == " ":
                continue
            ns_to_s_map[len(ns_chars)] = i
            ns_chars.append(c)
        ns_text = "".join(ns_chars)
        return (ns_text, ns_to_s_map)

    # We first tokenize `orig_text`, strip whitespace from the result
    # and `pred_text`, and check if they are the same length. If they are
    # NOT the same length, the heuristic has failed. If they are the same
    # length, we assume the characters are one-to-one aligned.
    tokenizer = BasicTokenizer(do_lower_case=do_lower_case)

    tok_text = " ".join(tokenizer.tokenize(orig_text))

    start_position = tok_text.find(pred_text)
    if start_position == -1:
        return orig_text
    end_position = start_position + len(pred_text) - 1

    (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text)
    (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text)

    if len(orig_ns_text) != len(tok_ns_text):
        return orig_text

    # We then project the characters in `pred_text` back to `orig_text` using
    # the character-to-character alignment.
    tok_s_to_ns_map = {}
    for (i, tok_index) in tok_ns_to_s_map.items():
        tok_s_to_ns_map[tok_index] = i

    orig_start_position = None
    if start_position in tok_s_to_ns_map:
        ns_start_position = tok_s_to_ns_map[start_position]
        if ns_start_position in orig_ns_to_s_map:
            orig_start_position = orig_ns_to_s_map[ns_start_position]

    if orig_start_position is None:
        return orig_text

    orig_end_position = None
    if end_position in tok_s_to_ns_map:
        ns_end_position = tok_s_to_ns_map[end_position]
        if ns_end_position in orig_ns_to_s_map:
            orig_end_position = orig_ns_to_s_map[ns_end_position]

    if orig_end_position is None:
        return orig_text

    output_text = orig_text[orig_start_position : (orig_end_position + 1)]
    return output_text


## most up to date version of code that scans for forward and reverse binding sites
import numpy as np
import pandas as pd
from termcolor import colored
from colorama import Fore, Style
from matplotlib import pyplot
import matplotlib.patches as mpatches
import re
## imports the table of genes
dna_sequences = pd.read_csv('gstd1_thor.csv', delimiter=',')
dna_sequences
## sets the transcription factor pfm used to analyze sequence
## this is the human pfm for ATF4 (matrix id: MA0833.1)
profile = {'A': [210, 169, 639, 0, 19, 888, 2, 1, 40, 879, 888, 0, 414], 'C': [71, 117, 202, 0, 0, 0, 372, 0, 838, 21, 3, 266, 272], 'G': [442, 409, 127, 7, 1090, 0, 9, 1104, 16, 0, 1, 62, 101], 'T': [170, 213, 3, 981, 11, 25, 470, 4, 113, 3, 4, 714, 78]}
profilerevcomp = {'T': [414, 0, 888, 879, 40, 1, 2, 888, 19, 0, 639, 169, 210], 'G': [272, 266, 3, 21, 838, 0, 372, 0, 0, 0, 202, 117, 71], 'C': [101, 62, 1, 0, 16, 1104, 9, 0, 1090, 7, 127, 409, 442], 'A': [78, 714, 4, 3, 113, 4, 470, 25, 11, 981, 3, 213, 170]}
## generates maximum binding score of transcription factor
max_binding_list = []
for i in range(len(profile['A'])):
    score_list = []
    score_list.append(profile['A'][i])
    score_list.append(profile['C'][i])
    score_list.append(profile['G'][i])
    score_list.append(profile['T'][i])
    max_binding_list.append(max(score_list))
max_score = sum(max_binding_list)
## goes through DNA sequences in data frame and analyzes each
DNA_column = dna_sequences['sequence']
gene_column = dna_sequences['gene']
for i in range(len(DNA_column)):
    ## accesses sequences from spreadsheet
    DNA = DNA_column.iloc[i]
    seq = str(DNA).lower()
    gene = gene_column.iloc[i]
    print('\n' + Fore.RED + '\033[5m' + gene + Style.RESET_ALL)
    ## reads DNA sequence by creating a binding score for a segement equal to the length of the transfac binding site then shifting over one base and creating a new binding score
    def transcription_factor (seq, profile):
        width = len(profile['T'])
        bindingscore = []
        for i in range(len(seq)-width):
            score = 0
            for j in range(width):
                letter = seq[i+j]
                score += profile[letter][j]
            bindingscore.append (score)
        return bindingscore
    upperseq = seq.upper()
    bindingscores = transcription_factor(upperseq, profile)
    revcompbindingscores = transcription_factor (upperseq, profilerevcomp)
    ## generates graphs of binding sites
    pyplot.plot (revcompbindingscores)
    pyplot.ylim (0.76 * max_score, max_score)
    pyplot.plot (bindingscores, "green")
    pyplot.ylim (0.76 * max_score, max_score)
    green_patch = mpatches.Patch(color='green', label='forward strand sites')
    blue_patch = mpatches.Patch(color='blue', label='reverse strand sites')
    pyplot.legend(handles=[green_patch, blue_patch])
    pyplot.show()
    ## accesses binding site information
    forsites = []
    forpairs = []
    revsites = []
    revpairs = []
    def binding_sites (binding, sites, pairs):
        for i in range (len(binding)):
            if binding[i] > 0.76 * max_score:
                number = binding[i]
                sequence = DNA[i:i + len(profile['A'])]
                sites.append(sequence)
                percent_max_score = int(number/max_score * 100.0)
                pair = sequence.upper() + ': ' + str(percent_max_score) + " percent of the max binding score"
                pairs.append(str(pair))
        return sites, pairs
    binding_sites(bindingscores, forsites, forpairs)
    binding_sites(revcompbindingscores, revsites, revpairs)
    ## color codes sites and displays them in the DNA sequence
    def replace_all(text, sites, color):
        for i in range (len(sites)):
            visible_sites = colored(sites[i].upper(), color, attrs=['bold'])
            text = text.replace(sites[i], visible_sites)
        return text
    forDNA = replace_all(seq, forsites, 'green')
    finalDNA = replace_all(forDNA, revsites, 'blue')
    print(finalDNA)
    ##presents binding scores for both original and revcomp
    allsites = forsites + revsites
    print('\n' + Fore.RED + str(len(allsites)) + ' binding sites')
    print(Fore.BLUE + '\n'.join(revpairs) + Fore.GREEN + '\n'.join(forpairs))


#!/bin/env python

import sys, os
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-do2016", help="do 2016", action='store_true')
parser.add_argument("-do2018", help="do 2018", action='store_true')
parser.add_argument("-add", help="add", action="store_true")
args = parser.parse_args()

# ~ csbins = ["inc", "cspos", "csneg"]
# ~ programs = ["makeFit.py", "makeGraph.py"]
programs = ["makeGraph.py"]
uncertainties = ["nominal","scaleup","scaledown","pileup","piledown","smeared","muonid"]

for unc in uncertainties:
	#cmd = "python signalYields.py -s %s"%unc
	cmd = "python signalYieldsSingleBin.py %s"%unc

	print (cmd)
	os.system(cmd)


# Copyright (c) 2023, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import json
import os
import uuid

def read_json(filename):
    assert os.path.isfile(filename), f"{filename} does not exist!"

    with open(filename, "r") as f:
        return json.load(f)

def write_json(data, filename):
    with open(filename, "w") as f:
        json.dump(data, f, indent=4)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--job", type=str, default="cifar10_fedavg", help="Path to job config.")
    parser.add_argument("--poc", action="store_true", help="Whether admin uses POC mode.")
    parser.add_argument(
        "--train_split_root", type=str, default="/tmp/cifar10_splits", help="Location where to save data splits."
    )
    parser.add_argument(
        "--alpha",
        type=float,
        default=0.0,
        help="Value controls the degree of heterogeneity. "
        "Lower values of alpha means higher heterogeneity."
        "Values of <= 0. means no data sampling. "
        "Assumes central training.",
    )
    args = parser.parse_args()

    # update alpha and split data dir
    job_name = os.path.basename(args.job)
    client_config_filename = os.path.join(args.job, job_name, "config", "config_fed_client.json")
    server_config_filename = os.path.join(args.job, job_name, "config", "config_fed_server.json")
    meta_config_filename = os.path.join(args.job, "meta.json")

    if args.alpha > 0.0:
        client_config = read_json(client_config_filename)
        server_config = read_json(server_config_filename)
        meta_config = read_json(meta_config_filename)
        print(f"Set alpha to {args.alpha}")
        token = str(uuid.uuid4())
        job_name = f"{job_name}_alpha{args.alpha}"
        server_config["alpha"] = args.alpha
        meta_config["name"] = job_name
        split_dir = os.path.join(args.train_split_root, f"{job_name}_{token}")
        print(f"Set train split root to {split_dir}")
        server_config["TRAIN_SPLIT_ROOT"] = split_dir
        client_config["TRAIN_SPLIT_ROOT"] = split_dir
        write_json(client_config, client_config_filename)
        write_json(server_config, server_config_filename)
        write_json(meta_config, meta_config_filename)
        print(f"Updated {meta_config_filename} to alpha={args.alpha}")
    else:
        print("Assuming centralized training.")

if __name__ == "__main__":
    main()


import Queue
class MedianFinder:
    def __init__(self):
        """
        Initialize your data structure here.
        """
        self.median = None
        self.MaxHeap, self.MinHeap = Queue.PriorityQueue(), Queue.PriorityQueue()

    def addNum(self, num):
        """
        Adds a num into the data structure.
        :type num: int
        :rtype: void
        """
        if not self.median:
            self.median = num
        else:
            if num > self.median:
                self.MinHeap.put(num)
            else:
                self.MaxHeap.put(-num)

            if self.MaxHeap.qsize() > self.MinHeap.qsize():
                self.MinHeap.put(self.median)
                self.median = -self.MaxHeap.get()
            if self.MaxHeap.qsize() + 1 < self.MinHeap.qsize():
                self.MaxHeap.put(-self.median)
                self.median = self.MinHeap.get()

    def findMedian(self):
        """
        Returns the median of current data stream
        :rtype: float
        """
        if not self.median:
            return float(0)
        else:
            if self.MaxHeap.qsize() == self.MinHeap.qsize():
                return self.median
            else:
                return (self.median + self.MinHeap.queue[0]) / float(2)

# Your MedianFinder object will be instantiated and called as such:
# mf = MedianFinder()
# mf.addNum(1)
# mf.findMedian()