#!/usr/bin/env python3
"""
:Created: 2013-12-30
:Updated: 2018-03-09

- Import old bookmarks from JSON, XML.

Commands:
  - list
  - add | modify | assert | show
  - remove
  - tags
  - urls
  - sync

  Database:
    - info | init | stats | clear
"""
from __future__ import print_function

__description__ = "bookmarks - ..."
__short_description__ = "..."
__version__ = '0.0.4-dev' # script-mpe
__db__ = '~/.bookmarks.sqlite'
__couch__ = 'http://localhost:5984/the-registry'
import os
if os.uname()[0] == 'Darwin':
    chrome_bookmarks_file= '~/Library/Application Support/Google/Chrome/Default/Bookmarks'
    chrome_history_path=   '~/Library/Application Support/Google/Chrome/Default/History'
elif os.uname()[0] == 'Linux':
    chrome_bookmarks_file= '~/.config/google-chrome/Default/Bookmarks'
    chrome_history_path=   '~/.config/google-chrome/Default/History'
#elif os.uname()[0] == 'Win'
    # C:\Users\xyz\AppData\Local\Google\Chrome\User Data\Default

__usage__ = """

Usage:
  bookmarks.py [-v... options] list [NAME] [ --tags=TAG... ]
  bookmarks.py [-v... options] (add|modify|assert|show) REF [ NAME ] [ TAGS... ]
  bookmarks.py [-v... options] remove REF
  bookmarks.py [-v... options] tags [TAGS...]
  bookmarks.py [-v... options] urls REF
  bookmarks.py [-v... options] sync
  bookmarks.py [-v... options] check [NAME]
  bookmarks.py [-v... options] webarchive [NAME]
  bookmarks.py [-v... options] (tag|href|domain) [NAME]
  bookmarks.py [-v... options] x [ARG...]
  bookmarks.py [-v... options] dlcs (parse|import FILE|export)
  bookmarks.py [-v... options] chrome (all|roots|groups|import) [--chrome-bookmarks-root NAME]
  bookmarks.py [-v... options] html (tree|stats|groups|import|export) HTML
  bookmarks.py [-v... options] sql (stats|couch|cleanup)
  bookmarks.py [-v... options] couch (sql|stats|list|update|init)
  bookmarks.py [-v... options] couch (add|modify) REF [ NAME [ TAGS... ] ]
  bookmarks.py [-v... options] info | init | stats | clear | memdebug
  bookmarks.py --background
  bookmarks.py -h|--help
  bookmarks.py help [CMD]
  bookmarks.py --version

Options:
  -s SESSION, --session-name SESSION
                should be bookmarks [default: default].
  -d REF, --dbref=REF
                SQLAlchemy DB URL [default: %s]
  --no-db       Don't initialize SQL DB connection.
  --couch=REF
                Couch DB URL [default: %s]
  --tag-offset INT
                Set import frequency-offset to exclude certain one-to-many
                relations if the usage is below given value.
                This entirely depends on usage.
                0 means to import everything [default: -1]
                Defaults to hiFreq * 0.1.
  --domain-offset INT
                Typical --*-offset, see before.
                Defaults to avgFreq. [default: -1]
  --chrome-bookmarks-file PATH
                [default: %s]
  --chrome-bookmarks-root GROUP
                Ie. bookmark_bar or other. [default: ]
  -O FMT, --output-format FMT
                json, outline, todo.txt, repr [default: rst]
  -i N, --interval N
                Be verbose at least every N records [default: 100]
  --interactive
                Prompt to resolve or override certain warnings.
                XXX: Normally interactive should be enabled if while process has a
                terminal on stdin and stdout.
  --batch
                Overrules `interactive`, exit on errors or strict warnings.
  --auto-commit N
                Auto-commit every N records.
  --no-commit   .
  --commit      [default: true].
  --clear-unknown-keys
                Delete unknown keys from CouchDB documents.
  --newer-only
                Only update entries in target if source update time is more
                recent.
  --no-partial-match
  --partial-match
                Treat input values as partial search values. This adds the
                equivalent of '*' around all the values (without '*').
  --update-hrefids
                Walk over URLs (.href) and rename if found as key.
  --update-tagstype
                Correct couch document 'tags->tag_list' attribute.
  --deleted
  --count       [default: false].
  --status STATUS[,STATUS]
  --days N
  --weeks N
  --from DATE
  --to DATE
  --on DAY
  --older-than SPEC
  --max-age SPEC
                Datetime query window. [Default: 1y] Queries are always on
                updated-time's.
  --added       Use added time instead of update in match with date/time.
  --matched-only
  --exact-match
  --struct-output
  --update
  --delete STATUS[,STATUS]
  --delete-error
  --include-null
  --ignore-last-seen
  --dry-run     Echo but don't make actual changes. This does all the
                document/record operations, but no commit.
  -v, --verbose  Increase verbosity, 3 is maximum.
  -q, --quiet   Turn off verbosity. Overrides verbosity flags.
  --print-memory
                Print memory usage just before program ends.
  -h --help     Show this usage description.
                For a command and argument description use the command 'help'.
  --version     Show version (%s).
""" % ( __db__, __couch__, chrome_bookmarks_file, __version__, )
import sys
import hashlib
import urllib
# import urllib2 TODO: rewrite to Py3
from datetime import datetime, timedelta

import uriref
from pydelicious_xml import dlcs_parse_xml
import bs4

from script_mpe.libhtd import *
from script_mpe.bookmarks_model import *
from script_mpe.res import bm, bm_chrome

ctx = Taxus(version='bookmarks')

cmd_default_settings = dict(
        verbose=1,
        partial_match=True,
        strict=True
    )

# XXX: old rsr bookmarks class
class bookmarks(rsr.Rsr):

    #zope.interface.implements(res.iface.ISimpleCommand)

    NAME = os.path.splitext(os.path.basename(__file__))[0]
    OPT_PREFIX = 'bm'
    OPTS_INHERIT = ('-v',)# '-q', '-i', '-c')

    DEFAULT_CONFIG_KEY = 'bm'

    #TRANSIENT_OPTS = Taxus.TRANSIENT_OPTS + ['']
    DEFAULT_ACTION = 'stats'

    DEPENDS = {
            'stats': ['rsr_session'],
            'list': ['rsr_session'],
            'add': ['rsr_session'],
            'add_lctr': ['rsr_session'],
            'list_lctr': ['rsr_session'],
            'add_lctr_ref_md5': ['rsr_session'],
            'add_ref_md5': ['rsr_session'],
            'moz_js_import': ['rsr_session'],
            'moz_js_group_import': ['rsr_session'],
            'moz_ht_import': ['rsr_session'],
            'dlcs_post_import': ['rsr_session'],
            'dlcs_post_read': ['rsr_session'],
            'dlcs_post_test': ['rsr_session'],
            'dlcs_post_test2': ['rsr_session'],
            'export': ['rsr_session']
        }

    @classmethod
    def get_optspec(Klass, inheritor):

        """
        Return tuples with optparse command-line argument specification.
        """
        p = inheritor.get_prefixer(Klass)
        return (
            # actions
                p(('-s', '--stats',), libcmd.cmddict()),
                p(('-l', '--list',), libcmd.cmddict()),
                p(('-a', '--add',), libcmd.cmddict()),
                p(('--list-lctr',), libcmd.cmddict()),

                (('--moz-js-import',), libcmd.cmddict()),
                (('--dlcs-post-import',), libcmd.cmddict()),
                (('--dlcs-post-test',), libcmd.cmddict()),

                p(('--export',), libcmd.cmddict(help="TODO: bm export")),

                (('--moz-js-group-import',), libcmd.cmddict()),

                (('--add-lctrs',), libcmd.cmddict(
                    help="Add locator records for given Locator's.")),
                (('--add-ref-md5',), libcmd.cmddict(
                    help="Add MD5-refs missing (on all locators). ")),

            # params
                p(('--public',), dict( action='store_true', default=False )),
                p(('--name',), dict( default=None, type="str" )),
                p(('--href',), dict( default=None, type="str" )),
                p(('--ext',), dict( default=None, type="str" )),
                p(('--tags',), dict( default=None, type="str" )),
                p(('--ref-md5',), dict( action='store_true',
                    default=False, help="Calculate MD5 for new locators. " )),
            )

    def assert_locator(self, sa=None, href=None, opts=None):
        lctr = Locator.fetch((Locator.global_id==href,), _sa=sa, exists=False)
        if not lctr:
            if len(href) > 255:
                log.err("Reference too long: %s", href)
                return
            lctr = Locator(
                    global_id=href,
                    date_added=datetime.now() )
            sa.add( lctr )
            if opts.rsr_auto_commit:
                sa.commit()
            if opts.bm_ref_md5:
                self.add_lctr_ref_md5(opts, sa, href)
        yield dict(lctr=lctr)

    def add_lctrs(self, sa=None, opts=None, *refs):
        if refs:
            for ref in refs:
                for ret in self.assert_locator( sa=sa, href=ref, opts=opts ):
                    yield ret['lctr']

    def add(self, sa=None, href=None, name=None, ext=None, public=False,
            tags=None, opts=None):
        "Create or update. alias --update?"
        lctr = [ r['lctr'] for r in self.assert_locator(sa=sa, href=href, opts=opts) ]
        if not lctr:
            yield dict( err="XXX Missed ref" )
        else:
            lctr = lctr.pop()
            assert lctr
            bm = Bookmark.fetch((Bookmark.ref==lctr,), _sa=sa, exists=False)
            if bm:
                # XXX: start local to bean dict
# XXXL name must be unique, must catch problems
                #if name != bm.name:
                #    bm.name = name
                #if lctr != bm.ref:
                #    bm.ref = lctr
                if ext != bm.extended:
                    bm.extended = ext
                if public != bm.public:
                    bm.public = public
                if tags != bm.tags:
                    bm.tags = tags
                bm.last_update = datetime.now()
            else:
                bm = Bookmark.fetch((Bookmark.name==name,), _sa=sa, exists=False)
                if bm:
                    log.err("Duplicate name %s", bm.name)
                    #bm.name = "%s (copy)" % name
                    bm = None
                else:
                    bm = Bookmark(
                            name=name,
                            ref=lctr,
                            extended=ext,
                            public=public,
                            tags=tags,
                        )
                    bm.init_defaults()
            if bm:
                assert bm.ref
                yield dict( bm=bm )
                sa.add(bm)
            if opts.rsr_auto_commit:
                sa.commit()

    def add_lctr_ref_md5(self, opts=None, sa=None, *refs):
        "Add locator and ref_md5 attr for Locators"
        if refs:
            if isinstance( refs[0], str ):
                opts.ref_md5 = True
                lctrs = [ ret['lctr'] for ret in self.add_lctrs(sa, opts, *refs) ]
                return
            else:
                assert isinstance( refs[0], Locator), refs
                lctrs = refs

            for lctr in lctrs:
                ref = lctr.ref or lctr.global_id
                ref_md5 = hashlib.md5( ref ).hexdigest()
                md5 = MD5Digest.fetch(( MD5Digest.digest == ref_md5, ),
                        exists=False)
                if not md5:
                    md5 = MD5Digest( digest=ref_md5,
                            date_added=datetime.now() )
                    sa.add( md5 )
                    log.info("New %s", md5)
                lctr.ref_md5 = md5
                sa.add( lctr )
                if opts.rsr_auto_commit:
                    sa.commit()
                log.note("Updated ref_md5 for %s to %s", lctr, md5)

    def add_ref_md5(self, opts=None, sa=None):
        "Add missing ref_md5 attrs. "
        lctrs = sa.query(Locator).filter(Locator.ref_md5_id==None).all()
        self.add_lctr_ref_md5( opts, sa, *lctrs )

    def moz_js_import(self, opts=None, sa=None, *paths):
        mozbm = res.bm.MozJSONExport()
        for path in paths:
            #for ref in mozbm.read_lctr(path):
            #    list(self.assert_locator(sa=sa, href=ref, opts=opts))
            for node in mozbm.read_bm(path):
                descr = [ a['value'] for a in node.get('annos', [] )
                        if a['name'] == 'bookmarkProperties/description' ]
                if 'uri' not in node or 'title' not in node or not node['title']:
                    log.warn("Illegal %s", node)
                else:
                    list(self.add(sa=sa,
                        href=node['uri'],
                        name=node['title'],
                        ext=descr and descr.pop() or None,
                        opts=opts))

    def moz_js_group_import(self, opts=None, sa=None, *paths):
        "Import groupnodes only. "
        mozbm = res.bm.MozJSONExport()
        for path in paths:
            nodes = {}
            roots = []
            #for ref in mozbm.read_lctr(path):
            #    list(self.assert_locator(sa=sa, href=ref, opts=opts))
            for node in mozbm.read(path):
                if 'title' in node and node['title'] and 'children' in node:
                    nodes[node['id']] = node
                    if 'root' in node:
                        roots.append(node)
            # TODO: store groups, but need to start at the root, sort out struct
            # XXX should need a tree formatter here
            print('Groups')
            for nid, node in nodes.items():
                #print repr(node['title']),
                if 'parent' in node:
                    parent = nodes[node['parent']]
                    self.rsr_add_group( node['title'], parent['title'],
                            sa=sa, opts=opts )
                else:
                    self.rsr_add_group( node['title'], None,
                            sa=sa, opts=opts )
            print('Roots')
            for root in roots:
                print(root['id'], root['title'])

### Commands

def cmd__dlcs_import(FILE, opts, g):

    """
    Import from (old pre-2009) del.icio.us posts XML export using
    pydelicious library.
    """
    global ctx

    importer = res.bm.BmImporter(ctx.sa_session)
    data = dlcs_parse_xml(open(FILE).read())

    # Validate URL, track tag/domain and create records where missing
    for post in data['posts']:
        href = post['href']
        dt = datetime.strptime(post['time'], ISO_8601_DATETIME)
        lctr = importer.init_locator(href, dt)
        if not lctr:
            continue

        tagcsv = unicode(post['tag'].replace(' ', ', '))
        bm = importer.init_bookmark(lctr, dt,
                post['description'], post['extended'], tagcsv )
        if not bm:
            continue

        # commit every x records (if not dry-run)
        importer.batch_flush(g)

    log.std("Checked %i locator references", len(data['posts']))
    # Commit (if not dry-run)
    importer.flush(g)

    # Commit (proc/fetch/init) Domain list, see --domain-offset.
    importer.flush_domains(g)

    # Commit (proc/fetch/init) Tag list, see  --tag-offset
    importer.flush_tags(g)

def cmd__html_groups(HTML, g):
    """
    TODO: work on just the groups (folders) extracted from the bookmark.html.
    """
    soup = bs4.BeautifulSoup(open(HTML), "html5lib")
    items = bm.TxtBmOutline.bm_html_soup_items_gen(soup)
    if not items:
        raise Exception("No definition lists in %s" % HTML)
    paths = []
    for lbl, attr in items:
        if 'href' in attr: continue
        if 'parent' in attr:
            path = attr['parent'] +" / "+ lbl
        else:
            path = lbl
        if path in paths: continue
        paths.append(path)
    print("\n".join(paths))

def cmd__html_tree(HTML, g):
    """
    Extract folder/bookmark item lines from HTML.
    """
    soup = bs4.BeautifulSoup(open(HTML), "html5lib")
    print(bm.html_soup_formatters[g.output_format](soup))

def cmd__html_import(HTML, g):
    """
    Import bookmarks from HTML
    """
    global ctx
    soup = bs4.BeautifulSoup(open(HTML), "html5lib")
    bm.TxtBmOutline.bm_html_soup_parse_to_sa(soup, ctx.sa_session)

def cmd__html_stats(HTML, g):
    """
    Show stats for bookmarks HTML
    """
    global ctx
    soup = bs4.BeautifulSoup(open(HTML), "html5lib")

    ctx.folders_cnt = 0
    def _folder(label, attrs):
        global ctx
        ctx.folders_cnt += 1

    ctx.items_cnt = 0
    def _item(label, attrs):
        global ctx
        ctx.items_cnt += 1

    items = bm.TxtBmOutline.bm_html_soup_items_gen(soup,
            folder_f=_folder, item_f=_item)
    if not items:
        raise Exception("No definition lists in %s" % HTML)
    for it in items: pass

    print("%i items" % ctx.items_cnt)
    print("%i folders" % ctx.folders_cnt)

def cmd__html_export(HTML, g):
    """
    Export bookmarks to HTML
    """
    global ctx

    bms_iter = Bookmark.all()
    for bm in bms_iter:
        print(bm)
        # TODO: save folders

def cmd__chrome_all(g):
    "List Chrome bookmarks (from JSON) in different formats"
    bms = bm_chrome.BookmarksJSON.load(g.chrome_bookmarks_file)
    if g.chrome_bookmarks_root:
        roots = [ g.chrome_bookmarks_root ]
    else:
        roots = dict(bms.roots()).keys()

    for root in roots:
        group = confparse.Values(bms.data['roots'][root])
        bm.moz_json_printer['item'][g.output_format](group)

def cmd__chrome_roots(g):
    bms = bm_chrome.BookmarksJSON.load(g.chrome_bookmarks_file)
    for k, v in bms.data['roots'].items():
        if v['type'] == 'folder': print(k)

def cmd__chrome_groups(g):
    """
    List Chrome bookmarks folder paths (from JSON). See also outline format with
    ``bookmarks.py chrome all``.
    """
    bms = bm_chrome.BookmarksJSON.load(g.chrome_bookmarks_file)
    for it in bm_chrome.flatten_norecurse(list(bms.groups())):
        print(it)

def cmd__chrome_import(g):
    """
    Import Chrome Bookmarks JSON to SQL.
    """

    global ctx

    importer = res.bm.BmImporter(ctx.sa_session)
    bms = bm_chrome.BookmarksJSON.load(g.chrome_bookmarks_file)

    if g.chrome_bookmarks_root:
        datagen = bms.urls_gen(bms.data['roots'][g.chrome_bookmarks_root])
    else:
        datagen = bms.urls_gen()

    # Validate URL, track tag/domain and create records where missing
    urlcnt, bmcnt = 0, 0
    for groups, bm in datagen:
        urlcnt += 1
        dt = res.dt.parse_chrome_microsecondstamp(int(bm['date_added']))
        lctr = importer.init_locator(bm['url'], dt)
        if not lctr:
            continue

        tagcsv = ', '.join(groups[1:])
        bm = importer.init_bookmark(lctr, dt, bm['name'], None, tagcsv)
        if not bm:
            continue
        bmcnt += 1

        # commit every x records (if not dry-run)
        importer.batch_flush(g)

    log.std("Checked %i urls", urlcnt)
    log.std("Added %i bookmarks", bmcnt)
    # Commit (if not dry-run)
    importer.flush(g)

    # Commit (proc/fetch/init) Domain list, see --domain-offset.
    importer.flush_domains(g)

    # Commit (proc/fetch/init) Tag list, see  --tag-offset
    importer.flush_tags(g)

def cmd__sql_stats(g):

    global ctx
    sa = ctx.sa_session

    for stat, label in (
                (sa.query(Locator).count(), "Number of Locators: %s"),
                (sa.query(Bookmark).count(), "Number of bookmarks: %s"),
                (sa.query(Domain).count(), "Number of domains: %s"),
                (sa.query(Tag).count(), "Number of tags: %s"),
            ):
        log.std(label, stat)

def cmd__sql_cleanup(g):

    global ctx
    sa = ctx.sa_session

    for bm in sa.query(Bookmark).filter(Bookmark.locator_id==None).all():
        bm.delete()
        log.note("Deleted bookmark without ref %s", bm)

def cmd__href(NAME, g):
    """List hyper-references"""
    sa = Locator.get_session(g.session_name, g.dbref)
    if NAME:
        rs = Locator.search(ref=NAME)
    else:
        rs = Locator.all()
    if not rs:
        log.stdout("{yellow}Nothing found{default}")
        if g.strict:
            return 1
    for r in rs:
        print(r.ref)

def cmd__domain(NAME, g):
    """List domains"""
    sa = Domain.get_session(g.session_name, g.dbref)
    if NAME:
        rs = Domain.search(name=NAME)
    else:
        rs = Domain.all()
    if not rs:
        log.stdout("{yellow}Nothing found{default}")
        if g.strict:
            return 1
    for r in rs:
        print(r.name)

def cmd__tag(NAME, g):
    """Get tag"""
    sa = Tag.get_session(g.session_name, g.dbref)
    if NAME:
        rs = Tag.search(name=NAME)
    else:
        rs = Tag.all()
    if not rs:
        log.stdout("{yellow}Nothing found{default}")
        if g.strict:
            return 1
    for r in rs:
        print(r.name)

def cmd__tags(TAGS, g):
    """
    Dump all tags found on bookmarks. Filter by given tag, fuzzy by default.
    Print all tags, ie. related tags (on the same record) too.
    """
    global ctx

    tags = Bookmark.unique_tags(TAGS, g, ctx)
    if tags:
        for t in tags:
            if g.matched_only:
                if g.exact_match:
                    if t in TAGS:
                        print(t)
                else:
                    for n in TAGS:
                        if n in t:
                            print(t)
            else:
                print(t)
        ctx.note("%i unique tags found on bookmarks", len(tags))

def cmd__urls(REF, g, opts):
    """List URLs from SQL.

    Unless --exact-match is given, inputs are turned into LIKE expressions for
    partial match.
    """
    global ctx

    filters = ctx.opts_to_filters(Locator)
    if REF:
        filters += ( sql_like_val(Locator.ref, REF, g), )
    if g.count:
        cnt = ctx.sa_session.query(Locator).filter(*filters).count()
        log.std("Records matched: %s", cnt)
        return

    rs = Bookmark.all(filters=filters)
    if not rs:
        log.stdout("{yellow}Nothing found{default}")
        if g.strict: return 1
    for r in rs:
        print(r)
    # FIXME ctx.lines_out(rs)

def cmd__list(NAME, g, opts):

    """
    List bookmarks given NAME, TAGS filter from SQL.

    All matches are partial (delimited by '%' and turned into LIKE expression),
    unless --exact-match is given.
    Asterisks '*' in any of the inputs are also handled as explicit LIKE expression.
    """
    global ctx

    filters = ctx.opts_to_filters(Bookmark)
    for f in filters:
        print(f)
    if g.tags:
        filters += tuple([ sql_like_val(Bookmark.tags, T) for T in g.tags ])
    if NAME:
        filters += ( sql_like_val(Bookmark.name, NAME), )

    if g.count:
        cnt = ctx.sa_session.query(Bookmark).filter(*filters).count()
        log.std("Records matched: %s", cnt)
        return

    rs = Bookmark.all(filters=filters)
    if not rs:
        log.stdout("{yellow}Nothing found{default}")
        if g.strict:
            return 1
        return

    render = ctx.get_renderer('bookmark')
    for r in rs:
        print(render(r.to_dict()))
    ctx.flush()
    print('%i found' % len(rs))

def cmd__add(REF, NAME, TAGS, g, sa=None):
    global ctx
    bm = Bookmark.forge(REF, NAME, TAGS, g, ctx.sa_session)
    # TODO: ctx.sa.commit
    print(ctx.get_renderer('bookmark')(bm.to_dict()))
    #if not g.dry_run: sa.commit()

def cmd__modify(REF, g):
    assert 'todo' # TODO: bookmarks.py modify HREF

def cmd__remove(REF, g):
    "Soft-delete Bookmark and Locator by Locator"
    global ctx
    sa = ctx.sa_session

    lctr = sa.query(Locator).filter(Locator.ref == REF).one()
    bm = sa.query(Bookmark).filter(Bookmark.location == lctr).one()

    bm.delete()
    lctr.delete()
    if not g.dry_run:
        sa.add(lctr)
        sa.add(bm)
        if not g.dry_run: sa.commit()
        ctx.note("Deleted %s", REF)
    print(lctr, bm)

def cmd__check(NAME, g):

    """
    Update last-seen time.

    Visit the Locator for each bookmark, and update the last-seen date if
    successful. Options below apply filters and specific actions.

    If no filters are provided, the default is to select Locators not seen in
    the last 52 weeks, and all those without status or last-seen fields.

    --deleted
    --weeks, --days
        Select only locators not seen within the last N weeks or days.
    --status [xxx]
        Select only locators with last recorded status.
    --include-null
        Applies to last-seen, dont exclude null (weeks, days, status)
    --delete [0,4,5,xxx]
        On resolve or HTTP error delete the Locator.

        To delete only on specific
        cases, pass 0 for just name resolution or network errors, a single
        digit for 4xx or 5xxx errors, or any specific HTTP status code.
    --delete-error
        Shortcut to --delete 0,4,5
    """

    sa = Locator.get_session(g.session_name, g.dbref)

    delete = []
    if opts.flags.delete_error:
        delete = [ 0, 4, 5 ]
    elif opts.flags.delete:
        delete = [ int(i) for i in opts.flags.delete.split(',') ]

    f = []

    if opts.flags.status:
        for s in opts.flags.status.split(','):
            if s > 99:
                if opts.flags.include_null:
                    f.append(or_(
                        Resource.status.is_(None),
                        Resource.status == s
                    ))
                else:
                    f.append( Resource.status == s )
            else:
                if opts.flags.include_null:
                    f.append(or_(
                        Resource.status.is_(None),
                        Resource.status >= s*100 and Resource.status < (s+1)*100
                    ))
                else:
                    f.append(
                        Resource.status >= s*100 and Resource.status < (s+1)*100
                    )

    if opts.flags.weeks:
        w = int(opts.flags.weeks)
        if opts.flags.include_null:
            f.append(or_(
                Resource.last_access.is_(None),
                Resource.last_access < (
                    datetime.utcnow() - timedelta(weeks=w))
            ))
        else:
            f.append( Resource.last_access < (
                datetime.utcnow() - timedelta(weeks=w)) )

    elif opts.flags.days:
        d = int(opts.flags.days)
        if opts.flags.include_null:
            f.append(sa.or_(
                Resource.last_access.is_(None),
                Resource.last_access < (datetime.utcnow() - timedelta(days=d))
            ))
        else:
            f.append( Resource.last_access < (
                datetime.utcnow() - timedelta(days=d)) )

    if not f:
        if opts.flags.include_null:
            f.append(or_(
                    Resource.last_access < (
                        datetime.utcnow() - timedelta(weeks=52)),
                    Resource.status.is_(None)
                ))
        else:
            f.append(or_(
                    Resource.last_access < (
                        datetime.utcnow() - timedelta(weeks=52)),
                    Resource.last_access.is_(None),
                    Resource.status.is_(None)
                ))

    if not opts.flags.deleted:
        f.append( Resource.deleted!=True )

    rs = Resource.all(f)
    print('%i Locator\'s to check' % len(rs))

    for i, r in enumerate(rs):
        ref = r.location.href()
        print(i, r.status, r.deleted, r.last_access, ref)
        if not g.dry_run:
            if i and ( i % 10 ) == 0:
                sa.commit()
                print('committed at %s items, %i to go' % ( i, len(rs)-i ))

        try:
            urlinfo = urllib2.urlopen(ref, timeout=9)
        except urllib2.HTTPError as e:
            r.status = e.code
            if e.code in delete:
                r.delete()
            sa.add(r)
            continue
        except urllib2.LocatorError as e:
            r.status = -2
            if 0 in delete or -1 in delete:
                r.delete()
            sa.add(r)
            continue
        except Exception as e:
            print(e, ref)
            r.status = -1
            if 0 in delete or -1 in delete:
                r.delete()
            sa.add(r)
            print(0, ref)
            continue

        info = urlinfo.info()
        status = urlinfo.getcode()
        if r.status == None or status != r.status:
            r.status = status
            sa.add(r)
            print(status, ref)

        if status == 200:
            r.last_access = datetime.now()
            sa.add(r)

        if status and ( status in delete or status/100 in delete ):
            r.delete()
            sa.add(r)

        #print i, r.status, r.deleted, r.last_access, ref
    if not g.dry_run:
        sa.commit()

def cmd__assert(REF, NAME, TAGS, g):

    """
    Create an Locator record if it does not exist yet.

    If NAME and optionally TAGS is given, create a bookmark too but only
    if it does not exist yet.
    """

    sa = Bookmark.get_session(g.session_name, g.dbref)
    bm = Bookmark.forge(REF, NAME, TAGS, g, sa)
    print(bm)
    if g.dry_run:
        log.std("Dry run")
    else:
        sa.commit()

def cmd__show(REF, NAME, TAGS, g):
    global ctx
    sa = ctx.sa_session
    filters = ()
    if not g.deleted:
        filters = ( Bookmark.deleted != True, )

    if REF:
        lctr = sa.query(Locator).filter(Locator.ref == REF).one()
        filters += ( Bookmark.location == lctr, )

    if TAGS:
        filters += tuple([ sql_like_val(Bookmark.tags, T, g) for T in TAGS ])

    if NAME:
        filters += ( sql_like_val(Bookmark.name, NAME, g), )

    if filters:
        rs = Bookmark.all(filters=filters)
    else:
        rs = Bookmark.all()
    if not rs:
        log.stdout("{yellow}Nothing found{default}")
        if g.strict:
            return 1
        return

    ctx.out(rs, 'bookmark')
    ctx.note("%i items", len(rs))

def cmd__webarchive(NAME, g):

    """
    Sort out and rewrite web.archive locators.

    http://web.archive.org/web/20030208015752/
    """
    global ctx
    sa = ctx.sa_session

    NS_WA = 'http://web.archive.org/web'
    ns_lctr = Locator.fetch((Locator.ref == NS_WA,), exists=False)
    if not ns_lctr:
        ns_lctr = Locator( ref=NS_WA )
        ns_lctr.init_defaults()
        sa.add(ns_lctr)

    rcres_type = Namespace.fetch(( Namespace.location == ns_lctr ,),
            exists=False)
    if not rcres_type:
        rcres_type = Namespace( location=ns_lctr )
        rcres_type.init_defaults()
        sa.add(rcres_type)

    if g.commit:
        sa.commit()

    rs = sa.query(Locator).filter(
            Locator.ref.like('%/web.archive.org/web/%'),
            Locator.deleted != True).all()
    if not rs:
        log.stdout("{yellow}Nothing found{default}")
        if g.strict:
            return 1
        return
    total_records = len(rs)

    i = 0
    for lctr in rs:
        assert 'web.archive.org/web' in lctr.ref

        i += 1
        bm = Bookmark.fetch((Bookmark.location == lctr,), exists=False)
        if not bm:
            continue

        cache_path = lctr.ref.split('web.archive.org/web/')[1]
        p = cache_path.index('/')
        res_ts = cache_path[:p]
        res_url = cache_path[p+1:]
        if not uriref.absoluteURI.match(res_url):
            res_url = 'http://'+res_url

        lctr_new = Locator.fetch((Locator.ref == res_url,), exists=False)
        if not lctr_new:
            lctr_new = Locator( ref=res_url )
            lctr_new.init_defaults()
            log.std("New: %s", lctr_new.href() )

        if res_ts.isdigit():
            DT_FMT='%Y%m%d%H%M%S'
            dt = datetime.strptime(res_ts, DT_FMT)
        else:
            log.err("Unknown date tag: %s", res_ts)
            continue

        rcres = RemoteCachedResource( location=lctr, rcres_type=rcres_type,
                last_access=bm.last_access,
                last_update=bm.last_update,
                last_modified=bm.last_modified,
                status=bm.status,
                date_added=lctr.date_added,
                date_deleted=lctr.date_deleted,
                date_updated=dt )

        if not g.dry_run:
            sa.add(lctr_new)
            sa.add(rcres)
            #sa.delete(lctr)
        if g.auto_commit and i and i % g.auto_commit == 0:
            if g.verbose > 1:
                ctx.note("Auto-commit at %i of %i", i, total_records)
            sa.commit()

        ctx.note('TODO %s', lctr.href())
    if g.commit:
        sa.commit()
    ctx.note("Found %i instances", i)

# TODO: bookmarks.py sync/update
def cmd__sync(g): pass
def cmd__update(g): pass
def cmd__x(g):

    """
    XXX: hacky hack hack
    """
    global ctx
    sa = ctx.sa_session

    bms = bm_chrome.BookmarksJSON.load(g.chrome_bookmarks_file)
    chrome_refs = []
    for groups, href in bms.urls_gen('url'):
        chrome_refs.append(href)

    for lctr in Locator.all():
        if lctr.href in chrome_refs:
            continue

        print(lctr)
        continue

        lctr = Locator.fetch((Locator.ref==href,), sa=sa, exists=False)
        if not lctr:
            print(href)
        continue

# WIP: bookmarks.py couchdb

def cmd__couch_stats(g):
    global ctx
    stats = ctx.couchconn.stats()
    print('# couchdb-stat current max min mean stddev sum description')
    for k in stats['couchdb']:
        print(k, end=' ')
        for k2 in 'current', 'max', 'min', 'mean', 'stddev', 'sum', 'description':
            print(stats['couchdb'][k][k2], end=' ')
        print()

couch_views = {
    '_design/bookmarks': dict( views=dict( list=dict( map= """function(doc) {

    if (doc.type && doc.type == 'bookmark') emit(doc.id, doc.href);

}"""))),
}

def cmd__couch_init(g):

    """
    /<db-name>/_design/<design-doc>/_view/<view-name>
    """
    global ctx

    for k, v in couch_views.items():
        if k in ctx.docs:
            if g.update:
                ctx.note("Updating view %s...", k)
                del ctx.docs[k]
        else:
            ctx.note("Adding view %s...", k)
        if k not in ctx.docs:
            ctx.docs[k] = v

def cmd__couch_list(g):
    global ctx
    for ls in ctx.docs.view('bookmarks/list'):
        print(ls.id, ls.value)

def cmd__couch_add(Locator, TITLE, TAGS, g):
    """
    Add record directly to Couch. NOTE: should be using SQL API instead.
    """
    global ctx
    #[Locator] = {
    #  'type': 'bookmark',
    #  'href': Locator,
    #  'tags': TAGS
    #}
    assert isinstance(TAGS, list), TAGS
    bm = bookmark.Bookmark(id=Bookmark.keyid(Locator), href=Locator, tag_list=TAGS)
    bm.store(ctx.docs)

def cmd__sql_couch(g):

    """
    Update SQL DB Bookmark records from CouchDB.
    """

    global ctx
    total_docs = len(ctx.docs)
    i, c = 0, 0

    sa = Bookmark.get_session(g.session_name, g.dbref)
    for idref in ctx.docs:
        bmdoc = bookmark.Bookmark.load(ctx.docs, idref)
        zope.interface.classImplements(bmdoc.__class__, taxus.iface.IPyDict)
        i += 1
        ctx.note("Processing %i of %i", i, total_docs, num=i)

        lctr = Locator.fetch((Locator.ref == bmdoc.href,), exists=False)
        if not lctr:
            lctr = Locator.forge(bmdoc, g, sa=sa)

        bm = Bookmark.fetch((Bookmark.location == lctr,), exists=False)
        if not bm:
            bm = Bookmark.fetch((Bookmark.name == bmdoc.name,), exists=False)
            if bm:
                ctx.note("Name already exists for other location: %r at %r vs %r",
                        bmdoc.name, bmdoc.href, bmdoc.href )
                continue
            bm = Bookmark.forge(dict(location=lctr, **bmdoc), g, sa=sa)

        if not bm.update_from(bmdoc, location=lctr):
            if g.verbose > 2:
                ctx.note("No-op: %s", bmdoc.href)
            continue

        c += 1
        if not g.dry_run:
            sa.add(bm)
        if not g.quiet:
            ctx.note(bmdoc.href)

        if g.auto_commit and c and c % g.auto_commit == 0:
            if g.verbose > 1:
                ctx.note("Auto-commit at %i of %i", c, total_docs)
            sa.commit()

    if g.commit:
        sa.commit()

def cmd__couch_sql(NAME, opts, g):

    """
    Update CouchDB bookmark-type documents from SQL.
    """

    global ctx
    total_docs = len(ctx.docs)
    c, i = 0, 0

    # Get records to sync
    rs = ctx.get_records(Bookmark, name=NAME)
    if not rs:
        log.stdout("{yellow}Nothing found{default}")
        if g.strict:
            return 1
        return
    total_records = len(rs)
    ctx.note('Going to sync %i SQL records to %s...', total_records, ctx.couch[1])

    n, u, e = 0, 0, 0
    for r in rs:
        d = r.to_doc()
        i += 1
        _id = d.id

        map_id = d.href in ctx.docs
        if map_id:
            e += 1
            log.stderr("HREF %s", d.href)
            continue

        if _id not in ctx.docs:

            # New entry
            if not g.dry_run:
                d.store(ctx.docs)
            n += 1
            ctx.note('new %s', d.href)

        else:
            bm = bookmark.Bookmark.load(ctx.docs, _id)
            updated = False

            # NOTE: not doing anything else than bookmarks
            if bm.type != 'bookmark':
                log.stderr((
                    "Document {0} exists but is not of required type: {1}".format(
                    href, bm['type'])))
                continue

            # We can stop right there.
            if r.deleted:
                if bm.deleted:
                    continue
                bm.deleted = true
                bm.date_deleted = r.date_deleted
                # Soft delete, for archival/cleanup by someone else.
                continue

            # Check on update time to things speed up
            if g.clear_unknown_keys:
                c_dt = iso8601_datetime_format(bm.date_updated)
                if c_dt >= r.date_updated:
                    continue

            # Delete missing keys
            #if g.clear_unknown_keys:
            #    for k in bm.keys():
            #        if k in [ '_rev', '_id', 'type' ]:
            #            continue
            #        if k not in d:
            #            del bm[k]
            #            updated = True

            #if r.update_from(bm)
            #raise NotImplementedError
            # Set new or changed values
            for k in d.keys():
              if k not in bm or bm[k] != d[k]:
                  print(k, d[k], bm[k])
                  bm[k] = d[k]
                  updated = True

            if updated:
                # XXX: should I take now? rather not increment on non-descr changes
                bm['date_updated'] = r.date_updated
                if not g.dry_run:
                    bm.store(ctx.docs)
                u += 1
                if g.verbose:
                    print('updated %s' % href)

        if u:
            ctx.note("Updated doc-from-record %i, at %i from %i...", u, i, total_records, num=u)
        elif i:
            ctx.note("No doc-from-record changes at %i from %i...", i, total_records, num=i)

    ctx.note('%i new', n)
    ctx.note('%i updated', u)

def cmd__couch_update(g):

    """
    Update couchdb, preping data and fixing common errors before doing actual
    syncs. Fixes are controlled through flags.

    Unless no-db is on, an initial second run is done using all records from
    SQL. This allows to fix records using data from SQL.
    """

    global ctx
    total_docs = len(ctx.docs)
    # doc num, record num, updated from SQL, updated, skip
    d, i, c, u, e = 0, 0, 0, 0, 0

    if not g.no_db:
        # Starting with records from SQL, check every one with couch
        rs = ctx.get_records(Bookmark)
        if not rs:
            log.stdout("{yellow}Nothing found{default}")
            if g.strict:
                return 1
            return
        total_records = len(rs)
        ctx.note('Going to sync %i SQL records to %s...', total_records, ctx.couch[1])

        for r in rs:
            i += 1

            ### Fix document, or skip further updates by continueing to next doc

            # ID rename cleanup
            if g.update_hrefids:
                map_id = r.href in ctx.docs
                if map_id:
                    u += 1
                    doc = r.to_doc()
                    _id = doc.id
                    if not g.dry_run:
                        del ctx.docs[doc.href]
                        assert doc.href not in ctx.docs, doc.href
                        doc.store(ctx.docs)
                    log.stderr("Renamed href at %i to %s", i, _id)
                    continue

            if g.update_tagstype:
                _id = Bookmark.keyid(r.href)
                doc = ctx.docs[_id]
                if 'tags' in doc and isinstance(doc['tags'], list):
                    tags_split_from_string_error = None
                    for t in doc['tags']:
                        if len(t) == 1:
                            tags_split_from_string_error = True
                        if len(t) != 1:
                            tags_split_from_string_error = False
                    if tags_split_from_string_error:
                        u += 1
                        tags = "".join(doc['tags'])
                        doc['tag_list'] = tags.split(', ')
                        del doc['tags']
                        ctx.docs[doc.id] = doc

                elif 'tags' in doc:
                    u += 1
                    tags = doc['tags']
                    assert isinstance(tags, str), tags
                    assert ', ' in tags, tags
                    doc['tag_list'] = tags.split(', ')
                    del doc['tags']
                    ctx.docs[doc.id] = doc

            # TODO: other generic SQL-to-Couch updates here
            if u:
                ctx.note("Updated doc-from-record %i, at %i from %i...", u, i, total_records, num=u)
            elif i:
                ctx.note("No doc-from-record changes at %i from %i...", i, total_records, num=i)

    if c:
        ctx.note("Updated %i from %i", c, total_records)
    else:
        ctx.note("Nothing to do for %i SQL records", total_records)

    # Now loop over ID's from database again
    log.std("Checking %i docs..", total_docs)
    for idref in ctx.docs:
        d += 1
        doc = ctx.docs[idref]

        ### Fix document, or skip further updates by continueing to next doc

        # ID rename cleanup
        if idref == doc['href']:
            if not g.update_hrefids:
                e += 1
                continue
            c += 1
            bm = bookmark.Bookmark.load(ctx.docs, idref)
            bm.id = bookmark.Bookmark.key(bm)
            if not g.dry_run:
                bm.store(ctx.docs)
        assert not doc['href'] in ctx.docs, doc['href']

        # Migrate tags
        if 'tags' in doc:
            if not g.update_tagstype:
                e += 1
                continue
            c += 1
            bm = bookmark.Bookmark.load(ctx.docs, idref)

            if g.verbose > 1:
                print(idref, doc['href'], bm.date_added )

            if isinstance(doc['tags'], list):
                bm.tag_list = bm.tags
                assert isinstance(bm.tags, list)

            else:
                assert ', ' in bm.tags
                bm.tag_list = bm.tags.split(', ')

            del bm['tags']
            if not g.dry_run:
                #del ctx.docs[idref]
                bm.store(ctx.docs)

        if c:
            ctx.note("Updated %i, at %i from %i...", d, i, total_docs, num=c)
        else:
            ctx.note("Nothing to do at %i from %i...", i, total_docs, num=i)

    if c:
        ctx.note("Updated %i from %i", d, total_docs)
    else:
        ctx.note("Nothing to do for %i documents", total_docs)

"""
TODO: sync shaarli either from SQL or couch.

Also google/firefox bookmarks, chrome outliner.
"""

from shaarli_client.client import ShaarliV1Client, InvalidEndpointParameters

def cmd__shaarli_update(g):
    """
    TODO: Update SQL DB Bookmark records from Shaarli.
    """

def cmd__shaarli_sync(NAME, opts, g):
    """
    TODO: Update Shaarli bookmark-type documents from SQL.
    """

def cmd__stats(g):
    cmd__sql_stats(g)

### Transform cmd__ function names to nested dict

commands = libcmd_docopt.get_cmd_handlers(globals(), 'cmd__')
commands.update(dict(
        help = libcmd_docopt.cmd_help,
        memdebug = libcmd_docopt.cmd_memdebug,
        info = db_sa.cmd_info,
        init = db_sa.cmd_init,
        clear = db_sa.cmd_reset
))

### Util functions to run above functions from cmdline

def defaults(opts, init={}):
    global cmd_default_settings, ctx
    libcmd_docopt.defaults(opts)
    opts.flags.update(cmd_default_settings)
    ctx.settings.update(opts.flags)
    opts.flags.update(ctx.settings)
    opts.flags.update(dict(
        commit = not opts.flags.no_commit and not opts.flags.dry_run,
        verbose = opts.flags.quiet and opts.flags.verbose or 1,
    ))
    if not opts.flags.interactive:
        if os.isatty(sys.stdout.fileno()) and os.isatty(sys.stdout.fileno()):
            opts.flags.interactive = True
    opts.flags.update(dict(
        partial_match = not opts.flags.exact_match,
        auto_commit = not opts.flags.no_commit and opts.flags.auto_commit,
        dbref = taxus.ScriptMixin.assert_dbref(opts.flags.dbref)
    ))
    if opts.flags.interval:
        opts.flags.interval = int(opts.flags.interval)
    if opts.flags.auto_commit:
        opts.flags.auto_commit = int(opts.flags.auto_commit)
    return init

def main(opts):

    """
    Execute using docopt-mpe options.
    """
    global ctx, commands

    ws = Homedir.require()
    ws.yamldoc('bmsync', defaults=dict(
            last_sync=None
        ))
    ctx.ws = ws
    ctx.settings = settings = opts.flags
    ctx.init()
    ret = libcmd_docopt.run_commands(commands, settings, opts)
    if settings.print_memory:
        libcmd_docopt.cmd_memdebug(settings)
    return ret

def get_version():
    return 'bookmarks.mpe/%s' % __version__

if __name__ == '__main__':
    #from importlib import reload
    #reload(sys)
    # sys.setdefaultencoding('utf-8') FIXME: see 'why we should not use ...'

    usage = __description__ +'\n\n'+ __short_description__ +'\n'+ \
            libcmd_docopt.static_vars_from_env(__usage__,
        ( 'BM_DB', __db__ ),
        ( 'COUCH_DB', __couch__ ) )

    db_sa.schema = sys.modules['__main__']
    db_sa.metadata = SqlBase.metadata

    opts = libcmd_docopt.get_opts(usage,
            version=get_version(), defaults=defaults)
    # TODO: mask secrets
    #log.std("Connecting to %s", opts.flags.dbref)
    #log.std("Connecting to %s", opts.flags.couch)
    sys.exit(main(opts))


import os
import time
import pytest

if __package__ is None:
    import sys
    from os import path
    sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))

from infra.test_base import TestBase, idparametrize
from tools.constants import LINK_SPEED_100M, LINK_SPEED_1G, LINK_SPEED_2_5G, LINK_SPEED_5G, LINK_SPEED_10G
from tools.command import Command
from tools.communicate import send_performance_results, PerfRecord
from trafficgen.traffic_gen import Packets

from dpdk_test_base import TestDPDKBase, PATH_TO_CONFIG_TXT, filter_match_get
from tools.utils import get_atf_logger

log = get_atf_logger()

def setup_module(module):
    # import tools._test_setup  # uncomment for manual test setup
    os.environ["TEST"] = "dpdk-perf"

class TestDPDKPerf(TestDPDKBase):

    @classmethod
    def setup_class(cls):
        super(TestDPDKPerf, cls).setup_class()

    def report(self, name=''):
        log.info('>> REPORT: ----------------------------------------------------------------------- ')
        log.info('>> | {:75s} | '.format(name))
        log.info('>> ------------------------------------------------------------------------------- ')
        log.info('>> |       Packet |      Mpps |              pps | line rate (%) | Bitrate (Mbps)| ')
        log.info('>> ------------------------------------------------------------------------------- ')

        results = []

        gmin = 1 << 31

        for k in sorted(self.reports.keys(), key=lambda x: float(x)):
            ps = int(k)
            pps = max(self.reports[k]) if len(self.reports[k]) > 0 else 0
            mpps = pps / 1000000.0
            bps = ps * pps * 8
            mbps = bps / 1000000.0
            lr = 100.0 * (mbps / 10000.0)
            msg = '>> | {:>12d} | {:9.2f} | {:>16d} | {:13.2f} | {:13.2f} | '.format(ps, mpps, pps, lr, mbps)
            log.info(msg)
            gmin = min(gmin, mbps)

            results.append(PerfRecord('Mbps', mbps, mbps, mbps, 1, 'Performance', '{{"packet_size": {}}}'.format(ps)))
            results.append(PerfRecord('Mpps', mpps, mpps, mpps, 1, 'Performance', '{{"packet_size": {}}}'.format(ps)))

        log.info('>> ------------------------------------------------------------------------------- ')
        name = TestBase.state.current_test if TestBase.state.current_test else "DPDK"
        send_performance_results(results, self.atf_os, self.job_id, name)

        assert gmin > 0, 'Bandwidth must be more 0'

    def test_performance_test_testpmd_multicore(self):
        self.reports = {}

        # generate config
        config = [
            'set verbose 1',
            'set log global 4',
            'set log pmd 4',

            'set portlist 0,1',
            'show config fwd',

            'port stop all',

            'port config all rxq 8',
            'port config all txq 8',

            'set stat_qmap rx 0 0 0',
            'set stat_qmap rx 0 1 1',
            'set stat_qmap rx 0 2 2',
            'set stat_qmap rx 0 3 3',
            'set stat_qmap rx 0 4 4',
            'set stat_qmap rx 0 5 5',
            'set stat_qmap rx 0 6 6',
            'set stat_qmap rx 0 7 7',

            'set stat_qmap tx 0 0 0',
            'set stat_qmap tx 0 1 1',
            'set stat_qmap tx 0 2 2',
            'set stat_qmap tx 0 3 3',
            'set stat_qmap tx 0 4 4',
            'set stat_qmap tx 0 5 5',
            'set stat_qmap tx 0 6 6',
            'set stat_qmap tx 0 7 7',

            'port config 0 rxq 0 ring_size 65535',
            'port config 0 rxq 1 ring_size 65535',
            'port config 0 rxq 2 ring_size 65535',
            'port config 0 rxq 3 ring_size 65535',
            'port config 0 rxq 4 ring_size 65535',
            'port config 0 rxq 5 ring_size 65535',
            'port config 0 rxq 6 ring_size 65535',
            'port config 0 rxq 7 ring_size 65535',

            'port config 0 txq 0 ring_size 65535',
            'port config 0 txq 1 ring_size 65535',
            'port config 0 txq 2 ring_size 65535',
            'port config 0 txq 3 ring_size 65535',
            'port config 0 txq 4 ring_size 65535',
            'port config 0 txq 5 ring_size 65535',
            'port config 0 txq 6 ring_size 65535',
            'port config 0 txq 7 ring_size 65535',

            'port config 1 rxq 0 ring_size 65535',
            'port config 1 rxq 1 ring_size 65535',
            'port config 1 rxq 2 ring_size 65535',
            'port config 1 rxq 3 ring_size 65535',
            'port config 1 rxq 4 ring_size 65535',
            'port config 1 rxq 5 ring_size 65535',
            'port config 1 rxq 6 ring_size 65535',
            'port config 1 rxq 7 ring_size 65535',

            'port config 1 txq 0 ring_size 65535',
            'port config 1 txq 1 ring_size 65535',
            'port config 1 txq 2 ring_size 65535',
            'port config 1 txq 3 ring_size 65535',
            'port config 1 txq 4 ring_size 65535',
            'port config 1 txq 5 ring_size 65535',
            'port config 1 txq 6 ring_size 65535',
            'port config 1 txq 7 ring_size 65535',

            'port config all burst 512',

            'port start 0',
            'port start 1',

            'port config all rss default',

            'set link-up port 0',
            'set link-up port 1',

            'show port info all',

            'set fwd macswap',
            'start'
        ]

        with open(PATH_TO_CONFIG_TXT, 'w') as f:
            for line in config:
                f.write(line + '\n')

        self.public_file(PATH_TO_CONFIG_TXT)

        for ps in self.PACKET_SIZE:
            packets_args = {
                'pktsize': ps
            }

            args = {
                'packets': Packets(**packets_args),
                'duration': 35
            }

            self.traffic_generator.start(**args)

            # run test
            result = self.run_app(command='./app/dpdk-testpmd',
                                  params_pmd = '-l 1-10 -n 4',
                                  params = '--portmask=0xf --nb-cores=8 --stats-period=3 --cmdline-file={}'.format(PATH_TO_CONFIG_TXT),
                                  timeout=35)

            arr = filter_match_get(result['output'], '\s*Rx-pps:\s+(\d+)')[5:-5]
            self.reports[str(ps)] = list(map(lambda x: int(x), arr))
            self.traffic_generator.stop()
            time.sleep(1)

        self.report('Testpmd: multicore (8 cores)')

    @idparametrize("speed", [LINK_SPEED_100M, LINK_SPEED_1G, LINK_SPEED_2_5G, LINK_SPEED_5G, LINK_SPEED_10G])
    def test_performance_test_testpmd(self, speed):
        self.lkp_ifconfig.set_link_speed(speed)

        self.reports = {}
        for ps in self.PACKET_SIZE:
            packets_args = {
                'pktsize': ps
            }

            args = {
                'packets': Packets(**packets_args),
                'duration': 35
            }

            self.traffic_generator.start(**args)

            # run test
            result = self.run_app(command='./app/dpdk-testpmd',
                                  params_pmd = '-l 5,6 -n 4',
                                  params = '--auto-start --stats-period=3 --portmask=0x3 --txd=2048 --rxd=2048 --txq=2 --rxq=2',
                                  timeout=35)

            arr = filter_match_get(result['output'], '\s*Rx-pps:\s+(\d+)')[5:-5]
            self.reports[str(ps)] = list(map(lambda x: int(x), arr))

            self.traffic_generator.stop()
            time.sleep(1)

        self.report('Testpmd: single cores: {}'.format(speed))

    def test_performance_test_loopback(self):
        self.reports = {}
        for ps in self.PACKET_SIZE:
            packets_args = {
                'pktsize': ps
            }

            args = {
                'packets': Packets(**packets_args),
                'duration': 35
            }

            self.traffic_generator.start(**args)

            result = self.run_app(command='./examples/dpdk-loopback',
                                  params_pmd = '-l 1,2,3,5 -n 2 -w 0000:{} -w 0000:{}'.format(
                                      self.machines['dut']['port0'], self.machines['dut']['port1']),
                                  params='-p 0xf --config="(0,0,1),(0,1,2),(1,0,3),(1,1,5)"',
                                  timeout=35)

            arr = filter_match_get(result['output'], 'rx: (\d+)pps/\d+Mbps tx: \d+pps/\d+Mbps drops: \d+')[5:-5]
            self.reports[str(ps)] = list(map(lambda x: int(x), arr))

            self.traffic_generator.stop()
            time.sleep(1)

        self.report('loopback')

if __name__ == "__main__":
    pytest.main([__file__, "-s", "-v"])


from sklearn.metrics import ranking, confusion_matrix
import matplotlib.pyplot as plt
import itertools
import numpy as np

def show_results(y_test, prob_test, name, show=True, output_folder='', maxFNR=0.03, thresh = None):
    auc = ranking.roc_auc_score(y_test, prob_test, average=None, sample_weight=None)

    fpr, tpr, thresholds = ranking.roc_curve(y_test, prob_test, pos_label=1, sample_weight=None)
    fnr = 1 - tpr

    eer = min(zip(fpr, fnr, thresholds), key=lambda x: abs(x[0] - x[1]))

    idx_fnr = np.where(fnr<maxFNR)[0][0]
    if thresh == None:
        target_fnr = thresholds[idx_fnr]
    else:
        target_fnr = thresh
    y_pred = [float(score>=target_fnr) for score in prob_test]

    #fig = plt.figure()

    # show ROC
    if show:
        plt.figure(221)
        plt.plot(fpr, tpr, linewidth=2)
        plt.ylim(0, 1)
        plt.xlim(0, 1)
        plt.xlabel('FPR')
        plt.ylabel('TPR')
        plt.title(name + ' - ROC curve, AUC = %f' % (auc))

        # show FPR-FNR vs threshold curves
        plt.figure(222)
        fnr_line, = plt.plot(thresholds, fnr * 100, linewidth=2, color='blue')
        fpr_line, = plt.plot(thresholds, fpr * 100, linewidth=2, color='red', linestyle='--')
        plt.legend([fnr_line, fpr_line], ['False Negative Rate (FNR)', 'False Positive Rate (FPR)'])
        plt.ylim(0, 100.001)
        plt.xlim(np.min(prob_test), np.max(prob_test))
        plt.title(name + ' - EER = %0.1f%% at t=%0.2f' % (100 * (eer[0] + eer[1]) / 2, eer[2]))
        plt.show()

    tn, fp, fn, tp = confusion_matrix(y_test, y_pred).ravel()

    print ('AUC = %.2f' % (auc))
    print ('Confusion matrix (absolute frequency) at threshold = %.2f' % (target_fnr))
    print ('+---------------+------------+------------+')
    print ('|               |          TRUTH          |')
    print ('+---------------+------------+------------+')
    print ('|   PREDICTED   |  LEGIT(1)  |  FAKE (0)  |')
    print ('+---------------+------------+------------+')
    print ('|    LEGIT (1)  |%12d|%12d|' % (tp, fp))
    print ('+---------------+------------+------------+')
    print ('|     FAKE (0)  |%12d|%12d|' % (fn, tn))
    print ('+---------------+------------+------------+')

    print ('Confusion matrix (relative to |LEGIT| and |FAKE|) at threshold = %.2f' % (target_fnr))
    print ('+---------------+------------+------------+')
    print ('|               |          TRUTH          |')
    print ('+---------------+------------+------------+')
    print ('|   PREDICTED   |  LEGIT(1)  |  FAKE (0)  |')
    print ('+---------------+------------+------------+')
    print ('|    LEGIT (1)  |%11.1f%%|%11.1f%%|' % (tp*100.0/(tp+fn), fp*100.0/(fp+tn)))
    print ('+---------------+------------+------------+')
    print ('|     FAKE (0)  |%11.1f%%|%11.1f%%|' % (fn*100.0/(tp+fn), tn*100.0/(fp+tn)))
    print ('+---------------+------------+------------+')

    return y_pred, target_fnr


from flask_restful import Resource, reqparse
from flask_jwt_extended import jwt_required, get_jwt_claims, get_jwt_identity, jwt_optional, fresh_jwt_required
from models.responsavel import ResponsavelModel

class Responsavel(Resource):
    parser = reqparse.RequestParser()
    parser.add_argument('telefone',
                        #type=int,
                        required=True,
                        help="O campo não pode esta vazio!"
                        )
    parser.add_argument('cpf',
                        #type=int,
                        required=True,
                        help="O campo não pode esta vazio!"
                        )
    parser.add_argument('msg',
                        #type=int,
                        required=True,
                        help="O campo não pode esta vazio!"
                        )
    parser.add_argument('user_id',
                        type=int,
                        required=True,
                        help="O campo não pode esta vazio!"
                        )

    @jwt_required  # No longer needs brackets
    def get(self, name):
        responsavel = ResponsavelModel.find_by_name(name)
        if responsavel:
            return responsavel.json()
        return {'message': 'Responsavel not found'}, 404

    @jwt_required
    def post(self, name):
        #if ResponsavelModel.find_by_name(name):
        #    return {'message': "An responsavel with name '{}' already exists.".format(name), "st":"2"}, 400

        data = Responsavel.parser.parse_args()

        responsavel = ResponsavelModel(name, **data)

        try:
            responsavel.save_to_db()
        except:
            return {"message": "Erro ao tentar enviar os dados", "st":"2"}, 500

        return (responsavel.json(), {'message': 'Responsável cadastrado com sucesso', 'st':'1'}), 201

class ResponsavelDelete(Resource):
    parser = reqparse.RequestParser()
    parser.add_argument('name',
                        #type=int,
                        required=True,
                        help="O campo não pode esta vazio!"
                        )
    parser.add_argument('telefone',
                        #type=int,
                        required=True,
                        help="O campo não pode esta vazio!"
                        )
    parser.add_argument('msg',
                        #type=int,
                        required=True,
                        help="O campo não pode esta vazio!"
                        )
    parser.add_argument('cpf',
                        #type=int,
                        required=True,
                        help="O campo não pode esta vazio!"
                        )
    parser.add_argument('user_id',
                        type=int,
                        required=True,
                        help="O campo não pode esta vazio!"
                        )

    @jwt_required
    def delete(self, id):

        responsavel = ResponsavelModel.find_by_id_unique(id)
        if responsavel:
            responsavel.delete_from_db()
            return {'message': 'Responsavel deleted.', 'st':'1'}, 201
        return {'message': 'Responsavel not found.'}, 404

    @jwt_required
    def put(self, id: int):
        data = ResponsavelDelete.parser.parse_args()

        responsavel = ResponsavelModel.find_by_id_unique(id)

        if responsavel:
            responsavel.name = data['name']
            responsavel.telefone = data['telefone']
            responsavel.cpf = data['cpf']
            responsavel.msg = data['msg']
            responsavel.user_id = data['user_id']
        else:
            responsavel = ResponsavelModel(id, **data)

        responsavel.save_to_db()

        return (responsavel.json(), {'message': 'Responsável alterado com sucesso', 'st':'1'}), 201

    @jwt_required  # No longer needs brackets
    def get(self, id: int):
        responsavel = ResponsavelModel.find_by_id_unique(id)
        if responsavel:
            return (responsavel.json(), {'message': 'Dados do Responsável', 'st':'1'}), 201

        return {'message': 'Responsavel not found', 'st':'1'}, 404

class ResponsavelList(Resource):
    @jwt_required
    def get(self):
        """
        Here we get the JWT identity, and then if the user is logged in (we were able to get an identity)
        we return the entire responsavel list.

        Otherwise we just return the responsavel names.

        This could be done with e.g. see orders that have been placed, but not see details about the orders
        unless the user has logged in.
        """
        user_id = get_jwt_identity()
        responsavels = [responsavel.json() for responsavel in ResponsavelModel.find_all()]
        if user_id:
            return {'responsavels': responsavels}, 200
        return {
            'responsavels': [responsavel['name'] for responsavel in responsavels],
            'message': 'More data available if you log in.'
        }, 200

class ResponsavelSelecao(Resource):
    @jwt_required
    #@classmethod
    def get(self):
        user_id = get_jwt_identity()
        responsavels = [responsavel.json() for responsavel in ResponsavelModel.find_by_id(user_id)]
        if user_id:
            return {'dependentes': responsavels, 'msg':'ok', 'st':'1'}, 201
        return {
            'responsavels': [responsavel['name'] for responsavel in responsavel],
            'message': 'More data available if you log in.'
        }, 200


import json

import config
from utils import tools, rdpool

async def gen_access_token(user_id, remote_ip):
    if user_id is None or remote_ip is None:
        return None

    token = tools.gen_id1()

    record = {'token': token, 'remote_ip': remote_ip}
    key = _key(user_id, token)
    rdpool.rds.set(key, json.dumps(record), config.AccessTokenTimeout)

    return token

async def verify_access_token(user_id, token, remote_ip):
    key = _key(user_id, token)

    data = rdpool.rds.get(key)
    if data is None:
        return False
    record = json.loads(data)

    if record['token'] == token and record['remote_ip'] == remote_ip:
        rdpool.rds.expire(key, config.AccessTokenTimeout)
        return True
    else:
        return False

async def remove_access_token(user_id, token):
    rdpool.rds.delete(_key(user_id, token))
    return True

def _key(user_id, token):
    if config.LoginMode == 'single':
        return 'safe-chat/user/login/{}'.format(user_id)
    else:
        return 'safe-chat/user/login/{}/{}'.format(user_id, token)


import os
import calendar
import shelve
import sys

script_path = os.path.dirname(os.path.realpath(__file__))
os.chdir(script_path)
# print(script_path)

year = int(input("Enter year: "))

if os.path.isdir(str(year)):
    os.chdir(str(year))
else:
    sys.exit("Invalid year.")

month = int(input("Enter month (1 to 12 inclusive): "))
month_name = str(calendar.month_name[month])

if not os.path.isfile(month_name + ".dat"):
    sys.exit("There is no such file in this directory.")

# day = int(input("Enter day: "))

database = shelve.open(month_name)

print("\nDatabase contents:")
for element in database:
    print(element)
    print(database[element])

database.close()
os.chdir(script_path)


# logic:to reach every path, then try to get the left leaves appended to a res=[]

# to verify the things need to implement at the sub tree level

def sumOfLeftLeaves(self, root):
    if not root:
        return 0
    res = []

    def summ(root):
        if root:
            if root.left and not root.left.left and not root.left.right:
                res.append(root.left.val)
            summ(root.left)
            summ(root.right)

    summ(root)
    # print(res)
    return sum(res)


from bmaclient import MonitoringAPI
from bmaclient.utils import get_api_key

def main():
    api = MonitoringAPI(api_key=get_api_key())
    users = api.fetch_users()
    print(users)

if __name__ == "__main__":
    main()


from django.shortcuts import render, redirect
from django.urls import reverse

from expenses.forms import ExpenseForm, FeebackForm
from . import models

def list_months(request):
    qs = models.Expense.objects.dates('date', 'month').order_by('-date')
    return render(request, "expenses/expense_months.html", {
        'dates': qs,
    })

def list(request, year=None, month=None):
    qs = models.Expense.objects.all()
    if year:
        qs = qs.filter(date__year=year)
    if month:
        qs = qs.filter(date__month=month)
    total = sum(
        o.amount for o in qs)  # there is a better way to do this (aggregation)
    return render(request, "expenses/expense_list.html", {
        'total': total,
        'objects': qs,
    })

def detail(request, id):
    o = models.Expense.objects.get(id=id)
    return render(request, "expenses/expense_detail.html", {
        'object': o,
    })

def create(request):
    if request.method == "POST":
        form = ExpenseForm(request.POST)
        if form.is_valid():
            o = form.save()
            return redirect(reverse("expenses:detail", args=(o.id,)))
    else:
        form = ExpenseForm()

    return render(request, "expenses/expense_form.html", {
        'form': form,
    })

def send_feedback(request):
    if request.method == "POST":
        form = FeebackForm(request.POST)
        if form.is_valid():
            print("feeback sent", form.cleaned_data)
            return redirect(reverse("expenses:list"))
    else:
        form = FeebackForm()

    return render(request, "expenses/feedback_form.html", {
        'form': form,
    })


import os
import shutil

import cv2

from config import (
    IMG_DIR,
    SAVE_IMG_EXTENSION,
    SAVE_VIDEO_EXTENSION,
    VIDEO_DIR,
    VIDEO_EXTENTIONS,
)

class VideoReader:
    def __init__(self, path):
        self.path = path
        self.read_video(self.path)

    def read_video(self, path):
        if os.path.isfile(path):
            if any(path.endswith(ext) for ext in VIDEO_EXTENTIONS):
                self.cap = cv2.VideoCapture(path)
            else:
                raise ValueError("File extension is not supported")
        else:
            raise ValueError("File does not exist")

    def get_size(self):
        return (
            self.cap.get(cv2.CAP_PROP_FRAME_WIDTH),
            self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT),
        )

    def get_fps(self):
        return self.cap.get(cv2.CAP_PROP_FPS)

    def get_frame_count(self):
        return int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT))

    def get_current_frame(self):
        return self.cap.get(cv2.CAP_PROP_POS_FRAMES)

    def get_name(self):
        return os.path.splitext(os.path.basename(self.path))[0]

def get_basename(file_path):
    return os.path.splitext(os.path.basename(file_path))[0]

def split_video(
    input_path, output_dir, rotate_direction: str = "None"
) -> None:
    video_reader = VideoReader(input_path)

    cap = video_reader.cap
    video_name = video_reader.get_name()

    output_dir = os.path.join(output_dir, video_name)
    os.makedirs(output_dir, exist_ok=True)

    while True:
        ret, frame = cap.read()
        if ret:
            frame_index = int(cap.get(cv2.CAP_PROP_POS_FRAMES))
            if rotate_direction == "left":
                frame = cv2.rotate(frame, cv2.ROTATE_90_COUNTERCLOCKWISE)
            elif rotate_direction == "right":
                frame = cv2.rotate(frame, cv2.ROTATE_90_CLOCKWISE)

            frame_name = f"{video_name}_{frame_index}{SAVE_IMG_EXTENSION}"
            output_path = os.path.join(output_dir, frame_name)
            cv2.imwrite(output_path, frame)
        else:
            break

    cap.release()
    return None

def process_video(
    video_reader: VideoReader,
    output_video_name: str,
    base_output_dir: str,
    start_frame: int,
    end_frame: int,
    rotate_direction: str = "none",
) -> None:
    cap = video_reader.cap
    fps = video_reader.get_fps()
    width, height = video_reader.get_size()

    video_dir = os.path.join(base_output_dir, VIDEO_DIR)
    img_dir = os.path.join(base_output_dir, IMG_DIR)
    os.makedirs(video_dir, exist_ok=True)
    os.makedirs(img_dir, exist_ok=True)

    fourcc = cv2.VideoWriter_fourcc(*"mp4v")
    video_output_path = os.path.join(
        video_dir, output_video_name + SAVE_VIDEO_EXTENSION
    )
    out = cv2.VideoWriter(
        video_output_path, fourcc, fps, (int(width), int(height))
    )

    cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame)

    while cap.isOpened():
        ret, frame = cap.read()
        if ret:
            frame_index = int(cap.get(cv2.CAP_PROP_POS_FRAMES))

            if rotate_direction == "right":
                frame = cv2.rotate(frame, cv2.ROTATE_90_COUNTERCLOCKWISE)
            elif rotate_direction == "left":
                frame = cv2.rotate(frame, cv2.ROTATE_90_CLOCKWISE)

            out.write(frame)
            frame_name = (
                f"{output_video_name}_{frame_index}{SAVE_IMG_EXTENSION}"
            )
            image_output_path = os.path.join(img_dir, frame_name)
            cv2.imwrite(image_output_path, frame)
            if frame_index == end_frame:
                break
        else:
            break

    cap.release()
    out.release()

def reconstruct_video(
    video_reader: VideoReader,
    output_video_name,
    start_frame: int,
    end_frame: int,
) -> None:
    cap = video_reader.cap
    fps = video_reader.get_fps()
    width, height = video_reader.get_size()

    fourcc = cv2.VideoWriter_fourcc(*"mp4v")
    output_path = output_video_name + SAVE_VIDEO_EXTENSION
    out = cv2.VideoWriter(output_path, fourcc, fps, (int(width), int(height)))

    cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame)

    while cap.isOpened():
        ret, frame = cap.read()
        if ret:
            out.write(frame)
            if cap.get(cv2.CAP_PROP_POS_FRAMES) == end_frame:
                break
        else:
            break

    cap.release()
    out.release()
    return None

def save_frames_as_images(
    video_reader: VideoReader,
    output_dir: str,
    start_frame: int,
    end_frame: int,
) -> None:
    video_name = video_reader.get_name()
    os.makedirs(output_dir, exist_ok=True)

    cap = video_reader.cap

    cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame)

    while cap.isOpened():
        ret, frame = cap.read()
        if ret:
            frame_index = int(cap.get(cv2.CAP_PROP_POS_FRAMES))
            if frame_index > end_frame:
                break

            frame_name = f"{video_name}_{frame_index}{SAVE_IMG_EXTENSION}"
            output_path = os.path.join(output_dir, frame_name)
            cv2.imwrite(output_path, frame)
        else:
            break

    cap.release()

def delete_img_dir(img_dir) -> None:
    shutil.rmtree(img_dir)


import numpy as np
from robot_mouse_track.mouse_track import MouseTrack
from robot_mouse_track.utils import small_runs

class LinearMotion:
    """
    斜直线运动（人手可能会画出横/竖直线，但是画不出来斜直线）

    :var int default=5 least_point: 这条线上至少有多少个点
    :var int default=200 th_length: 斜线运动长度超过多少，认为是风险，单位px
    :var float default=0.1 th_span: 两点之间的最大方向角变化幅度，低于这个值，则认为是风险
    """

    def __init__(self):
        self.least_point = 5  # 这条线上至少有多少个点
        self.th_length = 200  # 斜线运动长度超过多少，认为是风险，单位px
        self.th_span = 0.1

    def judge_risk(self, mouse_track: MouseTrack):
        """
        风险判定

        :param MouseTrack mouse_track: 鼠标轨迹对象
        :return: (have_risk, risk_level)
        :rtype: (bool, float)
        """

        # n个点算斜率，斜率相近表示平行
        feature_doa = mouse_track.get_feature_doa()

        # 斜率变化幅度较小则认为是斜线
        # 获取斜率变化幅度较小的片段的左右闭区间
        lst_small = small_runs(feature_doa, span=self.th_span)
        max_length = 0
        for left, right in lst_small:
            if right - left + 1 < self.least_point:
                # 如果该直线上的点的个数少，则不考虑
                continue
            point1 = mouse_track.arr_trace[left, :-1]
            point2 = mouse_track.arr_trace[right, :-1]
            length = np.sum((point2 - point1) ** 2) ** 0.5
            if length < self.th_length:
                # 如果该直线的长度小，则不考虑
                continue
            if length > max_length:  # 获取该轨迹中最长的直线长度
                max_length = length

        exceed_times_length = max_length / self.th_length
        if exceed_times_length > 1.0:
            return True, exceed_times_length
        return False, exceed_times_length


# -*- coding: utf-8 -*-
"""
Demonstrates GLVolumeItem for displaying volumetric data.

"""

## Add path to library (just for examples; you do not need this)
import initExample

from pyqtgraph.Qt import QtCore, QtGui
import pyqtgraph.opengl as gl

app = QtGui.QApplication([])
w = gl.GLViewWidget()
w.opts['distance'] = 200
w.show()
w.setWindowTitle('pyqtgraph example: GLVolumeItem')

#b = gl.GLBoxItem()
#w.addItem(b)
g = gl.GLGridItem()
g.scale(10, 10, 1)
w.addItem(g)

import numpy as np
## Hydrogen electron probability density
def psi(i, j, k, offset=(50,50,100)):
    x = i-offset[0]
    y = j-offset[1]
    z = k-offset[2]
    th = np.arctan2(z, (x**2+y**2)**0.5)
    phi = np.arctan2(y, x)
    r = (x**2 + y**2 + z **2)**0.5
    a0 = 2
    #ps = (1./81.) * (2./np.pi)**0.5 * (1./a0)**(3/2) * (6 - r/a0) * (r/a0) * np.exp(-r/(3*a0)) * np.cos(th)
    ps = (1./81.) * 1./(6.*np.pi)**0.5 * (1./a0)**(3/2) * (r/a0)**2 * np.exp(-r/(3*a0)) * (3 * np.cos(th)**2 - 1)

    return ps

    #return ((1./81.) * (1./np.pi)**0.5 * (1./a0)**(3/2) * (r/a0)**2 * (r/a0) * np.exp(-r/(3*a0)) * np.sin(th) * np.cos(th) * np.exp(2 * 1j * phi))**2

data = np.fromfunction(psi, (100,100,200))
positive = np.log(np.clip(data, 0, data.max())**2)
negative = np.log(np.clip(-data, 0, -data.min())**2)

d2 = np.empty(data.shape + (4,), dtype=np.ubyte)
d2[..., 0] = positive * (255./positive.max())
d2[..., 1] = negative * (255./negative.max())
d2[..., 2] = d2[...,1]
d2[..., 3] = d2[..., 0]*0.3 + d2[..., 1]*0.3
d2[..., 3] = (d2[..., 3].astype(float) / 255.) **2 * 255

d2[:, 0, 0] = [255,0,0,100]
d2[0, :, 0] = [0,255,0,100]
d2[0, 0, :] = [0,0,255,100]

v = gl.GLVolumeItem(d2)
v.translate(-50,-50,-100)
w.addItem(v)

ax = gl.GLAxisItem()
w.addItem(ax)

## Start Qt event loop unless running in interactive mode.
if __name__ == '__main__':
    import sys
    if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'):
        QtGui.QApplication.instance().exec_()


import os
import tempfile
import pytest
import time
import parse
import utils
from utils import ClientProcess

def get_num_running_clients(clients):
    count = 0
    for client in clients:
        if client.is_running():
            count += 1
    return count

@pytest.mark.timeout(300)
def test_pipeline_execution_positive(helpers, test_case, test_filename, generate):
    workdir_path = tempfile.TemporaryDirectory()
    clients = []
    is_max_frames_set = False
    num_of_concurrent_clients = 0
    for test in test_case["client"]:
        num_of_concurrent_clients += test.get("num_of_concurrent_clients", 1)
        if test["params"].get("max_frames"):
            is_max_frames_set = False

    if (len(test_case["client"]) > 1) and is_max_frames_set and test_case["golden_results"]:
        assert False, "Cannot limit number of video frames with multiple client test cases using ground truth"

    is_concurrent_test = num_of_concurrent_clients > 1
    helpers.run_server(test_case["server_params"], capture_log=is_concurrent_test)
    for test in test_case["client"]:
        for _ in range(test.get("num_of_concurrent_clients", 1)):
            output_file = "output{}.jsonl".format(len(clients))
            output_location = os.path.join(workdir_path.name, output_file)
            print("Spawning concurrent process with output location: {}".format(output_location))
            test["params"]["output_location"] = output_location
            client = ClientProcess(helpers, test["params"], is_concurrent_test)
            clients.append(client)

    if is_concurrent_test:
        sleep_duration = 0.25
        num_pipelines_running = 0
        num_pipelines_stopped = 0
        max_running_pipelines = 0
        while get_num_running_clients(clients) > 0 or num_pipelines_stopped != num_of_concurrent_clients:
            messages = helpers.get_server_log_messages()
            for message in messages:
                if parse.parse("Setting Pipeline {} State to RUNNING", message):
                    print("Test: " + message)
                    num_pipelines_running += 1

                if num_pipelines_running > max_running_pipelines:
                    max_running_pipelines = num_pipelines_running
                    print("Test: max_running_pipelines = {}".format(max_running_pipelines))

                if parse.parse("Pipeline {} Ended", message):
                    print("Test: " + message)
                    num_pipelines_running -= 1
                    num_pipelines_stopped += 1

            for client in clients:
                if not client.is_running():
                    assert client.has_correct_return_code(), "Incorrect return code"
            time.sleep(sleep_duration)

        if test_case["server_params"].get("max_running_pipelines", 0) > 1:
            assert max_running_pipelines == num_of_concurrent_clients, "Not all pipelines running in concurrent mode"
        else:
            assert max_running_pipelines == 1, "More than one concurrent pipeline running in serial mode"

    for client in clients:
        client.stop()
        utils.validate_output_against_schema(client.get_output_location())

    if test_case["golden_results"]:
        utils.golden_results(clients, test_case, generate, test_filename)


import random
number=random.randrange(1,10)
n1=random.randint(1,10)
n2=random.choice(['red','green','blue','yellow'])
character="abcdefghijklmnopqrstuvwxyz"
print(random.choice(character))
print(number)
print(n1)
print(n2)
for i in range(1,4):

    ans=[]
    ans=input("enter favorite  colors "  )
# for aj in range (1,22):
#     print(random.choice([ans[aj]]))


print('Gerador de PA')
print('-=' * 10)
pt = int(input('Digite o primeiro termo da PA: '))
razao = int(input('Digite a Razão da PA: '))
termo = pt
cont = 1
while cont <= 10:
    print('{} -> '.format(termo), end="")
    termo = termo + razao
    cont += 1

print('Fim')


import os
from subprocess import check_output

# Numpy/mpi4py must be installed prior to installing TACS
import numpy
import mpi4py

# Import distutils
from setuptools import setup
from distutils.core import Extension as Ext
from Cython.Build import cythonize

# Convert from local to absolute directories
def get_global_dir(files):
    tmr_root = os.path.abspath(os.path.dirname(__file__))
    new = []
    for f in files:
        new.append(os.path.join(tmr_root, f))
    return new

def get_mpi_flags():
    # Split the output from the mpicxx command
    args = check_output(["mpicxx", "-show"]).decode("utf-8").split()

    # Determine whether the output is an include/link/lib command
    inc_dirs, lib_dirs, libs = [], [], []
    for flag_ in args:
        try:
            flag = flag_.decode("utf-8")
        except:
            flag = flag_

        if flag[:2] == "-I":
            inc_dirs.append(flag[2:])
        elif flag[:2] == "-L":
            lib_dirs.append(flag[2:])
        elif flag[:2] == "-l":
            libs.append(flag[2:])

    return inc_dirs, lib_dirs, libs

inc_dirs, lib_dirs, libs = get_mpi_flags()

# Relative paths for the include/library directories
rel_inc_dirs = ["src"]
rel_lib_dirs = ["lib"]
libs.extend(["paropt"])

# Convert from relative to absolute directories
inc_dirs.extend(get_global_dir(rel_inc_dirs))
lib_dirs.extend(get_global_dir(rel_lib_dirs))

# Add the numpy/mpi4py directories
inc_dirs.extend([numpy.get_include(), mpi4py.get_include()])

# Add tacs-dev/lib as a runtime directory
runtime_lib_dirs = get_global_dir(["lib"])

exts = []
mod = "ParOpt"
exts.append(
    Ext(
        "paropt.ParOpt",
        sources=["paropt/ParOpt.pyx"],
        language="c++",
        include_dirs=inc_dirs,
        libraries=libs,
        library_dirs=lib_dirs,
        runtime_library_dirs=runtime_lib_dirs,
    )
)
exts.append(
    Ext(
        "paropt.ParOptEig",
        sources=["paropt/ParOptEig.pyx"],
        language="c++",
        include_dirs=inc_dirs,
        libraries=libs,
        library_dirs=lib_dirs,
        runtime_library_dirs=runtime_lib_dirs,
    )
)

for e in exts:
    e.cython_directives = {
        "language_level": "3",
        "embedsignature": True,
        "binding": True,
    }

optional_dependencies = {
    "testing": ["testflo>=1.4.7"],
    "docs": [
        "sphinx",
        "breathe",
        "sphinx-rtd-theme",
        "sphinxcontrib-programoutput",
        "sphinxcontrib-bibtex",
        "nbsphinx",
        "ipython",
    ],
}

# Add an optional dependency that concatenates all others
optional_dependencies["all"] = sorted(
    [
        dependency
        for dependencies in optional_dependencies.values()
        for dependency in dependencies
    ]
)

setup(
    name="paropt",
    version="2.1.3",
    description="Parallel interior-point optimizer",
    author="Graeme J. Kennedy",
    author_email="graeme.kennedy@ae.gatech.edu",
    install_requires=["numpy", "mpi4py>=3.1.1"],
    extras_require=optional_dependencies,
    ext_modules=cythonize(
        exts,
        language="c++",
        include_path=inc_dirs,
        compiler_directives={"language_level": "3"},
    ),
)


from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
import argparse
import datetime
import pickle
from config import Config
from shared_album import SharedAlbum

class PhotosApi:
    SCOPES = ['https://www.googleapis.com/auth/photoslibrary.readonly',
              'https://www.googleapis.com/auth/photoslibrary.sharing']

    def __init__(self):
        self.service = self.get_service(self.SCOPES)

    def get_service(self, scopes, creds=None):
        if not creds or not creds.valid:
            if (creds and creds.expired and creds.refresh_token):
                creds.refresh(Request())
            else:
                flow = InstalledAppFlow.from_client_secrets_file(Config.client_secrets_file, scopes)
                creds = flow.run_local_server(port=0)
            with open("token.pickle", "wb") as tokenFile:
                pickle.dump(creds, tokenFile)
        service = build('photoslibrary', 'v1', credentials=creds, static_discovery=False)
        return service

    def create_album(self, title):
        return self.service.albums().create(body={"album": {"title": title}}).execute()

    def share_album(self, album_id):
        return self.service.albums().share(albumId=album_id).execute()

    def create_shared_album(self, title):
        album_response = self.create_album(title)
        return self.share_album(album_response["id"])

    def get_shared_albums(self):
        return [SharedAlbum(service=self.service, **item) for item in
                self.service.sharedAlbums().list(pageSize=50).execute()["sharedAlbums"]]

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--start-date', type=str, required=False,
                        help='initial date for file range')
    parser.add_argument('--end-date', type=str, required=False,
                        help='end date for file range')
    parser.add_argument('--album-names', type=str, nargs="*", required=False,
                        help='which albums to download')

    args = parser.parse_args()
    start_date = datetime.strptime(args.start_date, "%Y-%m-%d") if args.start_date is not None else datetime.datetime(2000, 1, 1)
    end_date = datetime.datetime.strptime(args.end_date, "%Y-%m-%d")  if args.end_date is not None else None
    print(start_date, end_date)

    photos_api = PhotosApi()
    shared_albums = [album for album in photos_api.get_shared_albums() if album in args.album_names]

    for album in shared_albums:
        print(f"downloading from {album.title}")
        album.download_and_rename(Config.path + "/google_photo_downloads/")
    print("done!")


'''
题目：输入两个链表，找出它们的第一个公共结点
'''

'''
思路：共同节点，意味着从共同节点开始之后所有的节点数都是相同的，这是链表，只要有一个共同节点，那么之后所有的指向
也是重复的。
先依次遍历两个链表，记录两个链表的长度m和n，
如果 m > n，那么我们就先让长度为m的链表走m-n个结点，然后
两个链表同时遍历，当遍历到相同的结点的时候停止即可。对于 m < n，同理。

32ms
5632k
'''

# -*- coding:utf-8 -*-
# class ListNode:
#     def __init__(self, x):
#         self.val = x
#         self.next = None
class Solution:
    def getLength(self,pHead):
        L = 0
        while pHead is not None:
            pHead = pHead.next
            L += 1
        return L
    def FindFirstCommonNode(self, pHead1, pHead2):
        # write code here
        L1 = self.getLength(pHead1)
        L2 = self.getLength(pHead2)
        if L1>= L2:
            long = pHead1
            short = pHead2
        else:
            long = pHead2
            short = pHead1
        #让长链表先把差值走掉
        for i in range(abs(L1-L2)):
            long = long.next
        while long != short:
            long = long.next
            short = short.next
        return long

剑指offer/36 两个链表的第一个公共节点.py

36 两个链表的第一个公共节点.py


import logging
from typing import Generic

from common.messages import End, TripsStart
from common.messages.basic import (
    BasicStation,
    BasicTrip,
    BasicWeather,
)
from common.persistence import StatePersistor

from . import Phase, GenericJoinedTrip

class TripsPhase(Phase[GenericJoinedTrip], Generic[GenericJoinedTrip]):
    def handle_station(self, station: BasicStation) -> Phase[GenericJoinedTrip]:
        self.__warn("Station")
        return self

    def handle_weather(self, weather: BasicWeather) -> Phase[GenericJoinedTrip]:
        self.__warn("Weather")
        return self

    def handle_trips_start(self, start: TripsStart) -> Phase[GenericJoinedTrip]:
        self.__warn(f"TripsStart (host id: {start.host})")
        return self

    def handle_trip(self, trip: BasicTrip) -> Phase[GenericJoinedTrip]:
        joined_trip = self.joiner.handle_trip(trip)
        if joined_trip is not None:
            self.comms.send(self.job_id, joined_trip)
        self.state.count += 1
        return self

    def handle_end(self, end: End) -> Phase[GenericJoinedTrip]:
        super().handle_end(end)
        self.check_ends()
        return self

    def check_ends(self) -> None:
        if len(self.state.ends_received) < self.config.parsers_count:
            return

        logging.info(
            f"Job {self.job_id} | All parsers finished sending trips, waiting until all"
            " are joined"
        )
        self.comms.set_all_trips_done_callback(self.job_id, self._all_trips_done)

    def _all_trips_done(self) -> None:
        logging.info(
            f"Job {self.job_id} | Finished joining all trips. Total processed in this"
            f" node: {self.state.count}"
        )
        self.comms.stop_consuming_trips(self.job_id)
        StatePersistor().remove(self._control_store_key())
        StatePersistor().remove(self._joiner_store_key())
        self.on_finish(self.job_id)
        self.comms.send(self.job_id, End(self.comms.id), force_msg_id=None)

    def __warn(self, name: str) -> None:
        logging.warn(
            f"Job {self.job_id} | Unexpected {name} received while receiving trips"
        )


#! /usr/bin/python3
#----------------------------------------------------------------
# Dateiname: truetype_test.py
# Gefundene TrueType-Fonts werden ausgegeben.
#
# Michael Weigend
# Raspberry Pi programmieren mit Python, 4. Auflage, mitp 2019
# Kap. 12
# Michael Weigend 20. November 2018
#----------------------------------------------------------------
from PIL import Image, ImageDraw, ImageFont, ImageTk
from os import popen
from tkinter import Tk, Label
SIZE=20

fonts = popen("fc-list").readlines()
paths = [line.split(":")[0] for line in fonts]

w = SIZE * 25
h = SIZE * len(fonts) + 10

image = Image.new('1', (w, h))
draw = ImageDraw.Draw(image)
draw.rectangle((0,0,w, h), outline=0, fill=0)

y = 0
for path in paths:
    font = ImageFont.truetype(font=path, size=SIZE)
    name=path.split("/")[-1]
    draw.text((0, y), text=name, fill=255, font=font)
    y += SIZE

window = Tk()
img = ImageTk.PhotoImage(image)
label = Label(master=window, image=img )
label.pack()
window.mainloop()


class IpAddress:
    def __init__(self, str) -> None:
        self.str = str

    def _contains_abba(s):
        for i in range(0, len(s) - 3):
                first_char = s[i]
                second_char = s[i + 1]
                if second_char == first_char:
                    continue
                if s[i + 2] == second_char and s[i + 3] == first_char:
                    return True
        return False

    def _get_abas(s):
        for i in range(0, len(s) - 2):
                first_char = s[i]
                second_char = s[i + 1]
                if second_char == first_char:
                    continue
                if s[i + 2] == first_char:
                    yield s[i:i + 3]

    def _aba_to_bab(aba):
        return f'{aba[1]}{aba[0]}{aba[1]}'

    def _get_supernet_sequences(self):
        import re
        # replace all hypernet sequences and return the remaining elements
        return re.sub("\[.+?\]", " ", self.str).split(" ")

    def _get_hypernet_sequences(self):
        import re
        return re.findall("\[(.+?)\]", self.str)

    def supports_tls(self):
        abba_in_supernet = False

        for supernet_sequence in self._get_supernet_sequences():
            if IpAddress._contains_abba(supernet_sequence):
                abba_in_supernet = True
                break

        if not abba_in_supernet:
            return False

        abba_in_hypernet = False

        for hypernet_sequence in self._get_hypernet_sequences():
            if IpAddress._contains_abba(hypernet_sequence):
                abba_in_hypernet = True
                break

        return not abba_in_hypernet

    def supports_ssl(self):
        abas = []

        for supernet_seqeuence in self._get_supernet_sequences():
            for aba in IpAddress._get_abas(supernet_seqeuence):
                abas.append(aba)

        if len(abas) == 0:
            return False

        for hypernet_sequence in self._get_hypernet_sequences():
            for bab in (IpAddress._aba_to_bab(aba) for aba in abas):
                if bab in hypernet_sequence:
                    return True

        return False

def parse(file):
    with open(file, 'r') as f:
        return [IpAddress(l.strip()) for l in f]

def test():
    data = parse('test.txt')
    count = sum((i.supports_tls() for i in data))
    assert count == 2
    data = parse('test2.txt')
    count = sum((i.supports_ssl() for i in data))
    assert count == 3

def main():
    test()

    data = parse('input.txt')
    count = sum((i.supports_tls() for i in data))
    print(f'Pt1: {count}')
    count = sum((i.supports_ssl() for i in data))
    print(f'Pt2: {count}')

if __name__ == '__main__':
    main()


import turtle

turtle.getscreen()
# Personnalisation
turtle.title("Stock AMAZON")
turtle.speed(10)
turtle.color("black", "black")

# initialisation echelle
zoom = 0.2
echelle_x = 20
echelle_y = 10
n_affichage = 40

#Afficher l'axe x
s = turtle.screensize()
turtle.up()
turtle.setpos(-s[0], 0)
turtle.down()
turtle.setpos(2 * s[0], 0)

#Afficher l'axe y
turtle.up()
turtle.setpos(-s[0], -s[0])
turtle.down()
turtle.setpos(-s[0], 2 * s[0])

# Position du curseur
offset_x = -s[0]
turtle.up()
turtle.setpos(offset_x,0)
turtle.down()

turtle.color("red", "black")

with open("AMZN.csv", "r", encoding="utf-8") as f:
    data = f.readlines()
    final_data = []
    for i, l in enumerate(data):
        if i != 0:
            final_data.append(float(l.split(',')[1]))

for i, v in enumerate(final_data):
    x = zoom * i * echelle_x + offset_x
    y = zoom * v * echelle_y
    if i == 0:
        turtle.up()
        turtle.goto(x, y)
        turtle.down()
    else:
        turtle.goto(x, y)
    #Afficher valeurs
    if i % n_affichage == 0:
        turtle.color("green", "black")
        turtle.write(int(v), font=("Verdana", 12, "bold"))
        turtle.dot(5)
        turtle.color("red", "black")

turtle.done()


"""
filename: controllers.py
description: Controller for notes.
created by: Chris Lemelin (cxl8826@rit.edu)
created on: 04/02/18
"""

from flask_socketio import emit
from app.decorators import ensure_dict, get_user
from app import socketio, db
from app.actions.models import *
from app.committees.models import *
from app.charges.models import *
from app.notes.models import *
from app.users.models import Users
from app.notes.notes_response import Response

##
## @brief      Creates a note. (Must be admin user or committe head or assigned to action)
##
## @param      user_data  The user data required to create a committee.
##
##                        All the following fields are required:
##                        action      - id of action
##                        description - Description of new note
##
@socketio.on('create_note')
@ensure_dict
@get_user
def create_note(user, user_data):

    action = Actions.query.filter_by(id= user_data.get('action',-1)).first()

    if user is None:
        emit("create_note", Response.UsrDoesntExist)
        return;

    if action is None:
        emit("create_note", Response.ActionDoesntExist)
        return;

    charge = Charges.query.filter_by(id= action.charge).first()
    committee = Committees.query.filter_by(id= charge.committee).first()

    if (not user.is_admin and
        committee.head != user.id and
        committee not in user.committees):
        emit("create_note", Response.UsrNotAuth)
        return;

    note = Notes()
    note.action = action.id
    note.description = user_data.get('description',"")
    note.status = user_data.get('status',0)
    note.author = user.id
    note.hidden = False
    db.session.add(note)

    try:
        db.session.commit()
        emit('create_note', Response.AddSuccess)
        get_notes(action.id, broadcast = True)
    except Exception as e:
        db.session.rollback()
        db.session.flush()
        emit("create_note", Response.AddError)

##
## @brief      Gets notes from an action
##
## @param      action_id    - id of the action
##
@socketio.on('get_notes')
def get_notes(action_id, broadcast = False):
    notes = Notes.query.filter_by(action= action_id).all()
    note_ser = [
                    {
                        "id": c.id,
                        "author": "{} {}".format(Users.query.filter_by(id = c.author).first().first_name, Users.query.filter_by(id = c.author).first().last_name),
                        "action": c.action,
                        "description": c.description,
                        "status": c.status,
                        "action": c.action,
                        "created_at": c.created_at.isoformat(),
                        "hidden": c.hidden
                    }
                    for c in notes
                ]
    emit("get_notes", note_ser, broadcast = broadcast)

##
## @brief      Gets a note
##
## @param      id       - id of the note
##
@socketio.on('get_note')
def get_note(id, broadcast = False):

    note = Notes.query.filter_by(id= id).first()
    if note is not None:
        note_data = {
            "id": note.id,
            "author": note.author,
            "action": note.action,
            "description": note.description,
            "created_at": note.created_at,
            "hidden": note.hidden
        }
        emit('get_note', note_data, broadcast = broadcast)
    else:
        emit("get_note", {}, broadcast = broadcast)

##
## @brief      Edits a note (Must be admin user or committe head to hide,
##              only author can edit the description)
##
## @param      user_data  The user data to edit a note, must
##                        contain a token, an id and any of the following
##                        fields:
##                        - description
##                        - hidden
##
##                        Any other field will be ignored.
##
## @emit       Emits a success mesage if edited, errors otherwise.
##

@socketio.on('modify_note')
@ensure_dict
@get_user
def modify_note(user, user_data):

    if(user is None):
        emit('modify_note', Response.UsrDoesntExist)
        return
    note = Notes.query.filter_by(id= user_data.get('id',-1)).first()

    if(note is None):
        emit('modify_note', Response.NoteDoesntExist)
        return

    action = Actions.query.filter_by(id= note.action).first()
    charge = Charges.query.filter_by(id= action.charge).first()
    committee = Committees.query.filter_by(id= charge.committee).first()

    if(user.id == note.author):

        if "description" in user_data:
            note.description = user_data['description']

    if(user.id == committee.head or user.is_admin or user.id == note.author):

        if "hidden" in user_data:
            note.hidden = user_data['hidden']

        db.session.add(note)

        try:
            db.session.commit()
            emit('modify_note', Response.ModifySuccess)
            get_note(note.id, broadcast = True)
        except Exception as e:
            db.session.rollback()
            db.session.flush()
            emit("modify_note", Response.ModifyError)
    else:
        emit("modify_note", Response.UsrNotAuth)