#
# Copyright (c) 2012, 2013, 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from abc import ABCMeta
from abc import abstractmethod
import itertools
import re

from git_upstream import errors
from git_upstream import lib
from git_upstream.lib.pygitcompat import Commit
from git_upstream.lib.utils import GitMixin
from git_upstream.log import LogDedentMixin

class Searcher(GitMixin):
    """
    Base class that needs to be extended with the specific searcher on how to
    locate changes.
    """
    __metaclass__ = ABCMeta

    def __init__(self, branch="HEAD", *args, **kwargs):

        self._branch = branch

        self.filters = []
        self.commit = None

        super(Searcher, self).__init__(*args, **kwargs)

    @property
    def branch(self):
        """
        Default branch in the git repository to search.
        """
        return self._branch

    def add_filter(self, filter):
        if filter not in self.filters:
            self.filters.append(filter)

    @abstractmethod
    def find(self):
        """
        Implementation of this method should return a commit SHA1 that will be
        used by the generic list() method as the start commit to list commits
        from, to the tip of the branch or commit given in the constructor.

        In additional it must save a 'Commit' object for the found SHA1 as the
        self.commit property.
        """
        pass

    def _check_merge_is_previous(self, mergecommit, parent, last_merge,
                                 ancestry_commits):
        """Check if merge commit and parent describes previous import

        Method checks if the given merge and parent commits describe a
        previous import merge commit and returns a 3-tuple of a merge
        commit if it is a candidate for a previous import merge, a list
        of commits whose history should be excluded when looking for
        carried changes, and lastly a list of commits that should be
        removed individually from the final list while still retaining
        any history before such commit.
        """

        mergebase = self.git.merge_base(parent, self.commit,
                                        with_exceptions=False)
        self.log.debug(
            """
            previous upstream: %s
            merge commit: %s
            merge-base: %s
            parent: %s
            """, self.commit, mergecommit, mergebase, parent.hexsha)

        # if not a valid response from merge-base, we have an additional
        # branch with unrelated history, and should ignore all parents
        if not mergebase:
            self.log.info(
                """
                Found merge of additional branch:
                    %s
                """, mergecommit)
            return None, ["^%s" % mp for mp in mergecommit.parents], None

        # if the parent tree doesn't match the merge commit tree, we can
        # skip inspecting it as it and it's parent commit must be
        # included as it contributes changes to the tree.
        if (self.git.rev_parse("%s^{tree}" % parent) !=
                self.git.rev_parse("%s^{tree}" % mergecommit)):
            return None, [], None

        # otherwise we have a descendant commit with the same tree that
        # requires further inspection to determine if it is really the
        # previous import merge.

        # if the parent is not a descendent of the previous upstream, will
        # need to determine whether to exclude
        if mergebase != self.commit.hexsha:
            # if we're checking the last merge in the list, then looking at
            # the previous mainline that was replaced and should ignore
            if mergecommit == last_merge:
                # also means we've found the previous import
                self.log.debug("Merge is last merge to be inspected")
                return mergecommit, ["^%s" % parent], None

            # otherwise this an unusual state where we are looking at the
            # merge of the previous import with a change that landed on the
            # previous target mainline but was not included in the changes
            # that where on the previous import. This can occur due to a
            # change being approved/landed after the import was performed
            self.log.debug("Merge base is not the same as root merge base")
            return None, [], None

        other_parents = set(
            p.hexsha for p in mergecommit.parents if p != parent
        )
        self.log.debug("Merge base is same: other_parents: %s", other_parents)

        # if the merge base of the parent against any of the other parents
        # happens to be in a list of commits directly between the tip and
        # previous upstream import, then it is simply a merge of an obsolete
        # commit. These are very difficult to strip out normally
        exclude_commits = []
        for p in other_parents:
            if self.git.merge_base(
                    p, parent, with_exceptions=False) in ancestry_commits:
                exclude_commits.extend(
                    self.git.rev_list("%s..%s" % (parent, p)).splitlines())

        if exclude_commits:
            return None, [], exclude_commits

        # handle case where merges are always created in GitHub even when
        # not needed.
        if other_parents.issubset(ancestry_commits):
            return None, [], []

        # otherwise looking at the previous import merge commit and the parent
        # from the previous import branch, so exclude all other parents.
        return mergecommit, ["^%s" % ip for ip in other_parents], None

    def list(self, upstream=None):
        """
        Returns a list of Commit objects, between the '<commitish>' revision
        given in the constructor, and the commit object returned by the find()
        method. If given an upstream branch, uses --cherry-pick/--left-only to
        exclude commits that are identical to those already on the upstream
        branch.
        """
        if not self.commit:
            self.find()

        revision_spec = "{0}..{1}".format(self.commit.hexsha, self.branch)

        # search for previous import commit first, if found, wish to include
        # the discarded parents as part of the set of changes to ignore in the
        # final list.
        self.log.debug(
            """
            Searching for previous merges that exclude one side of the history
            since the last import.
                git rev-list --ancestry-path --merges %s
            """, revision_spec)

        merge_list = list(Commit.iter_items(self.repo, revision_spec,
                                            topo_order=True,
                                            ancestry_path=True, merges=True))

        # to handle special case where a merge of a commit introduces
        # nothing to the tree, need to spot when such commits are based
        # off of the import set, so as not to accidentally exclude
        strip_commits = set()
        ancestry_commits = [
            commit.hexsha
            for commit in Commit.iter_items(
                self.repo, revision_spec, topo_order=True,
                ancestry_path=True, merges=False)
        ]
        self.log.debug("Ancestry commits: %s", ancestry_commits)

        extra_args = []
        previous_import = None
        for mergecommit, parent in ((mc, p)
                                    for mc in merge_list
                                    for p in mc.parents):
            # inspect each
            (previous_import_candidate,
             ignores,
             prune_list) = self._check_merge_is_previous(
                mergecommit, parent, merge_list[-1], ancestry_commits)

            if ignores:
                self.log.debug(
                    """
                    Adding following to ignore list:
                        %s
                    """, "\n    ".join(ignores))
                extra_args.extend(ignores)

            if prune_list:
                self.log.debug(
                    """
                    Adding following commits to be pruned afterwards:
                        %s
                    """, "\n    ".join(prune_list))
                strip_commits.update(prune_list)

            if previous_import_candidate:
                previous_import = previous_import_candidate
                self.log.info(
                    """
                    Found possible previous import merge:
                        %s
                    """, previous_import)

        if previous_import:
            self.log.info(
                """
                Found possible previous import merge:
                    %s
                """, previous_import)

        # walk the tree and find all commits that lie in the path between the
        # commit found by find() and head of the branch in two steps, to
        # ensure a deterministic order between what is from the previous
        # upstream to the last import, and from that import to what is on
        # the tip of the head to avoid inversion where older commits
        # started before the previous import merge and approved afterwards
        # are not sorted by 'rev-list' predictably.
        commit_list = []
        if upstream is None:
            if previous_import:
                search_list = [
                    (previous_import, self.branch, None),
                    (self.commit, previous_import, None),
                ]
            else:
                search_list = [(self.commit, self.branch, None)]
            rev_spec = "{0}..{1}"
            git_args = {}
        else:
            if previous_import:
                search_list = [
                    (self.branch, upstream, "^%s" % previous_import),
                    (previous_import, upstream, "^%s~1" % previous_import)
                ]
            else:
                search_list = [(self.branch, upstream, None)]
            rev_spec = "{0}...{1}"
            git_args = {'cherry_pick': True,
                        'left_only': True,
                        'full_history': True,
                        }
            extra_args.append("^%s" % self.commit)

        for start, end, exclude in search_list:
            extra = list(extra_args)
            if exclude:
                extra.append(exclude)
            extra.extend(["--", "."])
            revision_spec = rev_spec.format(start, end)
            self.log.info(
                """
                Walking the changes between found commit and target, excluding
                those behind the previous import or merged as an additional
                branch during the previous import
                    git rev-list --topo-order %s %s %s
                """, ' '.join(self.git.transform_kwargs(**git_args)),
                revision_spec, " ".join(extra))

            commit_list.append(
                Commit._iter_from_process_or_stream(
                    self.repo,
                    self.git.rev_list(revision_spec,
                                      *extra,
                                      as_process=True,
                                      topo_order=True,
                                      **git_args)))

        # chain the filters as generators so that we don't need to allocate new
        # lists for each step in the filter chain.
        commit_list = itertools.chain(*commit_list)
        # strip commits that cannot be excluded through revision specifications
        # or without removing additional history
        commit_list = [c for c in commit_list if c.hexsha not in strip_commits]
        for f in self.filters:
            commit_list = f.filter(commit_list)

        commits = list(commit_list)

        self.log.debug(
            """
            commits found:
                %s
            """, ("\n" + " " * 4).join([c.hexsha for c in commits]))

        return commits

class NullSearcher(Searcher):
    """
    This searcher returns an empty list
    """

    def list(self):
        return []

class UpstreamMergeBaseSearcher(LogDedentMixin, Searcher):
    """
    Searches upstream references for a merge base with the target branch. By
    default this will search the 'upstream/*' namespace but can be overridden
    to search any namespace pattern.

    If not restricted to search specific remotes, it will search all
    available remote references matching the pattern for the most recent merge
    base available.
    """

    def __init__(self, branch, patterns=None, search_tags=False, remotes=None,
                 *args, **kwargs):

        super(UpstreamMergeBaseSearcher, self).__init__(branch, *args,
                                                        **kwargs)

        if not patterns:
            patterns = ["upstream/*"]
        self._patterns = set()
        self._revs = set()
        for rev in patterns:
            if "*" in rev:
                self._patterns.add(rev)
            elif self.is_valid_commit(rev):
                self._revs.add(rev)
            else:
                raise errors.GitUpstreamError(
                    "'%s' not recognized as valid commit(ish) or pattern" %
                    rev)

        self._references = ["refs/heads/{0}".format(ref)
                            for ref in self.patterns]

        if not remotes:
            remotes = []

        if remotes:
            self._references.extend(
                ["refs/remotes/{0}/{1}".format(s, ref)
                 for s in remotes for ref in self.patterns])
        else:
            self._references.extend(
                ["refs/remotes/*/{0}".format(ref) for ref in self.patterns])

        if search_tags:
            self._references.extend(
                ["refs/tags/{0}".format(ref) for ref in self.patterns])

    @property
    def patterns(self):
        """
        Pattern to limit which references are searched when looking for a
        merge base commit.
        """
        return self._patterns

    def find(self):
        """
        Searches the git history including local and remote branches, and tags
        if tag searching is enabled. References are included in the list to be
        checked if they match the pattern that was specified in the
        constructor.
        While 'git rev-list' supports a glob option to check all references, it
        isn't possible to anchor the pattern, so 'upstream/*' would match all
        of the following:
            refs/remotes/origin/upstream/master
            refs/heads/upstream/master
            refs/remotes/origin/other/upstream/area <--- undesirable

        Additional since 'git rev-list' doesn't accept patterns as commit refs
        it's better to make use of 'git for-each-ref' and how it does pattern
        matching in order to generate a list of references to pass to rev-list
        to walk.

        After determining all the references to look at, because of the
        overhead in using 'git merge-base' to determine the last commit from
        one of the upstream refs that was merged into the target branch, it is
        worth going to the additional effort of removing any reference that is
        reachable from another so that the calls to merge-base are minimized.
        """

        self.log.info(
            "Searching for most recent merge base with upstream branches")

        # process pattern given to get a list of refs to check
        if self._references:
            rev_list_args = self.git.for_each_ref(
                *self._references, format="%(refname:short)").splitlines()
        else:
            rev_list_args = []
        rev_list_args.extend(self._revs)
        self.log.info(
            """
            Upstream refs:
                %s
            """, "\n    ".join(rev_list_args)
        )

        # get the sha1 for the tip of each of the upstream refs we are going to
        #  search
        self.log.info(
            """
            Construct list of upstream revs to search:
                git rev-list --min-parents=1 --no-walk \\
                    %s
            """, (" \\\n" + " " * 8).join(rev_list_args))
        rev_list_args.append("--")
        search_list = set(self.git.rev_list(*rev_list_args,
                                            min_parents=1,
                                            no_walk=True).splitlines())
        rev_list_args = list(search_list)

        # construct a list of the parents of each ref so that we can tell
        # rev-list to ignore in the anything reachable from the list commits
        # which reduces the amount of revs to be searched with merge-base
        prune_list = []
        for rev in search_list:
            # only root commits won't have at least one parent which have been
            # excluded by the previous search
            commit = self.git.rev_list(rev, "--", parents=True,
                                       max_count=1).split()
            parents = commit[1:]
            prune_list.extend(parents)

        # We want to stop walking the tree and ignore all commits after each
        # time we encounter one from the prune_list, so make sure to set the
        # --not option followed by the list of revisions to exclude. Since
        # python-git may reorder options if given by way or keyword args, use
        # strings in the required order as '*args' are not reordered, and
        # order is critical here to ensure rev-list applies the '--not'
        # behaviour to the correct set.
        rev_list_args.append("--not")
        rev_list_args.extend(prune_list)
        self.log.info(
            """
            Retrieve minimal list of revs to check with merge-base by excluding
            revisions that are in the reachable from others in the list:
                git rev-list \\
                    %s
            """, " \\\n        ".join(rev_list_args))
        rev_list_args.append("--")
        revsions = self.git.rev_list(*rev_list_args).splitlines()

        # Running 'git merge-base' is relatively expensive to pruning the list
        # of revs to search since it needs to construct and walk a large
        # portion of the tree for each call. If the constructed graph was
        # retained betweens we could likely remove much of the code above.
        self.log.info(
            """
            Running merge-base against each found upstream revision and target
                git merge-base %s ${upstream_rev}
            """, self.branch)
        merge_bases = set()
        for rev in revsions:
            # ignore exceptions as there may be unrelated branches picked up by
            # the searching which would result in merge-base returning an error
            base = self.git.merge_base(self.branch, rev, with_exceptions=False)
            if base:
                merge_bases.add(base)

        if len(merge_bases) == 0:
            self.log.notice("Merge-base couldn't be found: it seems there " +
                            "is no common ancestor for the involved branches")
        else:
            self.log.info(
                """
                Order the possible merge-base commits in descendent order, to
                find the most recent one used irrespective of date:
                    git rev-list --topo-order --max-count=1 --no-walk \\
                        %s
                """, (" \\\n" + " " * 8).join(merge_bases))
            args = list(merge_bases) + ["--"]
            sha1 = self.git.rev_list(*args, topo_order=True, max_count=1)
            # now that we have the sha1, make sure to save the commit object
            self.commit = self.repo.commit(sha1)
            self.log.debug("Most recent merge-base commit is: '%s'",
                           self.commit.hexsha)

        if not self.commit:
            raise RuntimeError("Failed to locate suitable merge-base")

        return self.commit.hexsha

class CommitMessageSearcher(LogDedentMixin, Searcher):
    """
    This searcher returns a list of commits based on looking for commit message
    containing a specific message in the current branch.
    """

    def __init__(self, pattern, *args, **kwargs):
        self._pattern = pattern

        super(CommitMessageSearcher, self).__init__(*args, **kwargs)

    @property
    def pattern(self):
        """
        Pattern to search commit messages in the target branch for a match.
        """
        return self._pattern

    def find(self):
        """
        Searches the git history of the target branch for a commit message
        containing the pattern given in the constructor. This is used as a base
        commit from which to return a list of commits since this point.
        """

        commits = Commit.iter_items(self.repo, self.branch, grep=self.pattern,
                                    max_count=1, extended_regexp=True)

        self.commit = next(commits, None)
        if not self.commit:
            raise RuntimeError("Failed to locate a pattern match")

        self.log.debug("Commit matching search pattern is: '%s'",
                       self.commit.hexsha)

        return self.commit.hexsha

    def list(self, upstream=None, include=True):
        """
        Override parent implementation to permit inclusion of the found commit
        to be returned in the list of changes. This will help in cases where
        then commit is a specific merge including the additionally merged
        branches that would be returned by the generic upstream searcher.
        """

        commits = super(CommitMessageSearcher, self).list(upstream)
        if include:
            commits.append(self.commit)

        return commits

class CommitFilter(object):
    """
    CommitFilter instances are used to perform arbitrary filtering of commits
    returned by searchers.
    """
    __metaclass__ = ABCMeta

    def __init__(self, *args, **kwargs):
        super(CommitFilter, self).__init__(*args, **kwargs)

    @abstractmethod
    def filter(self, commit_iter):
        pass

class SupersededCommitFilter(LogDedentMixin, GitMixin, CommitFilter):
    """
    Prunes all commits that have a note with the "Superseded-by:" header
    containing a Change-Id present in upstream tracking branch

    :param string search_ref: git reference to search for ChangeIds (required).
    :param Commit limit: commit object to ignore searching history after
                        (optional).
    """

    def __init__(self, search_ref, limit=None, *args, **kwargs):

        super(SupersededCommitFilter, self).__init__(*args, **kwargs)

        if not self.is_valid_commit(search_ref):
            raise ValueError("Invalid value for 'search_ref': %s" % search_ref)
        self.search_ref = search_ref

        if limit:
            if not hasattr(limit, 'hexsha'):
                raise ValueError(
                    "Invalid object: no hexsha attribute for 'limit'")
            if not self.is_valid_commit(limit.hexsha):
                raise ValueError(
                    "'limit' object does not contain a valid SHA1")
        self.limit = limit

        self._regex = None

    def _get_rev_range(self):

        if self.limit:
            return "%s..%s" % (self.limit.hexsha, self.search_ref)
        else:
            return self.search_ref

    def _get_change_id(self, commit):
        """
        Returns the Change-Id string from the footer of the given commit.

        Will ignore any instances outside of the footer section
        """
        # read the commit message in reverse to access the
        # footer first but ignore subject and first blank line
        for line in reversed(commit.message.splitlines()[1:]):
            line = line.strip()
            # exit on the first blank line found since that indicates
            # we're reached the top of the footer section
            if not line:
                break

            cid = re.search('^Change-Id:\s*(.+)$', line, re.IGNORECASE)
            if cid:
                return cid.group(1)
        return

    def filter(self, commit_iter):

        self.log.info(
            """
            Filtering out all commits marked with a Superseded-by Change-Id
            which is present in '%s'
            """, self.search_ref)

        supersede_re = re.compile('^%s\s*(.+)\s*$' %
                                  lib.SUPERSEDE_HEADER,
                                  re.IGNORECASE | re.MULTILINE)

        for commit in commit_iter:
            commit_note = commit.note(note_ref=lib.IMPORT_NOTE_REF)
            # include non-annotated commits
            if not commit_note:
                yield commit
                continue

            # include annotated commits which don't have a SUPERSEDE_HEADER
            superseding_change_ids = supersede_re.findall(commit_note)
            if not superseding_change_ids:
                yield commit
                continue

            # search for all the change-ids in matches (egrep regex)
            commits_grep_re = '^Change-Id:\\s*\(%s\)\\s*$' % \
                              '\|'.join(superseding_change_ids)

            # retrieve all matching commits because we need to check
            # each match for whether the changeId is actually in
            # the footer or just included as a reference.
            matching_commits = Commit.iter_items(self.repo,
                                                 self._get_rev_range(),
                                                 regexp_ignore_case=True,
                                                 grep=commits_grep_re)

            for possible in matching_commits:
                change_id = self._get_change_id(possible)
                if change_id:
                    superseding_change_ids.remove(change_id)

            # include commits which have some superseding change-ids not
            # present in upstream
            if superseding_change_ids:
                self.log.debug(
                    """
                Including commit '%s %s'
                    because the following superseding change-ids have not been
                    found:
                    %s
                """, commit.short, commit.message.splitlines()[0],
                    '\n    '.join(superseding_change_ids))
                yield commit
                continue

            self.log.debug(
                """
                Filtering out commit '%s %s'
                    because it has been marked as superseded by the following
                    note:
                    %s
                """, commit.short, commit.message.splitlines()[0],
                commit_note)

class DroppedCommitFilter(LogDedentMixin, CommitFilter):
    """
    Prunes all commits that have a note with the Dropped: header
    """

    def filter(self, commit_iter):
        for commit in commit_iter:
            commit_note = commit.note(note_ref=lib.IMPORT_NOTE_REF)
            if not commit_note:
                yield commit
            elif not re.match('^%s.+' % lib.DROP_HEADER,
                              commit_note, re.IGNORECASE | re.MULTILINE):
                yield commit
            else:
                self.log.debug("Dropping commit '%s' as requested:", commit)
                self.log.debug(commit_note)

class MergeCommitFilter(CommitFilter):
    """
    Includes only those commits that have more than one parent listed (merges)
    """

    def filter(self, commit_iter):
        for commit in commit_iter:
            if len(commit.parents) >= 2:
                yield commit

class NoMergeCommitFilter(CommitFilter):
    """
    Prunes all that have more than one parent listed (merges)
    """

    def filter(self, commit_iter):
        for commit in commit_iter:
            if len(commit.parents) < 2:
                yield commit

class DiscardDuplicateGerritChangeId(LogDedentMixin, GitMixin, CommitFilter):
    """
    Filter out commit objects where the message footer contains a ChangeId
    string that is already available in the history of commit object provided
    to the constructor.

    ChangeId's are used by the Gerrit Code Review software to track changes
    through multiple amendments, http://code.google.com/p/gerrit/.

    :param string search_ref: git reference to search for duplicate ChangeIds
                             (required).
    :param Commit limit: commit object to ignore searching history after
                        (optional).
    """

    def __init__(self, search_ref, limit=None, *args, **kwargs):

        super(DiscardDuplicateGerritChangeId, self).__init__(*args, **kwargs)

        if not self.is_valid_commit(search_ref):
            raise ValueError("Invalid value for 'search_ref': %s" % search_ref)
        self.search_ref = search_ref

        if limit:
            if not hasattr(limit, 'hexsha'):
                raise ValueError(
                    "Invalid object: no hexsha attribute for 'limit'")
            if not self.is_valid_commit(limit.hexsha):
                raise ValueError(
                    "'limit' object does not contain a valid SHA1")
        self.limit = limit

        self._regex = None

    @property
    def regex(self):
        # compile the regex object on first usage
        if not self._regex:
            self._regex = re.compile("^Change-Id: ", re.I)
        return self._regex

    def _get_rev_range(self):

        if self.limit:
            return "%s..%s" % (self.limit.hexsha, self.search_ref)
        else:
            return self.search_ref

    def _get_change_id(self, commit):
        """
        Returns the Change-Id string from the footer of the given commit.

        Will ignore any instances outside of the footer section
        """
        # read the commit message in reverse to access the
        # footer first but ignore subject and first blank line
        for line in reversed(commit.message.splitlines()[1:]):
            line = line.strip()
            # exit on the first blank line found since that indicates
            # we're reached the top of the footer section
            if not line:
                break
            if self.regex.match(line):
                return line
        return

    def filter(self, commit_iter):

        self.log.info(
            """
            Filtering out all commits that have a Change-Id that matches one
            found in the given search ref: %s
            """, self.search_ref)

        for commit in commit_iter:
            change_id = self._get_change_id(commit)
            # if there is no change_id to compare against, return the commit
            if not change_id:
                self.log.debug(
                    """
                    Including change missing 'Change-Id'
                        Commit: %s %s
                        Message: %s
                    """, commit.short, commit.message.splitlines()[0],
                    commit.message)
                yield commit
                continue

            # retrieve all matching commits because we need to check
            # each match for whether the changeId is actually in
            # the footer or just included as a reference.
            matching_commits = Commit.iter_items(self.repo,
                                                 self._get_rev_range(),
                                                 regexp_ignore_case=True,
                                                 grep="^%s$" % change_id)

            duplicate_change_id = None
            for possible in matching_commits:
                duplicate_change_id = self._get_change_id(possible)
                if duplicate_change_id == change_id:
                    break

            if duplicate_change_id and duplicate_change_id == change_id:
                self.log.debug(
                    """
                    Skipping duplicate Change-Id in search ref
                        %s
                        Commit: %s %s
                    """, change_id, commit.short,
                    commit.message.splitlines()[0])
                continue

            # no match in the search ref, so include commit
            self.log.debug(
                """
                Including unmatched change
                    %s
                    Commit: %s %s
                """, change_id, commit.short,
                commit.message.splitlines()[0])
            yield commit

class TransformCommitToSHA1(CommitFilter):
    """
    Discard 'Commit' objects and simply return the SHA1 id's
    """

    def filter(self, commit_iter):
        for commit in commit_iter:
            yield commit.hexsha

class ReverseCommitFilter(LogDedentMixin, CommitFilter):
    """
    Reverses the list of commits passed.

    As this needs the complete list to work, it is recommended to only use
    once all other filtering is complete as otherwise it will remove the
    memory benefits of using generator behaviour when chaining multiple
    filters.
    """

    def filter(self, commit_iter):
        self.log.debug("Consuming generators to reverse commit list")
        return reversed(list(commit_iter))


"""
Dtella - DirectConnect Interface Module
Copyright (C) 2008  Dtella Labs (http://www.dtella.org)
Copyright (C) 2008  Paul Marks
Copyright (C) 2009  Dtella Cambridge (http://camdc.pcriot.com/)
Copyright (C) 2009  Andrew Cooper <amc96>, Ximin Luo <xl269> (@cam.ac.uk)

$Id$

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
"""

from twisted.protocols.basic import LineOnlyReceiver
from twisted.internet.protocol import ServerFactory, ClientFactory
from twisted.internet import reactor
from twisted.python.runtime import seconds
import twisted.python.log

from dtella.common.util import (validateNick, word_wrap, split_info,
                                split_tag, dc_unescape, dcall_discard,
                                format_bytes, dcall_timeleft, stdlines,
                                get_version_string, lock2key, CHECK,
                                parse_incoming_info, SSLHACK_filter_flags)
from dtella.client.dtellabot import DtellaBot
from dtella.common.ipv4 import Ad
import dtella.common.core as core
import dtella.local_config as local
import struct
import random
import re
import binascii
import socket

from zope.interface import implements
from zope.interface.verify import verifyClass
from dtella.common.interfaces import IDtellaStateObserver

# Login Procedure
# H>C $HubName
# H<C $ValidateNick
# H>C $Hello
# H<C $GetNickList + $MyINFO
# ...

LOCK_STR = "EXTENDEDPROTOCOLABCABCABCABCABCABC Pk=DTELLA"

class BaseDCProtocol(LineOnlyReceiver):

    delimiter='|'

    def connectionMade(self):
        try:
            self.transport.setTcpNoDelay(True)
        except socket.error:
            pass
        self.dispatch = {}

    def sendLine(self, line):
        #print "<:", line
        LineOnlyReceiver.sendLine(self, line.replace('|','|'))

    def lineReceived(self, line):
        #print ">:", line
        cmd = line.split(' ', 1)

        # Do a dict lookup to find the parameters for this command
        try:
            nargs, fn = self.dispatch[cmd[0]]
        except KeyError:
            # Unknown DC message
            return

        # Create the argument list
        if len(cmd) <= 1:
            args = []
        elif nargs < 0:
            nargs = -nargs
            args = cmd[1].split(' ', nargs-1)
        else:
            args = cmd[1].split(' ', nargs)

        if len(args) == nargs:
            try:
                fn(*args)
            except:
                twisted.python.log.err()

    def addDispatch(self, command, nargs, fn):
        self.dispatch[command] = (nargs, fn)

##############################################################################

# These next 3 classes implement a small subset of the DC client-client
# protocol, in order to impersonate a remote peer and generate an error.

class AbortTransfer_Factory(ClientFactory):

    def __init__(self, nick):
        self.nick = nick

    def buildProtocol(self, addr):
        p = AbortTransfer_Out(self.nick)
        p.factory = self
        return p

class AbortTransfer_Out(BaseDCProtocol):

    # if I initiate the connection:
    # send $MyNick + $Lock
    # (catch $Lock)
    # wait for $Key
    # -> send $Supports + $Direction + $Key + $Error

    def __init__(self, nick):

        self.nick = nick
        self.lock = ""

        # If we're not done in 5 seconds, something's fishy.
        def cb():
            self.timeout_dcall = None
            self.transport.loseConnection()

        self.timeout_dcall = reactor.callLater(5.0, cb)

    def connectionMade(self):
        BaseDCProtocol.connectionMade(self)

        self.addDispatch('$Lock',  2, self.d_Lock)
        self.addDispatch('$Key',  -1, self.d_Key)

        self.sendLine("$MyNick %s" % self.nick)
        self.sendLine("$Lock %s" % LOCK_STR)

    def d_Lock(self, lock, pk):
        self.lock = lock

    def d_Key(self, key):
        if self.lock.startswith("EXTENDEDPROTOCOL"):
            self.sendLine("$Supports ADCGet TTHL TTHF")
        self.sendLine("$Direction Upload 12345")
        self.sendLine("$Key %s" % lock2key(self.lock))
        self.sendLine("$Error File Not Available")
        self.transport.loseConnection()

    def connectionLost(self, reason):
        dcall_discard(self, 'timeout_dcall')

class AbortTransfer_In(BaseDCProtocol):

    # if I receive the connection:
    # receive $MyNick
    # wait for $Lock
    # -> send $MyNick + $Lock + $Supports + $Direction + $Key
    # wait for $Key
    # -> send $Error

    def __init__(self, nick, dch):

        self.nick = nick

        # Steal connection from the DCHandler
        self.factory = dch.factory
        self.makeConnection(dch.transport)
        self.transport.protocol = self

        # Steal the rest of the data
        self._buffer = dch._buffer
        dch.lineReceived = self.lineReceived

        # If we're not done in 5 seconds, something's fishy.
        def cb():
            self.timeout_dcall = None
            self.transport.loseConnection()

        self.timeout_dcall = reactor.callLater(5.0, cb)

    def connectionMade(self):
        BaseDCProtocol.connectionMade(self)

        self.addDispatch('$Lock', 2, self.d_Lock)

    def d_Lock(self, lock, pk):
        self.sendLine("$MyNick %s" % self.nick)
        self.sendLine("$Lock %s" % LOCK_STR)
        if lock.startswith("EXTENDEDPROTOCOL"):
            self.sendLine("$Supports ADCGet TTHL TTHF")
        self.sendLine("$Direction Upload 12345")
        self.sendLine("$Key %s" % lock2key(lock))

        self.addDispatch('$Key', -1, self.d_Key)

    def d_Key(self, key):
        self.sendLine("$Error File Not Available")
        self.transport.loseConnection()

    def connectionLost(self, reason):
        dcall_discard(self, 'timeout_dcall')

##############################################################################

class DCHandler(BaseDCProtocol):
    implements(IDtellaStateObserver)

    def __init__(self, main):
        self.main = main

    def initDataReceived(self, data):
        """Attempt to detect incoming clients not using DC."""
        #print ">>:", data
        dcall_discard(self, 'init_dcall')

        if data[0] == 'H':
            from dtella.client.adc import ADC_AbortConnection
            ADC_AbortConnection(self, data, "NMDC", "nmdc://localhost:%s or "
                "localhost:%s" % (self.main.state.clientport,
                self.main.state.clientport))
        else:
            # Passed all tests, let it through
            self.dataReceived = self._dataReceived
            self.dataReceived(data)

    def connectionMade(self):
        BaseDCProtocol.connectionMade(self)

        self.info = ''
        self.nick = ''
        self.bot = DtellaBot(self, '*Dtella')

        # Handlers which can be used before attaching to Dtella
        self.addDispatch('$ValidateNick',   1, self.d_ValidateNick)
        self.addDispatch('$GetNickList',    0, self.d_GetNickList)
        self.addDispatch('$MyINFO',        -3, self.d_MyInfo)
        self.addDispatch('$GetINFO',        2, self.d_GetInfo)
        self.addDispatch('',                0, self.d_KeepAlive)
        self.addDispatch('$KillDtella',     1, self.d_KillDtella)

        self.addDispatch('$MyNick',         1, self.d_MyNick)

        # Chat messages waiting to be sent
        self.chatq = []
        self.chat_counter = 99999
        self.chatRate_dcall = None

        # ['login_1', 'login_2', 'queued', 'ready', 'invisible']
        self.state = 'login_1'

        self.loginblockers = set(('MyINFO', 'GetNickList'))

        self.queued_dcall = None
        self.autoRejoin_dcall = None

        self.scheduleChatRateControl()

        # Set up the protocol trap
        self._dataReceived = self.dataReceived
        self.dataReceived = self.initDataReceived

        # If we're expecting a fake revconnect, delay the inital hub text.
        def cb():
            self.init_dcall = None
            self.sendLine("$Lock FOO Pk=BAR")
            self.pushTopic()

        if self.main.abort_nick:
            self.init_dcall = reactor.callLater(1.0, cb)
        else:
            cb()

    def isOnline(self):
        osm = self.main.osm
        return (self.state == 'ready' and osm and osm.syncd)

    def connectionLost(self, reason):

        self.main.removeDCHandler(self)

        dcall_discard(self, 'init_dcall')
        dcall_discard(self, 'chatRate_dcall')
        dcall_discard(self, 'autoRejoin_dcall')

    def fatalError(self, text):
        self.pushStatus("ERROR: %s" % text)
        self.transport.loseConnection()

    def d_KillDtella(self, key):
        from base64 import b32encode
        k = self.main.state.killkey
        if b32encode(k) == key:
            self.sendLine("$KillDtella 0 OK")
            def cb():
                reactor.stop()
            reactor.callLater(0, cb)
        else:
            self.sendLine("$KillDtella 1 BadKey")

    def d_MyNick(self, nick):
        # This is a fake RevConnect that we should terminate.

        if self.state != 'login_1':
            self.fatalError("$MyNick not expected.")
            return

        if not self.main.abort_nick:
            self.transport.loseConnection()
            return

        # Transfer my state to the connection abort handler
        AbortTransfer_In(self.main.abort_nick, self)
        self.main.abort_nick = None

    def d_ValidateNick(self, nick):

        if self.state != 'login_1':
            self.fatalError("$ValidateNick not expected.")
            return

        # Next, we expect $GetNickList+$MyINFO
        self.state = 'login_2'

        reason = validateNick(nick)

        if reason:
            self.pushStatus("Your nick is invalid: %s" % reason)
            self.pushStatus("Please fix it and reconnect.  Goodbye.")
            self.transport.loseConnection()
            return

        self.nick = nick

        self.pushHello(self.nick)

    def d_GetInfo(self, nick, _):

        if nick == self.bot.nick:
            dcinfo = "Local Dtella Bot$ $Bot\x01$$0$"
            self.pushInfo(nick, dcinfo)
            return

        if not self.isOnline():
            return

        try:
            n = self.main.osm.nkm.lookupNodeFromNick(nick)
        except KeyError:
            return

        if n.dcinfo:
            self.pushInfo(n.nick, n.dcinfo)

    def d_GetNickList(self):

        if self.state == 'login_1':
            self.fatalError("Got $GetNickList, expected $ValidateNick")
            return

        # Me and the bot are ALWAYS online
        nicks = [self.bot.nick, self.nick]

        # Add in the Dtella nicks, if we're fully online (DC and Dtella)
        if self.isOnline():
            nicks = set(nicks)
            nicks.update(self.main.osm.nkm.getNickList())
            nicks = list(nicks)

        nicks.sort()

        self.sendLine("$NickList %s$$" % '$$'.join(nicks))
        self.sendLine("$OpList %s$$" % self.bot.nick)

        if self.state == 'login_2':
            self.removeLoginBlocker('GetNickList')

    def d_MyInfo(self, _1, _2, info):

        if self.state == 'login_1':
            self.fatalError("Got $MyINFO, expected $ValidateNick")
            return

        # Save my new info
        self.info = info.replace('\r','').replace('\n','')

        if self.state == 'login_2':

            # detect passive
            try:
                chopstr = parse_incoming_info(SSLHACK_filter_flags(info))[0]
                chopstr = dc_unescape(split_info(chopstr)[0])
                chopstr = split_tag(chopstr)[1].split(' ')[1]

                if "M:P" in chopstr.split(','):
                    text = (
                        "You are currently in passive mode. This is less helpful "
                        "to the network, because passive mode users can't connect "
                        "to other passive mode users.",
                        "",
                        "Since your router/firewall (if you have one) is allowing "
                        "your connection to the Dtella network, it is likely that "
                        "active mode DC will work fine too, so please try to use "
                        "it. For most clients, this will involve something along "
                        "the lines of:",
                        "",
                        "File: Settings/Preferences: Connection: Incoming: "
                        "select \"Active mode\" or \"Direct connection\"",
                        "",
                        "To test whether active mode works, just try to get "
                        "someone's file list. If it doesn't seem to work, please "
                        "try to exhaust all possibilites (eg. check your firewall "
                        "settings, etc) before resorting to passive mode. Feel "
                        "free to ask for help in the main chat channel.",
                        ""
                        )

                    for par in text:
                        for line in word_wrap(par):
                            self.pushBotMsg(line)

            except ValueError:
                pass

            self.removeLoginBlocker('MyINFO')

        elif self.isOnline():
            self.main.osm.updateMyInfo()

    def removeLoginBlocker(self, blocker):

        CHECK(self.state == 'login_2')

        try:
            self.loginblockers.remove(blocker)
            if self.loginblockers:
                return
        except KeyError:
            return

        # None left, continue connecting...

        if self.main.dch is None:
            self.attachMeToDtella()

        elif self.main.pending_dch is None:
            self.state = 'queued'
            self.main.pending_dch = self

            def cb():
                self.queued_dcall = None
                self.main.pending_dch = None
                self.pushStatus("Nope, it didn't leave.  Goodbye.")
                self.transport.loseConnection()

            self.pushStatus(
                "Another DC client is already using Dtella on this computer.")
            self.pushStatus(
                "Waiting 5 seconds for it to leave.")

            self.queued_dcall = reactor.callLater(5.0, cb)

        else:
            self.pushStatus(
                "Dtella is busy with other DC connections from your "
                "computer.  Goodbye.")
            self.transport.loseConnection()

    def attachMeToDtella(self):

        CHECK(self.main.dch is None)

        if self.state == 'queued':
            self.queued_dcall.cancel()
            self.queued_dcall = None
            self.pushStatus(
                "The other client left.  Resuming normal connection.")

        dcall_discard(self, 'queued_dcall')

        # Add the post-login handlers
        self.addDispatch('$ConnectToMe',      2, self.d_NMDC_ConnectToMe)
        self.addDispatch('$RevConnectToMe',   2, self.d_NMDC_RevConnectToMe)
        self.addDispatch('$Search',          -2, self.d_Search)
        self.addDispatch('$To:',             -5, self.d_PrivateMsg)
        self.addDispatch("<%s>" % self.nick, -1, self.d_PublicMsg)

        # Announce my presence.
        # If Dtella's online too, this will trigger an event_DtellaUp.
        self.state = 'ready'
        self.main.addDCHandler(self)

    def formatMyInfo(self):
        # Build and return a hacked-up version of my info string.

        # Get version string
        ver_string = get_version_string()

        # Split info string
        try:
            info = split_info(self.info)
        except ValueError:
            # No info.  Just use the offline version tag
            return "<%s>" % ver_string

        # Split description into description and <tag>
        desc, tag = split_tag(info[0])

        # Update tag
        if tag:
            info[0] = "%s<%s,%s>" % (desc, tag, ver_string)
        else:
            info[0] = "%s<%s>" % (desc, ver_string)

        if local.use_locations:
            # Try to get my location name.
            try:
                ad = Ad().setRawIPPort(self.main.osm.me.ipp)
                loc = self.main.location[ad.getTextIP()]
            except (AttributeError, KeyError):
                loc = None

            # If I got a location name, splice it into my connection field
            if loc:
                # Append location suffix, if it exists
                suffix = self.main.state.suffix
                if suffix:
                    loc = '%s|%s' % (loc, suffix)

                info[2] = loc + info[2][-1:]

        info = '$'.join(info)

        if len(info) > 255:
            self.pushStatus("*** Your info string is too long!")
            info = ''

        return info

    def d_Search(self, addr_string, search_string):
        # Send a search request

        if not self.isOnline():
            self.pushStatus("Can't Search: Not online!")
            return

        if len(search_string) > 255:
            self.pushStatus("Search string too long")
            return

        osm = self.main.osm

        packet = osm.mrm.broadcastHeader('SQ', osm.me.ipp)
        packet.append(struct.pack('!I', osm.mrm.getPacketNumber_search()))

        packet.append(struct.pack('!B', len(search_string)))
        packet.append(search_string)

        osm.mrm.newMessage(''.join(packet), tries=4)

        # If local searching is enabled, send the search to myself
        if self.main.state.localsearch:
            self.push_NMDC_SearchRequest(osm.me.ipp, search_string)

    def d_PrivateMsg(self, nick, _1, _2, _3, text):

        text = dc_unescape(text)

        if nick == self.bot.nick:

            # No ! is needed for commands in the private message context
            if text[:1] == '!':
                text = text[1:]

            produced_output = [False]
            def out(text):
                if text is not None:
                    produced_output[0] = True
                    self.bot.say(text)

            self.bot.commandInput(out, produced_output, text)
            return

        if len(text) > 10:
            shorttext = text[:10] + '...'
        else:
            shorttext = text

        def fail_cb(detail):
            self.pushPrivMsg(
                nick,
                "*** Your message \"%s\" could not be sent: %s"
                % (shorttext, detail))

        if not self.isOnline():
            fail_cb("You're not online.")
            return

        try:
            n = self.main.osm.nkm.lookupNodeFromNick(nick)
        except KeyError:
            fail_cb("User doesn't seem to exist.")
            return

        n.event_PrivateMessage(self.main, text, fail_cb)

    def d_NMDC_ConnectToMe(self, nick, addr):

        osm = self.main.osm

        err_visible = True

        # Extract TCP port number from connect message.
        try:
            use_ssl = False
            port_str = addr[addr.rindex(':')+1:]

            # Some DC clients append an 'S' for SSL mode.
            if port_str.endswith('S'):
                use_ssl = True
                port_str = port_str[:-1]

            port = int(port_str)
            if not (1 <= port <= 65535):
                raise ValueError
        except ValueError:
            port = None

        def fail_cb(detail):
            if err_visible:
                self.pushStatus(
                    "*** Connection to <%s> failed: %s" % (nick, detail))

            # Don't try to abort SSL connections for now; too messy.
            if port and not use_ssl:
                reactor.connectTCP(
                    '127.0.0.1', port, AbortTransfer_Factory(nick))

        if not self.isOnline():
            fail_cb("you're not online.")
            return

        if not port:
            fail_cb("malformed address: <%s>" % addr)
            return

        try:
            n = osm.nkm.lookupNodeFromNick(nick)
        except KeyError:
            if nick == self.bot.nick:
                fail_cb("can't get files from yourself!")
            else:
                fail_cb("user doesn't seem to exist.")
            return

        if n.checkRevConnectWindow():
            # If we're responding to a RevConnect, disable errors
            err_visible = False

        elif self.isLeech():
            # I'm a leech
            err_visible = False
            fail_cb(None)
            return

        n.event_NMDC_ConnectToMe(self.main, port, use_ssl, fail_cb)

    def d_NMDC_RevConnectToMe(self, _, nick):

        osm = self.main.osm

        err_visible = True

        def fail_cb(detail):
            if err_visible:
                self.pushStatus(
                    "*** Connection to <%s> failed: %s" % (nick, detail))

            self.main.abort_nick = nick
            self.sendLine(
                "$ConnectToMe %s 127.0.0.1:%d"
                % (self.nick, self.factory.listen_port))

        if not self.isOnline():
            fail_cb("you're not online.")
            return

        try:
            n = osm.nkm.lookupNodeFromNick(nick)
        except KeyError:
            if nick == self.bot.nick:
                fail_cb("can't get files from yourself!")
            else:
                fail_cb("user doesn't seem to exist.")
            return

        if self.isLeech():
            # I'm a leech
            err_visible = False
            fail_cb(None)
            return

        n.event_NMDC_RevConnectToMe(self.main, fail_cb)

    def isLeech(self):
        # If I don't meet the minimum share, yell and return True

        osm = self.main.osm
        minshare = self.main.dcfg.minshare

        if osm.me.shared < minshare:
            self.pushStatus(
                "*** You must share at least %s in order to download!  "
                "(You currently have %s)" %
                (format_bytes(minshare), format_bytes(osm.me.shared)))
            return True

        return False

    def d_PublicMsg(self, text):

        text = dc_unescape(text)

        # Route commands to the bot
        if text[:1] == '!':

            produced_output = [False]
            def out(out_text):
                if out_text is not None:
                    produced_output[0] = True
                    self.pushStatus(out_text)

            if self.bot.commandInput(out, produced_output, text[1:], '!'):
                return

        if not self.isOnline():
            self.pushStatus("*** You must be online to chat!")
            return

        if self.main.osm.isModerated():
            self.pushStatus(
                "*** Can't send text; the chat is currently moderated.")
            return

        text = stdlines(text)

        for line in text.split('\n'):

            # Skip empty lines
            if not line:
                continue

            # Limit length
            if len(line) > 1024:
                line = line[:1024-12] + ' [Truncated]'

            flags = 0

            # Check for /me
            if len(line) > 4 and line[:4].lower() in ('/me ','+me ','!me '):
                line = line[4:]
                flags |= core.SLASHME_BIT

            # Check rate limiting
            if self.chat_counter > 0:

                # Send now
                self.chat_counter -= 1
                self.broadcastChatMessage(flags, line)

            else:
                # Put in a queue
                if len(self.chatq) < 5:
                    self.chatq.append( (flags, line) )
                else:
                    self.pushStatus(
                        "*** Chat throttled.  Stop typing so much!")
                    break

    def d_KeepAlive(self):
        # Doesn't do much really
        self.sendLine('')

    def pushChatMessage(self, nick, text, flags=0):
        self.sendLine("<%s> %s" % (nick, text))

    def pushInfo(self, nick, dcinfo):
        self.sendLine('$MyINFO $ALL %s %s' % (nick, dcinfo))

    def pushTopic(self, topic=None):
        if topic:
            self.sendLine("$HubName %s - %s" % (local.hub_name, topic))
        else:
            self.sendLine("$HubName %s" % local.hub_name)

    def pushHello(self, nick):
        self.sendLine('$Hello %s' % nick)

    def pushQuit(self, nick):
        self.sendLine('$Quit %s' % nick)

    def push_NMDC_ConnectToMe(self, ad, use_ssl):
        line = "$ConnectToMe %s %s" % (self.nick, ad.getTextIPPort())
        if use_ssl:
            line += 'S'
        self.sendLine(line)

    def push_NMDC_RevConnectToMe(self, nick):
        self.sendLine("$RevConnectToMe %s %s" % (nick, self.nick))

    def push_NMDC_SearchRequest(self, ipp, search_string):
        ad = Ad().setRawIPPort(ipp)
        self.sendLine("$Search %s %s" % (ad.getTextIPPort(), search_string))

    def pushPrivMsg(self, nick, text, flags=0):
        self.sendLine("$To: %s From: %s $<%s> %s"
                      % (self.nick, nick, nick, text))

    def pushBotMsg(self, text):
        self.sendLine("$To: %s From: %s $<%s> %s"
                      % (self.nick, self.bot.nick, self.bot.nick, text))

    def pushStatus(self, text):
        self.pushChatMessage(self.bot.nick, text)

    def scheduleChatRateControl(self):
        if self.chatRate_dcall:
            return

        def cb():
            self.chatRate_dcall = reactor.callLater(1.0, cb)

            if self.chatq:
                args = self.chatq.pop(0)
                self.broadcastChatMessage(*args)
            else:
                self.chat_counter = min(5, self.chat_counter + 1)

        cb()

    def broadcastChatMessage(self, flags, text):

        CHECK(self.isOnline())

        osm = self.main.osm

        if osm.isModerated():
            # If the channel went moderated with something in the queue,
            # wipe it out and don't send.
            del self.chatq[:]
            return

        packet = osm.mrm.broadcastHeader('CH', osm.me.ipp)
        packet.append(struct.pack('!I', osm.mrm.getPacketNumber_chat()))

        packet.append(osm.me.nickHash())
        packet.append(struct.pack('!BH', flags, len(text)))
        packet.append(text)

        osm.mrm.newMessage(''.join(packet), tries=4)

        # Echo back to the DC client
        if flags & core.SLASHME_BIT:
            nick = "*"
            text = "%s %s" % (osm.me.nick, text)
        else:
            nick = osm.me.nick

        self.pushChatMessage(nick, text)

    # Precompile a regex for pushSearchResult
    searchreply_re = re.compile(r"^\$SR ([^ |]+) ([^|]*) \([^ |]+\)\|?$")

    def pushSearchResult(self, data):

        m = self.searchreply_re.match(data)
        if not m:
            # Doesn't look like a search reply
            return

        nick = m.group(1)
        data = m.group(2)

        # If I get results from myself, map them to the bot's nick
        if nick == self.nick:
            nick = self.bot.nick

        self.sendLine("$SR %s %s (127.0.0.1:%d)"
                      % (nick, data, self.factory.listen_port))

    def remoteNickCollision(self):

        text = (
            "*** Another node on the network has reported that your nick "
            "seems to be in a conflicting state.  This could prevent your "
            "chat and search messages from reaching everyone, so it'd be "
            "a good idea to try changing your nick.  Or you could wait "
            "and see if the problem resolves itself."
            )

        for line in word_wrap(text):
            self.pushStatus(line)

    def doRejoin(self):
        if self.state != 'invisible':
            return

        dcall_discard(self, 'autoRejoin_dcall')

        self.state = 'ready'

        # This can trigger an event_DtellaUp()
        self.main.stateChange_ObserverUp()

    def isProtectedNick(self, nick):
        return (nick.lower() in (self.nick.lower(), self.bot.nick.lower()))

    def event_DtellaUp(self):
        CHECK(self.isOnline())
        self.d_GetNickList()

        # Grab the current topic from Dtella.
        tm = self.main.osm.tm
        self.pushTopic(tm.topic)
        if tm.topic:
            self.pushStatus(tm.getFormattedTopic())

    def event_DtellaDown(self):
        CHECK(self.isOnline())

        # Wipe out the topic
        self.pushTopic()

        # Wipe out my outgoing chat queue
        del self.chatq[:]

    def event_KickMe(self, lines, rejoin_time):
        # Sequence of events during a kick:
        # 1. event_RemoveNick(*)
        # 2. event_DtellaDown()
        # 3. event_KickMe()
        # 4. stateChange_ObserverDown()

        # Node will become visible again if:
        # - Dtella node loses its connection
        # - User types !REJOIN
        # - DC client reconnects (creates a new DCHandler)

        CHECK(self.state == 'ready')
        self.state = 'invisible'

        for line in lines:
            self.pushStatus(line)

        if rejoin_time is None:
            return

        # Automatically rejoin the chat after a timeout period.
        def cb():
            self.autoRejoin_dcall = None
            self.pushStatus("Automatically rejoining...")
            self.doRejoin()

        CHECK(self.autoRejoin_dcall is None)
        self.autoRejoin_dcall = reactor.callLater(rejoin_time, cb)

    def event_AddNick(self, n):
        if not self.isProtectedNick(n.nick):
            self.pushHello(n.nick)

    def event_RemoveNick(self, n, reason):
        if not self.isProtectedNick(n.nick):
            self.pushQuit(n.nick)

    def event_UpdateInfo(self, n):
        if n.dcinfo:
            self.pushInfo(n.nick, n.dcinfo)

    def event_ChatMessage(self, n, nick, text, flags):
        if flags & core.NOTICE_BIT:
            self.pushChatMessage(("*N# %s" % nick), text)
        elif flags & core.SLASHME_BIT:
            self.pushChatMessage("*", "%s %s" % (nick, text))
        else:
            self.pushChatMessage(nick, text)

verifyClass(IDtellaStateObserver, DCHandler)

# This class immediately sends the user a redirection sends disconnects them.
# Intended not for this module, but for other protocol modules that may want
# to redirect users to the correct connection address.

class AbortConnection(DCHandler):

    def __init__(self, dch, data, cprtl, caddr):
        DCHandler.__init__(self, dch.main)
        self.cprtl = cprtl
        self.caddr = caddr

        # Steal connection from the DCHandler
        self.factory = dch.factory
        self.makeConnection(dch.transport)
        self.transport.protocol = self

        # Steal the rest of the data
        self._buffer = dch._buffer

        # If we're not done in 5 seconds, something's fishy.
        def cb():
            self.timeout_dcall = None
            self.transport.loseConnection()

        self.timeout_dcall = reactor.callLater(5.0, cb)
        self.dataReceived(data)

    def attachMeToDtella(self):

        CHECK(self.main.dch is None)

        if self.state == 'queued':
            self.queued_dcall.cancel()
            self.queued_dcall = None
            self.pushStatus(
                "The other client left.  Resuming normal connection.")

        dcall_discard(self, 'queued_dcall')

        self.pushStatus("This node uses %s, not NMDC. Please connect to %s "
            "instead." % (self.cprtl, self.caddr))

        self.timeout_dcall.reset(0)

    def connectionLost(self, reason):
        dcall_discard(self, 'timeout_dcall')

##############################################################################

class DCFactory(ServerFactory):

    def __init__(self, main, listen_port):
        self.main = main

        self.listen_port = listen_port # spliced into search results

    def buildProtocol(self, addr):
        if addr.host != '127.0.0.1':
            return None

        p = DCHandler(self.main)
        p.protocol = core.PROTOCOL_NMDC
        p.factory = self
        return p

##############################################################################


import dejavu

if __name__ == '__main__':
    config = {"database": {
        "host":"127.0.0.1",
        "user":"local",
        "database":"dejavu"
    }}
    djv = dejavu.Dejavu(config)
    for i in range(0, 10):

        match = djv.recognize("../../podblocker/chunks_18/chunk_{}.mp3".format(i), group_id=0, known_song_id=0)
    print(match)


#!/usr/bin/env python
# coding: utf-8

# In[8]:

n =int(input())

l = [int(x) for x in input().split()]

o = [int(x) for x in l if x < 0]

if o == []:
    print(str(0))
else:
    print(abs(sum(o)))


import xbmc
import xbmcgui
import xbmcvfs
import resources.lib.tinyurl as tinyurl
import resources.lib.utils as utils
from resources.lib.dropbox import client, rest, session
from resources.lib.pydrive.auth import GoogleAuth
from resources.lib.pydrive.drive import GoogleDrive

class DropboxAuthorizer:
    APP_KEY = ""
    APP_SECRET = ""

    def __init__(self):
        self.APP_KEY = utils.getSetting('dropbox_key')
        self.APP_SECRET = utils.getSetting('dropbox_secret')

        self.setup()

    def setup(self):
        if(self.APP_KEY == '' and self.APP_SECRET == ''):
            #we can't go any farther, need these for sure
            xbmcgui.Dialog().ok(utils.getString(30010),utils.getString(30058),utils.getString(30059))
            return

    def isAuthorized(self):
        user_token_key,user_token_secret = self._getToken()

        return user_token_key != '' and user_token_secret != ''

    def authorize(self):
        result = True
        if(self.isAuthorized()):
            #delete the token to start over
            self._deleteToken()

        sess = session.DropboxSession(self.APP_KEY,self.APP_SECRET,"app_folder")

        token = sess.obtain_request_token()
        url = sess.build_authorize_url(token)

        #print url in log
        utils.log("Authorize URL: " + url)
        xbmcgui.Dialog().ok(utils.getString(30010),utils.getString(30056),utils.getString(30057),tinyurl.shorten(url))

        #if user authorized this will work
        user_token = sess.obtain_access_token(token)
        self._setToken(user_token.key,user_token.secret)

        return result;

    #return the DropboxClient, or None if can't be created
    def getClient(self):
        result = None
        sess = session.DropboxSession(self.APP_KEY,self.APP_SECRET,"app_folder")
        user_token_key,user_token_secret = self._getToken()

        if(user_token_key != '' and user_token_secret != ''):
            #create the client
            sess.set_token(user_token_key,user_token_secret)
            result = client.DropboxClient(sess)

            try:
                utils.log(str(result.account_info()))
            except:
                #this didn't work, delete the token file
                self._deleteToken()

        return result

    def _setToken(self,key,secret):
        #write the token files
        token_file = open(xbmc.translatePath(utils.data_dir() + "tokens.txt"),'w')
        token_file.write("%s|%s" % (key,secret))
        token_file.close()

    def _getToken(self):
        #get tokens, if they exist
        if(xbmcvfs.exists(xbmc.translatePath(utils.data_dir() + "tokens.txt"))):
            token_file = open(xbmc.translatePath(utils.data_dir() + "tokens.txt"))
            key,secret = token_file.read().split('|')
            token_file.close()

            return [key,secret]
        else:
            return ["",""]

    def _deleteToken(self):
        if(xbmcvfs.exists(xbmc.translatePath(utils.data_dir() + "tokens.txt"))):
            xbmcvfs.delete(xbmc.translatePath(utils.data_dir() + "tokens.txt"))

class GoogleDriveAuthorizer:
    CLIENT_ID = ''
    CLIENT_SECRET = ''

    def __init__(self):
        self.CLIENT_ID = utils.getSetting('google_drive_id')
        self.CLIENT_SECRET = utils.getSetting('google_drive_secret')

        self.setup()

    def setup(self):
        if(self.CLIENT_ID == '' and self.CLIENT_SECRET == ''):
            #we can't go any farther, need these for sure
            xbmcgui.Dialog().ok(utils.getString(30010),utils.getString(30058),utils.getString(30059))
            return

    def isAuthorized(self):
        return xbmcvfs.exists(xbmc.translatePath(utils.data_dir() + "google_drive.dat"))

    def authorize(self):
        result = True

        #create authorization helper and load default settings
        gauth = GoogleAuth(xbmc.validatePath(xbmc.translatePath(utils.addon_dir() + '/resources/lib/pydrive/settings.yaml')))
        gauth.LoadClientConfigSettings()

        settings = {"client_id":self.CLIENT_ID,'client_secret':self.CLIENT_SECRET}

        drive_url = gauth.GetAuthUrl(settings)

        utils.log("Google Drive Authorize URL: " + drive_url)

        xbmcgui.Dialog().ok(utils.getString(30010),utils.getString(30056),utils.getString(30102),tinyurl.shorten(drive_url))
        code = xbmcgui.Dialog().input(utils.getString(30103))

        gauth.Auth(code)
        gauth.SaveCredentialsFile(xbmc.validatePath(xbmc.translatePath(utils.data_dir() + 'google_drive.dat')))

        return result

    def getClient(self):
        #create authorization helper and load default settings
        gauth = GoogleAuth(xbmc.validatePath(xbmc.translatePath(utils.addon_dir() + '/resources/lib/pydrive/settings.yaml')))
        gauth.LoadClientConfigSettings()

        gauth.LoadCredentialsFile(xbmc.validatePath(xbmc.translatePath(utils.data_dir() + 'google_drive.dat')))

        result = GoogleDrive(gauth)

        return result


import logging
import multiprocessing
import threading
import os
import numpy as np
import tensorflow as tf

import Utils.utils
from Environment.environment import UnoEnvironment
from Environment.state import State
from Players.A3CPlayer import A3CPlayer
from Players.AgentPlaceholder import AgentPlaceholder
from Players.randomplayer import RandomPlayer
from Agents.ListMemory import ListMemory
from Agents.ActorCriticModel import ActorCriticModel

logging.basicConfig()
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logging.getLogger('matplotlib.font_manager').disabled = True

# Prirejeno po zgledu: https://blog.tensorflow.org/2018/07/deep-reinforcement-learning-keras-eager-execution.html
class A3CWorker(threading.Thread):
    GLOBAL_MAX_EPISODES = 1000
    global_model_update_frequency = 20
    global_best_episode_score = None
    global_total_episodes_across_all_workers = 0
    global_semaphore = threading.Lock()
    global_statistic = []

    def __init__(self, master_model, optimizer, worker_id, folder, beta, gamma, opponent_model_path=None):
        super(A3CWorker, self).__init__()
        self.master_model = master_model
        self.optimizer = optimizer
        self.worker_id = worker_id
        self.env = UnoEnvironment(False)
        self.state_size = State.STATE_SIZE
        self.action_size = State.ACTION_SIZE
        self.gamma = gamma  # discounting factor
        self.beta = beta
        self.worker_model = ActorCriticModel(self.action_size)
        self.memory = ListMemory()
        self.folder = folder
        self.episode_loss = 0
        self.episode_steps = 0
        self.episode_reward = 0
        self.episode_discounted_reward = 0
        self.steps = 0
        self.save = True
        self.agent_placeholder = AgentPlaceholder()
        if opponent_model_path is not None:
            self.opponent = A3CPlayer(opponent_model_path)
        else:
            self.opponent = RandomPlayer()

    def run(self):
        while A3CWorker.global_total_episodes_across_all_workers < A3CWorker.GLOBAL_MAX_EPISODES:
            A3CWorker.global_total_episodes_across_all_workers += 1
            logger.info("Starting episode " + str(A3CWorker.global_total_episodes_across_all_workers) + "/" +
                        str(A3CWorker.GLOBAL_MAX_EPISODES) + " with worker " + str(self.worker_id))

            done = False
            self.episode_steps = 0
            self.episode_loss = 0
            self.episode_reward = 0
            self.episode_discounted_reward = 0
            self.memory.empty()
            state = self.env.reset(self.agent_placeholder, self.opponent)

            while not done:
                action = self.get_action()
                next_state, reward, done = self.env.step_with_opp_step(action)
                self.episode_reward += reward
                self.memory.store(state, action, reward, next_state, done)
                state = next_state

                self.steps += 1
                self.episode_steps += 1

                if self.steps % A3CWorker.global_model_update_frequency == 0 or done:
                    self.sync_gradients()

                if done:
                    A3CWorker.global_statistic.append((self.episode_reward, self.episode_loss / self.episode_steps))
                    if A3CWorker.global_best_episode_score is None or \
                            self.episode_reward > A3CWorker.global_best_episode_score:
                        self.update_master_model()

    def get_action(self):
        # You are only interested in the legal policy logits
        policy_logits, values = self.worker_model(tf.convert_to_tensor(np.random.random((1, self.state_size)),
                                                                       dtype=tf.float32))
        legal_actions = self.env.get_legal_actions()
        legal_policy_logits = np.take(policy_logits, legal_actions)
        stochastic_legal_action_probabilities = tf.nn.softmax(legal_policy_logits)
        stochastic_legal_action_probabilities = stochastic_legal_action_probabilities.numpy()
        stochastic_legal_action_probabilities /= stochastic_legal_action_probabilities.sum()
        stochastic_policy_driven_legal_action = np.random.choice(legal_actions,
                                                                 p=stochastic_legal_action_probabilities)
        return stochastic_policy_driven_legal_action

    def update_master_model(self):
        A3CWorker.global_best_episode_score = self.episode_reward
        if self.save:
            with A3CWorker.global_semaphore:
                logger.info("Saving best model "
                            "- worker: " + str(self.worker_id) +
                            " episode: " + str(A3CWorker.global_total_episodes_across_all_workers) +
                            " reward: " + str(self.episode_reward) +
                            " discounted reward:" + str(self.episode_discounted_reward) +
                            " loss: " + str(self.episode_loss.numpy()))
                self.master_model.save_weights(os.path.join(self.folder, 'model_' + Utils.utils.get_timestamp() +'.h5'))

    def sync_gradients(self):
        with tf.GradientTape() as tape:
            total_loss = self.compute_loss()
        self.episode_loss += total_loss
        grads = tape.gradient(total_loss, self.worker_model.trainable_weights)
        self.optimizer.apply_gradients(zip(grads, self.master_model.trainable_weights))

        self.worker_model.set_weights(self.master_model.get_weights())
        self.memory.empty()

    def compute_loss(self):
        if self.memory.dones[-1]:
            reward_sum = 0.
        else:
            reward_sum = self.worker_model(tf.convert_to_tensor(self.memory.next_states[-1][None, :], dtype=tf.float32))[-1].numpy()[0]
        discounted_rewards = []
        for reward in self.memory.rewards[::-1]:
            reward_sum = reward + self.gamma * reward_sum
            discounted_rewards.append(reward_sum)
        discounted_rewards.reverse()

        self.episode_discounted_reward = discounted_rewards[0]

        logits, values = self.worker_model(tf.convert_to_tensor(np.vstack(self.memory.states), dtype=tf.float32))

        advantage = tf.convert_to_tensor(np.array(discounted_rewards)[:, None], dtype=tf.float32) - values # Baseline
        value_loss = advantage ** 2

        actions_one_hot = tf.one_hot(self.memory.actions, self.action_size, dtype=tf.float32)

        policy = tf.nn.softmax(logits)
        entropy = tf.reduce_sum(policy * tf.math.log(policy + self.beta), axis=1)

        # Entropija prepreč prehitro konvergiranje k suboptimalnem determinističnmem pravilniku
        policy_loss = tf.nn.softmax_cross_entropy_with_logits(labels=actions_one_hot, logits=logits)
        policy_loss *= tf.stop_gradient(advantage)
        policy_loss -= 0.01 * entropy
        total_loss = tf.reduce_mean((0.5 * value_loss + policy_loss))
        return total_loss


"""List billing information of orders."""
# :license: MIT, see LICENSE for more details.

import SoftLayer
from SoftLayer.CLI import environment
from SoftLayer.CLI import formatting
from SoftLayer import utils

import click

@click.command()
@click.option('--start_date', '-f', help='cost incurred from this start date')
@click.option('--end_date', '-e',
              help='cost incurred before this end date'
                   ' (default is current time stamp)')
@environment.pass_env
def cli(env, start_date, end_date):
    """List billing information for orders."""
    billing = SoftLayer.BillingManager(env.client)
    from_date = start_date
    to_date = end_date
    table = formatting.Table(['Order ID', 'Resource Name', 'Resource Type',
                              'cost', 'create_date'])
    resources = billing.list_resources(from_date, to_date)

    for resource in resources:
        resource = utils.NestedDict(resource)
        table.add_row([
            resource['id'],
            resource['hostName'],
            resource['resourceType'],
            resource['cost'],
            resource['createDate']
        ])

    return table


#!/bin/python

import pandas as pd
import datetime as dt
import numpy as np
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt
import seaborn as sn
import time
import pytz
import os
import shutil

import config
from bettor import Bettor, Team

bet_file_postfix = config.BET_FILE_POSTFIX
schedule_file = config.SCHEDULE_FILE
SEASON = config.SEASON
START_DATE = config.START_DATE
tz = pytz.timezone('EST')
today = dt.datetime.now(tz)
STR_TODAY = '%i/%i/%i' % (today.month, today.day, today.year)

def todays_games(person_list):
    team_bettor_dict = {team: person_obj.name for person_obj in person_list for team in person_obj.team_objs.keys()}
    team_bettor_dict['Blazers'] = team_bettor_dict['Trail Blazers']
    team_bettor_dict['76ers'] = team_bettor_dict['Sixers']
    # Upload the current schedule and convert date to timestamp
    nba_sched = pd.read_csv(schedule_file, index_col=0)
    nba_sched.index = pd.to_datetime(nba_sched.index)

    # Get today's date
    today = dt.datetime.strptime(STR_TODAY, '%m/%d/%Y').isoformat()

    # Grabs todays games
    nba_sched_today = nba_sched.loc[nba_sched.index == today]
    nba_sched_today['HomeShort'] = nba_sched_today['Home/Neutral'].str.split().str[-1]
    nba_sched_today['VisitorShort'] = nba_sched_today['Visitor/Neutral'].str.split().str[-1]

    games = ''
    for index, row in nba_sched_today.iterrows():
        ht = row['HomeShort']
        at = row['VisitorShort']

        games += html_table_add_row([ht + ' ('+team_bettor_dict[ht].title()+')',at + ' ('+team_bettor_dict[at].title()+')'])

    return games

def determine_current_pick_order(person_list):
    """
    Assign a new pick number for every team based on current win percentage.
    """
    win_pct = pd.DataFrame(columns=['win pct'])
    for person_obj in person_list:

        for team in person_obj.teams:

            team_obj = person_obj.team_objs[team]
            s = pd.Series(name=team_obj.name, data={'win pct': float(team_obj.wins())/float(team_obj.wins()+team_obj.losses())})
            win_pct = win_pct.append(s)

    win_pct.sort_values('win pct', inplace=True, ascending=False)
    win_pct['new pickno'] = range(1, 31)

    for person_obj in person_list:

        for team in person_obj.teams:

            person_obj.team_objs[team].new_pickno = win_pct.loc[team, 'new pickno']

    return

def plot_graph_all(person_list):
    """Creates bettor 30-day and all season Wins-Losses graphs and creates
       graphs that displays the Wins-Losses for a bettor's set of teams"""

    # Create a DF with every calendar day with the summed Wins-Losses for each bettor
    summed_diff_calendar_list = [reduce((lambda x, y: x+y),
        [person_obj.team_objs[team].diff_calendar() for team in person_obj.teams])
        for person_obj in person_list ]

     # Plot Wins-Losses as a function of time for last 30 days
    sn.set(color_codes=True)
    fig = plt.figure(figsize=(6.5, 6.5))
    ax = fig.add_subplot(111)
    for i in range(len(person_list)):
        summed_diff_calendar_list[i][today - dt.timedelta(days=30):today].plot()
    ax.legend(pick_order, loc='lower left')
    ax.set_ylabel('Wins - Losses', fontsize=12)
    fig.savefig('../images/win_percent_recent.png', bbox_inches='tight')
    plt.close(fig)

     # Plot Wins-Losses as a function of time for season so far
    sn.set(color_codes=True)
    fig = plt.figure(figsize=(6.5, 3))
    ax = fig.add_subplot(111)
    for i in range(len(person_list)):
        summed_diff_calendar_list[i].plot()
    ax.legend([person_obj.name for person_obj in person_list], loc='lower left')
    ax.set_ylabel('Wins - Losses', fontsize=12)
    fig.savefig('../images/win_percent_all.png', bbox_inches='tight')
    plt.close(fig)

    # Plot Wins-Losses for each bettor by team
    sn.set_palette(sn.color_palette("hls", 10))
    for order, person_obj in enumerate(person_list):
        # DF with every calendar day for each of the Bettor's teams
        team_diff_calendar_list = [person_obj.team_objs[team].diff_calendar() for team in person_obj.teams]
        fig = plt.figure(figsize=(10, 6.5))
        ax = fig.add_subplot(111)
        for iteam in person_obj.teams:
            person_obj.team_objs[iteam].diff_calendar().plot(label=iteam, ax=ax)
        ax.set_ylabel('Wins - Losses', fontsize=12)
        ax.legend()
        fig.savefig('../images/'+str(order+1)+'_teams.png', bbox_inches='tight')
        plt.close(fig)

    return

def make_html(person_list):
    """Populate main HTML page that summarizes bettors status and tonight's matchups"""
    with open('../template.html', 'r') as ftemp:
        temp = ftemp.read()

    # Person with the most wins
    current_winner = max([(person_obj.all_wins(), person_obj.name) for person_obj in person_list])[1]

    shutil.copyfile('../images/current_winner_'+current_winner+'.jpg', '../images/current_winner.jpg')

    # Use html_string to dump information into HTML page
    html_string = [current_winner, STR_TODAY, current_winner.title()]

    # Every bettor's quick stats
    wins, losses, remaining, draft_value = [],[],[],[]
    for person_obj in person_list:
        html_string += [person_obj.name.title()]
        wins.append(person_obj.all_wins())
        losses.append(person_obj.all_losses())
        remaining.append(person_obj.all_remaining()-person_obj.dup_remaining_games())
        draft_value.append(person_obj.all_draft_value())

    html_string += wins + losses + remaining + draft_value

    # Order of teams that were picked by a bettor
    # -> Bettor 1 pick 1, Bettor 2 pick 1, Bettor 3 pick 1, Bettor 1 pick 2, etc
    team_pickorder = [person_obj.name.title() for person_obj in person_list] + [person_obj.teams[i]+' ('+str(person_obj.team_objs[person_obj.teams[i]].new_pickno)+')' for i in range(10) for person_obj in person_list]

    html_string += team_pickorder

    # Today's games (by team and bettor)
    html_string += [todays_games(person_list)]
    updated = temp % tuple(html_string)

    with open('../bet.html', 'w') as findex:
        findex.write(updated)

    return

def make_bettor_html(person_list):
    """Make individual bettor page information that breaks down a bettor's teams"""
    with open('../template_bettor.html', 'r') as ftemp:
        temp = ftemp.read()
    html_string = [person_obj.name.title() for person_obj in person_list]
    for person_obj in person_list:

        team_list = ''
        # Summary of team information for a bettor
        for teamname in person_obj.teams:
            team_obj = person_obj.team_objs[teamname]
            team_list += html_table_add_row([teamname,
                                             team_obj.wins(),
                                             team_obj.losses(),
                                             team_obj.remaining(),
                                             team_obj.last10(),
                                             '{0:.2f}'.format(team_obj.expected_wins()),
                                             '{0:.2f}'.format(team_obj.draft_value())], align="center")

        html_string += [person_obj.name.title(), person_obj.all_wins(), person_obj.all_losses(),
                        person_obj.all_remaining()-person_obj.dup_remaining_games(),team_list]

    updated = temp % tuple(html_string)

    with open('../bettors.html', 'w') as findex:
        findex.write(updated)

    return

def html_table_add_row(element_list, align="left"):
    """Simple html table row"""
    connector = '    	'
    table_str = '
'
    table_str += connector.join(str(e) for e in element_list)
    table_str += '
'
    return table_str

if __name__ == '__main__':

    # Read in bettor pick order
    pick_order = np.loadtxt('pickorder.txt', dtype=str, delimiter='\t')

    # Create list of Bettor objects and initialize them
    person_list = []
    for order, bettor in enumerate(pick_order):
        bettor = bettor.lower()
        bettor_teams = np.loadtxt(bettor + bet_file_postfix, dtype=str, delimiter='\t')

        bettor_obj = Bettor(bettor, order, bettor_teams)

        person_list += [bettor_obj]

    # Add new pick number to each team
    determine_current_pick_order(person_list)

    # Create graphs and html pages
    plot_graph_all(person_list)
    make_html(person_list)
    make_bettor_html(person_list)


import cv2
import socket
import pickle
import base64

HOST = '192.168.1.27'
PORT = 9999

socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket.bind((HOST,PORT))
socket.listen()
conn,addr = socket.accept()
print(conn)

while True:
    try:
        datas = conn.recv(1000000)
        datas = base64.b64decode(datas)
        frame = pickle.loads(datas)
        frame = cv2.imdecode(frame, cv2.IMREAD_COLOR)
        cv2.imshow("deneme", frame)
        cv2.waitKey(1)
    except:
        continue


# uncompyle6 version 3.7.4
# Python bytecode 2.7 (62211)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /home/hvelarde/collective/lazysizes/src/collective/lazysizes/tests/test_robot.py
# Compiled at: 2016-10-22 10:15:51
from collective.lazysizes.testing import ROBOT_TESTING
from plone.testing import layered
import os, robotsuite, unittest

def test_suite():
    suite = unittest.TestSuite()
    current_dir = os.path.abspath(os.path.dirname(__file__))
    tests = [ doc for doc in os.listdir(current_dir) if doc.startswith('test_') and doc.endswith('.robot')
            ]
    suite.addTests([ layered(robotsuite.RobotTestSuite(t), layer=ROBOT_TESTING) for t in tests
                   ])
    return suite


from flask import Flask

app = Flask(__name__)

@app.route('/')
def home():
    return "Hello World"

@app.route('/python')
def python():
    html = """
        <html>
                {python_h1}
        </html>
    """
    python_h1 ="<h1>{title}</h1>".format(title="Python Programming Language")

    return html.format(python_h1=python_h1)


import csv
import numpy as np
from scipy import spatial
from sklearn import svm
from sklearn import metrics
vectorLength = 100

tags = {'PDT': 'PT', '.': '.', 'VB': 'VB', ':': ':', '#': '#', 'VBN': 'VN', 'RBR': 'RR', 'PRP$': 'PR', 'DT': 'DT', 'VBZ': 'VZ', 'CC': 'CC', 'TO': 'TO', 'LS': 'LS', 'SYM': 'SM', 'RBS': 'RS', 'JJ': 'JJ', 'EX': 'EX', 'WP': 'WP', 'POS': 'PS', 'WDT': 'WT', 'VBP': 'VP', 'WRB': 'WB', 'PRP': 'PP', 'JJR': 'JR', 'VBD': 'VD', 'NNPS': 'NQ', 'RB': 'RB', '-LRB-': 'L$', 'RP': 'RP', 'JJS': 'JS', 'CD': 'CD', '-RRB-': 'R$', 'NNP': 'NP', '$': '$', 'WP$': 'WP', 'FW': 'FW', 'VBG': 'VG', "''": "''", ',': ',', 'NN': 'NN', 'UH': 'UH', 'NNS': 'NS', 'MD': 'MD', '``': '``', 'IN': 'IN'}
inverseTags = dict()
for key, item in tags.items():
	inverseTags[item] = key
#missing = open("missingMSRLemmaUntaggedUnhyphenWords.txt",'w')
#missingSet = set()
with open('../vectors/lemmaTagVectors.vec', 'r') as vectorFile:
        vectors = dict()
        vectorFile.readline()
        for line in vectorFile:
                splitLine = line.split()
                start = len(splitLine) - vectorLength
                #for i in range(start, len(splitLine)):
                #        splitLine[i] = float(splitLine[i])
                vectors[" ".join(splitLine[0:start])] = np.array(splitLine[start:], dtype = float)

with open('missing/missingLEMMATAGVectors.txt', 'r') as missingFile:
	for line in missingFile:
		splitLine = line.split()
		start = len(splitLine) - vectorLength
		if " ".join(splitLine[0:start]) not in vectors:
			vectors[" ".join(splitLine[0:start])] = np.array(splitLine[start:], dtype = float)
		else:
			print("Weird!",splitLine)

#with open('../vectors/tagWordVectors.vec','r') as tagFile:
#	tagVectors = dict()
#	tagFile.readline()
#	for line in tagFile:
#		splitLine = line.split()
#		tagVectors[splitLine[0]] = np.array(splitLine[1:], dtype=float)

def getVector(sentence, vectors):
    resultVector = np.zeros(vectorLength)
    tagVector = np.zeros(10)
    splitLine = sentence.split()
    for word in splitLine:
        if word in vectors:
             resultVector = resultVector + vectors[word]
        else:
             print("Woah", word, splitLine)
    return resultVector
#        try:
#            if word[-2] == '_':
#                tempWord = word[:-2]
#                if tempWord in vectors:
#                    resultVector = resultVector + vectors[tempWord]
#                    tagVector = tagVector + tagVectors[inverseTags[word[-1]]]
#            elif word[-3] == '_':
#                tempWord = word[:-3]
#                if tempWord in vectors:
#                    resultVector = resultVector + vectors[tempWord]
#                    tagVector = tagVector + tagVectors[inverseTags[word[-2:]]]
#        except IndexError:
#            pass
#else:
#                print("Cray", word, splitLine)
#        else:
#            print("Woah", word, splitLine)
#    return resultVector
#        else:
#            tempWord = word
#            index = tempWord.find('-')
#            while index != -1:
#                current = tempWord[:index]
#                if current in vectors:
#                    resultVector = resultVector + vectors[current]
               #elif current != "" and current not in missingSet:
                #    print("Problem1", current)
                #    missing.write(current + '\n')
                #    missingSet.add(current)
#                tempWord = tempWord[index+1:]
#                index = tempWord.find('-')
#            if tempWord in vectors:
#                resultVector = resultVector + vectors[tempWord]
            #elif tempWord != "" and tempWord not in missingSet:
            #    print("Problem2", tempWord)
            #    missing.write(tempWord + '\n')
            #    missingSet.add(tempWord)
                #raise(ValueError)"""
#    return resultVector
#    return np.concatenate((resultVector,tagVector))

from sklearn import linear_model
from sklearn.metrics import mean_squared_error
from scipy.stats import spearmanr

with open('lemmaTagPPDBTrain.txt', 'r', encoding='utf8') as trainFile:
    distances = []
    truths = []
    fileSize = int(trainFile.readline())

    for i in range(fileSize):
        data1 = getVector(trainFile.readline()[:-1], vectors)
        data2 = getVector(trainFile.readline()[:-1], vectors)
        distances.append([spatial.distance.cosine(data1,data2)])
        truths.append(float(trainFile.readline()))

#    cutoff = len(distances)*8//10
#    best = 0

    model = linear_model.LinearRegression()
    model.fit(distances, truths)
    #for i in range(-4, 5):
    #    print(i)
    #    model = svm.SVC(kernel = 'linear',C=10**i)
    #    model.fit(distances[:cutoff], truths[:cutoff])

    #y_pred = model.predict(distances)

     #   f1 = metrics.f1_score(truths[cutoff:],y_pred[cutoff:])
     #   print(f1)
   #     if f1 > best:
   #         best = f1
   #         cvalue = i
#    model = svm.SVC(kernel='linear', C=10**cvalue)
#    model.fit(distances,truths)

#    outcome = [truths[i]==y_pred[i] for i in range(len(truths))]
    print("training error: ", mean_squared_error(model.predict(distances), truths))
    print("Training Spearman's: ", spearmanr(model.predict(distances),truths))
#    print("Training F1 score: ", metrics.f1_score(truths,y_pred))
#    print(cvalue)

with open('lemmaTagPPDBTest.txt', 'r', encoding='utf8') as testFile:
    distances = []
    truths = []
    fileSize = int(testFile.readline())
    for i in range(fileSize):
        data1 = getVector(testFile.readline()[:-1], vectors)
        data2 = getVector(testFile.readline()[:-1], vectors)
        distances.append([spatial.distance.cosine(data1,data2)])
        truths.append(float(testFile.readline()))

    y_pred = model.predict(distances)
    print("Test error: ", mean_squared_error(y_pred,truths))
    print("Test spearman's: ", spearmanr(y_pred, truths))
	#missing.close()


import copy

a = [1, 2, 3, 4, ['a', 'b']]
b = a
c = copy.copy(a)
d = copy.deepcopy(a)
a.append(5)
a[4].append('c')

print(a, b, c, d)

a = 1
try:
    a += 1
except:
    a += 1
else:
    a += 1
finally:
    a += 1

print(a)