# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import sys
import dis
import copy
import functools
import inspect

from unittest.mock import _must_skip

def _add_signature_validation(value, template, attr_name):
    if isinstance(template, StrictMock):
        if "__template" in template.__dict__:
            template = template.__template
        else:
            return value

    # This covers runtime attributes
    if not hasattr(template, attr_name):
        return value

    callable_template = getattr(template, attr_name)
    # FIXME decouple from _must_skip. It tells when self should be skipped
    # for signature validation.
    if _must_skip(template, attr_name, isinstance(template, type)):
        callable_template = functools.partial(callable_template, None)

    try:
        signature = inspect.signature(callable_template, follow_wrapped=False)
    except ValueError:
        signature = None

    def with_sig_check(*args, **kwargs):
        if signature:
            try:
                signature.bind(*args, **kwargs)
            except TypeError as e:
                raise TypeError(
                    "{}, {}: {}".format(repr(template), repr(attr_name), str(e))
                )
        return value(*args, **kwargs)

    return with_sig_check

class UndefinedBehavior(BaseException):
    """
    Tentative access of an attribute from a StrictMock that is not defined yet.
    Inherits from BaseException to avoid being caught by tested code.
    """

    __slots__ = ["strict_mock", "attr", "message"]

    def __init__(self, strict_mock, attr, message):
        super(UndefinedBehavior, self).__init__(strict_mock, attr, message)
        self.strict_mock = strict_mock
        self.attr = attr
        self.message = message

    def __str__(self):
        return (
            "{}:\n"
            "  Attribute '{}' has no behavior defined.\n"
            "  You can define behavior by assigning a value to it."
        ).format(repr(self.strict_mock), self.attr)

class NoSuchAttribute(BaseException):
    """
    Tentative of setting of an attribute from a StrictMock that is not present
    at the template class.
    Inherits from BaseException to avoid being caught by tested code.
    """

    __slots__ = ["strict_mock", "attr", "message"]

    def __init__(self, strict_mock, attr, message):
        super(NoSuchAttribute, self).__init__(strict_mock, attr, message)
        self.strict_mock = strict_mock
        self.attr = attr
        self.message = message

    def __str__(self):
        return ("{}:\n" "  No such attribute '{}'.\n" "  {}").format(
            repr(self.strict_mock), self.attr, self.message
        )

class _DescriptorProxy(object):
    def __init__(self, name):
        self.name = name
        self.attrs = {}

    def __get__(self, instance, _owner):
        if instance in self.attrs:
            return self.attrs[instance]
        else:
            raise AttributeError(
                "{}:\n  Object has no attribute '{}'".format(repr(instance), self.name)
            )

    def __set__(self, instance, value):
        self.attrs[instance] = value

    def __delete__(self, instance):
        if instance in self.attrs:
            del self.attrs[instance]

class _MethodProxy(object):
    def __init__(self, original_method, call):
        self.__dict__["_original_method"] = original_method
        self.__dict__["_call"] = call

    def __getattr__(self, name):
        return getattr(self.__dict__["_original_method"], name)

    def __setattr__(self, name, value):
        return setattr(self.__dict__["_original_method"], name, value)

    def __delattr__(self, name):
        return delattr(self.__dict__["_original_method"], name)

    def __call__(self, *args, **kwargs):
        return self.__dict__["_call"](*args, **kwargs)

class StrictMock(object):
    """
    Mock object that won't allow any attribute access or method call, unless its
    behavior has been explicitly defined. This is meant to be a safer
    alternative to Python's standard Mock object, that will always return
    another mock when referred by default.

    StrictMock is "safe by default", meaning that it will never misbehave by
    lack of configuration. It will raise in the following situations:

    - Get/Set attribute that's not part of the specification (template or
      runtime_attrs).
    - Get attribute that is part of the specification, but has not yet been
      defined.
    - Call a method with different signature from the template.

    When appropriate, raised exceptions inherits from BaseException, in order to
    let exceptions raise the test, outside tested code, so we can get a clear
    signal of what is happening: either the mock is missing a required behavior
    or the tested code is misbehaving.
    """

    TRIM_PATH_PREFIX = ""

    def __init__(self, template=None, runtime_attrs=None, name=None):
        """
        template: Template class to be used as a template for the mock. If the
        template class implements a context manager, empty mocks for __enter__()
        and __exit__() will be setup automatically.
        runtime_attrs: Often attributes are created within an instance's
        lifecycle, typically from __init__(). To allow mocking such attributes,
        specify their names here.
        name: an optional name for this mock instance.
        """
        if template:
            assert inspect.isclass(template), "Template must be a class."

        # avoid __getattr_ recursion
        self.__dict__["__template"] = template
        self.__dict__["__runtime_attrs"] = runtime_attrs or []
        self.__dict__["__name"] = name

        if sys.version_info.major >= 3 and sys.version_info.minor >= 6:
            frameinfo = inspect.getframeinfo(inspect.stack()[1][0])
            filename = frameinfo.filename
            lineno = frameinfo.lineno
        else:
            frame = inspect.stack()[1][0]
            filename = inspect.getsourcefile(frame) or inspect.getfile(frame)
            lineno = inspect.getframeinfo(frame).lineno

        if self.TRIM_PATH_PREFIX:
            split = filename.split(self.TRIM_PATH_PREFIX)
            if len(split) == 2 and not split[0]:
                filename = split[1]
        self.__dict__["__caller"] = "{}:{}".format(filename, lineno)

        if (
            self.__template
            and hasattr(self.__template, "__enter__")
            and hasattr(self.__template, "__exit__")
        ):
            self.__enter__ = lambda: self
            self.__exit__ = lambda exc_type, exc_value, traceback: None

    @property
    def __class__(self):
        return self.__template if self.__template is not None else type(self)

    @property
    def __template(self):
        import testslide.mock_constructor  # Avoid cyclic dependencies

        # If the template class was mocked with mock_constructor(), this will
        # return the mocked subclass, which contains all attributes we need for
        # introspection.
        return testslide.mock_constructor._get_class_or_mock(
            self.__dict__["__template"]
        )

    @property
    def __template_name(self):
        return self.__template.__name__ if self.__template else "None"

    @property
    def __runtime_attrs(self):
        return self.__dict__["__runtime_attrs"]

    def __get_class_init(self, klass):
        import testslide.mock_constructor  # Avoid cyclic dependencies

        if testslide.mock_constructor._is_mocked_class(klass):
            # If klass is the mocked subclass, pull the original version of
            # __init__ so we can introspect into its implementation (and
            # not the __init__ wrapper at the mocked class).
            mocked_class = klass
            original_class = mocked_class.mro()[1]
            return testslide.mock_constructor._get_original_init(
                original_class, instance=None, owner=mocked_class
            )
        else:
            return klass.__init__

    def __is_runtime_attr(self, name):
        if self.__template:
            for klass in self.__template.mro():
                template_init = self.__get_class_init(klass)
                if not inspect.isfunction(template_init):
                    continue
                for instruction in dis.get_instructions(template_init):
                    if (
                        instruction.opname == "STORE_ATTR"
                        and name == instruction.argval
                    ):
                        return True
        return False

    def __can_mock_attr(self, name):
        if not self.__template:
            return True
        return (
            hasattr(self.__template, name)
            or name in self.__runtime_attrs
            or name in getattr(self.__template, "__slots__", [])
            or self.__is_runtime_attr(name)
        )

    def __get_mock_value(self, name, value):
        if hasattr(self.__template, name):
            # If we are working with a callable we need to actually
            # set the side effect of the callable, not directly assign
            # the value to the callable
            if callable(getattr(self.__template, name)):
                if not callable(value):
                    raise ValueError(
                        "{}: Template class attribute '{}' attribute is callable and {} is not.".format(
                            repr(self), name, repr(value)
                        )
                    )
                value = _MethodProxy(
                    original_method=value,
                    call=_add_signature_validation(value, self.__template, name),
                )
        return value

    def __setattr__(self, name, value):
        if self.__can_mock_attr(name):
            if name in type(self).__dict__:
                type(self).__dict__[name].__set__(
                    self, self.__get_mock_value(name, value)
                )
            else:
                setattr(type(self), name, _DescriptorProxy(name))
                self.__setattr__(name, value)
        else:
            # If the template class has the attribute and we haven't yet defined its
            # behavior we use a different exception than when the attribute
            # doesn't event exist in the template class
            if hasattr(self.__template, name):
                raise UndefinedBehavior(
                    self,
                    name,
                    "The attribute {} is defined in the template class "
                    "{}, but its behavior is not yet defined in this "
                    "StrictMock".format(self.__template_name),
                )
            else:
                raise NoSuchAttribute(
                    self,
                    name,
                    "Can not set attribute {} that is neither "
                    "part of template class {} or runtime_attrs={}.".format(
                        name, self.__template_name, self.__runtime_attrs
                    ),
                )

    def __getattr__(self, attr):
        if attr in type(self).__dict__:
            try:
                return type(self).__dict__[attr].__get__(self, type(self))
            except AttributeError:
                pass

        if self.__can_mock_attr(attr):
            raise UndefinedBehavior(
                self,
                attr,
                "Can not getattr() an undefined StrictMock "
                "attribute. Use setattr() to define it.",
            )
        else:
            raise AttributeError(
                "{}: Can not getattr() an attribute '{}' that is neither part of "
                "template class {} or runtime_attrs={}.".format(
                    repr(self), attr, self.__template_name, self.__runtime_attrs
                )
            )

    def __delattr__(self, attr):
        if attr in type(self).__dict__:
            type(self).__dict__[attr].__delete__(self)

    def __repr__(self):
        template = (
            " template={}.{}".format(
                self.__template.__module__, self.__template.__name__
            )
            if self.__template
            else ""
        )
        if self.__dict__["__name"]:
            name = " name={}".format(repr(self.__dict__["__name"]))
        else:
            name = ""
        return "<StrictMock 0x{:02X}{name}{template} {caller}>".format(
            id(self), name=name, template=template, caller=self.__dict__["__caller"]
        )

    def __get_copy(self):
        return type(self)(template=self.__template, runtime_attrs=self.__runtime_attrs)

    def __get_instance_attr_items(self):
        items = []
        for name in type(self).__dict__:
            descriptor_proxy = type(self).__dict__[name]
            if type(descriptor_proxy) is not _DescriptorProxy:
                continue
            if self in descriptor_proxy.attrs:
                items.append((name, descriptor_proxy.attrs[self]))
        return items

    def __copy__(self):
        self_copy = self.__get_copy()
        for name, value in self.__get_instance_attr_items():
            setattr(self_copy, name, value)
        return self_copy

    def __deepcopy__(self, memo=None):
        if memo is None:
            memo = {}
        self_copy = self.__get_copy()
        memo[id(self)] = self_copy
        for name, value in self.__get_instance_attr_items():
            setattr(self_copy, name, copy.deepcopy(value, memo))
        return self_copy


import logicData

infile_path = "C:/Users/JarlVictor/Dropbox/Code/2017/capture/ticker_samples/"
inFile = infile_path + "trickle_2_chn.csv"
outFile = "output/ticker_trickle_output.csv"

def validate_intervals(data, imin, imax):
    imax = imin*(2^imax)
    interval = imin
    fail_count = 0
    print("Imin: ", imin, "Imax: ", imax)
    for time in data:
        if interval/2 <= time < interval:
            pass
        else:
            print("     test fail.", interval/2, "<=", time, " < ", interval, "not true.")
            fail_count += 1
        if interval*2 < imax:
            interval *= 2
        else:
            interval = imax
    print("Amount of anomalies in intervals: ", fail_count)

data = logicData.LogicData(inFile, "ms", 2)
data.set_decimal_points(2)
# logicData.capture(30, 1, inFile)
# delta = data.get_all_delta_times()
rx_range = (0.000222 * data.time_multiplier,
            0.000226 * data.time_multiplier)
channels = data.get_separated_data_for_channels()

delta1 = logicData.get_delta_times(channels[0])
delta2 = logicData.get_delta_times(channels[1])

intervals1 = logicData.calculate_tx_intervals(delta1, rx_range)
intervals2 = logicData.calculate_tx_intervals(delta2, rx_range)

# logicData.save(intervals1, outFile)
validate_intervals(intervals1,
                   (float(0xFF)/10000) * data.time_multiplier,
                   8)


'''
Created on Mar 16, 2017

@author: dan
'''

from keras.layers.merge import average
from keras.layers.core import Lambda
from keras.models import Model

import tensorflow as tf

def get_slice(data, idx, parts):
       # import tensorflow as tf
        shape = tf.shape(data)
        size = tf.concat(0, [ shape[:1] // parts, shape[1:] ])
        stride = tf.concat(0, [ shape[:1] // parts, shape[1:]*0 ])
        start = stride * idx
        return tf.slice(data, start, size)
def _merge_function(inputs):
        output = inputs[0]
        for i in range(1, len(inputs)):
            output += inputs[i]
        return output / len(inputs)
def make_parallel(model, gpu_count):
#    def get_slice(data, idx, parts):
#        shape = tf.shape(data)
#        size = tf.concat(0, [ shape[:1] // parts, shape[1:] ])
#        stride = tf.concat(0, [ shape[:1] // parts, shape[1:]*0 ])
#        start = stride * idx
#        return tf.slice(data, start, size)
    print(model.outputs,model.inputs)
    outputs_all = []

    #Place a copy of the model on each GPU, each getting a slice of the batch
    for i in range(gpu_count):
        with tf.device('/gpu:%d' % i):
            with tf.name_scope('tower_%d' % i) as scope:

                inputs = []
                #Slice each input into a piece for processing on this GPU
                for x in model.inputs:
                    input_shape = tuple(x.get_shape().as_list())[1:]
                    slice_n = Lambda(get_slice, output_shape=input_shape, arguments={'idx':i,'parts':gpu_count})(x)
                    inputs.append(slice_n)
                outputs = model(inputs)
#                 if not isinstance(outputs, list):
#                     outputs = [outputs]

                #Save all the outputs for merging back together later
                print(i,outputs)

                outputs_all.append(outputs)

    # merge outputs on CPU
    with tf.device('/cpu:0'):
#         merged = []
#         for outputs in outputs_all:
#             merged.append(merge(outputs, mode='concat', concat_axis=0))
        print(len(outputs_all))
#         for i,out in enumerate(outputs_all):
#             print(i,":",average(out))
        merged=average(outputs_all)
        print(merged)

        return Model(input=model.inputs, output=merged)


# Licensed to Modin Development Team under one or more contributor license agreements.
# See the NOTICE file distributed with this work for additional information regarding
# copyright ownership.  The Modin Development Team licenses this file to you under the
# Apache License, Version 2.0 (the "License"); you may not use this file except in
# compliance with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific language
# governing permissions and limitations under the License.

"""The module holds the factory which performs I/O using pandas on unidist."""

import io

import pandas

from modin.core.storage_formats.pandas.query_compiler import PandasQueryCompiler
from modin.core.execution.unidist.generic.io import UnidistIO
from modin.core.io import (
    CSVDispatcher,
    FWFDispatcher,
    JSONDispatcher,
    ParquetDispatcher,
    FeatherDispatcher,
    SQLDispatcher,
    ExcelDispatcher,
)
from modin.core.storage_formats.pandas.parsers import (
    PandasCSVParser,
    PandasFWFParser,
    PandasJSONParser,
    PandasParquetParser,
    PandasFeatherParser,
    PandasSQLParser,
    PandasExcelParser,
)
from modin.core.execution.unidist.common import UnidistWrapper, SignalActor
from ..dataframe import PandasOnUnidistDataframe
from ..partitioning import PandasOnUnidistDataframePartition

class PandasOnUnidistIO(UnidistIO):
    """Factory providing methods for performing I/O operations using pandas as storage format on unidist as engine."""

    frame_cls = PandasOnUnidistDataframe
    query_compiler_cls = PandasQueryCompiler
    build_args = dict(
        frame_partition_cls=PandasOnUnidistDataframePartition,
        query_compiler_cls=PandasQueryCompiler,
        frame_cls=PandasOnUnidistDataframe,
        base_io=UnidistIO,
    )

    def __make_read(*classes, build_args=build_args):
        # used to reduce code duplication
        return type("", (UnidistWrapper, *classes), build_args).read

    def __make_write(*classes, build_args=build_args):
        # used to reduce code duplication
        return type("", (UnidistWrapper, *classes), build_args).write

    read_csv = __make_read(PandasCSVParser, CSVDispatcher)
    read_fwf = __make_read(PandasFWFParser, FWFDispatcher)
    read_json = __make_read(PandasJSONParser, JSONDispatcher)
    read_parquet = __make_read(PandasParquetParser, ParquetDispatcher)
    to_parquet = __make_write(ParquetDispatcher)
    # Blocked on pandas-dev/pandas#12236. It is faster to default to pandas.
    # read_hdf = __make_read(PandasHDFParser, HDFReader)
    read_feather = __make_read(PandasFeatherParser, FeatherDispatcher)
    read_sql = __make_read(PandasSQLParser, SQLDispatcher)
    to_sql = __make_write(SQLDispatcher)
    read_excel = __make_read(PandasExcelParser, ExcelDispatcher)

    del __make_read  # to not pollute class namespace
    del __make_write  # to not pollute class namespace

    @staticmethod
    def _to_csv_check_support(kwargs):
        """
        Check if parallel version of ``to_csv`` could be used.

        Parameters
        ----------
        kwargs : dict
            Keyword arguments passed to ``.to_csv()``.

        Returns
        -------
        bool
            Whether parallel version of ``to_csv`` is applicable.
        """
        path_or_buf = kwargs["path_or_buf"]
        compression = kwargs["compression"]
        if not isinstance(path_or_buf, str):
            return False
        # case when the pointer is placed at the beginning of the file.
        if "r" in kwargs["mode"] and "+" in kwargs["mode"]:
            return False
        # encodings with BOM don't support;
        # instead of one mark in result bytes we will have them by the number of partitions
        # so we should fallback in pandas for `utf-16`, `utf-32` with all aliases, in instance
        # (`utf_32_be`, `utf_16_le` and so on)
        if kwargs["encoding"] is not None:
            encoding = kwargs["encoding"].lower()
            if "u" in encoding or "utf" in encoding:
                if "16" in encoding or "32" in encoding:
                    return False
        if compression is None or not compression == "infer":
            return False
        if any((path_or_buf.endswith(ext) for ext in [".gz", ".bz2", ".zip", ".xz"])):
            return False
        return True

    @classmethod
    def to_csv(cls, qc, **kwargs):
        """
        Write records stored in the `qc` to a CSV file.

        Parameters
        ----------
        qc : BaseQueryCompiler
            The query compiler of the Modin dataframe that we want to run ``to_csv`` on.
        **kwargs : dict
            Parameters for ``pandas.to_csv(**kwargs)``.
        """
        if not cls._to_csv_check_support(kwargs):
            return UnidistIO.to_csv(qc, **kwargs)

        signals = SignalActor.remote(len(qc._modin_frame._partitions) + 1)

        def func(df, **kw):  # pragma: no cover
            """
            Dump a chunk of rows as csv, then save them to target maintaining order.

            Parameters
            ----------
            df : pandas.DataFrame
                A chunk of rows to write to a CSV file.
            **kw : dict
                Arguments to pass to ``pandas.to_csv(**kw)`` plus an extra argument
                `partition_idx` serving as chunk index to maintain rows order.
            """
            partition_idx = kw["partition_idx"]
            # the copy is made to not implicitly change the input parameters;
            # to write to an intermediate buffer, we need to change `path_or_buf` in kwargs
            csv_kwargs = kwargs.copy()
            if partition_idx != 0:
                # we need to create a new file only for first recording
                # all the rest should be recorded in appending mode
                if "w" in csv_kwargs["mode"]:
                    csv_kwargs["mode"] = csv_kwargs["mode"].replace("w", "a")
                # It is enough to write the header for the first partition
                csv_kwargs["header"] = False

            # for parallelization purposes, each partition is written to an intermediate buffer
            path_or_buf = csv_kwargs["path_or_buf"]
            is_binary = "b" in csv_kwargs["mode"]
            csv_kwargs["path_or_buf"] = io.BytesIO() if is_binary else io.StringIO()
            storage_options = csv_kwargs.pop("storage_options", None)
            df.to_csv(**csv_kwargs)
            csv_kwargs.update({"storage_options": storage_options})
            content = csv_kwargs["path_or_buf"].getvalue()
            csv_kwargs["path_or_buf"].close()

            # each process waits for its turn to write to a file
            UnidistWrapper.materialize(signals.wait.remote(partition_idx))

            # preparing to write data from the buffer to a file
            with pandas.io.common.get_handle(
                path_or_buf,
                # in case when using URL in implicit text mode
                # pandas try to open `path_or_buf` in binary mode
                csv_kwargs["mode"] if is_binary else csv_kwargs["mode"] + "t",
                encoding=kwargs["encoding"],
                errors=kwargs["errors"],
                compression=kwargs["compression"],
                storage_options=kwargs.get("storage_options", None),
                is_text=not is_binary,
            ) as handles:
                handles.handle.write(content)

            # signal that the next process can start writing to the file
            UnidistWrapper.materialize(signals.send.remote(partition_idx + 1))
            # used for synchronization purposes
            return pandas.DataFrame()

        # signaling that the partition with id==0 can be written to the file
        UnidistWrapper.materialize(signals.send.remote(0))
        # Ensure that the metadata is syncrhonized
        qc._modin_frame._propagate_index_objs(axis=None)
        result = qc._modin_frame._partition_mgr_cls.map_axis_partitions(
            axis=1,
            partitions=qc._modin_frame._partitions,
            map_func=func,
            keep_partitioning=True,
            lengths=None,
            enumerate_partitions=True,
            max_retries=0,
        )
        # pending completion
        UnidistWrapper.materialize(
            [part.list_of_blocks[0] for row in result for part in row]
        )


import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.autograd import Variable
from progressbar import *

def getData(name='cifar10', train_bs=128, test_bs=1000):
    if name == 'svhn':
        train_loader = torch.utils.data.DataLoader(
    datasets.SVHN('../../data', split='extra', download=True,
                   transform=transforms.Compose([
                       transforms.ToTensor()
                   ])),
    batch_size=train_bs, shuffle=True)
        test_loader = torch.utils.data.DataLoader(
    datasets.SVHN('../../data', split='test', download=True,transform=transforms.Compose([
                       transforms.ToTensor()
                   ])),
    batch_size=test_bs, shuffle=False)

    if name == 'cifar10':
        transform_train = transforms.Compose([
        transforms.RandomCrop(32, padding=4),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
    ])

        transform_test = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
    ])

        trainset = datasets.CIFAR10(root='../../data', train=True, download=True, transform=transform_train)
        train_loader = torch.utils.data.DataLoader(trainset, batch_size=train_bs, shuffle=True)

        testset = datasets.CIFAR10(root='../../data', train=False, download=False, transform=transform_test)
        test_loader = torch.utils.data.DataLoader(testset, batch_size=test_bs, shuffle=False)

    if name == 'cifar100':

        transform_train = transforms.Compose([
        transforms.RandomCrop(32, padding=4),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
    ])

        transform_test = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
    ])

        trainset = datasets.CIFAR100(root='../../data', train=True, download=True, transform=transform_train)
        train_loader = torch.utils.data.DataLoader(trainset, batch_size=train_bs, shuffle=True)

        testset = datasets.CIFAR100(root='../../data', train=False, download=False, transform=transform_test)
        test_loader = torch.utils.data.DataLoader(testset, batch_size=test_bs, shuffle=False)

    if name == 'tinyimagenet':
        normalize = transforms.Normalize(mean=[0.44785526394844055, 0.41693055629730225, 0.36942949891090393],
                                     std=[0.2928885519504547, 0.28230994939804077, 0.2889912724494934])
        train_dataset = datasets.ImageFolder(
        '../../data/tiny-imagenet-200/train',
        transforms.Compose([
            transforms.RandomCrop(64, padding=4),
            transforms.RandomHorizontalFlip(),
            transforms.ToTensor(),
            normalize,
        ]))

        train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=train_bs, shuffle=True, num_workers=4, pin_memory=False)

        test_dataset = datasets.ImageFolder(
        '../../data/tiny-imagenet-200/val',
        transforms.Compose([
            transforms.ToTensor(),
            normalize,
        ]))

        test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=test_bs, shuffle=False)

    return train_loader, test_loader

def train(epoch, model, train_loader,optimizer, criterion=nn.CrossEntropyLoss()):
    model.train()
    print('\nTraing, Epoch: %d' % epoch)
    train_loss = 0
    correct = 0
    total = 0
    for batch_idx, (input, target) in enumerate(train_loader):
        input, target = input.cuda(), target.cuda()
        optimizer.zero_grad()
        output = model(input)
        loss = criterion(output, target)
        loss.backward()
        optimizer.step()

        train_loss += loss.item()
        _, predicted = output.max(1)
        total += target.size(0)
        correct += predicted.eq(target).sum().item()

        progress_bar(batch_idx, len(train_loader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                     % (train_loss/(batch_idx+1),
                        100.*correct/total, correct, total))

def test(model, test_loader):
    print('\nTesting')
    model.eval()
    correct = 0
    total_num = 0
    for data, target in test_loader:
        data, target = data.cuda(), target.cuda()
        output = model(data)
        pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability
        correct += pred.eq(target.data.view_as(pred)).cpu().sum().item()
        total_num += len(data)
    print('testing_correct: ', correct / total_num)
    return correct / total_num

def exp_lr_scheduler(epoch, optimizer, strategy=True, decay_eff=0.1, decayEpoch=[]):
    """Decay learning rate by a factor of lr_decay every lr_decay_epoch epochs"""
    print(strategy)

    if strategy=='normal':
        if epoch in decayEpoch:
            for param_group in optimizer.param_groups:
                param_group['lr'] *= decay_eff
    else:
        print('wrong strategy')
        raise ValueError('A very specific bad thing happened.')

    return optimizer

def fb_warmup(optimizer, epoch, baselr, large_ratio):
    for param_group in optimizer.param_groups:
        param_group['lr'] = epoch * (baselr * large_ratio-baselr) / 5. + baselr
    return optimizer

# useful
def group_add(params, update, lmbd=1):
    """
    params = params + update*alpha
    :param params: list of variable
    :param update: list of data
    :return:
    """
    for i,p in enumerate(params):
        params[i].add_(update[i]*lmbd+0.)
    return params

def group_product(xs, ys):
    """
    the inner product of two lists of variables xs,ys
    :param xs:
    :param ys:
    :return:
    """
    return sum([torch.sum(x*y) for (x, y) in zip(xs, ys)])

def get_p_g_m(opt, layers):
    i = 0
    paramlst = []
    grad = []
    mum = []

    for group in opt.param_groups:
        weight_decay = group['weight_decay']
        momentum = group['momentum']

        for p in group['params']:
            if p.grad is None:
                continue
            p.grad.data = p.grad.data
            d_p = p.grad.data
            if momentum != 0:
                param_state = opt.state[p]
                if 'momentum_buffer' not in param_state:
                    buf = param_state['momentum_buffer'] = torch.zeros_like(p.data)
                else:
                    buf = param_state['momentum_buffer']
            if i in layers:
                paramlst.append(p.data)
                grad.append(d_p+0.) # get grad
                mum.append(buf*momentum+0.)
            i += 1
    return paramlst, grad, mum

def manually_update(opt, grad):
    for group in opt.param_groups:
        weight_decay = group['weight_decay']
        momentum = group['momentum']
        dampening = group['dampening']
        nesterov = group['nesterov']

        for i,p in enumerate(group['params']):
            d_p = grad[i]
            if weight_decay != 0:
                d_p.add_(weight_decay, p.data)
            if momentum != 0:
                param_state = opt.state[p]
                if 'momentum_buffer' not in param_state:
                    buf = param_state['momentum_buffer'] = torch.zeros_like(p.data)
                    buf.mul_(momentum).add_(d_p)
                else:
                    buf = param_state['momentum_buffer']
                    buf.mul_(momentum).add_(1 - dampening, d_p)
                if nesterov:
                    d_p = d_p.add(momentum, buf)
                else:
                    d_p = buf
            p.data.add_(-group['lr'], d_p)

def change_lr_single(optimizer, best_lr):
    """change learning rate"""
    for param_group in optimizer.param_groups:
        param_group['lr'] = best_lr
    return optimizer

def bad_params(net):
    '''Init layer parameters.'''
    for m in net.modules():
        if isinstance(m, nn.Conv2d):
            nn.init.normal(m.weight, std=0.5)
            if m.bias is not None:
                nn.init.constant(m.bias, 0)
        elif isinstance(m, nn.Linear):
            nn.init.normal(m.weight, std=0.5)
            if m.bias is not None:
                nn.init.constant(m.bias, 0)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import logging
import pika
import json
import sys

LOG_FORMAT = ('%(levelname) -10s %(asctime)s %(name) -30s %(funcName) '
              '-35s %(lineno) -5d: %(message)s')
LOGGER = logging.getLogger(__name__)

class Publisher(object):
    EXCHANGE = 'mmy_default'
    EXCHANGE_TYPE = 'topic'

    def __init__(self):
        amqp_url = 'amqp://guest:guest@localhost:5672/%2F?connection_attempts=3&heartbeat_interval=3600'
        self._connection = None
        self._channel = None
        self._url = amqp_url
        self._message_body = None
        self._routing_key = None

    def connect(self):
        LOGGER.info('Connecting to %s', self._url)
        return pika.SelectConnection(pika.URLParameters(self._url),
                                     self.on_connection_open)

    def on_connection_open(self, unused_connection):
        LOGGER.info('Connection opened')
        self.add_on_connection_close_callback()
        self.open_channel()

    def add_on_connection_close_callback(self):
        LOGGER.info('Adding connection close callback')
        self._connection.add_on_close_callback(self.on_connection_closed)

    def on_connection_closed(self, connection, reply_code, reply_text):
        self._channel = None
        LOGGER.warning('Connection closed: (%s) %s',
                           reply_code, reply_text)

    def open_channel(self):
        LOGGER.info('Creating a new channel')
        self._connection.channel(on_open_callback=self.on_channel_open)

    def on_channel_open(self, channel):
        LOGGER.info('Channel opened')
        self._channel = channel
        self.add_on_channel_close_callback()
        self.setup_exchange(self.EXCHANGE)

    def add_on_channel_close_callback(self):
        LOGGER.info('Adding channel close callback')
        self._channel.add_on_close_callback(self.on_channel_closed)

    def on_channel_closed(self, channel, reply_code, reply_text):
        LOGGER.warning('Channel was closed: (%s) %s', reply_code, reply_text)
        self._connection.close()

    def setup_exchange(self, exchange_name):
        LOGGER.info('Declaring exchange %s', exchange_name)
        self._channel.exchange_declare(self.on_exchange_declareok,
                                       exchange_name,
                                       self.EXCHANGE_TYPE)

    def on_exchange_declareok(self, unused_frame):
        LOGGER.info('Exchange declared')
        self.publish_message()

    def publish_message(self):

        properties = pika.BasicProperties(delivery_mode=1,
                                          content_type='application/json')

        self._channel.basic_publish(self.EXCHANGE, self._routing_key,
                                    json.dumps(self._message_body, ensure_ascii=False),
                                    properties)
        LOGGER.info('Published message:%s', self._message_body)
        self._connection.close()

    def send_message(self, device, message):
        self._message_body = message
        self._routing_key = device
        # Create a new connection
        self._connection = self.connect()

        self._connection.ioloop.start()

    def close_channel(self):
        LOGGER.info('Closing the channel')
        if self._channel:
            self._channel.close()

    def close_connection(self):
        LOGGER.info('Closing connection')
        self._closing = True
        self._connection.close()

    def close(self):
        self.close_channel()
        self.close_connection()

def main():
    logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)

    # Connect to localhost:5672 as guest with the password guest and virtual host "/" (%2F)
    example = Publisher()
    try:
        test_device = 'device1'
        test_message = {u'type':u'any',
                        u'cmd':u'ls',
                        u'data':u'-l'}
        example.send_message(test_device, test_message)
    except KeyboardInterrupt:
        sys.stderr.write("Error occured\n")
        example.close()
        sys.exit(-1)

if __name__ == '__main__':
    main()


import itertools
from . import http
from .routes import *

class Router:
	"""
	Maps URLs to handlers. After creating a ``Router``, call the various
	methods to define your routes, and then call either `go` or `dispatch`.
	"""

	def __init__(self, web, consumed_path):
		self.web = web
		self._routes = []
		self._predef = {}
		self.consumed_path = consumed_path

	@property
	def routes(self):
		for r in itertools.chain(self._predef.values(), self._routes):
			if r is not None:
				yield r

	def route(self, *args, context=None, **kwargs):
		if context is None:
			context = self.web.executor.make_context(1)
		self._routes.append(ExecuteRoute(*args, context=context, **kwargs))

	def redirect(self, *args, **kwargs):
		self._routes.append(RedirectRoute(*args, **kwargs))

	def defer(self, *args, context=None, **kwargs):
		if context is None:
			context = self.web.executor.make_context(1)
		self._routes.append(DeferredRoute(*args, context=context, **kwargs))

	def file_link(self, *args, context=None, **kwargs):
		if context is None:
			context = self.web.executor.make_context(1)
		self._routes.append(FileSystemRoute(*args, context=context, **kwargs))

	def require_www(self, value=True):
		# TODO: this fails for TLDs with a dot, such as '.co.uk'
		# Is there any way to test for this without hardcoding a list?
		self._set_predef('require_www', RedirectRoute, value,
			((r'[^.]+\.[^.]+', '.*'), (r'www.\0', r'\0'), 301),
			((r'www\.(.+)', '.*'), (r'\1', r'\0'), 301))

	def trailing_slashes(self, value=True):
		self._set_predef('trailing_slashes', RedirectRoute, value,
			('(?:[^/]*/)*[^/.]+', r'\0/', 301),
			('(.+)/', r'\1', 301))

	def cleanup_slashes(self, value=True):
		def do_clean(parts):
			return {k: re.sub('/{2,}', '/', p.string) for k, p in parts.items()}
		self._set_predef('cleanup_slashes', RedirectRoute, value,
			('.*/{2,}.*', do_clean, 301),
			None)

	def _set_predef(self, name, cls, value, yes, no):
		if value is None:
			self._predef[name] = None
		else:
			args = yes if value else no
			self._predef[name] = cls(*args) if args else None

	def go(self, request, **kwargs):
		"""Handles the URL identified in the given request."""
		parts = dict(request.url_parts)
		if not parts['path'].startswith(self.consumed_path):
			raise ValueError('URL does not start with assumed prefix')
		parts['path'] = parts['path'][len(self.consumed_path):]
		return self.dispatch(parts, **kwargs)

	def dispatch(self, parts, *, error=True):
		"""
		Handles the URL passed via a dict containing URL components. All of the
		following must be specified: ``scheme``, ``host``, ``port``, ``path``,
		``query``, and ``method``.
		"""
		for route in self.routes:
			match = route.match(parts)
			if match:
				route.navigate(self.web, match, self.consumed_path)
				return True

		if error:
			if self.web.debug:
				self._debug_failure(parts)
			self.web.response.error(404)
		return False

	def _debug_failure(self, parts):
		with self.web.error_handler as handler:
			handler.write('A Router originating from {!r} failed to find a '
				'valid route for:\n'.format(self.consumed_path))
			handler.write('  ' + str(parts))
			handler.write('\nCurrently installed routes:')
			for route in self.routes:
				handler.write(' - ' + str(route))


#!/usr/bin/python

import spidev
import os
import time

# Define Axis Channels (channel 3 to 7 can be assigned for more buttons / joysticks)
swt_channel = 0
vrx_channel = 1
vry_channel = 2

#Time delay, which tells how many seconds the value is read out
delay = 0.5

# Spi oeffnen
spi = spidev.SpiDev()
spi.open(0,0)

# Function for reading the MCP3008 channel between 0 and 7
def readChannel(channel):
  val = spi.xfer2([1,(8+channel)<<4,0])
  data = ((val[1]&3) << 8) + val[2]
  return data

# endless loop
while True:

  # Determine position
  vrx_pos = readChannel(vrx_channel)
  vry_pos = readChannel(vry_channel)

  # SW determine
  swt_val = readChannel(swt_channel)

  # output
  print("VRx : {}  VRy : {}  SW : {}".format(vrx_pos,vry_pos,swt_val))

  # wait
  time.sleep(delay)


#!/usr/bin/env python

import os
import sys
import json

sys.path.append(os.environ['REPO_DIR'] + '/utilities/')
from metadata import *

stack = sys.argv[1]
input_dir = sys.argv[2]
output_dir = sys.argv[3]
filename_pairs = json.loads(sys.argv[4])
suffix = 'thumbnail'

parameter_dir = os.path.join(os.environ['REPO_DIR'], "preprocess/parameters")

rg_param = os.path.join(parameter_dir, "Parameters_Rigid.txt")
rg_param_mutualinfo = os.path.join(parameter_dir, "Parameters_Rigid_MutualInfo.txt")
rg_param_noNumberOfSamples = os.path.join(parameter_dir, "Parameters_Rigid_noNumberOfSpatialSamples.txt")
rg_param_requiredRatioOfValidSamples = os.path.join(parameter_dir, "Parameters_Rigid_RequiredRatioOfValidSamples.txt")

for fn_pair in filename_pairs:
	prev_fn = fn_pair['prev_fn']
	curr_fn = fn_pair['curr_fn']

	output_subdir = os.path.join(output_dir, curr_fn + '_to_' + prev_fn)

	if os.path.exists(output_subdir):
		sys.stderr.write('Result for aligning %s to %s already exists.\n' % (curr_fn, prev_fn))
		continue

	execute_command('rm -rf ' + output_subdir)
	os.makedirs(output_subdir)

	execute_command('%(elastix_bin)s -f %(fixed_fn)s -m %(moving_fn)s -out %(output_subdir)s -p %(rg_param)s' % \
			{'elastix_bin': os.environ['ELASTIX_BIN'],
			'rg_param': rg_param,
			'output_subdir': output_subdir,
			'fixed_fn': os.path.join(input_dir, prev_fn + '.tif'),
			'moving_fn': os.path.join(input_dir, curr_fn + '.tif')
			})


import click

# Note: While config files are in yaml, using yaml.dump adds a newline with an
# ellipsis. JSON will be compatible, but doesn't add fluff.
import json
import sys

from cloudify_tester.utils import NotInRepositoryError
try:
    from cloudify_tester.commands.utils import load_config
    from cloudify_tester.config import NotSet
except NotInRepositoryError as err:
    sys.stderr.write(
        'This must be run in the git repository that is being tested.\n'
        '{error}\n'.format(error=str(err))
    )
    sys.exit(1)

@click.group(
    name='config',
    short_help='Config tools- e.g. show schema or generate sample config.',
)
def config_group():
    pass

@config_group.command(
    short_help='Show the config schema.',
)
def schema():
    # TODO docstring
    config = load_config()

    show_entries(config.schema, False)

@config_group.command(
    short_help='Generate a sample config.',
)
def generate():
    # TODO docstring
    config = load_config()

    show_entries(config.schema, True)

@config_group.command(
    short_help='Validate the test config.',
)
# TODO: This is done several times. Make it better.
# also make the help message include default+envvar
@click.option(
    '--config', '-c',
    envvar='CLOUDIFY_PLUGIN_TESTS_CONFIG',
    default='test_config.yaml',
    help='Config file to validate.',
)
def validate(config):
    # TODO docstring
    config = load_config(config)

    namespaces = config.namespaces
    not_set_message = '{key} is not set and has no default!\n'
    magic_set_message = (
        '{key} is set. This will be ignored, as magic values cannot be '
        'configured.\n'
    )
    for namespace in namespaces:
        if namespace is None:
            check = config.items()
        else:
            check = config[namespace].items()
        for key, value in check:
            display_key = key
            if namespace is not None:
                display_key = '.'.join([namespace, key])
            if key not in namespaces:
                if namespace == 'magic':
                    if value is not NotSet:
                        sys.stderr.write(magic_set_message.format(
                            key=display_key,
                        ))
                else:
                    if value is NotSet:
                        sys.stderr.write(not_set_message.format(
                            key=display_key,
                        ))

def show_entries(schema, generate_sample_config=False, indent=''):
    sorted_config_entries = schema.keys()
    sorted_config_entries = [
        entry for entry in sorted_config_entries
        if isinstance(schema[entry], dict)
    ]
    sorted_config_entries.sort()

    namespaces = [
        entry for entry in sorted_config_entries
        if schema[entry].get('.is_namespace', False)
    ]
    if generate_sample_config and 'magic' in namespaces:
        namespaces.remove('magic')
        sorted_config_entries.remove('magic')
    root_config_entries = [
        entry for entry in sorted_config_entries
        if entry not in namespaces
    ]

    for config_entry in root_config_entries:
        details = schema[config_entry]
        if generate_sample_config:
            if 'default' in details.keys():
                print(indent + '{entry}: {default}'.format(
                    entry=config_entry,
                    default=json.dumps(details['default'])
                ))
            else:
                print(indent + '{entry}: '.format(entry=config_entry))
        else:
            line = '{entry}: {description}'
            if 'default' in schema[config_entry].keys():
                line = line + ' (Default: {default})'
            line = line.format(
                entry=config_entry,
                description=details['description'],
                default=json.dumps(details.get('default')),
            )
            print(indent + line)

    for namespace in namespaces:
        print(indent + namespace + ':')
        show_entries(
            schema[namespace],
            generate_sample_config,
            indent + '  ',
        )


'''
DO NOT RUN THIS FILE AGAIN
TASK HAS ALREADY BEEN COMPLETED!

(In the event that you do, however, simply change line 40 to divide instead of multiply and run again.)
'''

import numpy as np
from os import listdir
from .nameparser import parse_small_file

def kelvin_to_fahrenheit(kelvin):
    return (9.0/5)*(kelvin - 273) + 32

if __name__ == "__main__":

    SCALE = 86400*0.03937 # to convert mm/sec -> in/day

    FOLDERS = (
            "GFDL-files/",
            "IPSL-files/",
            # 'HadGEM2-files/',
            # 'MIROC5-files/',
            )

    for folder in FOLDERS:
        path = "small-isimip5-files/" + folder

        for count, file in enumerate(listdir(path)):

            model, ssp, variable, start_year, end_year = parse_small_file(filename=file)
            print(model, ssp, variable, start_year, end_year)

            #? INSERT ANY FILERTERING HERE.
            #? Ex: if variable == 'pr': continue (skip)
            if variable != 'pr': continue

            #************? Do meaningful work here now for each file! **************
            try:
                loaded_arr = np.loadtxt(path+file)
                loaded_arr *= SCALE
            except:
                print("error loading:", file)
                continue

            try:
                np.savetxt(path+file, loaded_arr)
            except:
                print("Error writing to file:", file)


from fs.get_fileinfo import get_fileinfo
from util.fatal_error import fatal_error
from db.queries import check_file_exists_in_database
import settings
import os

def search_file(filepath):
    print('Searching database for {0}'.format(filepath), end='\n\n')

    file_info = None
    try:
        file_info = get_fileinfo(filepath)
    except FileNotFoundError:
        fatal_error('Check file path; Unable to find file at {0}'.format())
    except PermissionError:
        fatal_error('Insufficient permissions to access {0}'.format(filepath))
    except BlockingIOError:
        fatal_error('Unable to access, another process has opened {0}'.format(filepath))

    row = check_file_exists_in_database(4, file_info['hashes']['sha1b32'])

    if row[0] is not '':
        print('\033[94mFile found! See {0}\033[0m'.format(os.path.join(settings.base_directory, row[0])))
        pass
    else:
        print('\033[94mFile not found in database.\033[0m')


from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import unittest
import time

class NewVisitorTest(unittest.TestCase):
    def setUp(self):
        self.browser = webdriver.Firefox()

    def tearDown(self):
        self.browser.quit()

    def test_can_start_a_list_and_retrieve_it_later(self):

        # Alice has heard about this awesome new to-do app that will
        # help her to get a thing done.
        # So she goes to check out its home page

        self.browser.get('http://localhost:8000')

        # She notices the page title and header mentions to-do lists
        self.assertIn('To-Do', self.browser.title)
        header_text = self.browser.find_element_by_tag_name('h1').text
        self.assertIn('To-Do', header_text)

        # She is invited to enter a to-do item rightway
        input_box = self.browser.find_element_by_id('id_new_item')
        self.assertEqual(
            input_box.get_attribute('place_holder'),
            'Enter a to-do item')

        # She enter "Get a New Futon from Ikea" since she just moved in her new
        # apartment
        input_box.send_keys('Get a New Futon from Ikea')

        # When she hits enter, now the page updates, and now the page lists
        # "1. Get a new Futon from Ikea"
        input_box.send_keys(Keys.ENTER)
        time.sleep(1)

        table = self.browser.find_element_by_id('id_list_table')
        rows = table.find_elements_by_tag_name('tr')
        self.assertTrue(
            any(row.text == '1. Get a New Futon from Ikea' for row in rows),
            'New to-do item does not appear in the list'
            )

        # There is still a text box invite her to enter a new item
        # So she enter, "Sign Up for Internet with Cox"
        # input_box = self.browser.find_element_by_id('id_new_item')
        # self.assertEqual(
        #     input_box.get_attribute('place_holder'),
        #     'Enter a to-do item')

        # The page updates again and now it shows both items on the list

        # Alice wonders if she will remember the list.
        # Then she sees that the site has generated an unique URL for her
        # There is some explanation text for the effect.

        # She visits her unique URL and her To-DO list is stil there

        # Satisfied, she goes back to sleep
        self.fail('Finish The Test!')
if __name__ == "__main__":
    unittest.main(warnings='ignore')


#!/usr/bin/env python

__author__    = "Radical.Utils Development Team (Andre Merzky, Ole Weidner)"
__copyright__ = "Copyright 2013, RADICAL@Rutgers"
__license__   = "MIT"

import os
import pytest
import radical.utils as ru

# ------------------------------------------------------------------------------
#
def test_config():

    base = os.path.abspath(os.path.dirname(__file__))
    path = '%s/data/resource_*.json' % base
    print(path)

    cfg1 = ru.Config(name=path)
    print(type(cfg1))

    import pprint
    pprint.pprint(cfg1)

    assert('bar' == cfg1.yale.query('grace.agent_launch_method'))
    assert('bar' == cfg1['yale']['grace']['agent_launch_method'])

    assert(None  is cfg1.query('yale.grace.no_launch_method'))
    with pytest.raises(KeyError):
        _ = cfg1['yale']['grace']['no_launch_method']                # noqa F841

    os.environ['FOO'] = 'GSISSH'

    cfg2 = ru.Config(name=path)
    assert('GSISSH' == cfg2.query('yale.grace.agent_launch_method'))
    assert(None     is cfg2.query('yale.grace.no_launch_method'))

    cfg3 = ru.Config(name=path, expand=False)
    assert('${FOO:bar}' == cfg3.query('yale.grace.agent_launch_method'))

    env  = {'FOO' : 'baz'}
    cfg4 = ru.Config(name=path, env=env)
    assert('baz' == cfg4.query('yale.grace.agent_launch_method'))

# ------------------------------------------------------------------------------
#
if __name__ == '__main__':

    test_config()

# ------------------------------------------------------------------------------


import argparse
import sys
import codecs

parser = argparse.ArgumentParser(description='Evaluation script for token overwrap based  macro-averaged F1 scores.')
parser.add_argument("-i", "--input-text-file", dest="file_inp", type=str, help="path to input text file")
parser.add_argument("-r", "--reference-file", dest="file_ref", type=str, help="path to reference file")
parser.add_argument("-s", "--system-output-file", dest="file_sys", type=str, help="path to system output file")
parser.add_argument("-l", "--length-limit", dest="len_limit", type=int, default=-1, help="Only evaluate sentences longer than this value. (default: -1)")
parser.add_argument("-a", "--all-score", dest="all_score", action='store_true', default=False, help="Print all scores. (default: False)")
args = parser.parse_args()

ref_file = codecs.open(args.file_ref, "r", encoding="utf8")
sys_file = codecs.open(args.file_sys, "r", encoding="utf8")
inp_file = codecs.open(args.file_inp, "r", encoding="utf8")

exact_match = 0
doc_len = 0
precision = 0
recall = 0
f = 0
comp_rate = 0

if args.all_score:
    print("Precision\tRecall\tF1\tCR")

for line_ref in ref_file:
    prec_cor = 0
    prec_all = 0
    recall_cor = 0
    recall_all = 0
    tok_all = 0
    tok_kept = 0
    line_sys = sys_file.readline()
    line_inp = inp_file.readline()
    line_ref = line_ref.rstrip("\r\n")
    line_sys = line_sys.rstrip("\r\n")
    line_inp = line_inp.rstrip("\r\n")
    toks_ref = line_ref.split(" ")
    toks_sys = line_sys.split(" ")
    toks_inp = line_inp.split(" ")
    if len(toks_inp) < args.len_limit:
        continue
    doc_len += 1
    if toks_ref == toks_sys:
        exact_match += 1
    dict_ref = {}
    dict_sys = {}
    for tok in toks_ref:
        if tok in dict_ref:
            dict_ref[tok] += 1
        else:
            dict_ref[tok] = 1
    for tok in toks_sys:
        if tok in dict_sys:
            dict_sys[tok] += 1
        else:
            dict_sys[tok] = 1
    for tok in dict_sys.keys():
        prec_all += dict_sys[tok]
        if tok in dict_ref:
            prec_cor += dict_sys[tok]
    for tok in dict_ref.keys():
        recall_all += dict_ref[tok]
        if tok in dict_sys:
            recall_cor += dict_ref[tok]
    prec_local = 0.0
    recall_local = 0.0
    f_local = 0.0
    cr_local = 0.0
    if prec_all > 0:
        prec_local = prec_cor / float(prec_all)
    recall_local = recall_cor / float(recall_all)
    if prec_all > 0 and (prec_cor != 0 or recall_cor != 0):
        f_local = 2 * (prec_cor / float(prec_all) * recall_cor / float(recall_all)) / (prec_cor / float(prec_all) + recall_cor / float(recall_all))
    cr_local = len(toks_sys) / len(toks_inp)
    # Print
    if args.all_score:
        print(str(prec_local) + "\t" + str(recall_local) + "\t" + str(f_local) + "\t" + str(cr_local))
    # Update
    if not args.all_score:
        precision += prec_local
        recall += recall_local
        f += f_local
        comp_rate += cr_local

if not args.all_score:
    precision /= float(doc_len)
    recall /= float(doc_len)
    f /= float(doc_len)
    comp_rate /= float(doc_len)
    exact_match /= float(doc_len)
    print("Line              : " + str(doc_len))
    print("F1                : " + str(f))
    print("Precision         : " + str(precision))
    print("Recall            : " + str(recall))
    print("Compression Ratio : " + str(comp_rate))
    print("Exact Match       : " + str(exact_match))


#-*- coding: utf-8 -*-

import sys
import network

DATA_FILE = '../data/SocialNetwork.txt'

def read_people(file):
    """ read people and construct name to id dict,
        also put all friend pairs into a list.
        name - people's name read from DATA_FILE, as string
        id - unique integer as people's identification
        return:
            name to id dict
            friend pair list
    """
    friend_pairs = []
    name_id_map = {}
    id = 0

    with open(file, 'rb') as datafile:
        for line in datafile:
            persons = line.strip('\n\r').split(',')
            friend_pairs.append(persons)
            for person in persons:
                if person not in name_id_map:
                    name_id_map[person] = id
                    id += 1
    return name_id_map, friend_pairs

def count_num_of_people(people):
    return len(people)

def print_help():
    help = """
    usage:
      python main.py count - get total number of people
      python main.py A B   - get distance between A and B
    """
    print(help)

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print('error: too few input parameters')
        print_help()
        raise SystemExit(1)

    if len(sys.argv) == 2 and sys.argv[1] == 'count':
        print(count_num_of_people(read_people(DATA_FILE)[0]))
    elif len(sys.argv) == 3:
        name_id_map, friend_pairs = read_people(DATA_FILE)
        network = network.Network(friend_pairs, name_id_map)
        print(network.distance(sys.argv[1], sys.argv[2]))


import os
import time
import matplotlib.pyplot as plt
import numpy as np
from mpl_toolkits.mplot3d import Axes3D
import cv2

def one_hot_encode(kmeans_result):
    """
    One hot encodes an image;

    Parameters
    ----------
    kmeans_result : ndarray
        Image after applying clusterization

    Returns
    -------
    encoded_image  : ndarray
        Array with k masks: one per each binary one-hot-encode mask
    """
    unique = np.unique(kmeans_result)
    colormap = {i:unique[i] for i in range(len(unique))}
    encoded_image = np.zeros(kmeans_result.shape[:2] + (len(colormap), ), dtype=np.float32)
    for i in range(encoded_image.shape[2]):
        encoded_image[:, :, i] = np.all(kmeans_result.reshape((-1, 1)) == colormap[i],
                                        axis=1).reshape(kmeans_result.shape[:2])
    return encoded_image

def binarize_data(img=None, k=None, bilateral_filter=None,
                  d=None, sigmaColor=None, sigmaSpace=None):
    """
    Apply clusterization to segment data;

    Parameters
    ----------
    img : ndarray
        Image to segment
    k : int
        Num of clusters
    bilateral_filter : boolean
        whether to apply bilateral filter
    d, sigmaColor, sigmaSpace : int
        bilateral filter params

    Returns
    -------
    segmented_image, porosity  : ndarray
        results of prediction
    """
    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 200, 0.99)

    #reshape to appropriate for KMeans shape
    pixel_vals = img.reshape((-1,1))

    # Convert to float type
    pixel_vals = np.float32(pixel_vals)
    if bilateral_filter:
        bilateral = cv2.bilateralFilter(pixel_vals, d, sigmaColor, sigmaSpace)
        retval, labels, centers = cv2.kmeans(bilateral, k, None, criteria, 10, cv2.KMEANS_RANDOM_CENTERS)
    else:
        retval, labels, centers = cv2.kmeans(pixel_vals, k, None, criteria, 10, cv2.KMEANS_RANDOM_CENTERS)
    centers = np.uint8(centers)
    segmented_data = centers[labels.flatten()]

    segmented_image = one_hot_encode(segmented_data.reshape((img.shape)))[:,:, 1]

    porosity = np.sum(segmented_image==0)/np.prod(segmented_image.shape)

    return segmented_image, porosity

def binarize_3d(img_3d, k = 2, bilateral_filter = None,
               d=None, sigmaColor=None, sigmaSpace=None):
    """
    Apply clusterization to segment 3D image;

    Parameters
    ----------
    img_3d : ndarray
        3D image to segment
    k : int
        Num of clusters
    bilateral_filter : boolean
        whether to apply bilateral filter
    d, sigmaColor, sigmaSpace : int
        bilateral filter params

    Returns
    -------
    result_3d, porosity_arr  : ndarray
        results of prediction
    """
    x_dim, y_dim, z_dim = img_3d.shape
    result_3d = np.zeros((x_dim, y_dim, z_dim))
    porosity_arr = np.zeros((x_dim,))
    for x in range(x_dim):
        result_3d[x, :, :], porosity_arr[x] = binarize_data(img = img_3d[x, :, :],
                                                            k = 2,
                                                            bilateral_filter = bilateral_filter,
                                                            d=d,
                                                            sigmaColor=sigmaColor,
                                                            sigmaSpace=sigmaSpace)
    return result_3d, porosity_arr

def plot_central_planes(image):
    """
    Plot central planes of volumetric data

    Parameters
    ----------
    img : ndarray
        3D image
    img_title : string
        plot title

    """
    n_x, n_y, n_z = image.shape
    fig, axs = plt.subplots(1,3, figsize = (15, 10))
    axs[0].imshow(image[n_x//2, :, :], cmap = 'gray'), axs[0].set_title('X central plane')
    axs[1].imshow(image[:, n_y//2, :], cmap = 'gray'), axs[1].set_title('Y central plane')
    axs[2].imshow(image[:, :, n_z//2], cmap = 'gray'), axs[2].set_title('Z central plane')
    plt.show()

def plot_3d(image, img_title=''):
    """
    Plot boundary planes of volumetric data

    Parameters
    ----------
    img : ndarray
        3D image
    img_title : string
        plot title

    """
    n_x, n_y, n_z = image.shape
    yy, zz = np.mgrid[0:n_y, 0:n_z]
    xx, zz = np.mgrid[0:n_x, 0:n_z]
    xx, yy = np.mgrid[0:n_x, 0:n_y]

    x_center_loc, y_center_loc, z_center_loc = n_x-1, 0, n_z-1

    # plot 3 orthogonal slices
    X, Y, Z = image[x_center_loc, :, :], image[:, y_center_loc, :], image[:, :, z_center_loc]

    fig = plt.figure(figsize = (25,10))
    ax1 = fig.add_subplot(121, projection='3d')
    ax1.set_title(img_title, fontsize = 17)
    ax1.contourf(X, xx, zz,  zdir='x', offset=n_x-1, cmap='gray')
    ax1.contourf(xx, Y, zz,  zdir='y', offset=y_center_loc, cmap='gray')
    ax1.contourf(xx, yy, Z,  zdir='z', offset=n_z-1, cmap='gray')
    plt.show()

def create_dir(dir_to_save):
    """
    creates folder in defined directory

    Parameters
    ----------
    dir_to_save : str
        path

    """
    if not os.path.exists(dir_to_save):
        os.makedirs(dir_to_save)

def PlotSlice(image, sliceNumber):
    plt.imshow(image[:,:,sliceNumber], cmap="gray")
    plt.gca().axes.get_yaxis().set_visible(False)
    plt.gca().axes.get_xaxis().set_visible(False)
    plt.title("Slice: " + str(sliceNumber), size=20)

def PlotImage(image):
    plt.figure(figsize=(30,10))
    plt.subplot(1,3,1)
    PlotSlice(image, 30)
    plt.subplot(1,3,2)
    PlotSlice(image, 250)
    plt.subplot(1,3,3)
    PlotSlice(image, 500)


# coding: utf-8

import StringIO
import shlex
import os
import sys
import json
import datetime
import ConfigParser
from minecraft_query import MinecraftQuery
from PIL import Image, ImageDraw, ImageFont
from subprocess import PIPE, Popen

ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__)))

#Doc
#http://www.pythonware.com/library/pil/handbook/imagedraw.htm
#http://stackoverflow.com/questions/4011705/python-the-imagingft-c-module-is-not-installed (_imaging)

"""
map = world
motd = BEST-HOSTING s.r.o.
hostname = 194.8.253.48
numplayers = 6
gametype = SMP
maxplayers = 40
hostport = 25565

map = world
motd = BEST-HOSTING s.r.o.
hostport = 25565
numplayers = 6
gametype = SMP
players = ['janice158', 'nitramek2013', 'batman', 'Fox24', 'adelka', 'HunySss']
version = 1.4.5
maxplayers = 40
plugins = ['WorldEdit 5.4.2', 'iConomy 7.0', 'HawkEye 1.1.0', 'Vault 1.2.13-b148', 'PermissionsEx 1.19.1', 'LWC 4.3.1 (b767-git-MANUAL) (November 15, 2012)', 'WorldGuard 644-6df9c36', 'Essentials 2.9.3', 'dynmap 0.70.3-1298', 'Dynmap-WorldGuard 0.30', 'EssentialsProtect 2.9.3', 'AuthMe 2.6.7b5', 'Dynmap-Essentials 0.30', 'EssentialsSpawn 2.9.3', 'DragonTravel 1.7.1', 'ChestShop 3.46', 'EssentialsChat 2.9.3']
raw_plugins = CraftBukkit on Bukkit 1.4.5-R0.2: WorldEdit 5.4.2; iConomy 7.0; HawkEye 1.1.0; Vault 1.2.13-b148; PermissionsEx 1.19.1; LWC 4.3.1 (b767-git-MANUAL) (November 15, 2012); WorldGuard 644-6df9c36; Essentials 2.9.3; dynmap 0.70.3-1298; Dynmap-WorldGuard 0.30; EssentialsProtect 2.9.3; AuthMe 2.6.7b5; Dynmap-Essentials 0.30; EssentialsSpawn 2.9.3; DragonTravel 1.7.1; ChestShop 3.46; EssentialsChat 2.9.3
game_id = MINECRAFT
hostip = 194.8.253.48
software = CraftBukkit on Bukkit 1.4.5-R0.2

"""

class MinecraftWidget(object):

    def __init__(self, data, profile="hb"):
        self.data = data
        self.profile = profile

        self.init_config()
        self.init_colors()
        self.init_sizes()

    def init_config(self):
        self.config = ConfigParser.RawConfigParser(allow_no_value=True)
        self.config.read(os.path.join(ROOT, "script.ini"))

    def _parse(self, value):
        return tuple([int(x.strip()) for x in value.split(",")])

    def init_colors(self):
        self.COLOR_BG = self._parse(self.config.get(self.profile, "color_bg"))
        self.COLOR_TEXT = self._parse(self.config.get(self.profile, "color_text"))
        self.COLOR_TEXT_VALUE = self._parse(self.config.get(self.profile, "color_text_value"))
        self.COLOR_BAR_MAX = self._parse(self.config.get(self.profile, "color_bar_max"))
        self.COLOR_BAR_VALUE = self._parse(self.config.get(self.profile, "color_bar_value"))
        self.COLOR_GRAPH_MAX = self._parse(self.config.get(self.profile, "color_graph_max"))
        self.COLOR_GRAPH_VALUE = self._parse(self.config.get(self.profile, "color_graph_value"))
        self.COLOR_GRAPH_TEXT = self._parse(self.config.get(self.profile, "color_graph_text"))
        self.COLOR_GRAPH_LINE = self._parse(self.config.get(self.profile, "color_graph_line"))

    def init_sizes(self):
        self.SIZE_NAME = self._parse(self.config.get(self.profile, "name"))
        self.SIZE_IP = self._parse(self.config.get(self.profile, "ip"))
        self.SIZE_COUNT = self._parse(self.config.get(self.profile, "count"))
        self.SIZE_LOAD = self._parse(self.config.get(self.profile, "load"))
        self.SIZE_MEMORY = self._parse(self.config.get(self.profile, "memory"))
        self.SIZE_BAR_LOAD = self._parse(self.config.get(self.profile, "bar_load"))
        self.SIZE_BAR_MEMORY = self._parse(self.config.get(self.profile, "bar_memory"))
        self.SIZE_GRAPH = self._parse(self.config.get(self.profile, "graph"))

    def draw_name(self, x ,y, two_lines=False):
        self.draw.text((x, y), unicode("Server"), font=self.font, fill=self.COLOR_TEXT)
        if two_lines:
            offset_x = 0
            offset_y = 15
        else:
            offset_x = 42
            offset_y = 0
        self.draw.text((x+offset_x, y+offset_y), unicode(self.data["name"]), font=self.font, fill=self.COLOR_TEXT_VALUE)

    def draw_ip(self, x ,y, two_lines=False):
        if two_lines:
            self.draw.text((x, y), unicode("IP"), font=self.font, fill=self.COLOR_TEXT)
        #+20
        if two_lines:
            offset_x = 0
            offset_y = 15
        else:
            offset_x = 0
            offset_y = 0
        self.draw.text((x+offset_x, y+offset_y), unicode("%s:%s" % (self.data["ip"], self.data["port"])), font=self.font, fill=self.COLOR_TEXT_VALUE)

    def draw_count(self, x ,y, two_lines=False):
        self.draw.text((x, y), unicode("Hráči"), font=self.font, fill=self.COLOR_TEXT)
        if two_lines:
            offset_x = 0
            offset_y = 15
        else:
            offset_x = 32
            offset_y = 0
        self.draw.text((x+offset_x, y+offset_y), unicode(self.data["count"]), font=self.font, fill=self.COLOR_TEXT_VALUE)

    def draw_load(self, x ,y, load, two_lines=False):
        self.draw.text((x, y), unicode("Zatížení"), font=self.font, fill=self.COLOR_TEXT)
        if two_lines:
            offset_x = 0
            offset_y = 15
        else:
            offset_x = 50
            offset_y = 0
        self.draw.text((x+offset_x, y+offset_y), unicode("%.2f %%" % load), font=self.font, fill=self.COLOR_TEXT_VALUE)

    def draw_memory(self, x ,y, two_lines=False):
        self.draw.text((x, y), unicode("Paměť"), font=self.font, fill=self.COLOR_TEXT)
        if two_lines:
            offset_x = 0
            offset_y = 15
        else:
            offset_x = 40
            offset_y = 0
        self.draw.text((x+offset_x, y+offset_y), unicode("%s/%s MB" % (self.data["memory"], self.data["memory_max"])), font=self.font, fill=self.COLOR_TEXT_VALUE)

    def draw_graphs(self, x, y, values, height=40, multiplicator=1, two_lines=False):
        width=len(values)
        ceil = max(values)*1.2
        if not ceil: ceil = 1
        self.draw.rectangle((x, y, x+width*(multiplicator+1)+2, y+height+2), fill=self.COLOR_GRAPH_MAX)
        offset = 1
        for i, value in enumerate(values):
            for offset_local in range(multiplicator):
                self.draw.line((x+1+i+offset, y+height-1, x+1+i+offset, y+height-1-(float(value)/ceil*height)), fill=self.COLOR_GRAPH_VALUE)
                offset += 1
        self.draw.text((x+2, y+height-14), unicode("0"), font=self.font, fill=self.COLOR_GRAPH_TEXT)
        self.draw.text((x+2, y+2), unicode("%.1f" % ceil), font=self.font, fill=self.COLOR_GRAPH_TEXT)
        self.draw.line((x+2, y+height, x+width+width*multiplicator, y+height), fill=self.COLOR_GRAPH_LINE)
        self.draw.line((x+2, y+2, x+width+width*multiplicator, y+2), fill=self.COLOR_GRAPH_LINE)
        #self.draw.text(((x+(width+width*multiplicator)/2-8), y+height), unicode("24h"), font=self.font, fill=self.COLOR_GRAPH_TEXT)

    def draw_bar(self, x, y, percent, width=118, two_lines=False):
        self.draw.rectangle((x, y, x+width, y+14), fill=self.COLOR_BAR_MAX)
        self.draw.rectangle((x+1, y+1, x+((width-1)/100.0)*percent, y+13), fill=self.COLOR_BAR_VALUE)

    def get_image(self):
        self.im = Image.open(os.path.join(ROOT, "bg", self.config.get(self.profile, "bg")))
        self.draw = ImageDraw.Draw(self.im)
        self.font = ImageFont.truetype(os.path.join(ROOT, "PT_Sans-Web-Regular.ttf"), 12, encoding="unicode")

        two_lines = self.config.getboolean(self.profile, "two_lines")

        self.draw_name(self.SIZE_NAME[0], self.SIZE_NAME[1], two_lines)
        self.draw_ip(self.SIZE_IP[0], self.SIZE_IP[1], two_lines)
        self.draw_count(self.SIZE_COUNT[0], self.SIZE_COUNT[1], two_lines)
        load = float(self.data["load"])/float(self.data["cpu_count"])*100
        self.draw_load(self.SIZE_LOAD[0], self.SIZE_LOAD[1], load, two_lines)
        self.draw_memory(self.SIZE_MEMORY[0], self.SIZE_MEMORY[1], two_lines)
        self.draw_bar(self.SIZE_BAR_LOAD[0], self.SIZE_BAR_LOAD[1], int(load), self.SIZE_BAR_LOAD[2])
        self.draw_bar(self.SIZE_BAR_MEMORY[0], self.SIZE_BAR_MEMORY[1], float(self.data["memory"])/float(self.data["memory_max"])*100, self.SIZE_BAR_MEMORY[2])
        if self.data["history"]:
            self.draw_graphs(self.SIZE_GRAPH[0], self.SIZE_GRAPH[1], self.data["history"], self.SIZE_GRAPH[2], self.SIZE_GRAPH[3])

        output = StringIO.StringIO()
        self.im.save(output, "PNG")
        return output.getvalue()

#def main(hostname, port):
#    collector = MinecraftWidgetCollector(hostname, port)
#    collector.save_data()
#    for profile in ('vb', 'vw', 'hb', 'hw'):
#        widget = MinecraftWidget(collector.get_last(), profile)
#        with open(os.path.join(config["data_dir"], "www", "widget_%s_%s_%d.png" % (profile, collector.addr[0], collector.addr[1])), "w") as f:
#            f.write(widget.get_image())

#if __name__ == "__main__":
#    main("194.8.253.48", 25565)


import urllib.request
import json
import bs4
from tournamentClasses import *
import warnings

warnings.filterwarnings("ignore", category=UserWarning, module='bs4')

def get_html(yr):
    print('...')
    url = 'http://www.sports-reference.com/cbb/postseason/{}-ncaa.html'.format(yr)
    ls = [urllib.request.urlopen(url).read().decode('utf-8')]
    url = 'http://www.sports-reference.com/cbb/seasons/{}-polls.html'.format(yr)
    ls.append(urllib.request.urlopen(url).read().decode('utf-8'))
    print('done with {}'.format(yr))
    return ls

# html_dict = {yr:get_html(yr) for yr in range(1985,2017)}  # uncomment to rebuild html_dict
# json.dump(html_dict, open("html_dict_file.txt", "w"))

html_dict = json.load(open("html_dict_file.txt", "r"))
html_dict = {int(yr): html_dict[yr] for yr in html_dict.keys()}

def get_teams(yr):
    soup = bs4.BeautifulSoup(html_dict[yr][1])
    data = []
    table = soup.find(lambda tag: tag.name == 'table' and tag.has_key('id') and tag['id'] == 'ap')
    rows = table.findAll('tr')
    for row in rows:
        cols = row.findAll('td')
        cols = [ele.text.strip() for ele in cols]
        data.append(data.append([ele for ele in cols]))
    data = [ele for ele in data if isinstance(ele, list) and len(ele) >= 1 and ele[0] == 'Final']
    data = [[ele for i, ele in enumerate(team) if i not in [0, 3, 4]] for team in data]
    data = [[ele[:-2] if i == 1 else ele for i, ele in enumerate(team)] for team in data]

    # replace all names that change within the bracket:
    replacement_dict = {'North Carolina': 'UNC', 'Louisiana State': 'LSU', 'Nevada-Las Vegas': 'UNLV',
                        'Virginia Commonwealth': 'VCU', 'Texas-El Paso': 'UTEP', 'Pittsburgh': 'Pitt',
                        'Connecticut': 'UConn', 'East Tennessee State': 'ETSU', 'Southern Mississippi': 'Southern Miss',
                        'Southern California': 'SCU', 'Massachusetts': 'UMass', "Saint Joseph's": "St. Joseph's",
                        'Mississippi': 'Ole Miss', 'Brigham Young': 'BYU',
                        'Southern Methodist': 'SMU'}  # add replacements here`
    data = [[ele if ele not in replacement_dict.keys() else replacement_dict[ele] for ele in team] for team in data]

    '''rank_dict = {ele[1]: [int(ele[0]), ele[2]] for ele in data}
    return rank_dict'''
    return [Team(ele[1], ele[0], ele[2]) for ele in data]

def find_team_by_name(name, teams):
    for team in teams:
        if team.name == name:
            return team
    return None

region_dict = {}
for yr in range(1985, 2017):
    if yr != 2011 and (yr in range(2007, 2017)) or (yr in range(1998, 2004)):
        region_dict[yr] = ['east', 'west', 'south', 'midwest', 'national']
    elif yr in range(1985, 1998):
        region_dict[yr] = ['east', 'west', 'southeast', 'midwest', 'national']
region_dict[2004] = ['stlouis', 'atlanta', 'eastrutherford', 'phoenix', 'national']
region_dict[2005] = ['syracuse', 'albuquerque', 'austin', 'chicago', 'national']
region_dict[2006] = ['minneapolis', 'atlanta', 'oakland', 'washington', 'national']
region_dict[2011] = ['east', 'southeast', 'southwest', 'west', 'national']

def gen_year_dict(start, end):
    year_dict = {}
    url_tag_dict = {}
    for yr in range(start, end + 1):
        yr_url_tags = []
        teams = get_teams(yr)
        soup = bs4.BeautifulSoup(html_dict[yr][0])
        brackets = soup.find("div", {"id": "brackets"})

        conference_and_record_dict = json.load(open('conferences_doc_full.txt','r'))
        conference_and_record_dict = {int(key):conference_and_record_dict[key] for key in conference_and_record_dict.keys()}
        regions = region_dict[yr]
        games = []

        for region in regions:
            bracket = brackets.find("div", {"id": "{}".format(region)})
            bracket_results = bracket.find_all(['a', 'span'])
            url_tag_data = [str(item)[22:str(item)[22:].find('/') + 22] for item in bracket_results if
                            str(item)[1:7] == 'a href' and str(item)[22:str(item)[22:].find('/') + 22] != 's']
            results = [info.text for info in bracket_results if info.text[:2] != 'at']
            results = [results[6 * i:6 * i + 6] for i in range(16)]
            if region != "national":
                num_games = 15
            else:
                num_games = 3
            for game_i in range(num_games):
                if region != 'national':
                    if game_i in range(8):
                        tourn_round = "Round of 64"
                    elif game_i in range(8, 12):
                        tourn_round = "Round of 32"
                    elif game_i in range(12, 14):
                        tourn_round = "Sweet 16"
                    elif game_i == 14:
                        tourn_round = "Elite 8"
                else:
                    if game_i in range(2):
                        tourn_round = 'Final 4'
                    else:
                        tourn_round = 'Championship'
                if not find_team_by_name(results[game_i][1], teams):
                    team1 = Team(results[game_i][1], 'NR', '???', results[game_i][0], region)
                    teams.append(team1)
                else:
                    team1 = find_team_by_name(results[game_i][1], teams)
                    team1.seed = results[game_i][0]
                if region != 'national':
                    team1.region = region
                if not find_team_by_name(results[game_i][4], teams):
                    team2 = Team(results[game_i][4], 'NR', '???', results[game_i][3], region)
                    teams.append(team2)
                else:
                    team2 = find_team_by_name(results[game_i][4], teams)
                    team2.seed = results[game_i][3]
                    if region != 'national':
                        team2.region = region
                if tourn_round == 'Round of 64':
                    team1.url_tag = url_tag_data[game_i * 2]
                    team2.url_tag = url_tag_data[game_i * 2 + 1]
                    yr_url_tags.append(team1.url_tag)
                    yr_url_tags.append(team2.url_tag)
                    team1.conf = conference_and_record_dict[yr][team1.url_tag][0]
                    team1.record = conference_and_record_dict[yr][team1.url_tag][1]
                    team2.conf = conference_and_record_dict[yr][team2.url_tag][0]
                    team2.record = conference_and_record_dict[yr][team2.url_tag][1]
                games.append(Game(tourn_round, team1, results[game_i][2], team2, results[game_i][5]))
        year_dict[yr] = games
        url_tag_dict[yr] = yr_url_tags
        print(str(yr) + " completed")
    # uncomment next line to recreate the url_tag_dictionary file
    # json.dump(url_tag_dict,open('url_tags_by_yr.txt','w'))
    return year_dict

def write_csv(start, end, filename):
    with open(filename, 'w') as f:
        year_dict = gen_year_dict(start, end)
        for yr in range(start, end + 1):
            f.write(str(yr) + '\n')
            for game in year_dict[yr]:
                f.write('{}\n'.format(",".join(game.results()[0])))
                f.write('{}\n\n'.format(",".join(game.results()[1])))

#write_csv(1985, 2016, 'output.csv')
#gen_year_dict(1985, 2016)


from bs4 import BeautifulSoup
import requests
import logging
from fake_useragent import UserAgent
import sys
from tqdm import tqdm

AFISHA_URL = 'https://www.afisha.ru/novosibirsk/schedule_cinema/'
AFISHA_PARAMS = {'view': 'list'}

KINOPOISK_URL = 'https://www.kinopoisk.ru{}'
KINOPOISK_PARAMS = {
    'level': '7',
    'from': 'forma',
    'result': 'adv',
    'm_act[from]': 'forma',
    'm_act[what]': 'content',
    'm_act[find]': '',
    'm_act[year]': '',
}

def getCinemasLogger():
    logging.basicConfig(
        filename='cinemas.log',
        level=logging.DEBUG,
        filemode='w',
        format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
    )
    cinemasLogger = logging.getLogger('CinemasLogger')
    return cinemasLogger

def get_proxies_list():
    response = requests.get(
        'http://www.freeproxy-list.ru/api/proxy?anonymity=false&token=demo',
    )
    proxies_list = response.text.split('\n') * 5
    proxies_list.insert(0, '')
    return proxies_list

def fetch_page(url, user_agent, proxies_list=('',), params=None):
    for proxy in proxies_list:
        try:
            response = requests.get(
                url,
                headers={'User-Agent': user_agent},
                params=params,
                proxies={'https': proxy},
                timeout=10,
            )
            if 'showcaptcha' in response.url:
                continue
            return response.text, response.url
        except (
                requests.exceptions.ConnectionError,
                requests.exceptions.ConnectTimeout,
                requests.exceptions.ReadTimeout,
        ):
            continue
    return

def parse_afisha_list(raw_html):
    movies_list = []
    soup = BeautifulSoup(raw_html, 'html.parser')
    for item_info in soup.find_all('div', {'class': 'new-list__item-info'}):
        movies_list.append(
            {
                'name': item_info.h3.text,
                'year': item_info.find(
                    'div', {'class': 'new-list__item-status'}
                ).text[:4],
            }
        )
    return movies_list

def fetch_movie_rating(raw_html, url):
    soup = BeautifulSoup(raw_html, 'html.parser')
    try:
        if 'index.php' in url:
            element_most_wanted = soup.find('div',
                                            {'class': 'element most_wanted'})
            movie_rating = float(element_most_wanted.div.div.text)
        else:
            block_rating = soup.find('div', {'id': 'block_rating'})
            movie_rating = float(block_rating.div.div.a.span.text)
    except (AttributeError, ValueError) as e:
        logger.error(e)
        return 0
    if not movie_rating:
        return 0
    return movie_rating

def output_movies_to_console(movies):
    for movie in sorted(movies, key=lambda m: m['rating'], reverse=True):
        if movie['rating'] == 0:
            rating = 'Рейтинг недоступен (мало оценок)'
        else:
            rating = movie['rating']
        print('{} / {}'.format(movie['name'], rating))

if __name__ == '__main__':
    logger = getCinemasLogger()
    logger.info('Script started')
    user_agent = UserAgent()
    proxies_list = get_proxies_list()
    afisha_raw_html, afisha_url = fetch_page(
        url=AFISHA_URL,
        user_agent=user_agent.random,
        params=AFISHA_PARAMS,
    )

    if not afisha_raw_html:
        message = 'No info at Afisha.ru or connection error'
        logger.error('Script finished with "{}"'.format(message))
        sys.exit(message)
    else:
        logger.info("Afisha's content loaded")

    movies = parse_afisha_list(afisha_raw_html)

    if not movies:
        message = 'No movies today'
        logger.error('Script finished with "{}"'.format(message))
        sys.exit(message)
    else:
        logger.info('Movies names list fetched')

    progressbar = iter(tqdm(
        movies,
        desc='Getting movies info',
        leave=False,
    ))
    for movie in movies:
        next(progressbar)
        KINOPOISK_PARAMS['m_act[find]'] = movie['name']
        KINOPOISK_PARAMS['m_act[year]'] = movie['year']
        kinopoisk_raw_html, kinopoisk_movie_url = fetch_page(
            url=KINOPOISK_URL.format('/index.php'),
            user_agent=user_agent.random,
            proxies_list=proxies_list,
            params=KINOPOISK_PARAMS,
        )

        if not kinopoisk_raw_html:
            logger.error('Error page "{}" fetching'.format(movie['name']))
            continue
        else:
            logger.info('Page "{}" fetched'.format(movie['name']))
        movie['rating'] = fetch_movie_rating(
            kinopoisk_raw_html,
            kinopoisk_movie_url,
        )
        if not movie['rating']:
            logger.error(
                "Error {}'s rating fetching".format(movie['name']))
            movie_rating = 0
        else:
            logger.info('Movie "{}" rating fetched'.format(movie['name']))
    try:
        next(progressbar)
    except StopIteration:
        pass
    output_movies_to_console(movies)
    logger.info('Script finished')


import random
import socket

class Player:
    name = ''
    hostname = ''
    messages = []
    opponentIP = ''
    ready = False

    def __init__(self, h):
        self.hostname = h
        self.messages = []
        self.opponentIP = ''

    def sendMessages(self, client):
        message = ''
        print('Player messages: ', self.messages)
        for m in self.messages:
            message += m + '-'
        client.send(message.encode())
        self.messages = []

    def addMessage(self, msg):
        self.messages.append(msg)

class Server():
    playerList = [Player('') for count in range(0)]
    serverSock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    hostname = ''
    port = 0
    running = False

    def __init__(self):

        self.hostname = '10.24.131.127'
        self.getPort(self.serverSock)
        self.serverSock.listen(32)
        self.serverSock.setblocking(False)
        self.start()

    def run(self):
        self.connect()

    def start(self):
        self.run()

    def connect(self):
        print("Listening on", self.hostname, ":", self.port, "\n-------------------------------------------")
        while True:
            try:
                client, address = self.serverSock.accept()
                player = self.isInList(address[0])
                print('Connected to', address[0], ":", address[1])
                if player is not None:
                    print("Player is in the list")
                    if self.hasEnemy(player.hostname):  # Player has enemy
                        self.handlePlayerWithOpponent(player, client)
                    else:  # Player has no enemy
                        self.handlePlayerWithoutOpponent(player, client)
                else:  # New Player
                    self.handleNewPlayer(client, address)
                client.close()
                print("Client lost\n--------------------------------------------------------------")

            except:
                pass

    def getPort(self, socket):
        passed = False
        while not passed:
            try:
                port = random.randint(0, 1500) + 5000
                self.serverSock.bind((self.hostname, port))
                self.port = port
                passed = True
            except:
                pass

    def handlePlayerWithOpponent(self, player, client):
        print("Player has enemy")
        enemy = self.getEnemy(player.hostname)
        print('Enemy IP:', enemy.hostname)
        self.handleMessage(player, enemy, client)

    def handlePlayerWithoutOpponent(self, player, client):
        print("Player does not have enemy")
        for other in self.playerList:  # Search for opponent
            if self.connectPlayers(player, other) == True:  # Opponent found
                break
        self.handleNullMessage(player, client)

    def handleNewPlayer(self, client, address):
        print('New Player')
        player = Player(address[0])
        self.playerList.append(player)
        player.name = client.recv(1024).decode()[5:-1]
        player.addMessage('NULL')
        player.sendMessages(client)

    def handleMessage(self, player, enemy, client):
        message = client.recv(1024).decode()[:-2]
        print("Message from player: ", message)

        if 'LEAVE' in message:
            enemy.addMessage('LEAVE')
            player.addMessage('NULL')
        elif 'CLOSE' in message:
            enemy.addMessage('CLOSE')
            player.addMessage('CLOSE')
        elif 'FINISHED_PLACING_SHIPS' in message:
            player.ready = True
            if player.ready and enemy.ready:
                player.addMessage('SET_TURN-FALSE')
                enemy.addMessage('SET_TURN-TRUE')
            else:
                player.addMessage('NULL')
        elif 'SET_TURN' in message:
            turn = message[9:-1]
            enemy.addMessage('SET_TURN')
            enemy.addMessage(turn)
            player.addMessage('NULL')
        elif 'RESULT' in message:
            if 'HIT' in message:
                enemy.addMessage('RESULT-HIT')
                enemy.addMessage('SET_TURN-TRUE')
                player.addMessage('SET_TURN-FALSE')
            elif 'MISS' in message:
                enemy.addMessage('RESULT-MISS')
                enemy.addMessage('SET_TURN-FALSE')
                player.addMessage('SET_TURN-TRUE')
            elif 'SINK' in message:
                enemy.addMessage(message)
                enemy.addMessage('SET_TURN-TRUE')
                player.addMessage('SET_TURN-FALSE')
        elif 'MOVE' in message:
            print('Move: ', message)
            enemy.addMessage(message)
            player.addMessage('NULL')
        elif 'END_GAME' in message:
            enemy.addMessage('END_GAME')
            player.addMessage('END_GAME')
        else:
            enemy.addMessage('NULL')
            player.addMessage('NULL')

        player.sendMessages(client)

    def handleNullMessage(self, player, client):
        message = client.recv(1024).decode()[:-2]
        print(message)
        player.addMessage('NULL')
        player.sendMessages(client)

    def isInList(self, hostname) -> Player:
        for p in self.playerList:
            if p.hostname == hostname:
                return p
        return None

    def hasEnemy(self, hostname) -> bool:
        for p in self.playerList:
            if p.opponentIP == hostname:
                return True
        return False

    def getEnemy(self, hostname) -> Player:
        for p in self.playerList:
            if p.opponentIP == hostname:
                return p
        return None

    def getPlayer(self, hostname) -> Player:
        for p in self.playerList:
            if p.hostname == hostname:
                return p
        return None

    def findEnemy(self, hostname) -> Player:
        for p in self.playerList:
            if p.opponentIP == '' and p.hostname != hostname:
                return p
        return None

    def connectPlayers(self, player1, player2) -> bool:
        print("Attempting to connect players")
        if player1.opponentIP == '' and player2.opponentIP == '':
            if player1.hostname != player2.hostname:
                player1.opponentIP = player2.hostname
                player2.opponentIP = player1.hostname
                player1.addMessage('SET_OPPONENT_NAME')
                player1.addMessage(player2.name)
                player2.addMessage('SET_OPPONENT_NAME')
                player2.addMessage(player1.name)
                print('Connected players!')
                return True
        print("Failed to connect players")
        return False

s= Server()


from tweet_cleaner import tweet_obrabiarka
import math
import numpy as np

"""
Module which should be used for feature extracting.
Outside of module only function:
    features_from_corpus() <--- MAIN
    list_of_all_words_base()
should be used
"""

# Przyjmuje caly tekst jako string
def _term_count_in_sentence_raw(sent):
    """
    This function calculate number of occurences for each word.
    :param sent: one tweet as a String (raw tweet)
    :return: dictionary of pairs like (word, count(word))
    """
    sent = tweet_obrabiarka(sent, hashowac=0, stemmer=1)
    freq_table = {}
    for word in sent:
        if word in freq_table:
            freq_table[word] += 1
        else:
            freq_table[word] = 1

    return freq_table

# Przyjmuje pokawalkowany string :)
def _term_count_in_sentence(sent):
    """
    This function calculate number of occurences for each word.
    :param sent: one tweet as a list of words (tweet after tokenization)
    :return: dictionary of pairs like (word, count(word))
    """
    freq_table = {}
    for word in sent:
        if word in freq_table:
            freq_table[word] += 1
        else:
            freq_table[word] = 1

    return freq_table

# Przyjmuje zbior tweetow jako calych tekstow w postaci string
def _term_count_in_corpus_raw(sentences):
    """
    This function calculate number of occurences for each word in whole corpus.
    :param sentences: List of tweets (called "corpus") -  (where each tweet is a String)
    :return: dictionary of pairs like (word, count(word)) for whole corpus
    """
    whole_dict = {}
    for sent in sentences:
        new_dict = _term_count_in_sentence_raw(sent)

        dict3 = {**whole_dict, **new_dict}
        for key, value in dict3.items():
            if key in whole_dict and key in new_dict:
                dict3[key] = value + whole_dict[key]

        whole_dict = dict3
    return whole_dict

# Przyjmuje zbior pokawalkowanych stringow :)
def _term_count_in_corpus(sentences):
    """
    This function calculate number of occurences for each word in whole corpus.
    :param sentences: List of tweets (called "corpus") -  (where each tweet is a list of words)
    :return: dictionary of pairs like (word, count(word)) for whole corpus
    """
    nowy = []
    for sent in sentences:
        nowy.append(tweet_obrabiarka(sent, hashowac=0, stemmer=1))
    dictionary = _term_count_in_corpus_raw(nowy)
    return dictionary

def calculate_tf(sentence):
    """
    This function calculate feauture called term frequency.
    :param sentence: Tweet as a list of word
    :return: Dictionary of pairs like (word, word frequency)
    """
    tf_list = {}
    count_dict = _term_count_in_sentence(sentence)
    for word in sentence:
        tf_list[word] = (count_dict[word] / len(sentence))
    return tf_list

def ile_zawiera(szukana, korpus):
    """
    This function calculate how many time word (szukana) occures in whole corpus (set of tweets).
    :param szukana: word, which occurences we want to calculate as a String
    :param korpus: list of tweets (tokenized)
    :return: number of occurences + 1 (in purpose to avoid /0)
    """
    licznik = 0
    for document in korpus:
        stan = 0
        for word in document:
            if word == szukana:
                stan = 1
        if stan == 1:
            licznik = licznik + 1
    # Żeby uniknąć dzielenia przez 0
    return licznik + 1

def number_of_sentence_in_document(corpus):
    """
    This function calculate number of tweets in corpus.
    :param corpus: List of tweets
    :return: Number of tweets in corpus
    """
    return len(corpus)

def calculate_idf(cutted_sentence, corpus):
    """
    This function calculate feature called inversed document (corpus) frequency for one of tweet
    :param cutted_sentence: Sentence (as a List of words) for which we want to find idf
    :param corpus: List of all tweets (each tweet as a list of words)
    :return: Dictionary of pairs (word, idf(word))
    """
    idf_list = {}
    for word in cutted_sentence:
        idf_list[word] = (math.log(number_of_sentence_in_document(corpus) / ile_zawiera(word, corpus)))
    return idf_list

def calculate_tf_idf(tweet, corpus):
    """
    This function calculate feature called tf*idf for one tweet from corpus
    :param tweet: Tweet as a list of words
    :param corpus: List of all tweets (each tweet as a list of words)
    :return: Dictionary of pairs (word, tfidf(word))
    """
    tf_idf = {}
    for word in tweet:
        tf = calculate_tf(sentence=tweet)[word]
        idf = calculate_idf(cutted_sentence=tweet, corpus=corpus)[word]
        tf_idf[word] = tf * idf
    return tf_idf

def list_of_all_words_base(corpus):
    """
    This function let to find all words in whole corpus
    :param corpus: List of all tweets (each tweet as a list of words)
    :return: set of words in corpus
    """
    lista = set()
    for tweet in corpus:
        for word in tweet:
            lista.add(word)
    return lista

def vector_maker(feature_dict, all_word_list):
    """
    This function prepare vector of features.
    :param feature_dict: Dictionary of pairs like (word, some_feature(word))
    :param all_word_list: Set of all words in corpus
    :return: vector of values of features, values have same order as all word list.
    """
    vector = []
    for word in all_word_list:
        wart = feature_dict[word] if word in feature_dict else 0
        vector.append(wart)
    return np.array(vector)

def features_from_corpus(corpus, method):
    """
    This function prepare array of feature vectors. This function should be called outside.
    :param corpus: List of all tweets (each tweet as a list of words)
    :param method: Choice of feature extracting method
    :return: np.array of vectors with feature values for each words in corpus
    """
    list_of_features = []
    list_of_all_words = list_of_all_words_base(corpus)
    for tweet in corpus:
        if method == 'tf_idf':
            dict = calculate_tf_idf(tweet, corpus)
        if method == 'bag_of_word':
            dict = _term_count_in_sentence(tweet)
        list_of_features.append(vector_maker(dict, list_of_all_words))
    return np.array(list_of_features)


from django.contrib.auth.models import User
from django.http import HttpResponseForbidden

def account_ownership_required(func): #계정의 소유권 판단 남의 계정으로 다른사람 계정을 탈퇴시키는 경우 방지
    def decorated(request ,*args,**kwargs): #안에 user 가 존재
        #target user 는 DB 에서 가져와서 사용
        target_user = User.objects.get(pk = kwargs['pk']) #kwargs pk 의 정보가 담겨져있음 + user.objects.get() 하나의 인자만 얻고 싶을때
        if target_user == request.user:
            return func(request,*args,**kwargs)
        else:
            return HttpResponseForbidden() #접근 금지 경고창을 띄운다.

    return decorated