"""
Simple but highly configurable plotting functions for spiketrains in neo format.
While building on the matplotlib libary the functions lay an emphasis on clear,
pleasant-to-look-at visualizations.
"""

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from elephant.statistics import mean_firing_rate
from math import log10, floor

def _round_to_1(x):
    rounded = round(x, -int(floor(log10(abs(x)))))
    return rounded, rounded > x

def _get_attributes(spiketrains, key_list):
    """
    This function returns attribute_array which is of an array of shape
    (len(spiketrains), len(key_list)) and consists of numerical ids for each
    value of each key for each spike train.

    Passed spike trains must be already sorted according to key_list and
    key_list must not be empty.
    """

    key_count = len(key_list)
    attribute_array = np.zeros((len(spiketrains), len(key_list)))
    # count all group sizes for all keys in key_list:
    while key_count > 0:
        key_count -= 1
        group_key = key_list[key_count]
        i = 0
        if group_key in spiketrains[i].annotations:
            current_value = spiketrains[i].annotations[group_key]
        else:
            # use placeholder value when key is not in annotations
            # of the current spike train
            current_value = '####BLANK####'
        ref_value = current_value
        values = np.array([])
        # count all group sizes for values of current key:
        while i < spiketrains.__len__():
            if not len(values) or current_value not in values:
                values = np.append(values, current_value)
            # count group size for a value of the current key:
            while i < len(spiketrains) and current_value == ref_value:
                attribute_array[i][key_count] = \
                np.where(values == current_value)[0][0]
                i += 1
                if i < len(spiketrains):
                    if group_key in spiketrains[i].annotations:
                        current_value = spiketrains[i].annotations[
                            group_key]
                    else:
                        current_value = '####BLANK####'
            ref_value = current_value
    return attribute_array

def rasterplot(spiketrain_list,
               key_list=[],
               groupingdepth=0,
               spacing=[8, 3],
               colorkey=0,
               pophist_mode='color',
               pophistbins=100,
               right_histogram=mean_firing_rate,
               righthist_barwidth=1.01,
               filter_function=None,
               histscale=.1,
               labelkey=None,
               markerargs={'markersize':4,'marker':'.'},
               separatorargs=[{'linewidth':2, 'linestyle':'--', 'color':'0.8'},
                              {'linewidth':1, 'linestyle':'--', 'color':'0.8'}],
               legend=False,
               legendargs={'loc':(.98,1.), 'markerscale':1.5, 'handletextpad':0},
               ax=None,
               rasterized=False,
               style='ticks',
               palette=None,
               context=None,  # paper, poster, talk
               ):

    """
    This function plots the dot display of spike trains alongside its
    population histogram and the mean firing rate (or a custom function).

    Optional visual aids are offered such as sorting, grouping and color coding
    on the basis of the arrangement in list of spike trains and spike train
    annotations.
    Changes to optics of the dot marker, the separators and the legend can be
    applied by providing a dict with the respective parameters. Changes and
    additions to the dot display itself or the two histograms are best realized
    by using the returned axis handles.

    :param spiketrain_list: list
        List can either contain Neo SpikeTrains object or lists of Neo
        SpikeTrains objects.
    :param key_list: str | list of str
        Annotation key(s) for which the spike trains should be ordered.
        When list of keys is given the spike trains are ordered successively
        for the keys.
        By default the ordering by the given lists of spike trains have
        priority. This can be bypassed by using an empty string '' as list-key
        at any position in the key_list.
    :param groupingdepth: 0 | 1 | 2
        * 0: No grouping (default)
        * 1: grouping by first key in key_list.
             Note that when list of lists of spike trains are given the first
             key is by the list identification key ''. If this is unwanted
             the empty string '' can be placed at a different position in
             key_list.
        * 2: additional grouping by second key respectively
        The groups are separated by whitespace specified in the spacing
        parameter and optionally by a line specified by the the separatorargs.
    :param spacing: int | [int] | [int, int]
        Size of whitespace separating the groups in units of spike trains.
        When groupingdepth == 2 a list of two values can specify the distance
        between the groups in level 1 and level 2. When only one value is given
        level 2 spacing is set to half the spacing of level 1.
        Default: [5, 3]
    :param colorkey: str | int  (default 0)
        Contrasts values of a key by color. The key can be defined by its
        namestring or its position in key_list. Note that position 0 points to
        the list identification key ('') when list of lists of spike trains are
        given, if not otherwise specified in key_list!
    :param pophist_mode: 'color' (default) | 'total'
         * total: One population histogram for all drawn spike trains
         * color: Additionally to the total population histogram,
                  a histogram for each colored subset is drawn (see colorkey).
    :param pophistbins: int (default 100)
        Number of bins used for the population histogram.
    :param right_histogram: function
        The function gets ONE neo.SpikeTrain object as argument and has to
        return a scalar.
        For example the functions in the elephant.statistics module can
        be used. (default: mean_firing_rate)
        When a function is applied is is recommended to set the axis label
        accordingly by using the axis handle returned by the function:
        axhisty.set_xlabel('Label Name')
    :param filter_function: function
        The function gets ONE neo.SpikeTrain object as argument and if the
        return is True the spike train is included; if False it is exluded.
    :param histscale: float (default .1)
        Portion of the figure used for the histograms on the right and upper
        side.
    :param labelkey: 0 | 1 | '0+1' (default) | 'annotation key' | None
        * 0, 1: Set label according to first or second key in key_list.
                Note that the first key is by default the list identification
                key ('') when list of lists of spike trains are given.
        * '0+1': Two level labeling of 0 and 1
        * annotation-key: Labeling each spike train with its value for given key
        * None: No labeling
        Note that only groups (-> see groupingdepth) can be labeled as bulks.
        Alternatively you can color for an annotation key and show a legend.
    :param markerargs: dict
        Arguments dictionary is passed on to matplotlib.pyplot.plot()
    :param separatorargs: dict | [dict, dict] | None
        If only one dict is given and groupingdepth == 2 the arguments are
        applied to the separator of both level. Otherwise the arguments are
        of separatorargs[0] are applied to the level 1 and [1] to level 2.
        Arguments dictionary is passed on to matplotlib.pyplot.plot()
        To turn of separators set it to None.
    :param legend: boolean
        Show legend?
    :param legendargs: dict
        Arguments dictionary is passed on to matplotlib.pyplot.legend()
    :param ax: matplotlib axis | None (default)
        The axis onto which to plot. If None a new figure is created.
        When an axis is given, the function can't handle the figure settings.
        Therefore it is recommended to call seaborn.set() with your preferred
        settings before creating your matplotlib figure in order to control
        your plotting layout.
    :param rasterized: bool (default: False)
        Passed as 'rasterized' keyword to the matplotlib plot function for the dot display.
    :param style: str
        seaborn style setting. Default: 'ticks'
    :param palette: string | sequence
        Define the color palette either by its name or use a custom palette in
        a sequence of the form ([r,g,b],[r,g,b],...).
    :param context: 'paper'(default) | 'talk' | 'poster'
        seaborn context setting which controls the scaling of labels. For the
        three options the parameters are scaled by .8, 1.3, and 1.6
        respectively.
    :return: ax, axhistx, axhisty   <matplotlib axis handle>
        * ax is handle of the dot display plot
        * axhistx is handle of the histogram plot above the the dot display
        * axhisty is handle of the histogram plot on the right hand side

    *Basic Example:*
        >>> from elephant.spike_train_generation import homogeneous_poisson_process as HPP
        >>> from quantities import Hz
        >>> import matplotlib.pyplot as plt
        >>>
        >>> st_list = [HPP(rate=10*Hz) for _ in range(100)]
        >>> rasterplot(st_list)
        >>> plt.show()

    *Grouping Example:*
        >>> from elephant.spike_train_generation import homogeneous_poisson_process as HPP
        >>> from elephant.spike_train_generation import homogeneous_gamma_process as HGP
        >>> from quantities import Hz
        >>> import matplotlib.pyplot as plt
        >>>
        >>> st_list1 = [HPP(rate=10*Hz) for _ in range(100)]
        >>> st_list2 = [HGP(a=3, b=10*Hz) for _ in range(100)]
        >>>
        >>> # plot visually separates the two lists
        >>> rasterplot([st_list1, st_list2])
        >>>
        >>> # add annotations to spike trains
        >>> for i, (st1, st2) in enumerate(zip(st_list1, st_list2)):
        >>>     if i.__mod__(2):
        >>>         st1.annotations['parity'] = 'odd'
        >>>         st2.annotations['parity'] = 'odd'
        >>>     else:
        >>>         st1.annotations['parity'] = 'even'
        >>>         st2.annotations['parity'] = 'even'
        >>>
        >>> # plot separates the lists and the annotation values within each list
        >>> rasterplot([st_list1, st_list2], key_list=['parity'],
        >>>            groupingdepth=2, labelkey='0+1')
        >>>
        >>> # '' key can change the priority of the list grouping
        >>> rasterplot([st_list1, st_list2], key_list=['parity', ''],
        >>>            groupingdepth=2, labelkey='0+1')
        >>>
        >>> # groups can also be emphasized by an explicit color code
        >>> rasterplot([st_list1, st_list2], key_list=['', 'parity'],
        >>>            groupingdepth=1, labelkey=0, colorkey='parity',
        >>>            legend=True)
        >>>
        >>> plt.show()

    """

    # Initialize plotting canvas
    sns.set_style(style)

    if context is not None:
        sns.set_context(context)

    if palette is not None:
        sns.set_palette(palette)
    else:
        palette = sns.color_palette()

    if ax is None:
        fig, ax = plt.subplots()
        # axis must be created after sns.set() command for style to apply!

    margin = 1 - histscale
    left, bottom, width, height = ax.get_position().bounds
    ax.set_position([    left,                  bottom,
                         margin * width,        margin * height])
    axhistx = plt.axes([left,                   bottom + margin * height,
                        margin * width,         histscale * height])
    axhisty = plt.axes([left + margin * width,  bottom,
                        histscale * width,      margin * height])

    sns.despine(ax=axhistx)
    sns.despine(ax=axhisty)

    # Whitespace margin around dot display = 2%
    ws_margin = 0.02

    # Control of user entries
    if groupingdepth > 2:
        raise ValueError("Grouping is limited to two layers.")
    groupingdepth = int(groupingdepth)

    list_key = "%$\@[#*&/!"  # unique key to be added to annotations to store
                             # list ordering information.

    if type(key_list) == 'str':
        key_list = [key_list]

    if '' not in key_list:
        key_list = [list_key] + key_list
    else:
        key_list = [list_key if not key else key for key in key_list]

    if type(spacing) == list:
        if len(spacing) == 1:
            spacing = [spacing[0], spacing[0]/2.]
    else:
        spacing = [spacing, spacing/2.]
    if spacing[0] < spacing[1]:
        raise DeprecationWarning("For reasonable visual aid, spacing between" \
                               + " top level group (spacing[0]) must be larger" \
                               + " than for subgroups (spacing[1]).")

    if type(colorkey) == int and len(key_list):
        if colorkey >= len(key_list):
            raise IndexError("An integer colorkey must refer to a position in" \
                           + " key_list.")
        colorkey = key_list[colorkey]
    else:
        if not colorkey:
            colorkey = list_key
        elif colorkey not in key_list:
            raise AttributeError("colorkey must be in key_list.")

    if legend and not key_list:
        raise AttributeError("Legend requires a non empty key_list.")

    if labelkey == '':
        labelkey = list_key

    if type(separatorargs) == list:
        if len(separatorargs) == 1:
            separatorargs += separatorargs
        for args in separatorargs:
            if type(args) != dict:
                raise TypeError("The parameters must be given as dict.")
    else:
        separatorargs = [separatorargs, separatorargs]

    for i, args in enumerate(separatorargs):
        if 'c' in args:
            separatorargs[i]['color'] = args['c']
        elif 'color' not in args:
            separatorargs[i]['color'] = '0.8'

    markerargs['linestyle'] = ''

    # Flatten list of lists while keeping the grouping info in annotations
    if isinstance(spiketrain_list[0], list):
        for list_nbr, st_list in enumerate(spiketrain_list):
            for st in st_list:
                st.annotations[list_key] = "set {}".format(list_nbr)
        spiketrain_list = [st for sublist in spiketrain_list for st in sublist]
    else:
        for st in spiketrain_list:
            st.annotations[list_key] = "set {}".format(0)
        key_list.remove(list_key)
        key_list.append(list_key)

    # Input checks on flattened lists
    if len(key_list) < groupingdepth:
        raise ValueError("Can't group more as keys in key_list.")

    # Filter spike trains according to given filter function
    if filter_function is not None:
        filter_index = []
        for st_count, spiketrain in enumerate(spiketrain_list):
            if filter_function(spiketrain):
                filter_index += [st_count]
        spiketrain_list = [spiketrain_list[i] for i in filter_index]

    # Initialize plotting parameters
    t_lims = [(st.t_start, st.t_stop) for st in spiketrain_list]
    tmin = min(t_lims, key=lambda f: f[0])[0]
    tmax = max(t_lims, key=lambda f: f[1])[1]
    period = tmax - tmin
    ax.set_xlim(tmin - ws_margin*period, tmax + ws_margin*period)
    yticks = np.zeros(len(spiketrain_list))

    # Sort spike trains according to keylist
    def sort_func(x):
        return ['' if key not in x.annotations
                else x.annotations[key] for key in key_list]

    spiketrain_list = sorted(spiketrain_list, key=lambda x: sort_func(x))
    if len(key_list) > 1:
        attribute_array = _get_attributes(spiketrain_list, key_list)
    elif len(key_list) == 1:
        attribute_array = np.zeros((len(spiketrain_list), 2))
        attribute_array[:,0] = _get_attributes(spiketrain_list, key_list)[:,0]
    else:
        attribute_array = np.zeros((len(spiketrain_list), 1))

    # Define colormap
    if not len(key_list):
        nbr_of_colors = 1
        colorkey = None
    else:
        colorkey = np.where(colorkey == np.array(key_list))[0][0]
        nbr_of_colors = int(max(attribute_array[:, colorkey]) + 1)

    colormap = sns.color_palette(palette, nbr_of_colors)

    # Draw population histogram (upper side)
    colorkeyvalues = np.unique(attribute_array[:, colorkey])

    if pophist_mode == 'color' and len(colorkeyvalues)-1:
        if len(sns.color_palette()) < len(colorkeyvalues):
            print("\033[31mWarning: There are more subsets than can be " +
                  "separated by colors in the color palette which might lead " +
                  "to confusion!\033[0m")
        max_y = 0
        for value in colorkeyvalues:
            idx = np.where(attribute_array[:, colorkey] == value)[0]
            histout = axhistx.hist([stime for strain in
                                      [spiketrain_list[i] for i in idx]
                                      for stime in strain],
                                     pophistbins, histtype='step', linewidth=1,
                                     color=colormap[int(value)])
            max_y = np.max([max_y, np.max(histout[0])])

    else: # pophist_mode == 'total':
        if len(colorkeyvalues)-1:
            sum_color = separatorargs[0]['color']
        else:
            sum_color = sns.color_palette()[0]

        histout = axhistx.hist([stime for strain in spiketrain_list
                                      for stime in strain],
                               pophistbins, histtype='step', linewidth=1,
                               color=sum_color)
        max_y = np.max(histout[0])

    # Set ticks and labels for population histogram
    axhistx_ydim, up = _round_to_1(max_y)
    if max_y > axhistx.get_ylim()[-1]:
        axhistx.set_ylim(0, max_y)
    if up and axhistx_ydim > max_y:
        axhistx.set_ylim(0, axhistx_ydim)
    axhistx.set_yticks([axhistx_ydim])
    axhistx.set_yticklabels(['{:.0f}'.format(axhistx_ydim)])
    axhistx.set_ylabel('count')

    # Legend for colorkey
    if legend:
        __, index = np.unique(attribute_array[:, colorkey], return_index=True)
        legend_labels = [spiketrain_list[i].annotations[key_list[colorkey]]
                         for i in index]
        legend_handles = [0] * len(index)

    # Reshape list into sublists according to groupingdepth
    if groupingdepth > 0:
        value1, index1, counts1 = np.unique(attribute_array[:, 0],
                                            return_index=True,
                                            return_counts=True)
        for v1, i1, c1 in zip(value1, index1, counts1):
            v1 = int(v1)
            spiketrain_list[v1:v1 + c1] = [spiketrain_list[v1:v1 + c1]]
            if groupingdepth > 1:
                __, counts2 = np.unique(attribute_array[i1:i1 + c1, 1],
                                        return_counts=True)
                for v2, c2 in enumerate(counts2):
                    v2 = int(v2)
                    spiketrain_list[v1][v2:v2+c2] = [spiketrain_list[v1][v2:v2+c2]]
            else:
                spiketrain_list[v1] = [spiketrain_list[v1]]
    else:
        spiketrain_list = [[spiketrain_list]]

    # HIERARCHIE:
    # [ [ []..[] ] .... [ []..[] ] ] spiketrain_list
    # [ []..[] ] LIST
    # [] list
    # spike train

    # Loop through lists of lists of spike trains
    for COUNT, SLIST in enumerate(spiketrain_list):

        # Separator depth 1
        if COUNT and separatorargs is not None:
            linepos = ypos + len(spiketrain_list[COUNT-1][-1]) \
                      + spacing[0]/2. - 0.5
            ax.plot(ax.get_xlim(), [linepos] * 2, **separatorargs[0])

        # Loop through lists of spike trains
        for count, slist in enumerate(SLIST):
            nbr_of_drawn_sts = int(sum([len(sl) for SL in
                                        spiketrain_list[:COUNT] for sl in SL])\
                                 + sum([len(sl) for sl in SLIST[:count]]))

            # Calculate postition of next spike train to draw
            prev_spaces = np.sum([len(SLIST_it) - 1
                                  for SLIST_it in spiketrain_list[:COUNT]])
            ypos = nbr_of_drawn_sts \
                 + int(bool(groupingdepth)) * COUNT * spacing[0] \
                 + groupingdepth/2 * count * spacing[1] \
                 + groupingdepth/2 * prev_spaces * spacing[1]

            # Separator depth 2
            if count and separatorargs is not None:
                linepos = ypos - (spacing[1] + 1) / 2.
                ax.plot(ax.get_xlim(), [linepos] * 2, **separatorargs[1])

            # Loop through spike trains
            for st_count, st in enumerate(slist):
                current_st = nbr_of_drawn_sts + st_count
                annotation_value = int(attribute_array[current_st, colorkey])
                color = colormap[annotation_value]

                # Dot display
                handle = ax.plot(st.times.magnitude,
                                 [st_count + ypos] * st.__len__(),
                                 color=color, **markerargs, rasterized=rasterized)
                if legend:
                    legend_handles[annotation_value] = handle[0]

                # Right side histogram bar
                barvalue = right_histogram(st)
                barwidth = righthist_barwidth
                axhisty.barh(y=st_count + ypos, # - barwidth/2.,
                             width=barvalue,
                             height=barwidth,
                             color=color,
                             edgecolor=color)

            # Append positions of spike trains to tick list
            ycoords = np.arange(len(slist)) + ypos
            yticks[nbr_of_drawn_sts:nbr_of_drawn_sts+len(slist)] = ycoords

    # Plotting axis
    yrange = yticks[-1] - yticks[0]
    ax.set_ylim(yticks[0] - ws_margin*yrange,
                yticks[-1] + ws_margin*yrange)
    axhistx.set_xlim(ax.get_xlim())
    axhisty.set_ylim(ax.get_ylim())
    ax.set_xlabel('t [{}]'.format(spiketrain_list[0][0][0].units.dimensionality))
    axhistx.get_xaxis().set_visible(False)
    axhisty.get_yaxis().set_visible(False)

    # Set ticks and labels for right side histogram
    axhisty_xdim, up = _round_to_1(axhisty.get_xlim()[-1])
    if up:
        axhistx.set_ylim(0, axhistx_ydim)
    axhisty.set_xticks([axhisty_xdim])
    axhisty.set_xticklabels(['{}'.format(axhisty_xdim)])

    # Y labeling
    if key_list and labelkey in key_list + [0, 1, '0+1']:
        if labelkey == key_list[0]:
            if groupingdepth > 0:
                labelkey = 0
        elif len(key_list) > 1 and labelkey == key_list[1]:
            if groupingdepth > 1:
                labelkey = 1

        if type(labelkey) == int or labelkey == '0+1':
            labelpos = [[] for label_level in range(2)]
            labelname = [[] for label_level in range(2)]

            # Labeling depth 1 + 2
            if groupingdepth:
                values1, index1, counts1 = np.unique(attribute_array[:, 0],
                                                     return_index=True,
                                                     return_counts=True)

                for v1, i1, c1 in zip(values1, index1, counts1):
                    st = spiketrain_list[int(v1)][0][0]
                    if key_list[0] in st.annotations:
                        labelname[0] += [st.annotations[key_list[0]]]
                        if labelkey == '0+1':
                            labelname[0][-1] += ' ' * 5
                    else:
                        labelname[0] += ['']

                    labelpos[0] += [(yticks[i1] + yticks[i1+c1-1])/2.]

                    # Labeling depth 2
                    if groupingdepth / 2 and labelkey and len(key_list) - 1:
                        __, index2, counts2 = np.unique(attribute_array[i1:i1+c1, 1],
                                                        return_index=True,
                                                        return_counts=True)

                        for v2, (i2, c2) in enumerate(zip(index2, counts2)):
                            st = spiketrain_list[int(v1)][int(v2)][0]
                            if key_list[1] in st.annotations:
                                labelname[1] += [st.annotations[key_list[1]]]
                            else:
                                labelname[1] += ['']

                            labelpos[1] += [(yticks[i1+i2] + yticks[i1+i2+c2-1])/2.]

            # Set labels according to labelkey
            if type(labelkey) == int:
                ax.set_yticks(labelpos[1] if labelkey else labelpos[0])
                ax.set_yticklabels(labelname[1] if labelkey else labelname[0])

            elif labelkey == "0+1":
                ax.set_yticks(labelpos[0] + labelpos[1])
                ax.set_yticklabels(labelname[0] + labelname[1])

        else:
            # Annotatation key as labelkey
            labelname = []
            for COUNT, SLIST in enumerate(spiketrain_list):
                for count, slist in enumerate(SLIST):
                    for st_count, st in enumerate(slist):
                        if labelkey in st.annotations:
                            labelname += [st.annotations[labelkey]]
                        else:
                            labelname += ['']
            ax.set_yticks(yticks)
            ax.set_yticklabels(labelname)

    else:
        ax.set_yticks([])
        ax.set_yticklabels([''])

    # Draw legend
    if legend:
        ax.legend(legend_handles, legend_labels, **legendargs)

    # Remove list_key from annotations
    for SLIST in spiketrain_list:
        for slist in SLIST:
            for st in slist:
                st.annotations.pop(list_key, None)

    return ax, axhistx, axhisty


##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Configuration support for **zpkg**.

The syntax of the configuration files is incredibly simple, but is
intended to be a strict subset of the `ZConfig`_.  This allows us to
switch to `ZConfig`_ in the future if we decide the dependency is
worth it.

.. _ZConfig:  http://www.zope.org/Members/fdrake/zconfig/

:undocumented: boolean non_empty_string \*_STRINGS
"""

import os
import urllib

from zpkgsetup import cfgparser
from zpkgtools import locationmap

TRUE_STRINGS =("yes", "true", "on")
FALSE_STRINGS = ("no", "false", "off")

def boolean(string):
    s = string.lower()
    if s in FALSE_STRINGS:
        return False
    if s in TRUE_STRINGS:
        return True
    raise ValueError("unknown boolean value: %r" % string)

def non_empty_string(string):
    if not string:
        raise ValueError("value cannot be empty")
    return string

SCHEMA = cfgparser.Schema(
    ({"resource-map": non_empty_string,
      "include-support-code": boolean,
      }, [], None),
    )

class Configuration:
    """Configuration settings for **zpkg**.

    :Ivariables:
      - `locations`: Mapping of resource identifiers to location URLs.

      - `location_maps`: List of location maps to load.

      - `include_support_code`: Indicates whether support code should
        be included in distributions.
    """

    def __init__(self):
        """Initialize a new `Configuration` object."""
        self.location_maps = []
        self.locations = locationmap.LocationMap()
        self.include_support_code = True

    def finalize(self):
        """Load the location maps into `locations`."""
        for loc in self.location_maps:
            self.locations = locationmap.fromPathOrUrl(loc,
                                                       mapping=self.locations)

    def loadPath(self, path):
        """Load configuration from a file.

        :param path: Path of the file to load.

        :raises IOError: If the `path` cannot be found, is not a file,
          or cannot be read.
        """
        basedir = os.path.dirname(path)
        f = open(path, "rU")
        try:
            self.loadStream(f, path, basedir)
        finally:
            f.close()

    def loadStream(self, f, path, basedir):
        """Load configuration from an open stream.

        :Parameters:
          - `f`: The stream, which must have been opened for reading
            in text mode.

          - `path`: Path with which to identify the stream in error
            messages.

          - `basedir`: Base directory with which to join relative
            paths found in the configuration file.

        """
        p = cfgparser.Parser(f, path, SCHEMA)
        cf = p.load()
        for value in cf.resource_map:
            type, rest = urllib.splittype(value)
            if basedir and not type:
                # local path references are relative to the file
                # we're loading
                value = os.path.join(basedir, value)
            self.location_maps.append(value)
        if len(cf.include_support_code) > 1:
            raise cfgparser.ConfigurationError(
                "include-support-code can be specified at most once")
        if cf.include_support_code:
            self.include_support_code = cf.include_support_code[0]

def defaultConfigurationPath():
    """Return the path name of the configuration file.

    This returns different things for Windows and POSIX systems.

    :return: Path to the default configuration file.  The directory or
      file may not exist.
    """
    zpkgdir = "zpkg"
    if os.name == "posix":
        zpkgdir = "." + zpkgdir
    name = os.path.join("~", zpkgdir, "zpkg.conf")
    path = os.path.expanduser(name)
    if os.path.exists(path):
        path = os.path.realpath(path)
    return path


#
# Reluzy : A Lazy NN-with-Relu Solver
#
# Author : Ahmed Irfan
#          <irfan@cs.stanford.edu>
#

import logging

from nnet2smt import Nnet2Smt

from pysmt.shortcuts import Solver, And, Real, Max, Equals, GE, LE, Implies

class Reluzy:

    def __init__(self, filename, violationfile, logger):
        self.logger = logger
        self.nnet2smt = Nnet2Smt(filename, violationfile)
        self.nnet2smt.convert(True)
        self.input_vars = self.nnet2smt.input_vars
        self.output_vars = self.nnet2smt.output_vars
        self.formulae = self.nnet2smt.formulae
        self.relus = self.nnet2smt.relus
        self.relus_level = self.nnet2smt.relus_level
        self.solver = Solver(name='yices')
        self.sat_checker = Solver(name='yices')
        self.init()

    def init(self):
        self.solver.add_assertion(And(self.formulae))
        self.sat_checker.add_assertion(And(self.formulae))
        for r1, r2 in self.relus:
            self.sat_checker.add_assertion(Equals(r1, Max(r2, Real(0))))
        #lemmas = self.refine_zero_lb(False)
        #self.solver.add_assertion(And(lemmas))
        #lemmas = self.refine_slope_lb(False)
        #self.solver.add_assertion(And(lemmas))

    def solve(self):
        while True:
            self.logger.info('Solving')
            res = self.solver.solve()
            if not res:
                print('unsat')
                break
            else:
                lemmas = self.refine()
                if not lemmas:
                    print('sat')
                    break
                else:
                    self.solver.add_assertion(And(lemmas))

    def check_sat(self):
        self.logger.info('Checking for Sat')
        self.sat_checker.push()
        for v in self.input_vars:
            self.sat_checker.add_assertion(Equals(v, self.solver.get_value(v)))
        res = self.sat_checker.solve()
        if res:
            for x in self.input_vars:
                print(v, self.sat_checker.get_value(x))
            return True
        else:
            self.sat_checker.pop()
            return False

    def refine(self):
        self.logger.info('Refining')
        lemmas = self.refine_zero_lb()
        if not lemmas:
            lemmas = self.refine_slope_lb()

        if not lemmas:
            lemmas = self.refine_zero_ub()

        if not lemmas:
            lemmas = self.refine_slope_ub()

        if not lemmas:
            for v in self.input_vars:
                print(v, self.solver.get_value(v))
        #elif self.check_sat():
        #    return []

        return lemmas

    def refine_zero_lb(self, check=True):
        lemmas = []
        zero = Real(0)
        for r1, _ in self.relus:
            l = GE(r1, zero)
            if check:
                tval = self.solver.get_value(l)
                if tval.is_false():
                    lemmas.append(l)
                    self.logger.debug('Adding %s' % l )
            else:
                lemmas.append(l)
        return lemmas

    def refine_zero_ub(self):
        lemmas = []
        zero = Real(0)
        for s in self.relus_level:
            for r1, r2 in s:
                l = Implies(LE(r2, zero), LE(r1, zero))
                tval = self.solver.get_value(l)
                if tval.is_false():
                    lemmas.append(l)
                    self.logger.debug('Adding %s' % l )
            if lemmas:
                break
        return lemmas

    def refine_slope_lb(self, check=True):
        lemmas = []
        for r1, r2 in self.relus:
            l = GE(r1, r2)
            if check:
                tval = self.solver.get_value(l)
                if tval.is_false():
                    lemmas.append(l)
                    self.logger.debug('Adding %s' % l )
            else:
                lemmas.append(l)
        return lemmas

    def refine_slope_ub(self):
        lemmas = []
        zero = Real(0)
        for s in self.relus_level:
            for r1, r2 in s:
                l = Implies(GE(r2, zero), LE(r1, r2))
                tval = self.solver.get_value(l)
                if tval.is_false():
                    lemmas.append(l)
                    self.logger.debug('Adding %s' % l )
            if lemmas:
                break
        return lemmas


"""
	Construct account, product and review features given a review dataset.
"""
import nltk
import nltk.tokenize.punkt
from nltk.tokenize import word_tokenize

import numpy as np

import codecs
import re
import string
from datetime import datetime

import json
import pickle
import gzip
import sys
sys.path.insert(0, '../')
from iohelper import *

date_time_format_str = '%Y-%m-%d'

def MNR(data):
	"""
		Normalized maximum number of reviews in a day for a user/product
		Args:
			data is a dictionary with key=u_id or p_id and value = tuples of (neighbor id, rating, label, posting time)
		Return:
			dictionary with key = u_id or p_id and value = MNR
	"""
	# maximum number of reviews written in a day for user / product
	feature = {}
	for i, d in data.items():
		# key = posting date; value = number of reviews
		frequency = {}
		for t in d:
			if t[3] not in frequency:
				frequency[t[3]] = 1
			else:
				frequency[t[3]] += 1
		feature[i] = max(frequency.values())

	# normalize it
	normalizer = max(feature.values())

	for k in feature.keys():
		feature[k] /= normalizer
	return feature

def PR_NR(data):
	"""
		Ratio of positive and negative reviews of a user or product
		Args:
			data is a dictionary with key=u_id or p_id and value = tuples of (neighbor id, rating, label, posting time)
		Return:
			dictionary with key = u_id or p_id and value = (PR, NR)
	"""
	feature = {}

	for i, d in data.items():
		positives = [1 for t in d if t[1] > 3]
		negatives = [1 for t in d if t[1] < 3]
		feature[i] = (float(len(positives)) / len(d), float(len(negatives)) / len(d))
	return feature

def avgRD(user_data, product_data):
	"""
		Average rating deviation of each user / product.
		For a user i, avgRD(i) = average(r_ij - avg_j | for all r_ij of the user i)
		For a product j, avgRD(j) = average(r_ij - avg_j | for all r_ij of the user i) = 0!?
		Return:
			average rating deviation on users, as defined in the paper
				Detecting product review spammers using rating behaviors, CIKM, 2010
			The paper does not define avgRD for the products.
	"""
# find the average rating of each product
	p_avg = {}
	for i, d in product_data.items():
		p_avg[i] = np.mean(np.array([t[1] for t in d]))

	# find average rating deviation of each user
	u_avgRD = {}
	for i, d in user_data.items():
		u_avgRD[i] = np.mean(np.array([abs(t[1] - p_avg[t[0]]) for t in d]))

	return u_avgRD

def BST(user_data):
	""" Burstiness of reviews by users. Spammers are often short term
		members of the site: so BST(i) = 0, if L(i) - F(i) > tau else BST(i) = 1 -
		(L(i) - F(i))/tau, where, L(i) - F(i) are number of days between first and
		last review of i, tau = 28 days

		Args:
			user_data is a dictionary with key=u_id value = tuples of (prod_id, rating, label, posting time)
		Return:
			dictionary with key = u_id and value = BST
	"""
	bst = {}
	tau = 28.0	# 28 days
	for i, d in user_data.items():
		post_dates = sorted([datetime.strptime(t[3], date_time_format_str) for t in d])
		delta_days = (post_dates[-1] - post_dates[0]).days
		if delta_days > tau:
			bst[i] = 0.0
		else:
			bst[i] = 1.0 - (delta_days / tau)
	return bst

def ERD(data):
	"""
		Entropy of the rating distribution of each user (product)
	"""
	erd = {}
	for i, d in data.items():
		ratings = [t[1] for t in d]
		h, _ = np.histogram(ratings, bins = np.arange(1,7))
		h = h / h.sum()
		h = h[np.nonzero(h)]
		erd[i] = (- h * np.log2(h)).sum()
	return erd

def ETG(data):
	"""
		Entropy of the gaps between any two consecutive ratings.
	"""
	etg = {}
# [0,1) -> 1
# [1,3) -> 2
# ...
# [17, 33) -> 6
# anything larger than 33 will be discarded.
	bins = [0,1,3,5,9,17,33]
	for i, d in data.items():

		# if there is only one posting time, then entropy = 0
		if len(d) <= 1:
			etg[i] = 0
			continue
		# sort posting dates from the past to the future
		posting_dates = sorted([datetime.strptime(t[3], date_time_format_str) for t in d])
		# find the difference in days between two consecutive dates
		#delta = posting_dates[1:] - posting_dates[:-1]
		delta_days = [(posting_dates[i+1] - posting_dates[i]).days for i in range(len(posting_dates) - 1)]

		# bin to the 6 bins, discarding any differences that are greater than 33 days
		h, _ = np.histogram(delta_days, bins = bins)
		#h, _ = np.histogram(delta_days, bins = bins)[:-1]
		if h.sum() == 0:
			etg[i] = 0
			continue
		h = h / h.sum()
		h = h[np.nonzero(h)]
		etg[i] = (- h * np.log2(h)).sum()
	return etg

def RL(data, text_features, isUser = True):
	"""
		Average review length of each user / product
		Args:
			data: a dictionary with key=u_id or p_id and value = tuples of (neighbor id, rating, label, posting time)
			text_features: dictionary with key=(u_id, p_id) and value = dict of feature_name:feature_value
			isUser: is dealing with user data or product data
		Return:
			dictionary with key = u_id or p_id and value = (RL)
	"""
	rl = {}
	for i, d in data.items():
		if isUser:
			# d[0] is product id
			lengths = [text_features[(i, t[0])]['L'] for t in d]
		else:
			lengths = [text_features[(t[0], i)]['L'] for t in d]
		rl[i] = np.mean(lengths)
	return rl

def RD(product_data):
	"""Calculate the deviation of the review ratings to the product average.

		Args:
			prod_data:
		Return:
			a dictionary with key = (u_id, p_id), value = deviation of the rating of this review to the average rating of the target product
	"""
	rd = {}
	for i, d in product_data.items():
		avg = np.mean(np.array([t[1] for t in d]))
		for t in d:
			rd[(t[0], i)] = abs(t[1] - avg)
	return rd

def EXT(product_data):
	"""
		Whether a rating is extreme or not
		Args:
			product_data is a dictionary with key=p_id and value = tuples of (u_id, rating, label, posting time)
		Return:
			a dictionary with key = (u_id, p_id) and value = 0 (not extreme) / 1 (extreme)
	"""
	ext = {}
	for i, d in product_data.items():
		for t in d:
			if int(t[1]) == 5 or int(t[1]) == 1:
				ext[(t[0], i)] = 1
			else:
				ext[(t[0], i)] = 0
	return ext

def DEV(product_data):
	"""
		Deviation of each rating from the average rating of the target product.
		Need to use "recursive minimal entropy partitioning" to find beta_1
		Args:
			product_data is a dictionary with key=p_id and value = tuples of (neighbor id, rating, label, posting time)
		Return:
			a dictionary with key = (u_id, p_id) and value = (RD_ij, RD_ij / 4 > 0.63 ? 1: 0)
			where RD_ij = |r_ij - average rating of product j|
	"""
	beta_1 = 0.63
	dev = {}
# i is a product id
	for i, d in product_data.items():
		# find the average rating of each product
		p_avg_rating = np.mean(np.array([t[1] for t in d]))
		for t in d:
			u_id = t[0]	# user id
			if (abs(p_avg_rating - t[1]) / 4.0 > 0.63):
				dev[(u_id, i)] = 1	# absolute difference between current rating and product average rating
			else:
				dev[(u_id, i)] = 0	# absolute difference between current rating and product average rating
	return dev

def ETF(product_data):
	"""
		Binary feature: 0 if ETF_prime <= beta_3, 1 otherwise.
		Need to use "recursive minimal entropy partitioning" to find beta_1
		ETF_prime = 1 - (date of last review of user i on product p from the date of the first review of the product / 7 months)
	"""

	beta_3 = 0.69

# for each product j, find the time of the earliest review F(j)
	first_time_product = {}
	for i,d in product_data.items():
		for t in d:
			if i not in first_time_product:
				first_time_product[i] = datetime.strptime(t[3], date_time_format_str)
			elif datetime.strptime(t[3], date_time_format_str) < first_time_product[i]:
				first_time_product[i] = datetime.strptime(t[3], date_time_format_str)

	etf = {}	# key = (u_id, p_id), value = maximum difference between reviews (u_id, p_id) and first review of the product
	for i, d in product_data.items():
		for t in d:
			td = datetime.strptime(t[3], date_time_format_str) - first_time_product[i]
			if (t[0], i) not in etf:
				etf[(t[0], i)] = td
# find the largest td for the review
			elif td > etf[(t[0], i)]:
				etf[(t[0], i)] = td

	for k, v in etf.items():
		if v.days > 7*30:
			etf[k] = 0
		elif v.days / (7 * 30) > beta_3:
			etf[k] = 1
		else:
			etf[k] = 0
	return etf

def ISR(user_data):
	"""
		Check if a user posts only one review
	"""
	isr = {}
	for i, d in user_data.items():
		# go through all review of this user
		for t in d:
			if len(d) == 1:
				isr[(i, t[0])] = 1
	return isr

def low_level_text_features(review_filename, output_filename):
	"""
		Since extracting low level text feature can take some time, we extract and write the features to files for later use.
		Output to file lines of review features, in the same order as the review text file.

		List of features:
			1. word counts in reviews
			2. ratio of all Capital words to all words
			3. ratio of capital letters to all letters
			4. PP1: number of first person names.
			5. RES: ratio of sentences with exclamations.
		Args:
			review_filename: the review text file
			output_filename: the json file that contains the dictionary = {key = (uid, pid): value = dict(key = text feature name, value = feature value)}
	"""
# prepare various lexicons
	PP1_filename = '1PP-words.txt'

	PP1 = set()

	with open(PP1_filename, 'r') as f:
		for line in f:
			PP1.add(line.strip())

	tag = ['CC', 'CD', 'DT', 'EX', 'FW', 'IN', 'TO', 'UH', 'PDT', 'SYM', 'RP']
	noun = ['NN', 'NNS', 'NP', 'NPS']
	adj = ['JJ', 'JJR', 'JJS']
	pronoun = ['PP', 'PP$', 'WP', 'WP$']
	verb = ['VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']
	adverb = ['RB', 'RBR', 'RBS', 'WRB']

	# = 'SentiWordNet_3.0.0_20100705.txt'
	#swn = SentiWordNetCorpusReader(swn_filename)

	review_text_features = dict()

# use the rt mode to read strings instead of binary
	infile = gzip.open(review_filename, 'rt')
	line_num = 0
	for line in infile:
		line_num += 1
		if line_num % 10000 == 0:
			print (line_num)

		items = re.split("\s+", line, 3)
		u_id = items[0]
		p_id = items[1]
		time = items[2]
		texts = items[3]

		words = nltk.word_tokenize(texts)

		lower_words = texts.lower()

		wordcount = float(len(words))

		allCapitalCount = 0

		PP1Count = 0

# count the number of capital letters
		countCapital = sum(1 for c in line if c.isupper())
		countAllChars = sum(1 for c in line)

		for w in words:
			# count the numebr of words that have all capital letters
			if w.isupper():
				allCapitalCount += 1
			# count the number of 1st person words
			if w in PP1:
				PP1Count += 1

# find ratios of subjective and objective words

		#objCount = 0
		#subCount = 0
		#for w in word_tag:
		#	word = w[0]
		#	pos_tag = w[1]
		#	if not pos_tag in tag:
		#		if pos_tag in noun:
		#			pos_Char = 'n'
		#		if pos_tag in adj:
		#			pos_Char = 'a'
		#		if pos_tag in pronoun:
		#			pos_Char = 'p'
		#		if pos_tag in verb:
		#			pos_Char = 'v'
		#		if pos_tag in adverb:
		#			pos_Char = 'r'
		#		else:
		#			pos_Char = 'none'
		#	try:
		#		if pos_Char == 'none':
		#			s = swn.senti_synsets(word)
		#		else:
		#			s = swn.senti_synsets(word, pos_Char)
		#		scores = list(s)[0]
		#		if score.obj_score > 0.5:
		#			objCount += 1
		#		elif score.neg_score + score.pos_score > 0.5:
		#			subCount += 1
		#	except:
		#		pass
		#
		#if objCount + subCount > 0:
		#	ratioObj = float(objCount) / (objCount + subCount)
		#	ratioSub = float(subCount) / (objCount + subCount)
		#else:
		#	ratioObj = 0.0
		#	ratioSub = 0.0

		sents = nltk.sent_tokenize(texts)
		countExc = 0
		countSent = 0
		for sentence in sents:
		    countSent += 1
		    if '!' in sentence:
		        countExc += 1
		if countSent > 0:
		    ratioExcSent = float(countExc)/countSent
		else:
		    ratioExcSent = 0.0

		# feature vector for a review: number of words, ratio of allCap words, ratio of allCap Chars, number of 1st persons, number of 2nd persons, ratio of sentences end with exclamations
		review_text_features[(u_id, p_id)] = {"L":wordcount,
			"PCW" : allCapitalCount / wordcount,
			"PC" : float(countCapital) / countAllChars,
			"PP1" : PP1Count,
			"RES" : ratioExcSent}

	with open(output_filename, 'wb') as out_f:
		pickle.dump(review_text_features, out_f)

	print ('%d lines processed.' % line_num)
	print ('Text features written to %s' % output_filename)

def add_feature(existing_features, new_features, feature_names):
	"""
		Add feature(s) of a set of nodes of the same type to the existing feature(s).
		Args:
			existing_features: a dictionary {node_id:dict{feature_name:feature_value}}
			new_features: new feature(s) to be added. A dict {node_id: list of feature values}
			feature_names: the name of the new feature. A list of feature names, in the same order of the list of feature values in new_features
	"""

	for k, v in new_features.items():
		# k is the node id and v is the feature value
		if k not in existing_features:
			existing_features[k] = dict()
		# add the new feature to the dict of the node
		for i in range(len(feature_names)):
			if len(feature_names) > 1:
				existing_features[k][feature_names[i]] = v[i]
			else:
				existing_features[k][feature_names[i]] = v

#def construct_all_features(metadata_filename, text_feature_filename, user_feature_filename, prod_feature_filename, review_feature_filename):
def construct_all_features(user_data, prod_data, text_features):
	"""
		Main entry to feature construction.
		Args:
			metadata_filename:
			text_feature_filename:
		Return:
			user, product and review features
	"""

	UserFeatures = {}	# key = user id, value = dict of {feature_name: feature_value}
	ProdFeatures = {}	# key = product id, value = dict of {feature_name: feature_value}

# go through feature functions
	print ('\nadding user and product features......\n')
	uf = MNR(user_data)
	add_feature(UserFeatures, uf, ["MNR"])
	pf = MNR(prod_data)
	add_feature(ProdFeatures, pf, ["MNR"])

	uf = PR_NR(user_data)
	add_feature(UserFeatures, uf, ["PR", "NR"])
	pf = PR_NR(prod_data)
	add_feature(ProdFeatures, pf, ["PR", "NR"])

	uf = avgRD(user_data, prod_data)
	add_feature(UserFeatures, uf, ["avgRD"])

	uf = BST(user_data)
	add_feature(UserFeatures, uf, ["BST"])

	uf = ERD(user_data)
	add_feature(UserFeatures, uf, ["ERD"])
	pf = ERD(prod_data)
	add_feature(ProdFeatures, pf, ["ERD"])

	uf = ETG(user_data)
	add_feature(UserFeatures, uf, ["ETG"])
	pf = ETG(prod_data)
	add_feature(ProdFeatures, pf, ["ETG"])

	uf = RL(user_data, text_features)
	add_feature(UserFeatures, uf, ['RL'])
	pf = RL(prod_data, text_features, isUser = False)
	add_feature(ProdFeatures, pf, ['RL'])

# go through review features
	print ('\nadding review features......\n')
	ReviewFeatures = {}
	rf = RD(prod_data)
	add_feature(ReviewFeatures, rf, ['RD'])

	rf = EXT(prod_data)
	add_feature(ReviewFeatures, rf, ['EXT'])

	rf = DEV(prod_data)
	add_feature(ReviewFeatures, rf, ['DEV'])

	rf = ETF(prod_data)
	add_feature(ReviewFeatures, rf, ['ETF'])

	rf = ISR(prod_data)
	add_feature(ReviewFeatures, rf, ['ISR'])

# add low level text features
	print ('\nadding low level review features......\n')
	for k, v in text_features.items():
		# k is a tuple (u_id, p_id)
		# v is a dict (key=feature name, value = feature value)
		names = [name for name, value in v.items()]
		values = [value for name, value in v.items()]

# recall that the arguments to add_feature
		add_feature(ReviewFeatures, {k:values}, names)

	return UserFeatures, ProdFeatures, ReviewFeatures

def calculateNodePriors(node_features, when_suspicious):
	"""
		Calculate priors of nodes P(y=1|node) using node features.
		Args:
			node_features: a dictionary with key = node_id and value = dict of feature_name:feature_value
			when_suspicious: a dictionary with key = feature name and value = 'H' (the higher the more suspicious) or 'L' (the opposite)
		Return:
			A dictionary with key = node_id and value = S score (see the SpEagle paper for the definition)
	"""
# return value
	priors = {}

	feature_names = set()
	for k, v in node_features.items():
		for fn, fv in v.items():
			# only include those in when_suspicious
			if fn in when_suspicious:
				feature_names.add(fn)

	feature_names = list(feature_names)
	n_features = len(feature_names)

	print ('number of features: %d' % n_features)

	feature_values = {name:[] for name in feature_names}

	# go through the instances
	for k, v in node_features.items():
		# go through the features
		for fn in feature_names:
			if fn not in v:
				feature_values[fn].append(0)		# add 0 by defaulty
			else:
				feature_values[fn].append(v[fn])	# add the non-zero feature values

	for k,v in feature_values.items():
		assert len(v) == len(node_features), 'number of feature values is different from number of instances'

	#def ecdf(x):
	#    x = np.sort(x)
	#	def result(v):
	#			        return np.searchsorted(x, v, side='right') / x.size
	#				    return result

	# compute CDF for each feature
	all_cdfs = {}
	for fn in feature_names:
		print (fn)
		if fn not in feature_values:
			print (fn + ' not in feature values')
			continue
		bins = sorted(list(set(feature_values[fn])))
		#if fn == 'DEV':
		print (bins[0])
		if len(bins) == 1:
			print (fn + ' has only one value.')
# bin1=[a,b), bin2=[b,c), ... last_bin = [y,z]
		h, d = np.histogram(feature_values[fn], bins = bins)
		cdf = np.cumsum(h)
		normalizer= 1.0 / cdf[-1]
		cdf = normalizer * cdf

# for later lookup
		print (len(bins), len(d), len(cdf))
		all_cdfs[fn] = {d[i]:cdf[i] for i in range(1, len(d)-1)}
		all_cdfs[fn][d[0]] = 0	# the least value has cdf 0
		all_cdfs[fn][d[-1]] = 1	# the least value has cdf 0

	#print (list(all_cdfs['BST'].keys()))
# for each node, find the S score for each feature
	for k, v in node_features.items():
		s = 0
		for fn in feature_names:
			x = 0 if fn not in v else v[fn]
			c = all_cdfs[fn][x]
			fx = 1.0 - c if when_suspicious[fn] == '+' else c
			s += fx * fx
		s = 1 - np.sqrt(s / n_features)
		priors[k] = s

	return priors

if __name__ == '__main__':

# path to the folder containing the files
	#prefix = '/home/sihong/data/yelp_reviews/'
	prefix = '/Users/bon/Documents/data mining/datasets/yelp_reviews/nyc/'

# raw data file names
	metadata_filename = prefix + 'metadata.gz'
	review_filename = prefix + 'reviewContent.gz'

# feature file names
	user_feature_filename = prefix + 'UserFeatures.pickle'
	prod_feature_filename = prefix + 'ProdFeatures.pickle'
	review_feature_filename = prefix + 'ReviewFeatures.pickle'

# prior file names
	user_prior_filename = prefix + 'UserPriors.pickle'
	prod_prior_filename = prefix + 'ProdPriors.pickle'
	review_prior_filename = prefix + 'ReviewPriors.pickle'

# output file names
	text_feature_filename = prefix + 'text_features.pickle'

# feature configuration
	feature_suspicious_filename = '../feature_configuration.txt'

# low level text features
	# just need to construct text features once
	#print ('Starting constructing low level text features\n')
	#low_level_text_features(review_filename, text_feature_filename)

	# can just load the constructed low level text features
	#tf = load_text_feature(text_feature_filename)
	#print ('Finished constructing low level text features\n')

# all high level features
	print ('Starting constructing high level user, product and review features\n')
	user_data, prod_data = read_graph_data(metadata_filename)
	text_features = load_text_feature(text_feature_filename)
	UserFeatures, ProdFeatures, ReviewFeatures = construct_all_features(user_data, prod_data, text_features)
	with open(user_feature_filename, 'wb') as f:
		pickle.dump(UserFeatures, f)

	with open(prod_feature_filename, 'wb') as f:
		pickle.dump(ProdFeatures, f)

	with open(review_feature_filename, 'wb') as f:
		pickle.dump(ReviewFeatures, f)
	print ('Finished constructing high level user, product and review features\n')

# Priors
	print ('Start calculating user, product and review priors.\n')

	feature_config = load_feature_config(feature_suspicious_filename)
	#print (feature_config)

	with open(user_feature_filename, 'rb') as f:
		user_features = pickle.load(f)
	user_priors = calculateNodePriors(user_features, feature_config)
	with open(user_prior_filename, 'wb') as f:
		pickle.dump(user_priors, f)

	with open(prod_feature_filename, 'rb') as f:
		prod_features = pickle.load(f)
	prod_priors = calculateNodePriors(prod_features, feature_config)
	with open(prod_prior_filename, 'wb') as f:
		pickle.dump(prod_priors, f)

	with open(review_feature_filename, 'rb') as f:
		review_features = pickle.load(f)
	review_priors = calculateNodePriors(review_features, feature_config)
	with open(review_prior_filename, 'wb') as f:
		pickle.dump(review_priors, f)
	print ('Finished calculating user, product and review priors.\n')


x=int(input())
c=0
y=[]
ll=list(map(int,input().split()))
for i in ll:
    y.append(i)
ll.sort()
for i in range(len(ll)):
    if ll[i]==y[i]:
        c+=1
if c==len(y):
    print("yes")
else:
    print("no")


'''
This file containts the scripts for contour generation under the elastica theory.

The elastica energy is estimated using the integration method outlined in Sharon et al.

I.e. it's assumed to be a polynomial f = function, and

author: sanderkeemink@gmail.com
'''

from __future__ import division
import holoviews as hv
import pylab as pl
from pylab import exp,cos,sin,pi,tan

def mises(a,k,ref,x):
    ''' basic von mises function
    Inputs
    -----------------------
    - a: determines magnitude
    - k: determines width (low k is wider width)
    - ref: reference angle
    - x: input angle
    '''
    return a*exp(k*(cos(2*(ref-x))))

def findE(c,f,X):
    ''' find E, the approximated sharon energy
    Inputs
    ----------
    - c: current bar orientation
    - f: flanker orientation
    - X: relative flanker distance and position orientation
    '''
    # define x and y
    x = X[0]
    y = X[1]

    # flanker positional angle
    theta = pl.arctan2(x,y)

    # find and return D
    Ba = pl.arctan(tan(0.5*(-f+theta)))*2
    Bb = pl.arctan(tan(0.5*(c-theta)))*2
    return  4*(Ba**2 + Bb**2 - Ba*Bb)

def E(c,f,X):
    ''' find the direction invariant approximated sharon energy
    Inputs
    ----------
    - c: current bar orientation (can be double or array, if array D returns an array of D values)
    - f: flanker orientation  (double)
    - X: relative flanker distance and position orientation
    '''

    # check if c is an array, and assign length (note, f cannot be array)
    if isinstance(c,pl.ndarray):
        length = len(c)
    else:
        length = 1

    # find D candiates across all directions
    c = c - pi/2
    f = f - pi/2
    E = pl.zeros((length,4))
    E[:,0] = findE(c,f,X)
    E[:,1] = findE(c+pi,f,X)
    E[:,2] = findE(c,f+pi,X)
    E[:,3] = findE(c+pi,f+pi,X)

    # return the minimum energy
    return E.min(1)

class scene:
    """ Scene class. Define a scene, for which the neural responses to each bar can then be found """
    def __init__(self,N,O,X,dim,Kc,Ac,a,E0):
        self.n = len(O) # number of bars
        self.N = N      # number of neurons per location
        self.O = O      # bar orientations
        self.X = X      # bar locations
        self.dim = dim  # image dimension [x,y]
        self.ang = pl.linspace(-pi/2,pi/2-pi/N,N) # orientations
        self.Kc = Kc    # determines width of tuning to bars in location (lower Kc is wider tuning)
        self.Ac = Ac    # determines response magnitude
        self.a  = a     # determines strength of modulation
        self.E0 = E0    # offset for modulation
        self.FRc = pl.zeros((self.n,N))   # 'firing rates' or 'probabilities' for local bars
        self.FRs = pl.zeros((self.n,N))   # modulation from smoothest targets (net effect)
        self.FR = pl.zeros((self.n,N))   # final 'firing rates' or 'probabilities' for each bar
        self.est = pl.zeros(self.n)   # to store the orientation estimates

    def popvec(self,X):
        ''' Population vector for the set of responses X, with each value in
        the vector X corresponding to an angle in self.ang
        X is a 1D vector of length len(self.ang)
        Returns the angle of the population vector.
        '''
        # define vector coordinates
        v = pl.zeros((2,self.N))
        v[0,:] = cos(2*self.ang)
        v[1,:] = sin(2*self.ang)

        # find population vector
        vest0 = pl.sum(((X-min(X))/max(X))*v,1)

        # return the angle of the population vector
        return 0.5*pl.arctan2(vest0[1],vest0[0])

    def plotscene(self,length=1,oriens = 'NA',alphas='NA',colors='off'):
        ''' Plot the scene

        Inputs
        ----------------
        - length: length of each plotted bar
        - oriens: orientations of each bar, if 'NA' use the orientations as defined in self.O
        - alphas: the alpha values for each bar, if 'NA' set all to one
        - colors: the color of each bar, if 'NA' all are black
        oriens and alphas should all be arrays of length self.n if 'off'

        '''
        # check for alphas/oriens inputs
        if alphas == 'NA': alphas = pl.ones(self.n)
        if oriens == 'NA': oriens = self.O

        # initiate image
        img = hv.Curve([])

        for i in range(self.n):
            # get bar location
            x = self.X[i,0]
            y = self.X[i,1]
            # get bar orientation
            f = oriens[i]
            # check colors
            if colors == 'off':
                c = 'k'
            else:
                if alphas[i]<pl.mean(alphas): # if below average make blue
                    c = 'b'
                else: # otherwise make red
                    c = 'r'
            # plot the bar
            img *= hv.Curve(zip([x-sin(f)*length,x+sin(f)*length],[y-cos(f)*length,y+cos(f)*length]))#,color=c,linewidth=4,alpha = alphas[i]/max(alphas))

        # return img object
        return img

    def findE(self,c,f,X):
        ''' find E, the approximated sharon energy
        Inputs
        ----------
        - c: current bar orientation
        - f: flanker orientation
        - X: relative flanker distance and position orientation
        '''
        # define x and y
        x = X[0]
        y = X[1]

        # flanker positional angle
        theta = pl.arctan2(x,y)

        # find and return D
        Ba = pl.arctan(tan(0.5*(-f+theta)))*2
        Bb = pl.arctan(tan(0.5*(c-theta)))*2
        return  4*(Ba**2 + Bb**2 - Ba*Bb)

    def E(self,c,f,X):
        ''' find the direction invariant approximated sharon energy
        Inputs
        ----------
        - c: current bar orientation (can be double or array, if array D returns an array of D values)
        - f: flanker orientation  (double)
        - X: relative flanker distance and position orientation
        '''

        # check if c is an array, and assign length (note, f cannot be array)
        if isinstance(c,pl.ndarray):
            length = len(c)
        else:
            length = 1

        # find D candiates across all directions
        E = pl.zeros((length,4))
        E[:,0] = self.findE(c,f,X)
        E[:,1] = self.findE(c+pi,f,X)
        E[:,2] = self.findE(c,f+pi,X)
        E[:,3] = self.findE(c+pi,f+pi,X)

        # return the minimum energy
        return E.min(1)

    def plotlocalmod(self,iLoc,torus='on'):
        ''' Plot contributions to local modulation
        iLoc: location id (which bar to look at)
        torus: whether image is on a torus or not
        '''
        # make mask to select all locations except iLoc
        mask = pl.ones(self.n)
        mask[iLoc] = 0

        # find orientations at all other locations
        F = self.O[mask==1]

        # find positions of all other locations
        X = self.X[mask==1,:]  ##### set up properly, now based on [x,y], should be [r,theta]
        # and set coordinates to be relative to iLoc
        X[:,0]-=self.X[iLoc,0] # update x coordinate to make current the center
        X[:,1]-=self.X[iLoc,1] # update y coordinate to make current the center

        # transform other coordinates according to torus
        # does not work when there are only two bars present!
        if torus == 'on':
            mx = max(self.X[:,0])
            X[X[:,0]<-mx,0]+=pl.sqrt(self.n)*mx/pl.floor(pl.sqrt(self.n)/2)
            X[X[:,1]<-mx,1]+=pl.sqrt(self.n)*mx/pl.floor(pl.sqrt(self.n)/2)
            X[X[:,0]>mx,0]-=pl.sqrt(self.n)*mx/pl.floor(pl.sqrt(self.n)/2)
            X[X[:,1]>mx,1]-=pl.sqrt(self.n)*mx/pl.floor(pl.sqrt(self.n)/2)

        # find the distances from every other bar to the 'center'
        R = pl.sqrt(X[:,0]**2+X[:,1]**2)

        # initiate image
        img = hv.Curve([])

        # find the total modulation, and plot each individual contribution
        for i in range(self.n-1):
            # find curvature energy
            E  =  self.E(self.ang,F[i],X[i,:])
            # find resulting modulation
            h  = exp(-self.a*(E-self.E0)/R[i])
            # plot the modulation curve
            img *= hv.Curve(zip(self.ang/pi,h))#,alpha=0.25,label=i)

        # return img
        return img

    def simulate(self,iLoc,torus='on'):
        '''
        For a location iLoc simulate the responses given the scene:
        - find the orientation at iLoc to get the drive
        - find the orientation and relative position of all other bars, to determine the modulation

        Updates FRc, FRs, FR, S , est for iLoc

        Inputs
        -------------------------
        - iLoc: the index of the location to simulate
        - torus: if 'on', put the scene on a torus
        '''
        # find the orientation of the bar in iLoc
        c = self.O[iLoc] # 'centre' flanker

        # make mask to select all locations except iLoc
        mask = pl.ones(self.n)
        mask[iLoc] = 0

        # find orientations at all other locations
        F = self.O[mask==1]

        # find positions of all other locations
        X = self.X[mask==1,:]  ##### set up properly, now based on [x,y], should be [r,theta]
        # and set coordinates to be relative to iLoc
        X[:,0]-=self.X[iLoc,0] # update x coordinate to make current the center
        X[:,1]-=self.X[iLoc,1] # update y coordinate to make current the center

        # transform other coordinates according to torus, if necessary
        # does not work when there are only two bars present!
        if torus == 'on':
            mx = max(self.X[:,0])
            X[X[:,0]<-mx,0]+=pl.sqrt(self.n)*mx/pl.floor(pl.sqrt(self.n)/2)
            X[X[:,1]<-mx,1]+=pl.sqrt(self.n)*mx/pl.floor(pl.sqrt(self.n)/2)
            X[X[:,0]>mx,0]-=pl.sqrt(self.n)*mx/pl.floor(pl.sqrt(self.n)/2)
            X[X[:,1]>mx,1]-=pl.sqrt(self.n)*mx/pl.floor(pl.sqrt(self.n)/2)

        # find the distances from every other location to iLoc
        R = pl.sqrt(X[:,0]**2+X[:,1]**2)

        # find the drive from the orientation in iLoc
        rc =  mises(self.Ac,self.Kc,c,self.ang)

        # find the modulation from all other lcoations
        rs=1
        for i in range(self.n-1): # for all locations except iLoc
            # find D across all preferred orientations
            E =  self.E(self.ang,F[i],X[i,:])

            # update the modulation
            rs*=exp(-self.a*(E-self.E0)/R[i])

        # find final response in location iLoc across preferred oreintations
        rf = rc*rs

        # update relevant variables
        self.FRc[iLoc,:] = rc#/sum(rc)
        self.FRs[iLoc,:] = rs
        self.FR[iLoc,:] = rf#/sum(rf)
        self.est[iLoc] = self.popvec(rf)

    def simulate_all(self):
        '''
        Simulate all locations
        '''
        for i in range(self.n):
            self.simulate(i)

    def saliency(self,base):
        '''
        Calculate the saliency for all location either based on:
        - base = 'mean': the mean responses
        - base = 'max' : the max responses
        '''
        sal = pl.zeros(self.n)
        if base == 'mean':
            for i in range(self.n):
                sal[i] = pl.mean(self.FR[i,:])
            self.sal = sal/max(sal)

        if base == 'max':
            for i in range(self.n):
                sal[i] = max(self.FR[i,:])
            self.sal = sal/max(sal)


from .googlesuggest import g_suggest
import json
import re
#import requests
import collections as cl

def main(phrase):
    vol = 1000
    ret = g_suggest (phrase)
    #ret = [['データサイエンティスト 資格', 'データサイエンティスト 資格 難易度', 'データサイエンティスト 資格 emc', 'データサイエンティスト 資格 費用', 'データサイエンティスト 資格 価格', 'データサイエンティスト 転職 資格'], ['データサイエンティスト 転職', 'データサイエンティスト 転職 エージェント', 'データサイエンティスト 転職 資格'], ['データサイエンティスト 勉強'], ['データサイエンティスト とは'], ['データサイエンティスト アナリスト', 'データサイエンティスト アナリスト 違い'], ['データサイエンティスト アメリカ', 'データサイエンティスト 大学院 アメリカ', 'アメリカ データサイエンティスト 年収']]
    idnum = 0
    d_nodes = {}
    nodeslist = []
    linkslist = []
    data = {}
    idnum2=0
    blank_d = re.compile('  ')
    blank_d2 = re.compile(' ')
    name_list = ["nodes"]
    colors = ['#F3C759','#C0D860','#5EC84E','#3DB680','#40BFB0','#42AAC7','#44A5CB','#6A8CC7','#8B90BE','#CE579B','#C35B9D','#F9DB57','#FFEE55','#E4EC5B','#45A1CF','#E6855E','#D45D87','#9499C5','#7F96CA','#6C9CCF','#6C9CCF','#6C9CCF','#6C9CCF']

    data["size"] = 150
    data["color"] = '#DA6272'
    data["keyword"] = phrase
    nodeslist.append(data)

    for j,ret_list in enumerate(ret[1:20]):
        dict_sg = {}
        color = colors[j]
        for i,suggest in enumerate(ret_list):
            data = {}
            links = {}
            if i == 0:
                suggest = secondword = suggest[len(phrase):]
                secondword = blank_d.sub('', secondword)
                secondword = blank_d2.sub('', secondword)
                size =60
                links["source"] = 0
                links["target"] = idnum+1
            else:
                pattern = re.compile(phrase)
                pattern2 = re.compile(secondword)
                suggest = pattern.sub('',suggest)
                suggest = pattern2.sub('',suggest)
                size =10
                links["source"] = 1+idnum2
                links["target"] = idnum+1

            if suggest != '':
                idnum += 1
                dict_sg[suggest] = idnum
                data["size"] = size
                data["color"] = color
                data["keyword"] = suggest
                data["number"] = idnum
                nodeslist.append(data)
                linkslist.append(links)
        idnum2 = idnum

    d_nodes["graph"] = []
    d_nodes["links"] = linkslist
    d_nodes["nodes"] = nodeslist
    d_nodes["directed"] = "false"
    d_nodes["multigraph"] = "false"

    fw = open('static/jsonfiles/s_graph.json','w')
    json.dump(d_nodes,fw,ensure_ascii=False,indent=4)
    return


import scrapy
from datetime import date

class DafitiSpider(scrapy.Spider):

    name = 'dafiti'

    start_urls = ['https://www.dafiti.com.br/calcados/adidas/',
                  'https://www.dafiti.com.br/calcados/asics/',
                  'https://www.dafiti.com.br/calcados/mizuno/',
                  'https://www.dafiti.com.br/calcados/new-balance/',
                  'https://www.dafiti.com.br/calcados/nike/']

    download_delay = 2.0

    def parse(self, response):

        diaAtual = date.today()

        dafiti_xpath = '//div[contains(@class, "product-box-detail")]'

        for itens in response.xpath(dafiti_xpath):

            Loja = 'Dafiti'

            Nome = itens.xpath('.//p/text()').extract_first()

            Preco = itens.xpath('.//div[3]/span/text()').extract_first()

            Data = diaAtual

            Marca = itens.xpath('.//div/text()').extract_first()

            if Marca == 'adidas':
                Marca = 'Adidas'

            if (Marca == None) or (Nome == None):
                Marca = None
                Nome  = None
                Preco = None
                Loja  = None
                Data  = None

            yield dict(Nome = Nome , Preco = Preco , Data = Data, Loja = Loja, Marca = Marca)

            link_next = response.xpath('//a[contains(., "next")]/@href').extract_first()
            if link_next:
                yield scrapy.Request(response.urljoin(link_next))


import pygame
from direction import *

class Robot(pygame.sprite.Sprite):

    def __init__(self, imagepath, width, height, x, y, surface):
        pygame.sprite.Sprite.__init__(self)

        self.imagepath = imagepath
        self.image = pygame.image.load(imagepath)

        self.width = width
        self.height = height

        self.rect = self.image.get_rect()
        self.rect.x = x
        self.rect.y = y


# This is your project's main settings file that can be committed to your
# repo. If you need to override a setting locally, use settings_local.py

from funfactory.settings_base import *

# Name of the top-level module where you put all your apps.
# If you did not install Playdoh with the funfactory installer script
# you may need to edit this value. See the docs about installing from a
# clone.
PROJECT_MODULE = 'source'

# Bundles is a dictionary of two dictionaries, css and js, which list css files
# and js files that can be bundled together by the minify app.
MINIFY_BUNDLES = {
    'css': {
        'sourceapp_css': (
            'css/bootstrap.css',
            'css/font-awesome.css',
            'css/app.css',
        ),
    },
    'js': {
        'sourceapp_js': (
            'js/app.js',
        ),
    }
}

# Defines the views served for root URLs.
ROOT_URLCONF = '%s.urls' % PROJECT_MODULE

INSTALLED_APPS = list(INSTALLED_APPS) + [
    'django.contrib.admin',
    'django.contrib.flatpages',
    'django.contrib.sites',
    '%s.base' % PROJECT_MODULE,
    '%s.articles' % PROJECT_MODULE,
    '%s.code' % PROJECT_MODULE,
    '%s.people' % PROJECT_MODULE,
    'caching',
    'haystack',
    'sorl.thumbnail',
    'south',
    'taggit',
]

TEMPLATE_CONTEXT_PROCESSORS = list(TEMPLATE_CONTEXT_PROCESSORS) + [
    'source.base.context_processors.http_protocol',
]

SUPPORTED_NONLOCALES = ['media', 'admin', ]

STATIC_URL = '/static/'

# Adding to standard funfactory middleware classes. Need to insert the
# UpdateCacheMiddleware early on, then append FetchFromCacheMiddleware
MIDDLEWARE_CLASSES = list(MIDDLEWARE_CLASSES)
MIDDLEWARE_CLASSES.insert(2, 'django.middleware.cache.UpdateCacheMiddleware')
MIDDLEWARE_CLASSES.append('django.contrib.flatpages.middleware.FlatpageFallbackMiddleware')
MIDDLEWARE_CLASSES.append('django.middleware.cache.FetchFromCacheMiddleware')

CACHE_MIDDLEWARE_SECONDS = 120

# Search with django-haystack
HAYSTACK_CONNECTIONS = {
    'default': {
        'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
        'URL': 'http://127.0.0.1:9200/',
        'INDEX_NAME': 'haystack',
    },
}

# Because Jinja2 is the default template loader, add any non-Jinja templated
# apps here:
JINGO_EXCLUDE_APPS = [
    'admin',
    'registration',
]

# dev is under https and live is (currently) only on http - quick hack to ensure
# we embed the disqus code from the right protocol
HTTP_PROTOCOL = 'http'

# sorl-thumbnail settings
DEFAULT_IMAGE_SRC = 'img/missing.png'

# Tells the extract script what files to look for L10n in and what function
# handles the extraction. The Tower library expects this.
DOMAIN_METHODS['messages'] = [
    ('%s/**.py' % PROJECT_MODULE,
        'tower.management.commands.extract.extract_tower_python'),
    ('%s/**/templates/**.html' % PROJECT_MODULE,
        'tower.management.commands.extract.extract_tower_template'),
    ('templates/**.html',
        'tower.management.commands.extract.extract_tower_template'),
],

# # Use this if you have localizable HTML files:
# DOMAIN_METHODS['lhtml'] = [
#    ('**/templates/**.lhtml',
#        'tower.management.commands.extract.extract_tower_template'),
# ]

# # Use this if you have localizable JS files:
# DOMAIN_METHODS['javascript'] = [
#    # Make sure that this won't pull in strings from external libraries you
#    # may use.
#    ('media/js/**.js', 'javascript'),
# ]

LOGGING = dict(loggers=dict(playdoh={'level': logging.DEBUG}))


import json, requests
import coordinates, cadnums
url_point = 'http://212.26.144.110/kadastrova-karta/find-Parcel'
url_data = 'http://212.26.144.110/kadastrova-karta/get-parcel-Info'

#cadnum='0524587000:01:003:0358'
cadnum = input('Вставьте кадастровый номер: ')

params_point = dict(
    cadnum=cadnum,
)

resp = (json.loads(requests.get(url=url_point, params=params_point).text)['data'])[0]
params_data = dict(
    koatuu=cadnum.split(':')[0],
    zone=cadnum.split(':')[1],
    quartal=cadnum.split(':')[2],
    parcel=cadnum.split(':')[3],
)
resp.update((json.loads(requests.get(url=url_data, params=params_data).text)['data'])[0])
coordinates.addCoordinates(resp)

if __name__ == '__main__':
    try:
        print(resp)
    except:
        print("Нет данных по {}".format(cadnum))


from urllib.request import urlopen
from urllib.error import HTTPError
from bs4 import BeautifulSoup

url = 'http://www.pythonscraping.com/pages/warandpeace.html'

def gettitle(url_info):
    try:
        html = urlopen(url_info)
    except HTTPError as e:
        return None
    else:
        try:
            bsobj = BeautifulSoup(html.read())
            title = bsobj.find_all('span', {'class':{'red', 'green'}})
        except AttributeError as e:
            return None
        else:
            return title

# 获取span标签下属性为green或red的内容
title_list = gettitle(url)
for title_info in title_list:
    if title_info == None:
        print('Title could not be found')
    else:
        print(title_info)


#!/usr/bin/python

from PIL import Image
import Modules
import optparse
import pyscreenshot

def main():
    # Parse arguments
    parser = optparse.OptionParser()
    parser.add_option('-i', '--image',
                      action="store", dest="imageFile",
                      help="Path to an image file", default="")
    parser.add_option('-d', '--database',
                      action="store", dest="database",
                      help="Path to the database", default="")
    options, args = parser.parse_args()

    # Calculate and write values to db
    if options.imageFile != "":
        # No database given -> return
        if options.database == "":
            print("Error: No database file given!")
            return
        # Load image
        sprites = Image.open(options.imageFile)
        # Open database
        f = open(options.database, 'a')
        spriteHeight = int(sprites.height / 2)
        # Seek to the second byte of the file
        f.seek(2)
        # Write name to database
        string = options.imageFile.split("/")[-1].split(".")[0]
        f.write(string + ":")
        i = 0
        j = 1
        # Calculate all images and write values to database
        while i < sprites.width:
            # Crop image to current sprite
            image = image.crop(box=(i, spriteHeight,
                                    Modules.image.spriteWidth + i, image.height))
            # Do image processing
            values = image_processing(image)
            # Write to file
            for d in values:
                # <number>=<hu moment>,
                f.write(str(j) + "=" + str(d) + ",")
                j += 1
            i += Modules.image.spriteWidth
            j = 1
        # Write newline and close file
        f.write("\n")
        f.close()
    # Game loop
    else:
        # Do game loop
        gameLoop()

def gameLoop():
    # Get all adjacent cells
    cells = Modules.image.getNearCells()
    # Calculate cells
    for c in cells:
        image = c.im
        image.show()

def imageProcessing(image):
    # Calculate image values
    return Modules.image_processing.imageValues(image)

if __name__ == "__main__":
    main()


import numpy as np
import pickle
import math
from tabulate import tabulate
from scipy.stats import kendalltau
from scipy.stats import pearsonr, spearmanr

def get_pickle(file_path):
    with open(file_path, 'rb') as fp:
        x = pickle.load(fp)
    return x

def print_score_ranges(sd):
    metrics_list = get_metrics_list(sd)
    print_list = []
    headers = ["min", "25-perc", "median", "75-perc", "max", "mean"]
    for m in metrics_list:
        scores = [s['scores'][m] for d in sd.values() for s in d['system_summaries'].values()]
        print_list.append([m,
                           np.min(scores),
                           np.percentile(scores, 25),
                           np.median(scores),
                           np.percentile(scores, 75),
                           np.max(scores),
                           np.mean(scores)])
    print(tabulate(print_list, headers=headers, floatfmt=".4f", tablefmt="pretty"))

def get_metrics_list(sd):
    """
    Does each system summary dict have same all_metrics?
    :param sd: scores dict
    :return: list of all_metrics in the scores dict
    """
    metrics_tuple_set = set(
        [tuple(sorted(list(x['scores'].keys())))
         for d in sd.values() for x in d['system_summaries'].values()])
    assert len(metrics_tuple_set) == 1, "all system summary score dicts should have the same set of all_metrics"
    metrics_list = list(list(metrics_tuple_set)[0])
    return metrics_list

def print_ktau_matrix(metrics, percentile, sd, cutoff_metric='bert_f_score', y_type='ktau'):
    high_pvals = 0
    print(metrics)
    for min_j, mx in enumerate(metrics):
        mean_ktaus = np.zeros((len(percentile), len(metrics)))
        for i, perc in enumerate(percentile):
            for j, my in enumerate(metrics):
                if mx == my or j <= min_j:
                    continue
                ktaus_pvals = [get_doc_y_val(isd, mx, my, y_type=y_type, cutoff_metric=cutoff_metric,
                                             percentile=perc)
                               for isd in sd.values()]
                ktaus = []
                for ktau, pval in ktaus_pvals:
                    # ktaus.append(ktau)
                    if pval <= 0.05:
                        ktaus.append(ktau)
                    else:
                        high_pvals += 1
                mean_ktaus[i, j] = np.mean(ktaus)

        print(mean_ktaus)
        print()

    total_ktaus = (len(metrics) / 2) * (len(metrics) - 1) * len(percentile) * len(sd)
    print(f"total {high_pvals}/{total_ktaus} = {high_pvals * 100 / total_ktaus}% values ignored")

def get_pairwise_correlation(scores_data, key_a, key_b, pval_threshold,
                             filter_metric=None, filter_score=None, top=None, return_ktau_d=False):
    '''
    get the kendall's tau (a correlation ranking score) for the metric pair 'key_a' and 'key_b'
    '''
    kendall_scores = []
    num_ignored = 0
    ktau_d = {}
    for doc_id, isd in scores_data.items():  # isd = "i-th scoe dict, for the i-th doc"
        sumdicts_to_iterate = isd['system_summaries'].values()
        if filter_metric:
            sumdicts_to_iterate = [summary_dict for summary_dict in sumdicts_to_iterate
                                   if summary_dict['scores'][filter_metric] > filter_score]

        if top is not None:
            top_key_a = sorted([(summary_dict['scores'][key_a], summary_dict) for summary_dict in sumdicts_to_iterate],
                               key=lambda x: x[0], reverse=True)[:top]
            top_key_b = sorted([(summary_dict['scores'][key_b], summary_dict) for summary_dict in sumdicts_to_iterate],
                               key=lambda x: x[0], reverse=True)[:top]
            sumdicts_to_iterate = top_key_a + top_key_b
            sumdicts_to_iterate = [summary_dict[1] for summary_dict in sumdicts_to_iterate]

        key_a_scores = [summary_dict['scores'][key_a] for summary_dict in sumdicts_to_iterate]
        key_b_scores = [summary_dict['scores'][key_b] for summary_dict in sumdicts_to_iterate]

        # kendalltau is not reliable if we have less than 4 ranks to compare
        if (len(key_a_scores) < 4) or (len(key_b_scores) < 4):
            num_ignored += 1
            continue

        ktau, pval = kendalltau(key_a_scores, key_b_scores, nan_policy="raise")
        if pval > pval_threshold or math.isnan(ktau):
            num_ignored += 1
            continue

        kendall_scores.append(ktau)
        if doc_id % 100 == 0:
            print(f"done {doc_id}/{len(scores_data)}", end="\r")
        if return_ktau_d:
            ktau_d[doc_id] = ktau
    if return_ktau_d:
        return np.mean(kendall_scores), num_ignored, ktau_d
    else:
        return np.mean(kendall_scores), num_ignored

def filter_summaries(isd, cutoff_metric, percentile):
    if cutoff_metric == 'nas':
        for sys_name in isd['system_summaries']:
            scores = isd['system_summaries'][sys_name]['scores']
            if 'nas' in scores.keys():
                break
            keys = [key for key in scores.keys() if 'recall' in key] + ['mover_score']
            # print(keys)
            scores['nas'] = np.mean([isd['system_summaries'][sys_name]['normed_scores'][m] for m in keys])

    c_scores = [summdict['scores'][cutoff_metric] for summdict in isd['system_summaries'].values()]
    cutoff_score_min = np.percentile(c_scores, percentile[0])
    cutoff_score_max = np.percentile(c_scores, percentile[1])
    filtered_sumdicts_l = [summdict for summdict in isd['system_summaries'].values()
                           if (
                                   (summdict['scores'][cutoff_metric] >= cutoff_score_min)
                                   and (summdict['scores'][cutoff_metric] <= cutoff_score_max)
                           )
                           ]
    return filtered_sumdicts_l

doc_y_types = ['ktau', 'pearson', 'spearman', 'm']

def get_doc_y_val(isd, m1, m2, y_type, cutoff_metric=None, percentile=None):
    assert (y_type in doc_y_types)

    if y_type == 'm':
        return isd['mean_scores'][m1], 0

    filtered_summaries = isd['system_summaries'].values()
    if cutoff_metric is not None:
        filtered_summaries = filter_summaries(isd, cutoff_metric, percentile)

    m1_scores = [summdict['scores'][m1] for summdict in filtered_summaries]
    m2_scores = [summdict['scores'][m2] for summdict in filtered_summaries]

    if y_type == 'ktau':
        ktau, pval = kendalltau(m1_scores, m2_scores, nan_policy="raise")
        if np.isnan(ktau):
            # return high pvalue to ignore
            return 0, 1
        return ktau, pval

    elif y_type == 'pearson':
        pearson_corr, pval = pearsonr(m1_scores, m2_scores)
        return pearson_corr, pval

    elif y_type == 'spearman':
        spearman_corr, pval = spearmanr(m1_scores, m2_scores)
        return spearman_corr, pval


# sam import naprawia sciezki
import path_fixing

import unittest
from graph_tool.all import *

from node import Node
from algorithms.color6 import Color6
from graph_generators import createRandomTree
from basics import get_children_ids

class Color6_Tests(unittest.TestCase):
    """ Testy do klasy Color6 z modulu algorithms.
    """

    def test_update_outbox(self):
        g, sim_nodes = createRandomTree(5)

        c6instance = Color6(g, sim_nodes)
        root_node = sim_nodes[0]

        root_color = 15
        root_node.memory['color'] = root_color
        root_vertex = g.vertex(0)

        root_children_ids = get_children_ids(root_vertex, 0)

        c6instance._update_outbox(root_node)

        self.failUnless(len(root_children_ids) == len(root_node.outbox))

        for msg in root_node.outbox:
            self.failUnless(msg.source_id == 0)
            self.failUnless(msg.target_id in root_children_ids)
            self.failUnless(msg.data == root_color)

    def test_color_graph(self):
        g, sim_nodes = createRandomTree(500)

        c6instance = Color6(g, sim_nodes)
        c6instance.color_graph()

        # tak dodatkowo...
        from visualizations import saveColoredGraph
        saveColoredGraph(g, sim_nodes, '/home/julka/testowe')

        for node in sim_nodes:
            self.failUnless(node.memory['color'] <= 5)

def main():
    unittest.main()

if __name__ == '__main__':
    main()


# -*- encoding: utf-8 -*-
##############################################################################
#
#    OpenERP - MongoDB backend
#    Copyright (C) 2011 Joan M. Grande <totaler@gmail.com>
#    Thanks to Sharoon Thomas for the operator mapping code
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from __future__ import absolute_import
import copy

from osv.osv import module_class_list, module_list, class_pool
from . import orm_mongodb

class osv_mongodb(orm_mongodb.orm_mongodb):
    #__metaclass__ = inheritor
    def __new__(cls):
        module = str(cls)[6:]
        module = module[:len(module) - 1]
        module = module.split('.')[0][2:]
        if not hasattr(cls, '_module'):
            cls._module = module
        module_class_list.setdefault(cls._module, []).append(cls)
        class_pool[cls._name] = cls
        if module not in module_list:
            module_list.append(cls._module)
        return None

    def createInstance(cls, pool, module, cr):
        parent_name = hasattr(cls, '_inherit') and cls._inherit
        if parent_name:
            parent_class = pool.get(parent_name).__class__
            assert pool.get(parent_name), \
                "parent class %s does not exist in module %s !" % (
                    parent_name, module
                )
            nattr = {}
            for s in ('_columns', '_defaults'):
                pool_obj = pool.get(parent_name)
                if not hasattr(pool_obj, s):
                    continue
                new = copy.copy(getattr(pool_obj, s))
                if hasattr(new, 'update'):
                    new.update(cls.__dict__.get(s, {}))
                else:
                    new.extend(cls.__dict__.get(s, []))
                nattr[s] = new
            name = hasattr(cls, '_name') and cls._name or cls._inherit
            if cls in parent_class.mro():
                cls = parent_class
            else:
                cls = type(name, (cls, parent_class), nattr)
        obj = object.__new__(cls)
        obj.__init__(pool, cr)
        return obj
    createInstance = classmethod(createInstance)

    def __init__(self, pool, cr):
        pool.add(self._name, self)
        self.pool = pool
        orm_mongodb.orm_mongodb.__init__(self, cr)


# -*- coding: utf-8 -*-
"""
Dubalu Framework
~~~~~~~~~~~~~~~~

:author: Dubalu Framework Team. See AUTHORS.
:copyright: Copyright (c) 2013-2014, deipi.com LLC. All Rights Reserved.
:license: See LICENSE for license details.

"""
from __future__ import absolute_import, unicode_literals, print_function

import os
import datetime

# For django to work, we need to populate apps and models as soon as possible
# If the apps have not yet been populated by now (as is the case when using uwsgi),
# populate.
from django.db.models.loading import cache
cache._populate()

from django.conf import settings
from django.conf.urls import patterns, include, url
from django.conf.urls.static import static
from django.utils.functional import memoize

from django.views.decorators.http import last_modified
from django.views.decorators.cache import cache_page
from django.views.i18n import javascript_catalog

# from dfw.sections.urls_base import get_urlresolver as get_sections_urlresolver

# Admin:
# from django.contrib import admin
# admin.autodiscover()

handler400 = settings.HANDLER400
handler403 = settings.HANDLER403
handler404 = settings.HANDLER404
handler500 = settings.HANDLER500

JSI18N_DATE_FILE = getattr(settings, 'JSI18N_DATE_FILE', '/tmp/jsi18n')
try:
    JSI18N_DATE = os.path.getmtime(JSI18N_DATE_FILE)
except OSError:
    JSI18N_DATE = float(datetime.datetime.now().strftime('%s'))
    open(JSI18N_DATE_FILE, 'a').close()
jsi18n_date = datetime.datetime.fromtimestamp(JSI18N_DATE)

# The value of JSI18N_DATE must change when translations change.
@last_modified(lambda req, **kw: jsi18n_date)
@cache_page(86400, key_prefix='js18n-%s' % JSI18N_DATE)
def cached_javascript_catalog(request, domain='djangojs', packages=None):
    if packages is None:
        packages = tuple(settings.INSTALLED_APPS) + ('django.conf',)
    return javascript_catalog(request, domain, packages)

# Default base urlpatterns:
urlpatterns = patterns('')

if settings.DEBUG:
    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
    urlpatterns += static(settings.STATIC_URL, view='django.contrib.staticfiles.views.serve')

urlpatterns += patterns('',
    # url(r'^autocomplete/', include('dfw.autocomplete.urls')),
    # url(r'^impersonate/', include('dfw.impersonation.urls')),

    url(r'^i18n/', include('django.conf.urls.i18n')),
    url(r'^i18n/js/$', cached_javascript_catalog, name='javascript_catalog'),

    # url(r'^morsels/(?P<__params__>[^/]+)/', include('dfw.morsels.urls_base', app_name='morsel')),
)

# urlconf_module = include('dfw.notification.urls')[0]
# urlpatterns += getattr(urlconf_module, 'urlpatterns', urlconf_module)

# urlconf_module = include('dfw.pluggables.urls_base')[0]
# urlpatterns += getattr(urlconf_module, 'urlpatterns', urlconf_module)

if settings.PROJECT_NAME:
    urlconf_module = include('%s.urls' % settings.PROJECT_NAME)[0]
    urlpatterns += getattr(urlconf_module, 'urlpatterns', urlconf_module)

urlconf_module = include('dfw.contrib.pages.urls')[0]
urlpatterns += getattr(urlconf_module, 'urlpatterns', urlconf_module)

def get_urls(project_name, project_suffix):
    return type(b'ProjectUrls', (object,), dict(
        handler400=handler400,
        handler403=handler403,
        handler404=handler404,
        handler500=handler500,
        urlpatterns=urlpatterns,
    ))()
get_urls._cache = {}
get_urls = memoize(get_urls, get_urls._cache, 2)


import numpy as np
import tensorflow as tf
from sklearn import datasets
from tensorflow.examples.tutorials.mnist import input_data

def convert_to_1_hot(in_data, max_val):
    a = np.array(in_data)
    out_data = np.zeros((len(in_data),max_val))
    out_data[np.arange(len(in_data)), a] = 1
    return out_data

def get_diabetes(train_test_ratio):
    diabetes = datasets.load_diabetes()
    x = diabetes.data
    y = diabetes.target

    y = np.reshape(y,(y.shape[0],1))
    cutoff = int(x.shape[0] * train_test_ratio)

    x = x.astype(np.float32)
    y = y.astype(np.float32)

    x_train = x[0:cutoff, :]
    x_test = x[cutoff:, :]
    y_train = y[:cutoff]
    y_test = y[cutoff:]
    return x_train, y_train, x_test, y_test

def get_boston(train_test_ratio):
    boston = datasets.load_boston()
    x = boston.data
    y = boston.target

    y = np.reshape(y,(y.shape[0],1))
    cutoff = int(x.shape[0] * train_test_ratio)

    x = x.astype(np.float32)
    y = y.astype(np.float32)

    x_train = x[0:cutoff, :]
    x_test = x[cutoff:, :]
    y_train = y[:cutoff]
    y_test = y[cutoff:]
    return x_train, y_train, x_test, y_test

def get_covertype(train_test_ratio):
    covertype = datasets.fetch_covtype()
    x = covertype.data

    y = convert_to_1_hot(covertype.target - 1, 7)
    cutoff = int(x.shape[0] * train_test_ratio)

    x = x.astype(np.float32)
    y = y.astype(np.float32)

    x_train = x[0:cutoff, :]
    x_test = x[cutoff:, :]
    y_train = y[0:cutoff]
    y_test = y[cutoff:]
    return x_train, y_train, x_test, y_test

def get_iris(train_test_ratio):
    iris = datasets.load_iris()
    x = iris.data
    y = convert_to_1_hot(iris.target, 3)

    cutoff = int(x.shape[0] * train_test_ratio)

    x = x.astype(np.float32)
    y = y.astype(np.float32)

    x_train = x[0:cutoff, :]
    x_test = x[cutoff:, :]
    y_train = y[:cutoff]
    y_test = y[cutoff:]
    return x_train, y_train, x_test, y_test

def get_mnist(train_test_ratio):
    # mnist = tf.examples.tutorials.mnist.input_data.read_data_sets("MNIST_data/", one_hot=True)
    digits = datasets.load_digits()
    x = digits.data

    y = convert_to_1_hot(digits.target, 10)
    cutoff = int(x.shape[0] * train_test_ratio)

    x = x.astype(np.float32)
    y = y.astype(np.float32)

    x_train = x[0:cutoff, :] / 16.
    x_test = x[cutoff:, :] / 16.
    y_train = y[0:cutoff]
    y_test = y[cutoff:]
    return x_train, y_train, x_test, y_test

def generate_noisy_polinomial_data(mu, sigma, n_samples, f0, f1, f2, low=-100., high=100. ):
    if mu == 0. and sigma == 0.:
        noise = np.zeros(n_samples)
    else:
        noise = np.random.normal(mu, sigma, n_samples)

    x = (high - low) * np.random.random_sample(n_samples) + low

    y = f0 * x**2 + f1 * x**1 + f2 + noise
    x = np.expand_dims(x, 1)
    y = np.expand_dims(y, 1)

    return x, y

def generate_noisy_linear_data(mu,sigma,n_samples, w, b, low=-100.,high=100.):
    if mu == 0. and sigma == 0.:
        noise = np.zeros(n_samples)
    else:
        noise = np.random.normal(mu, sigma, n_samples)
    x = (high - low) * np.random.random_sample(n_samples) + low

    y = x * w + b + noise

    x = np.expand_dims(x,1)
    y = np.expand_dims(y,1)

    return x, y

if __name__ == '__main__':
    mnist = get_mnist()


"""adding date imgaed

Revision ID: d1e5eef0bd70
Revises: f2d46bc0e813
Create Date: 2020-01-02 12:11:26.489005

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'd1e5eef0bd70'
down_revision = 'f2d46bc0e813'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('precise_filters', sa.Column('_date_img', sa.Date(), nullable=True))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('precise_filters', '_date_img')
    # ### end Alembic commands ###