import os
import json
import re
import string

from threading import Thread

from rpitts import say
from gmusicplayer import playgmusicplaylist, updategmusiclibrary, stopgmusicplayer, creategmusicplaylist, generategmusicplaylist, searchgmusic

#Find loop playback or not
def getloopstatus(querystring):
   regexobj = (re.match( r"^((loop|shuffle)?\s*(and)?\s*(loop|shuffle))\s*(.*)", querystring, re.I))
   if regexobj:
      if (regexobj.group(2) == 'loop' or regexobj.group(4) == 'loop'):
          return True
      else:
          return False
   else:
      return False

#Find shuffle playback or not
def getshufflestatus(querystring):
   regexobj = (re.match( r"^((loop|shuffle)?\s*(and)?\s*(loop|shuffle)?)", querystring, re.I))
   if regexobj:
      if (regexobj.group(2) == 'shuffle' or regexobj.group(4) == 'shuffle'):
          return True
      else:
          return False
   else:
      return False

#Parse music query string and identify song, artist etc.
def parse_music_query(querystring):
    if (re.search( r"(.*)\s*play all (songs)$", querystring, re.I)):
       return 'PLAYALLSONGS',None,None,None
    elif (re.search( r"(.*)\s*play\s*(song|songs)?\s*(.*)?\s*(from)?\s*playlist (.*)$", querystring, re.I)):
          regexobj=re.search( r"(.*)\s*play\s*(song|songs)?\s*(.*)?\s*(from)?\s*playlist (.*)$", querystring, re.I)
          return regexobj.group(3),None,None,regexobj.group(5),None,None
    elif (re.search( r"(.*)\s*play\s*(songs)?\s*(from)?\s*station (.*)$", querystring, re.I)):
          regexobj=re.search( r"(.*)\s*play\s*(songs)?\s*(from)?\s*station (.*)$", querystring, re.I)
          return None,None,None,None,regexobj.group(4),None
    elif (re.search( r"(.*)\s*play\s*(from)?\s*podcast (.*)$", querystring, re.I)):
          regexobj=re.search( r"(.*)\s*play\s*(from)?\s*podcast (.*)$", querystring, re.I)
          return None,None,None,None,None,regexobj.group(3)
    elif (re.search(r"play\s*(song|songs)?\s*(.*)?\s*(by|of)\s*(artist)\s*(.*)\s*(from|in)\s*(album)\s*(.*)", querystring, re.I)):
          regexobj=re.search(r"play\s*(song|songs)?\s*(.*)?\s*(by|of)\s*(artist)\s*(.*)\s*(from|in)\s*(album)\s*(.*)", querystring, re.I)
          return regexobj.group(2),regexobj.group(5),regexobj.group(8),None,None,None
    elif (re.search(r"play\s*(song|songs)?\s*(.*)?\s*(from|in)\s*(album)\s*(.*)\s*(by|of)\s*(artist)\s*(.*)", querystring, re.I)):
          regexobj=re.search(r"play\s*(song|songs)?\s*(.*)?\s*(from|in)\s*(album)\s*(.*)\s*(by|of)\s*(artist)\s*(.*)", querystring, re.I)
          return regexobj.group(2),regexobj.group(8),regexobj.group(5),None,None,None
    elif (re.search(r"play\s*(song(s)?)\s*(.*)?\s*(from|in)\s*(album)\s*(.*)", querystring, re.I)):
          regexobj=re.search(r"play\s*(song(s)?)\s*(.*)?\s*(from|in)\s*(album)\s*(.*)", querystring, re.I)
          return regexobj.group(3),None,regexobj.group(6),None,None,None
    elif (re.search(r"play\s*(song(s)?)\s*(.*)?\s*(by|of)\s*(artist)\s*(.*)", querystring, re.I)):
          regexobj=re.search(r"play\s*(song(s)?)\s*(.*)?\s*(by|of)\s*(artist)\s*(.*)", querystring, re.I)
          return regexobj.group(3),regexobj.group(6),None,None,None,None
    elif (re.search(r"play\s*(album)\s*(.*)\s*(of|by)\s*(artist)\s*(.*)", querystring, re.I)):
          regexobj=re.search(r"play\s*(album)?\s*(.*)\s*(of|by)\s*(artist)?\s*(.*)", querystring, re.I)
          return None,regexobj.group(5),regexobj.group(2),None,None,None
    elif (re.search(r"play\s*(artist)\s*(.*)\s*(from|in)\s*(album)\s*(.*)", querystring, re.I)):
          regexobj=re.search(r"play\s*(artist)?\s*(.*)\s*(from|in)\s*(album)?\s*(.*)", querystring, re.I)
          return None,regexobj.group(2),regexobj.group(3),None,None,None
    elif (re.search(r"play\s*(album)\s*(.*)", querystring, re.I)):
          regexobj=re.search(r"play\s*(album)\s*(.*)", querystring, re.I)
          return None,None,regexobj.group(2),None,None,None
    elif (re.search(r"play\s*(artist)\s*(.*)", querystring, re.I)):
          regexobj=re.search(r"play\s*(artist)\s*(.*)", querystring, re.I)
          return None,regexobj.group(2),None,None,None,None
    elif (re.search(r"play\s*(song(s)?)?\s*(.*)", querystring, re.I)):
          regexobj=re.search(r"play\s*(song(s)?)?\s*(.*)", querystring, re.I)
          return regexobj.group(3),None,None,None,None,None
    else:
          return None,None,None,None,None,None

#Update music library in database
def updatemedialibraries():
    medialibraryupdated=updategmusiclibrary()
    return medialibraryupdated

def playmedia(**kwargs):
    player = kwargs.get('player', 'gmusic')
    loop = kwargs.get('loop', False)
    shuffle = kwargs.get('shuffle', False)
    playgmusicplaylist(loop=loop,shuffle=shuffle)

#Stop media players
def stopmediaplayer(*args):
    stopgmusicplayer()

#Create media playlist from query
def createmediaplaylist(**kwargs):
    myquery = kwargs.get('query', None)
    player = kwargs.get('player', 'gmusic')
    song,artist,album,playlist,station,podcast=parse_music_query(myquery)
    if not song:
       song = None
    if not artist:
       artist = None
    if not album:
       album = None
    if not playlist:
       playlist = None
    if not station:
       station = None
    if not podcast:
       podcast = None
    response = "Looking for"
    if song is not None:
       response = response + " song "+song
    if artist is not None:
       response = response + " by artist "+artist
    if album is not None:
       response = response + " from album "+album
    if playlist is not None:
       response = response + " playlist "+playlist
    if station is not None:
       response = response + " station "+station
    if podcast is not None:
       response = response + " podcast "+podcast
    say(response)
    if (player == 'gmusic'):
        if (station is not None):
            generategmusicplaylist(station=station)
        elif (podcast is not None):
            generategmusicplaylist(podcast=podcast)
        else:
            creategmusicplaylist(song=song,artist=artist,album=album,playlist=playlist)
        if not os.path.isfile("gmusicplaylist.json"):
           searchgmusic(song=song,artist=artist,album=album,playlist=playlist)

def mediaplayer(query):
    createmediaplaylist(query=query,player='gmusic')
    loopstatus=getloopstatus(query)
    shufflestatus=getshufflestatus(query)
    gmusicplayerthread = Thread(target=playmedia, kwargs={'player':'gmusic','shuffle':shufflestatus,'loop':loopstatus})
    gmusicplayerthread.start()


import os
import glob
import pickle
import requests
import concurrent.futures
import numpy as np
from sklearn.metrics import roc_curve, roc_auc_score
from misc.utils import htmd_featurizer
from misc.ligand_extract import PocketFromLigandDetector

import logging
logger = logging.getLogger(__name__)

class Prospeccts:
    """ ProSPECCTs dataset by Ehrt et al (http://www.ccb.tu-dortmund.de/ag-koch/prospeccts/) """

    dbnames = ['P1', 'P1.2', 'P2', 'P3', 'P4', 'P5', 'P5.2', 'P6', 'P6.2', 'P7']

    def __init__(self, dbname):
        self.dbname = dbname

    @staticmethod
    def _extract_pocket_and_get_uniprot(pdbpath):
        fname = os.path.basename(pdbpath).split('.')[0]
        if '_' in fname:
            return None, None

        # 1) Extract the pocket
        detector = PocketFromLigandDetector(include_het_resname=False, save_clean_structure=True, keep_other_hets=False, min_lig_atoms=1, allowed_lig_names=['LIG'])
        detector.run_one(pdbpath, os.path.dirname(pdbpath))

        # 2) Attempt to map to Uniprots (fails from time to time, return 'None' in that case)
        pdb_code = fname[:4].lower()
        query_chain_id = fname[4].upper() if len(fname)>4 else ''
        result = set()
        try:
            r = requests.get('http://www.ebi.ac.uk/pdbe/api/mappings/{}/{}'.format('uniprot', pdb_code))
            fam = r.json()[pdb_code]['UniProt']
        except Exception as e:
            # this logically fails for artificial proteins not in PDB, such as in decoys (P3, P4), but that's fine.
            logger.warning('PDB not found {} {} {}'.format(e, pdb_code, query_chain_id))
            return fname, 'None'
        for fam_id in fam.keys():
            for chain in fam[fam_id]['mappings']:
                if not query_chain_id:
                    result.add(fam_id)
                elif chain['chain_id'] == query_chain_id:
                    if len(result) > 0 and fam_id != next(iter(result)):
                        logger.warning('Duplicate chain {} {}'.format(fam_id, result))
                    result.add(fam_id)
        if not result:
            logger.warning('Chain not found {}'.format(pdb_code))
        return fname, result

    def preprocess_once(self):
        logger.info('Preprocessing: extracting pockets and obtaining uniprots, this will take time.')
        all_pdbs = glob.glob(os.environ['STRUCTURE_DATA_DIR'] + '/prospeccts/**/*.pdb', recursive=True)
        code_to_uniprot = {}
        with concurrent.futures.ProcessPoolExecutor() as executor:
            for code, uniprot in executor.map(Prospeccts._extract_pocket_and_get_uniprot, all_pdbs):
                if code:
                    code_to_uniprot[code] = uniprot
        pickle.dump(code_to_uniprot, open(os.environ['STRUCTURE_DATA_DIR'] + '/prospeccts/code_to_uniprot.pickle', 'wb'))

        htmd_featurizer(self.get_structures(), skip_existing=True)

    def _prospeccts_paths(self):
        if self.dbname== 'P1':
            dir1, dir2, listfn = 'identical_structures', 'identical_structures', 'identical_structures.csv'
        elif self.dbname== 'P1.2':
            dir1, dir2, listfn = 'identical_structures_similar_ligands', 'identical_structures_similar_ligands', 'identical_structures_similar_ligands.csv'
        elif self.dbname== 'P2':
            dir1, dir2, listfn = 'NMR_structures', 'NMR_structures', 'NMR_structures.csv'
        elif self.dbname== 'P3':
            dir1, dir2, listfn = 'decoy', 'decoy_structures', 'decoy_structures5.csv'
        elif self.dbname== 'P4':
            dir1, dir2, listfn = 'decoy', 'decoy_shape_structures', 'decoy_structures5.csv'
        elif self.dbname== 'P5':
            dir1, dir2, listfn = 'kahraman_structures', 'kahraman_structures', 'kahraman_structures80.csv'
        elif self.dbname== 'P5.2':
            dir1, dir2, listfn = 'kahraman_structures', 'kahraman_structures', 'kahraman_structures.csv'
        elif self.dbname== 'P6':
            dir1, dir2, listfn = 'barelier_structures', 'barelier_structures', 'barelier_structures.csv'
        elif self.dbname== 'P6.2':
            dir1, dir2, listfn = 'barelier_structures', 'barelier_structures_cofactors', 'barelier_structures.csv'
        elif self.dbname== 'P7':
            dir1, dir2, listfn = 'review_structures', 'review_structures', 'review_structures.csv'
        else:
            raise NotImplementedError
        return dir1, dir2, listfn

    def get_structures(self):
        """ Get list of PDB structures with metainfo """
        dir1, dir2, listfn = self._prospeccts_paths()
        root = os.path.join(os.environ.get('STRUCTURE_DATA_DIR'), 'prospeccts', dir1)
        npz_root = os.path.join(os.environ.get('STRUCTURE_DATA_DIR'), 'processed/htmd/prospeccts', dir1)

        db_pdbs = set()
        with open(os.path.join(root, listfn)) as f:
            for line in f.readlines():
                tokens = line.split(',')
                db_pdbs.add(tokens[0])
                db_pdbs.add(tokens[1])

        try:
            code5_to_uniprot = pickle.load(open(os.path.join(os.environ.get('STRUCTURE_DATA_DIR'), 'prospeccts', 'code_to_uniprot.pickle'), 'rb'))
        except FileNotFoundError:
            logger.warning('code_to_uniprot.pickle not found, please call preprocess_once()')
            code5_to_uniprot = None

        entries = []
        for pdb in db_pdbs:
            entries.append({'protein': root + '/{}/{}_clean.pdb'.format(dir2, pdb),
                     'pocket': root + '/{}/{}_site_1.pdb'.format(dir2, pdb),
                     'ligand': root + '/{}/{}_lig_1.pdb'.format(dir2, pdb),
                     'protein_htmd': npz_root + '/{}/{}_clean.npz'.format(dir2, pdb),
                     'code5': pdb,
                     'code': pdb[:4],
                     'uniprot': code5_to_uniprot[pdb] if code5_to_uniprot else 'None'
                     })

        return entries

    def evaluate_matching(self, descriptor_entries, matcher):
        """
        Evaluate pocket matching on one Prospeccts dataset. The evaluation metrics is AUC.
        :param descriptor_entries: List of entries
        :param matcher: PocketMatcher instance
        :return:
        """

        target_dict = {d['code5']: d for d in descriptor_entries}
        pairs = []
        positives = []

        dir1, dir2, listfn = self._prospeccts_paths()
        root = os.path.join(os.environ.get('STRUCTURE_DATA_DIR'), 'prospeccts', dir1)

        with open(os.path.join(root, listfn)) as f:
            for line in f.readlines():
                tokens = line.split(',')
                id1, id2, cls = tokens[0], tokens[1], tokens[2].strip()
                if id1 in target_dict and id2 in target_dict:
                    pairs.append((target_dict[id1], target_dict[id2]))
                    positives.append(cls=='active')
                else:
                    logger.warning('Detection entry missing for {},{}'.format(id1, id2))

        scores = matcher.pair_match(pairs)

        goodidx = np.flatnonzero(np.isfinite(np.array(scores)))
        if len(goodidx) != len(scores):
            logger.warning('Ignoring {} pairs'.format(len(scores) - len(goodidx)))
            positives_clean, scores_clean = np.array(positives)[goodidx],  np.array(scores)[goodidx]
        else:
            positives_clean, scores_clean = positives, scores

        fpr, tpr, roc_thresholds = roc_curve(positives_clean, scores_clean)
        auc = roc_auc_score(positives_clean, scores_clean)

        results = {'auc': auc, 'fpr': fpr, 'tpr': tpr, 'th_roc': roc_thresholds,
                   'pairs': pairs, 'scores': scores, 'pos_mask': positives}
        return results


# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python2, python3
"""Partial differential equation kernels to be consumed by grid_stepper."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
from tf_quant_finance.math.pde import time_marching_schemes

class ParabolicDifferentialEquationSolver(object):
  """Solver for parabolic differential equations.

  Creates a solver (see, e.g.,  [Forsyth, Vetzal(2002)][3]) for a
  parabolic differential equation of the form

  ```none
   V_{t} + 1 / 2 * a(t, x) * V_{xx} + b(t, x) * V_{x} - c(t, x) * V = 0
  ```

  Here `V = V(t, x)` is a solution to the 2-dimensional PDE. `V_{t}` is the
  derivative over time and `V_{x}` and `V_{xx}` are the first and second
  derivatives over the space component. For a solution to be well-defined, it is
  required for `a` to be positive on its domain. Henceforth,
  `a(t, x)`, `b(t, x)`, and `c(t, x)` are referred to as quadratic, linear and
  shift coefficients, respectively. Note that the backward Kolmogorov equation
  is of the above form with a(t, x) as the drift, b(t, x) the diffusion
  constant, and c = 0. See, e.g., [1] or [Oskendal(2010)][2].

  #### References
  [1]: Kolmogorov_backward_equations. Wikipedia/
    https://en.wikipedia.org/wiki/Kolmogorov_backward_equations_(diffusion)
  [2]: Brent Oksendal. Stochastic Differential Equations: An Introduction with
    Applications. Springer. 2010.
  [3]: P.A. Forsyth, K.R. Vetzal. Quadratic Convergence for Valuing American
    Options Using A Penalty Method. Journal on Scientific Computing, 2002.
    http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.28.9066&rep=rep1&type=pdf
  """

  def __init__(self,
               quadratic_coeff_fn,
               lower_boundary_fn,
               upper_boundary_fn,
               linear_coeff_fn=None,
               shift_coeff_fn=None,
               time_marching_scheme=None,
               name=None):
    """Initializes the solver.

    Args:
      quadratic_coeff_fn: A Python callable returning the quadratic coefficient
        `a(t, x)` at the given time `t`. The callable accepts the time `t` and a
        grid of values `x` at which the coefficient is to be computed. Should
        return a `Tensor` of float dtype and of shape broadcastable with
        `[num_equations, num_grid_points]`, where `num_equations` is the number
        of PDE equations being solved and `num_grid_points` is the number of
        grid points over the space coordinate `x`.
      lower_boundary_fn: A Python callable returning a `Tensor` of the same
        `dtype` as the output of `quadratic_coeff_fn` and of shape
        `[num_equations]`. Represents Dirichlet boundary condition for the lower
        value of `x`, i.e., `V(t, x_min)`.
      upper_boundary_fn: A Python callable returning a `Tensor` of the same
        `dtype` as the output of `quadratic_coeff_fn` and of shape
        `[num_equations]`. Represents Dirichlet boundary condition for the upper
        value of `x`, i.e., `V(t, x_max)`.
      linear_coeff_fn:  Python callable returning the linear coefficient `b(t,
        x)` at the given time `t`. The callable accepts the time `t` and a grid
        of values `x` at which the coefficient is to be computed. Should return
        a `Tensor` of the same dtype as the output of `quadratic_coeff_fn` and
        of shape broadcastable with `[num_equations, num_grid_points]`. Defaults
        to `linear_coeff_fn(t, x) = 0.0`.
      shift_coeff_fn:  Python callable returning the shift coefficient `c(t, x)`
        at the given time `t`. The callable accepts the time `t` and a grid of
        values `x` at which the coefficient is to be computed. Should return a
        `Tensor` of the same dtype as the output of `quadratic_coeff_fn` and of
        shape broadcastable with `[num_equations, num_grid_points]`. Defaults to
        `shift_coeff_fn(t, x) = 0.0`.
      time_marching_scheme: an implementation of TimeMarchingScheme (see
        pde_time_marching_schemes.py) defining a time marching scheme used:
          implicit, explicit, Crank-Nicolson, etc. By default, Crank-Nicolson
          scheme is used for all steps except the first. The first step is done
          using the extrapolation scheme (see [2]) to damp the oscillations
          typical for Crank-Nicolson solution with non-smooth initial
          conditions.
      name: Python str. The name prefixed to the ops created by this class. If
        not supplied, the default name 'CrankNicolson' is used.
    ### References:
    [1]: P.A. Forsyth, K.R. Vetzal. Quadratic Convergence for Valuing American
      Options Using A Penalty Method. Journal on Scientific Computing, 2002.
      http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.28.9066&rep=rep1&type=pdf
    [2]: D. Lawson, J & Ll Morris, J. The Extrapolation of First Order Methods
      for Parabolic Partial Differential Equations. I. 1978 SIAM Journal on
      Numerical Analysis. 15. 1212-1224.
      https://epubs.siam.org/doi/abs/10.1137/0715082
    """
    self._name = name
    self._quadratic_coeff_fn = quadratic_coeff_fn

    self._lower_boundary_fn = lower_boundary_fn
    self._upper_boundary_fn = upper_boundary_fn

    self._linear_coeff_fn = (linear_coeff_fn or (lambda *args: 0.0))
    self._shift_coeff_fn = (shift_coeff_fn or (lambda *args: 0.0))
    self.time_marching_scheme = (
        time_marching_scheme or
        time_marching_schemes.crank_nicolson_with_oscillation_damping())

  @property
  def quadratic_coeff_fn(self):
    return self._quadratic_coeff_fn

  @property
  def lower_boundary_fn(self):
    return self._lower_boundary_fn

  @property
  def upper_boundary_fn(self):
    return self._upper_boundary_fn

  @property
  def linear_coeff_fn(self):
    return self._linear_coeff_fn

  @property
  def shift_coeff_fn(self):
    return self._shift_coeff_fn

  @property
  def name(self):
    return self._name

  def one_step(self, state):
    """Performs one step backwards in time.

    Given a space discretization grid `{x_1,.., x_N}`, time step `dt` the method
    updates values of the current estimates for the PDE solution
    {V^k(x_1),.., V^k(x_N)} using the explicit constraint timestep method
    (see (5.3) in [Forsyth, Vetzal][1]). Note, the method can propagate a batch
    of PDE's at the same time.

    ### Example. European call option pricing.
    ```python
    num_equations = 2  # Number of PDE
    num_grid_points = 1024  # Number of grid points
    dtype = np.float64
    # Build a log-uniform grid
    s_min = 0.01
    s_max = 300.
    grid = grids.uniform_grid(minimums=[s_min], maximums=[s_max],
                              sizes=[num_grid_points],
                              dtype=dtype)
    # Specify volatilities and interest rates for the options
    volatility = tf.constant([[0.3], [0.15]], dtype)
    rate = tf.constant([[0.01], [0.03]], dtype)
    expiry = 1.0
    strike = tf.constant([[50], [100]], dtype)
    # Define parabolic equation coefficients. In this case the coefficients
    # can be computed exactly but the same functions as below can be used to
    # get approximate values for general case.
    def quadratic_coeff_fn(t, x):
      del t
      return tf.square(volatility) * tf.square(x)

    def linear_coeff_fn(t, x):
      del t
      return rate * x

    def shift_coeff_fn(t, x):
      del t, x
      return rate

    def lower_boundary_fn(t):
      del t
      return dtype([0.0, 0.0])

    def upper_boundary_fn(t):
      return tf.squeeze(s_max - strike * tf.exp(-rate * (expiry - t)))

    solver_kernel = pde_kernels.ParabolicDifferentialEquationSolver(
        quadratic_coeff_fn,
        lower_boundary_fn,
        upper_boundary_fn,
        linear_coeff_fn, shift_coeff_fn)

    def time_step_fn(state):
      del state
      return tf.constant(0.01, dtype=dtype)
    def payoff_fn(state):
      option_bound = tf.nn.relu(state.coordinate_grid.locations[0] - strike)
      # Broadcast to the shape of value dimension, if necessary.
      option_bound = option_bound + tf.zeros_like(state.value_grid)
      return option_bound

    bgs = grid_stepper.BackwardGridStepper(
        expiry, solver_kernel.one_step, grid,
        time_step_fn=time_step_fn,
        value_dim=num_equations,
        dtype=dtype)
    bgs.transform_values(payoff_fn)
    bgs.step_back_to_time(0.0)
    estimate = bgs.state()
    grid_locations = estimate.coordinate_grid.locations[0]
    # Estimated prices of the option at the grid locations
    value_grid_first_option = estimate.value_grid[0, :]
    value_grid_second_option = estimate.value_grid[1, :]

    ```

    ### References:
    [1]: P.A. Forsyth, K.R. Vetzal. Quadratic Convergence for Valuing American
      Options Using A Penalty Method. Journal on Scientific Computing, 2002.
      http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.28.9066&rep=rep1&type=pdf

    Args:
      state: An instance of `GridStepperState` or a namedtuple containing the
        following attributes. `time`.  A scalar `Tensor` of float dtype. The
        current time. `time_step`. A positive scalar `Tensor` of float dtype.
        The next time step to take. `value_grid`. The function values on the
        grid. A tensor of shape `[num_equations, num_grid_points]`, where
        `num_grid_points` is the number of points on the grid and
        `num_equations` is the number of PDE equations to be propagated.
        `coordinate_grid`. Grid discretization along the space coordinate. An
        instance of `GridSpec` or a namedtuple containing the following
        attributes. `deltas`. The difference between consecutive elements of the
        grid. Represented as an array of `Tensor`s. Each `Tensor` is of a shape
        that will broadcast with a `Tensor` of shape broadcastable with
        `[num_equations, num_grid_points]`, where `num_grid_points` is the
        number of points on the grid and `num_equations` is the number of PDE
        equations to be propagated. `locations`. The full grid of coordinates.
        The grid is a single real `Tensor` of shape broadcastable with
        `[num_equations, num_grid_points]`, where `num_grid_points` is the
        number of points on the grid and `num_equations` is the number of PDE
        equations to be propagated.

    Returns:
      A `Tensor` of the same shape and dtype as value_grid. Corresponds to the
      grid values after one iteration of the solver.
    """
    with tf.compat.v1.name_scope(
        self.name, 'ParabolicDifferentialEquationSolver_one_step', [state]):
      matrix_constructor = (
          lambda t: self._construct_tridiagonal_matrix(state, t))

      return self.time_marching_scheme.apply(
          state.value_grid,
          state.time - state.time_step,
          state.time,
          state.num_steps_performed,
          matrix_constructor,
          self.lower_boundary_fn,
          self.upper_boundary_fn,
          backwards=True)

  def _construct_tridiagonal_matrix(self, state, t):
    """Constructs tridiagonal matrix to pass to the time marching scheme."""
    state_grid = state.coordinate_grid.locations[0][..., 1:-1]

    # Extract broadcasted grid deltas.
    diff = tf.broadcast_to(
        state.coordinate_grid.deltas[0],
        tf.shape(state.coordinate_grid.locations[0][..., 1:]))[..., 1:-1]

    # Get forward, backward and total differences.

    # Create forward paddings [[0, 0], [0, 0], .. [0, 1]]
    # Forward differences
    forward_paddings = tf.concat([
        tf.zeros([tf.rank(diff) - 1, 2], dtype=tf.int32),
        tf.constant([[0, 1]], dtype=tf.int32)
    ], 0)
    forward_diff = tf.pad(
        diff, forward_paddings, constant_values=tf.reduce_min(diff[..., -1]))
    # Create backward paddings [[0, 0], [0, 0], .. [1, 0]]
    backward_paddings = tf.concat([
        tf.zeros([tf.rank(diff) - 1, 2], dtype=tf.int32),
        tf.constant([[1, 0]], dtype=tf.int32)
    ], 0)
    # Backward differences
    backward_diff = tf.pad(
        diff, backward_paddings, constant_values=tf.reduce_min(diff[..., 0]))
    # Note that the total difference = 2 * central difference.
    total_diff = forward_diff + backward_diff

    # 3-diagonal matrix construction. See matrix `M` in [Forsyth, Vetzal][1].
    #  The `tridiagonal` matrix is of shape
    # `[value_dim, 3, num_grid_points]`.

    # Get the PDE coefficients and broadcast them to the shape of value grid.
    broadcast_shape = tf.shape(state.value_grid[..., 1:-1])

    quadratic_coeff = tf.convert_to_tensor(
        self._quadratic_coeff_fn(t, state_grid), dtype=state_grid.dtype)
    quadratic_coeff = tf.broadcast_to(quadratic_coeff, broadcast_shape)
    linear_coeff = tf.convert_to_tensor(
        self._linear_coeff_fn(t, state_grid), dtype=state_grid.dtype)
    linear_coeff = tf.broadcast_to(linear_coeff, broadcast_shape)
    shift_coeff = tf.convert_to_tensor(
        self._shift_coeff_fn(t, state_grid), dtype=state_grid.dtype)
    shift_coeff = tf.broadcast_to(shift_coeff, broadcast_shape)

    # The 3-diagonal matrix involves coefficients `gamma` and `beta` which
    # are referred to as `dxdx_coef` and `dx_coef`, respectively. This is done
    # to reflect that `dxdx_coef` is coming from the 2-nd order discretization
    # of `V_{xx}` and `dx_coef` is from 1-st order discretization of `V_{x}`,
    # where `V` is a solution to the PDE.

    temp = quadratic_coeff / total_diff
    dxdx_coef_1 = temp / forward_diff
    dxdx_coef_2 = temp / backward_diff

    dx_coef = linear_coeff / total_diff

    # The 3 main diagonals are constructed below. Note that all the diagonals
    # are of the same length
    upper_diagonal = (-dx_coef - dxdx_coef_1)

    lower_diagonal = (dx_coef - dxdx_coef_2)

    diagonal = shift_coeff - upper_diagonal - lower_diagonal

    return diagonal, upper_diagonal, lower_diagonal


# This is a check and review to make sure nothing was "lost" over break

# Aaron Fuller
# Period 1

# Variables declaratio and assignments
# Example
myVar = "Hello"
# Try, declare two variables 1 string and 1 number, and assign values
strVar = "This is a string variable"
numVar = 489
# While loop
# Example
x = 10
while x > 0:
	print(x)
	x = x - 1

# Try, print your name 100 times
# Don't do this
# print("Name\n"*100)
s = 100
while s > 0:
	print("Aaron")
	s -= 1

# String Concetenation
name = "Aaron"
print("Hello " + name)
# Make a variable with your favorite movie
# Print "My favorite movie is " and then the value stored in the variable
favMovie = input("What's your favorite movie?")
print("My favorite movie is " + favMovie)

# Inputs
# Example
myName = input("What is your name:")
print("Your name is: " + myName)
#Prompt for favorite song and print "your favorite song is: "
favSong = input("What is your favorite song: ")
print("Your favorite song is: " + favSong)

# Casting, changing the type of a variable
myNum = 489
print("My number is: " + str(myNum))
num1 = input("Enter number: ")
num1 = int(num1) + 10
print("num1 + 10 is " + str(num1))
# Ask for two numbers, add them, and print the answer
myNum1 = input("What is one number: ")
myNum2 = input("What is another number: ")
print("myNum1 + myNum2 = " + (myNum1 + myNum2))

# If/Else
# Example
num3 = int(input("Type a number: "))
if num3 > 100:
	print("Your number is greater than 100")

elif num3 == 100:
	print("Your number is equal to 100")

else:
	print("Your number is less than to 100")

#Ask if today is your birthday, if it is print Happy Birthday
day = input("Is it your birthday?")
if day =="Yes":
	print("YAY! Happy birthday")
else:
	print("Ok. Have a nice day")
inf = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100


num = int(input())
dic = []
record=[]

def contain(element, lists):
    for x in lists:
        if x[0] == element:
            return lists.index(x)
    return -1

for i in range(0, num):
    tempList = list(input().split(" "))
    element = [tempList[0], int(tempList[1])]
    if contain(tempList[0], dic) == -1:
        dic.append(element)
        record.append(element)
    else:
        element1 = dic[contain(tempList[0], dic)]
        element = [tempList[0], element[1] + element1[1]]
        record.append(element)
        dic.remove(element1)
        dic.append(element)

res = dic[0]
multipleMax=False
for el in dic:
    if res[1] < el[1]:
        res = el

for el in dic:
    if res[1]==el[1] and res!=el:
        multipleMax=True

if multipleMax:
    for el in record:
        if res[1]<=el[1]:
            res=el
            break

print(res[0])


from django.core.management.base import BaseCommand
from datetime import timedelta, datetime, time
from gastos.models import Categoria, Gasto
import random

def rango_de_fechas(fecha_inicio, fecha_fin):
    for n in range(int ((fecha_fin - fecha_inicio).days)):
        yield fecha_inicio + timedelta(days=n)

def es_laborable(dia):
    return dia.weekday() in [1,2,3,4,5,6]

def get_categoria_random():
    categoria = Categoria.objects.all()
    indice = random.randint(0, len(categoria)-1)
    return categoria[indice]

def get_descripcion_random():
    descripciones = ['pago', 'arreglo', 'mejora', 'inversion']
    indice = random.randint(0, len(descripciones)-1)
    return descripciones[indice]

def generar_gasto(dia):
    gasto = Gasto()
    gasto.fecha = dia
    gasto.categoria = get_categoria_random()
    gasto.monto = random.uniform(100000, 1000)
    gasto.descripcion = str(get_descripcion_random())
    gasto.save()

class Command(BaseCommand):
    def handle(self, *args, **options):
        fecha_inicio = datetime(2017, 1, 1)
        fecha_fin = datetime(2018, 1, 1)
        cargados = []
        for dia_actual in rango_de_fechas(fecha_inicio, fecha_fin):
            i = 0
            if es_laborable(dia_actual):
                while i < 5:
                    generar_gasto(dia_actual)
                    i += 1


from collections import OrderedDict
from pyramid.view import view_config
from pyramid.response import Response
from snovault.util import simple_path_ids
from urllib.parse import (
    parse_qs,
    urlencode,
)

import csv
import io
import json

import logging

log = logging.getLogger(__name__)

def includeme(config):
    config.add_route('batch_download', '/batch_download/{search_params}')
    config.add_route('metadata', '/metadata/{search_params}/{tsv}')
    config.add_route('peak_metadata', '/peak_metadata/{search_params}/{tsv}')
    config.scan(__name__)

# includes concatenated properties
_tsv_mapping = OrderedDict([
    ('File accession', ['files.title']),
    ('File format', ['files.file_type']),
    ('Output type', ['files.output_type']),
    ('Experiment accession', ['accession']),
    ('Assay', ['assay_term_name']),
    ('Biosample term id', ['biosample_term_id']),
    ('Biosample term name', ['biosample_term_name']),
    ('Biosample type', ['biosample_type']),
    ('Biosample life stage', ['replicates.library.biosample.life_stage']),
    ('Biosample sex', ['replicates.library.biosample.sex']),
    ('Biosample organism', ['replicates.library.biosample.organism.scientific_name']),
    ('Biosample treatments', ['replicates.library.biosample.treatments.treatment_term_name']),
    ('Biosample subcellular fraction term name', ['replicates.library.biosample.subcellular_fraction_term_name']),
    ('Biosample phase', ['replicates.library.biosample.phase']),
    ('Biosample synchronization stage', ['replicates.library.biosample.fly_synchronization_stage',
                                         'replicates.library.biosample.worm_synchronization_stage',
                                         'replicates.library.biosample.post_synchronization_time',
                                         'replicates.library.biosample.post_synchronization_time_units']),
    ('Experiment target', ['target.name']),
    ('Antibody accession', ['replicates.antibody.accession']),
    ('Library made from', ['replicates.library.nucleic_acid_term_name']),
    ('Library depleted in', ['replicates.library.depleted_in_term_name']),
    ('Library extraction method', ['replicates.library.extraction_method']),
    ('Library lysis method', ['replicates.library.lysis_method']),
    ('Library crosslinking method', ['replicates.library.crosslinking_method']),
    ('Experiment date released', ['date_released']),
    ('Project', ['award.project']),
    ('RBNS protein concentration', ['files.replicate.rbns_protein_concentration', 'files.replicate.rbns_protein_concentration_units']),
    ('Library fragmentation method', ['files.replicate.library.fragmentation_method']),
    ('Library size range', ['files.replicate.library.size_range']),
    ('Biosample Age', ['files.replicate.library.biosample.age_display']),
    ('Biological replicate(s)', ['files.biological_replicates']),
    ('Technical replicate', ['files.replicate.technical_replicate_number']),
    ('Read length', ['files.read_length']),
    ('Run type', ['files.run_type']),
    ('Paired end', ['files.paired_end']),
    ('Paired with', ['files.paired_with']),
    ('Derived from', ['files.derived_from.accession']),
    ('Size', ['files.file_size']),
    ('Lab', ['files.lab.title']),
    ('md5sum', ['files.md5sum']),
    ('File download URL', ['files.href']),
    ('Assembly', ['files.assembly']),
    ('Platform', ['files.platform.title'])
])

def get_file_uuids(result_dict):
    file_uuids = []
    for item in result_dict['@graph']:
        for file in item['files']:
            file_uuids.append(file['uuid'])
    return list(set(file_uuids))

def get_biosample_accessions(file_json, experiment_json):
    for f in experiment_json['files']:
        if file_json['uuid'] == f['uuid']:
            accession = f.get('replicate', {}).get('library', {}).get('biosample', {}).get('accession')
            if accession:
                return accession
    accessions = []
    for replicate in experiment_json.get('replicates', []):
        accession = replicate['library']['biosample']['accession']
        accessions.append(accession)
    return ', '.join(list(set(accessions)))

def get_peak_metadata_links(request):
    if request.matchdict.get('search_params'):
        search_params = request.matchdict['search_params']
    else:
        search_params = request.query_string

    peak_metadata_tsv_link = '{host_url}/peak_metadata/{search_params}/peak_metadata.tsv'.format(
        host_url=request.host_url,
        search_params=search_params
    )
    peak_metadata_json_link = '{host_url}/peak_metadata/{search_params}/peak_metadata.json'.format(
        host_url=request.host_url,
        search_params=search_params
    )
    return [peak_metadata_tsv_link, peak_metadata_json_link]

@view_config(route_name='peak_metadata', request_method='GET')
def peak_metadata(context, request):
    param_list = parse_qs(request.matchdict['search_params'])
    param_list['field'] = []
    header = ['assay_term_name', 'coordinates', 'target.label', 'biosample.accession', 'file.accession', 'experiment.accession']
    param_list['limit'] = ['all']
    path = '/region-search/?{}&{}'.format(urlencode(param_list, True),'referrer=peak_metadata')
    results = request.embed(path, as_user=True)
    uuids_in_results = get_file_uuids(results)
    rows = []
    json_doc = {}
    for row in results['peaks']:
        if row['_id'] in uuids_in_results:
            file_json = request.embed(row['_id'])
            experiment_json = request.embed(file_json['dataset'])
            for hit in row['inner_hits']['positions']['hits']['hits']:
                data_row = []
                coordinates = '{}:{}-{}'.format(hit['_index'], hit['_source']['start'], hit['_source']['end'])
                file_accession = file_json['accession']
                experiment_accession = experiment_json['accession']
                assay_name = experiment_json['assay_term_name']
                target_name = experiment_json.get('target', {}).get('label') # not all experiments have targets
                biosample_accession = get_biosample_accessions(file_json, experiment_json)
                data_row.extend([assay_name, coordinates, target_name, biosample_accession, file_accession, experiment_accession])
                rows.append(data_row)
                if assay_name not in json_doc:
                    json_doc[assay_name] = []
                else:
                    json_doc[assay_name].append({
                        'coordinates': coordinates,
                        'target.name': target_name,
                        'biosample.accession': list(biosample_accession.split(', ')),
                        'file.accession': file_accession,
                        'experiment.accession': experiment_accession
                    })
    if 'peak_metadata.json' in request.url:
        return Response(
            content_type='text/plain',
            body=json.dumps(json_doc),
            content_disposition='attachment;filename="%s"' % 'peak_metadata.json'
        )
    fout = io.StringIO()
    writer = csv.writer(fout, delimiter='\t')
    writer.writerow(header)
    writer.writerows(rows)
    return Response(
        content_type='text/tsv',
        body=fout.getvalue(),
        content_disposition='attachment;filename="%s"' % 'peak_metadata.tsv'
    )

@view_config(route_name='metadata', request_method='GET')
def metadata_tsv(context, request):
    param_list = parse_qs(request.matchdict['search_params'])
    if 'referrer' in param_list:
        search_path = '/{}/'.format(param_list.pop('referrer')[0])
    else:
        search_path = '/search/'
    param_list['field'] = []
    header = []
    file_attributes = []
    for prop in _tsv_mapping:
        header.append(prop)
        param_list['field'] = param_list['field'] + _tsv_mapping[prop]
        if _tsv_mapping[prop][0].startswith('files'):
            file_attributes = file_attributes + [_tsv_mapping[prop][0]]
    param_list['limit'] = ['all']
    path = '{}?{}'.format(search_path, urlencode(param_list, True))
    results = request.embed(path, as_user=True)
    rows = []
    for row in results['@graph']:
        if row['files']:
            exp_data_row = []
            for column in header:
                if not _tsv_mapping[column][0].startswith('files'):
                    temp = []
                    for c in _tsv_mapping[column]:
                        c_value = []
                        for value in simple_path_ids(row, c):
                            if str(value) not in c_value:
                                c_value.append(str(value))
                        if c == 'replicates.library.biosample.post_synchronization_time' and len(temp):
                            if len(c_value):
                                temp[0] = temp[0] + ' + ' + c_value[0]
                        elif len(temp):
                            if len(c_value):
                                temp = [x + ' ' + c_value[0] for x in temp]
                        else:
                            temp = c_value
                    exp_data_row.append(', '.join(list(set(temp))))
            f_attributes = ['files.title', 'files.file_type',
                            'files.output_type']
            for f in row['files']:
                if 'files.file_type' in param_list:
                    if f['file_type'] not in param_list['files.file_type']:
                        continue
                f['href'] = request.host_url + f['href']
                f_row = []
                for attr in f_attributes:
                    f_row.append(f[attr[6:]])
                data_row = f_row + exp_data_row
                for prop in file_attributes:
                    if prop in f_attributes:
                        continue
                    path = prop[6:]
                    temp = []
                    for value in simple_path_ids(f, path):
                        temp.append(str(value))
                    if prop == 'files.replicate.rbns_protein_concentration':
                        if 'replicate' in f and 'rbns_protein_concentration_units' in f['replicate']:
                            temp[0] = temp[0] + ' ' + f['replicate']['rbns_protein_concentration_units']
                    if prop == 'files.paired_with':
                        # chopping of path to just accession
                        if len(temp):
                            new_values = [t[7:-1] for t in temp]
                            temp = new_values
                    data = list(set(temp))
                    data.sort()
                    data_row.append(', '.join(data))
                rows.append(data_row)
    fout = io.StringIO()
    writer = csv.writer(fout, delimiter='\t')
    writer.writerow(header)
    writer.writerows(rows)
    return Response(
        content_type='text/tsv',
        body=fout.getvalue(),
        content_disposition='attachment;filename="%s"' % 'metadata.tsv'
    )

@view_config(route_name='batch_download', request_method='GET')
def batch_download(context, request):
    # adding extra params to get required columns
    param_list = parse_qs(request.matchdict['search_params'])
    param_list['field'] = ['files.href', 'files.file_type']
    param_list['limit'] = ['all']
    path = '/search/?%s' % urlencode(param_list, True)
    results = request.embed(path, as_user=True)
    metadata_link = '{host_url}/metadata/{search_params}/metadata.tsv'.format(
        host_url=request.host_url,
        search_params=request.matchdict['search_params']
    )
    files = [metadata_link]
    if 'files.file_type' in param_list:
        for exp in results['@graph']:
            for f in exp['files']:
                if f['file_type'] in param_list['files.file_type']:
                    files.append('{host_url}{href}'.format(
                        host_url=request.host_url,
                        href=f['href']
                    ))
    else:
        for exp in results['@graph']:
            for f in exp['files']:
                files.append('{host_url}{href}'.format(
                    host_url=request.host_url,
                    href=f['href']
                ))
    return Response(
        content_type='text/plain',
        body='\n'.join(files),
        content_disposition='attachment; filename="%s"' % 'files.txt'
    )


from src.Messages.Message import Message
from src.Protocols.Paxos import PaxosMsg

class FigPaxos:

    name = "FigPaxos Protocol"

    def __init__(self, **kwargs):
        self.env = kwargs['env']
        self.n = self.env.get_n()
        self.id = -1
        self.current_belief = None
        self.current_ballots = {}
        self.maximum_ballot_no = (-1, -1)
        self.last_accepted = None
        self.last_accepted_ballot_no = (-1, -1)
        self.current_proposals = {}
        self.current_proposal_ballot_nos = {}
        self.reply_lists = {}
        self.accept_counts = {}
        self.decision = None

    # Received some msg
    def recv(self, msg):
        self.id = self.env.get_id(self)
        msgtype = msg.get_extraction()[0]
        if msgtype is PaxosMsg.INIT:
            self.init_ballot(msg)
        elif msgtype is PaxosMsg.PREPARE:
            self.prepare(msg)
        elif msgtype is PaxosMsg.REPLY:
            self.reply(msg)
        elif msgtype is PaxosMsg.PROPOSE:
            self.propose(msg)
        elif msgtype is PaxosMsg.ACCEPT:
            self.accept(msg)
        elif msgtype is PaxosMsg.DECIDE:
            self.decide(msg)
        else:
            raise RuntimeError

    # I am the leader to initiate a new ballot
    def init_ballot(self, msg):
        new_ballot_no = (self.maximum_ballot_no[0] + 1, self.id)
        if self.last_accepted is None:
            new_proposal = msg.get_extraction()[1]
        else:
            new_proposal = self.last_accepted
        new_msg = Message(self.id, [PaxosMsg.PREPARE, new_ballot_no, new_proposal])
        if self.current_belief != new_proposal:
            self.current_ballots.clear()
            self.current_belief = new_proposal
        self.maximum_ballot_no = new_ballot_no
        self.current_ballots[new_ballot_no] = None
        self.current_proposals[new_ballot_no] = msg.get_extraction()[1]
        self.current_proposal_ballot_nos[new_ballot_no] = (-1, -1)
        self.reply_lists[new_ballot_no] = []
        self.env.put_broadcast(self, self.id, new_msg)

    # Received a prepare message
    def prepare(self, msg):
        ballot_no = msg.get_extraction()[1]
        assumption = msg.get_extraction()[2]
        if ballot_no <= self.maximum_ballot_no:
            if assumption == self.current_belief:
                if ballot_no not in self.current_ballots:
                    self.current_ballots[ballot_no] = None
                new_msg = Message(self.id, [PaxosMsg.REPLY, ballot_no, self.last_accepted, self.last_accepted_ballot_no])
                self.env.put_packet(self, self.id, new_msg, msg.get_sender())
        else:
            if assumption != self.current_belief:
                self.current_ballots.clear()
                self.current_belief = assumption
            self.maximum_ballot_no = ballot_no
            self.current_ballots[ballot_no] = None
            new_msg = Message(self.id, [PaxosMsg.REPLY, ballot_no, self.last_accepted, self.last_accepted_ballot_no])
            self.env.put_packet(self, self.id, new_msg, msg.get_sender())

    # Received a reply message
    def reply(self, msg):
        content = msg.get_extraction()
        ballot_no = content[1]
        last_acc = content[2]
        last_acc_no = content[3]
        if ballot_no in self.current_ballots:
            if len(self.reply_lists[ballot_no]) * 2 <= self.n:
                if self.current_proposal_ballot_nos[ballot_no] < last_acc_no:
                    self.current_proposals[ballot_no] = last_acc
                    self.current_proposal_ballot_nos[ballot_no] = last_acc_no
                self.reply_lists[ballot_no].append(msg.get_sender())
                if len(self.reply_lists[ballot_no]) * 2 > self.n:
                    if self.current_proposals[ballot_no] != self.current_belief:
                        self.current_ballots.pop(ballot_no)
                    else:
                        self.accept_counts[ballot_no]= 0
                        new_msg = Message(self.id, [PaxosMsg.PROPOSE, ballot_no])
                        for id in self.reply_lists[ballot_no]:
                            self.env.put_packet(self, self.id, new_msg, id)

    # Received a propose message
    def propose(self, msg):
        content = msg.get_extraction()
        ballot_no = content[1]
        if ballot_no in self.current_ballots:
            new_msg = Message(self.id, [PaxosMsg.ACCEPT, ballot_no])
            if self.last_accepted_ballot_no is None or ballot_no > self.last_accepted_ballot_no:
                self.last_accepted = self.current_belief
                self.last_accepted_ballot_no = ballot_no
            self.env.put_packet(self, self.id, new_msg, msg.get_sender())

    # Received an accept message
    def accept(self, msg):
        ballot_no = msg.get_extraction()[1]
        if ballot_no in self.current_ballots:
            self.accept_counts[ballot_no] += 1
            if self.accept_counts[ballot_no] == len(self.reply_lists[ballot_no]):
                new_msg = Message(self.id, [PaxosMsg.DECIDE, self.current_belief])
                self.env.put_broadcast(self, self.id, new_msg)

    # Received a decide message
    def decide(self, msg):
        proposal = msg.get_extraction()[1]
        if self.decision is not None and self.decision is not proposal:
            raise RuntimeError
        if self.decision is None:
            self.decision = proposal
            self.env.put_output(self, proposal)


# -*- coding:utf-8 -*-

import numpy as np
import json
from sklearn.externals import joblib
import os
import sys

if __name__ == "__main__":
    if len(sys.argv) != 5:
        print("Please enter the model path data path avg_overdue path result filename")
        sys.exit(-1)
    model_path = sys.argv[1]
    data_path = sys.argv[2]
    avg_overdue_path = sys.argv[3]
    result_fn = sys.argv[4]
    if not (os.path.exists(model_path) and os.path.exists(data_path) and os.path.exists(avg_overdue_path)):
        print("model path or data path or avg_overdue path is not exist!")
        sys.exit(-1)

    model = joblib.load(model_path)
    data = np.load(data_path)
    print(data.shape)
    score = model.predict_proba(data)
    risk_score = score[:, 1]
    sorted_risk_score = sorted(risk_score.tolist())
    data_num = len(sorted_risk_score)
    avg_overdue = json.load(open(avg_overdue_path))
    f = open(result_fn, 'w')
    head = "pass_rate\tthreshold\tavg_score\tpre_overdue"
    print(head)
    f.write(head + "\n")
    for i in np.arange(0.3, 1.0, 0.03):
        cut = int(i*data_num)
        cut_value = sorted_risk_score[cut]
        cut_avg = np.average(sorted_risk_score[:cut])
        pre_overdue = -1.0
        for j in range(len(avg_overdue)):
            if cut_avg < avg_overdue[j][0]:
                up = float(cut_avg - avg_overdue[j-1][0])
                down = float(avg_overdue[j][0] - cut_avg)
                total = float(avg_overdue[j][0] - avg_overdue[j-1][0])
                pre_overdue = (up/total) * avg_overdue[j][1] + (down/total) * avg_overdue[j-1][1]
                break
        item = "%f\t%f\t%f\t%f" % (i, cut_value, cut_avg,pre_overdue)
        print(item)
        f.write(item + "\n")
    f.close()


Pick_a_number = input("pick a number: ")

def character_name():
    char_name = input("Please enter your name: ")
    print(f"Hello {char_name}. Welocme to the game.")

if(Pick_a_number == "5"):
    character_name()
else:
    print("You did not pick the right number.")

character_name()


'''
    Создать (программно) текстовый файл, записать в него программно
    набор чисел, разделенных пробелами. Программа должна
    подсчитывать сумму чисел в файле и выводить ее на экран.
'''

try:
    with open('task_5.txt', 'w+') as file:
        line = input('Введите цифры: ')
        file.writelines(line)
        num = line.split()
        print(sum(map(int, num)))
except IOError:
    print('Ошибка в файле')
except ValueError:
    print('Ошибка в синтаксисе! Можно вводить только числа!')


# 4. Создать (не программно) текстовый файл со следующим содержимым:
# One — 1
# Two — 2
# Three — 3
# Four — 4
# Необходимо написать программу, открывающую файл на чтение и считывающую построчно данные.
# При этом английские числительные должны заменяться на русские.
# Новый блок строк должен записываться в новый текстовый файл.
f_obj = open('../practical5/task4.txt', encoding='utf-8')
lines = f_obj.readlines()

for i, items in enumerate(lines):
    items = items.rstrip('\n')
    items = items.split(' — ')
    item = items[0]
    if items[1] == '1':
        item = 'Один - '
    elif items[1] == '2':
        item = 'Два - '
    elif items[1] == '3':
        item = 'Три - '
    elif items[1] == '4':
        item = 'Четыре - '
    lines[i] = item + items[1]
print(lines)

# Образец решения:
with open('../practical5/task4.txt', encoding='utf-8') as f:
    lines = f.readlines()

with open('../practical5/task4rus.txt', 'w', encoding='utf-8') as f:
    for line in lines:
        if '1' in line:
            line = line.replace('One', 'Один')
        elif '2' in line:
            line = line.replace('Two', 'Два')
        elif '3' in line:
            line = line.replace('Three', 'Три')
        elif '4' in line:
            line = line.replace('Four', 'Четыре')
        f.write(line)

# на заметку:
# не правильно выполнил задание, нужно было не вывести на экран а сохранить в новый файл(


#!/usr/bin/env python
'''
@author Luke Campbell <LCampbell@ASAScience.com>
@date Wed Aug 15 10:13:02 EDT 2012
@file ion/processes/data/transforms/mux.py
@brief Multiplexing and Demultiplexing Transforms
'''
from ion.core.process.transform import TransformStreamListener
from pyon.ion.stream import StreamPublisher
from pyon.util.log import log

class DemuxTransform(TransformStreamListener):
    def on_start(self): #pragma no cover
        log.info('Starting Demuxer')
        TransformStreamListener.on_start(self)
        log.info('----------')
        self.output_streams = self.CFG.get_safe('process.out_streams')

        if self.output_streams is None or not isinstance(self.output_streams, list):
            log.error('(%s) Failed to demux, I/O configuration is incorrect: (%s)', self.id, self.output_streams)
            return
        self.publishers = []
        for stream in self.output_streams:
            log.info("   -> %s", stream)
            self.publishers.append(StreamPublisher(process=self, stream_id=stream))

    def on_quit(self): #pragma no cover
        TransformStreamListener.on_quit(self)
        for publisher in self.publishers:
            publisher.close()

    def recv_packet(self, msg, stream_route, stream_id):
        log.info('Rebroadcasting message')

        self.publish(msg,None)

    def publish(self, msg, to_name):
        for publisher in self.publishers:
            publisher.publish(msg)


import os
import numpy as np
import copy
from sklearn.manifold import TSNE
from sklearn.decomposition import PCA, FastICA, NMF
from sklearn.metrics import normalized_mutual_info_score, adjusted_mutual_info_score
from sklearn.cluster import KMeans, AffinityPropagation, AgglomerativeClustering
from sklearn.neighbors import NearestNeighbors
import scipy.spatial
import scipy.sparse
import community
import networkx
import logging

# if syslog is True，append log to /var/log/syslog
def create_logger(app_name, logfilename=None, level=logging.INFO,
                  console=False, syslog=False):
    """ Build and return a custom logger. Accepts the application name,
    log filename, loglevel and console logging toggle and syslog toggle """

    log = logging.getLogger(app_name)
    log.setLevel(level)
    # Add file handler
    if logfilename is not None:
        log.addHandler(logging.FileHandler(logfilename))

    if syslog:
        log.addHandler(logging.handlers.SysLogHandler(address='/dev/log'))

    if console:
        log.addHandler(logging.StreamHandler())

    # Add formatter
    for handle in log.handlers:
        formatter = logging.Formatter('%(asctime)s : %(levelname)-8s - %(message)s',
                                      datefmt='%Y-%m-%d %H:%M:%S')
        handle.setFormatter(formatter)
    return log

def cluster_acc(y_true, y_pred):
    """
    Calculate clustering accuracy. Require scikit-learn installed

    # Arguments
        y: true labels, numpy.array with shape `(n_samples,)`
        y_pred: predicted labels, numpy.array with shape `(n_samples,)`

    # Return
        accuracy, in [0,1]
    """
    y_true = y_true.astype(np.int64)
    y_pred = y_pred.astype(np.int64)
    assert y_pred.size == y_true.size, 'y_pred.size {} y_true.size {}'.format(y_pred.size, y_true.size)
    D = max(y_pred.max(), y_true.max()) + 1
    w = np.zeros((D, D), dtype=np.int64)
    for i in range(y_pred.size):
        w[y_pred[i], y_true[i]] += 1
    from sklearn.utils.linear_assignment_ import linear_assignment
    ind = linear_assignment(w.max() - w)
    return sum([w[i, j] for i, j in ind]) * 1.0 / y_pred.size

class SingleCellData(object):

    def __init__(self, geoid):
        self.data_path = os.path.join('dataset', geoid)
        if not os.path.exists(self.data_path):
            raise FileNotFoundError("don't find single cell {} data".format(geoid))

        self.cell_num = None
        self.gene_num = None
        self.label_num = None
        self.id2gene = None
        self.gene2id = None
        self.id2cell = None
        self.cell2id = None
        self.id2label = None
        self.label2id = None
        self.expression_matrix = None
        self.labels = None
        self.__load_data()

    def __load_data(self):
        # load all cell name and statistic the number of genes
        with open(os.path.join(self.data_path, 'data.txt')) as f:
            cell_line = f.readline().strip()
            all_cells = cell_line.split('\t')[1:]
            self.cell_num = len(all_cells)
            self.id2cell = all_cells
            self.cell2id = {c_name:i for i, c_name in enumerate(self.id2cell)}
            gene_num = 0
            for l in f:
                l = l.strip()
                if l:
                    gene_num += 1
            self.gene_num = gene_num
            self.expression_matrix = np.zeros((self.cell_num, self.gene_num))
        self.id2gene = []
        with open(os.path.join(self.data_path, 'data.txt')) as f:
            f.readline()
            for i in range(self.gene_num):
                l = f.readline().strip()
                items = l.split('\t')
                gene_name = items[0]
                self.id2gene.append(gene_name)
                assert len(items[1:]) == self.cell_num
                for j, v in enumerate(items[1:]):
                    v = float(v)
                    self.expression_matrix[j, i] = float(v)
        self.gene2id = {g_name:i for i, g_name in enumerate(self.id2gene)}
        self.expression_matrix = np.log(self.expression_matrix + 1)

        with open(os.path.join(self.data_path, 'label.txt')) as f:
            label_names = []
            for l in f:
                l = l.strip()
                if l:
                    label_names.append(l)
            assert len(label_names) == self.cell_num
            self.id2label = list(set(label_names))
            self.label2id = {l_name:i for i, l_name in enumerate(self.id2label)}
            self.label_num = len(self.id2label)
            self.labels = np.array([self.label2id[l_name] for l_name in label_names], dtype=np.int64)

    @staticmethod
    def get_shrunk_data_by_std(single_cell_dataset, gene_num):
        assert isinstance(single_cell_dataset, SingleCellData)
        if single_cell_dataset.gene_num <= gene_num:
            print('given number is less than data\'s gene num')
            return single_cell_dataset
        gene_std = np.std(single_cell_dataset.expression_matrix, axis=0)
        selected_gene_id_lst = np.argsort(-gene_std)[:gene_num]
        id2gene = [single_cell_dataset.id2gene[i] for i in selected_gene_id_lst]
        gene2id = {g_name: i for i, g_name in enumerate(id2gene)}
        expression_matrix = np.ascontiguousarray(single_cell_dataset.expression_matrix[:, selected_gene_id_lst])
        data_copy = copy.deepcopy(single_cell_dataset)
        data_copy.id2gene = id2gene
        data_copy.gene2id = gene2id
        data_copy.gene_num = len(id2gene)
        data_copy.expression_matrix = expression_matrix
        return data_copy

class FeatureTransform(object):

    def __init__(self):
        self.verbose = True
        self.random_state = 0
        self.raw_feat = None
        self.transformed_feat = None
        self.labels = None
        self.tsne_feat = None

    def fit_transform(self, feat):
        raise NotImplementedError('not implement')

    def set_random_state(self, random_state):
        self.random_state = random_state

    def set_verbose(self, verbose):
        self.verbose = verbose

    def set_feature(self, raw_feat):
        self.raw_feat = raw_feat
        self.transformed_feat = self.fit_transform(raw_feat)

    def set_labels(self, labels):
        self.labels = labels

    def dump_distance_matrix(self, dumpdir):
        distance = cal_distance_matrix(self.transformed_feat, self.labels)
        np.savetxt(os.path.join(dumpdir, 'distance_matrix.txt'), distance)

    def dump_results(self, dumpdir):
        # self.dump_transformed_feat(os.path.join(dumpdir, 't_feat.txt'))
        self.dump_tsne_feat(os.path.join(dumpdir, 'v_feat.txt'))
        # self.dump_distance_matrix(dumpdir)
        with open(os.path.join(dumpdir, 'labels.txt'),'w') as f:
            for l in self.labels:
                f.write("{}\n".format(int(l)))

    def dump_transformed_feat(self, filename):
        assert self.transformed_feat is not None
        np.savetxt(filename, self.transformed_feat)

    def dump_tsne_feat(self, filename):
        if self.tsne_feat is None:
            assert self.transformed_feat is not None
            tsne = TSNE(n_components=2, verbose=self.verbose, random_state=self.random_state)
            tsne_feat = tsne.fit_transform(self.transformed_feat)
            self.tsne_feat = tsne_feat
        np.savetxt(filename, self.tsne_feat)

class IdentityTransform(FeatureTransform):

    def __init__(self, n_components=-1):
        super().__init__()
        self.n_components = n_components

    def fit_transform(self, feat):
        return feat

class PCAFeatureTransform(FeatureTransform):

    def __init__(self, n_components):
        super().__init__()
        self.n_components = n_components
        self.pca = PCA(n_components=n_components, random_state=self.random_state)

    def fit_transform(self, feat):
        t_feat = self.pca.fit_transform(feat)
        return t_feat

class ICAFeatureTransform(FeatureTransform):

    def __init__(self, n_components):
        super().__init__()
        self.n_components = n_components
        self.ica = FastICA(n_components=n_components, random_state=self.random_state)

    def fit_transform(self, feat):
        t_feat = self.ica.fit_transform(feat)
        return t_feat

class NMFFeatureTransform(FeatureTransform):

    def __init__(self, n_components):
        super().__init__()
        self.n_components = n_components
        self.nmf = NMF(n_components=n_components, random_state=self.random_state, verbose=True)

    def fit_transform(self, feat):
        t_feat = self.nmf.fit_transform(feat)
        return t_feat

class ClusteringModel(object):

    def __init__(self):
        self.random_state = 0
        self.verbose = True
        self.raw_feat = None
        self.pred_labels = None
        self.labels = None

    def fit_predict(self, feat):
        raise NotImplementedError('not implement')

    def set_random_state(self, random_state):
        self.random_state = random_state

    def set_verbose(self, verbose):
        self.verbose = verbose

    def set_feature(self, raw_feat):
        self.raw_feat = raw_feat
        pred = self.fit_predict(self.raw_feat)
        self.pred_labels = pred

    def set_labels(self, labels):
        self.labels = labels

    def dump_distance_matrix(self, dumpdir):
        distance = cal_distance_matrix(self.raw_feat, self.pred_labels)
        np.savetxt(os.path.join(dumpdir, 'distance_matrix.txt'), distance)

    def dump_results(self, dumpdir):
        assert self.pred_labels is not None
        # self.dump_distance_matrix(dumpdir)
        # np.savetxt(os.path.join(dumpdir, 'pred.txt'), self.pred_labels)
        with open(os.path.join(dumpdir, 'pred.txt'), 'w') as f:
            for l in self.pred_labels:
                f.write("{}\n".format(int(l)))
        if self.labels is not None:
            # np.savetxt(os.path.join(dumpdir, 'labels.txt'), self.labels)
            with open(os.path.join(dumpdir, 'labels.txt'), 'w') as f:
                for l in self.labels:
                    f.write("{}\n".format(int(l)))
            acc = cluster_acc(self.labels, self.pred_labels)
            nmi = normalized_mutual_info_score(self.labels, self.pred_labels)
            ari = adjusted_mutual_info_score(self.labels, self.pred_labels)
            with open(os.path.join(dumpdir, 'scores.txt'), 'w') as f:
                f.write('acc: {} nmi: {} ari: {}\n'.format(acc, nmi, ari))

def cal_distance_matrix(latent, labels=None):
    if labels is None:
        order_latent = latent
    else:
        order_latent = np.vstack([x for _, x in sorted(zip(labels, latent), key=lambda pair: pair[0])])
    distance = scipy.spatial.distance_matrix(order_latent, order_latent)
    return distance

class KMeansModel(ClusteringModel):

    def __init__(self, n_clusters):
        super().__init__()
        self.n_clusters = n_clusters
        self.kmeans = KMeans(n_clusters=self.n_clusters,
                             random_state=self.random_state,
                             verbose=self.verbose,
                             n_init=5, n_jobs=5)

    def fit_predict(self, feat):
        pred = self.kmeans.fit_predict(feat)
        return pred

class HierarchicalModel(ClusteringModel):

    def __init__(self, n_clusters):
        super().__init__()
        self.n_clusters = n_clusters
        self.hierachical = AgglomerativeClustering(n_clusters=n_clusters)

    def fit_predict(self, feat):
        pred = self.hierachical.fit_predict(feat)
        return pred

class APModel(ClusteringModel):

    def __init__(self, n_clusters):
        super().__init__()
        self.n_clusters = n_clusters
        self.ap = AffinityPropagation(verbose=True)

    def fit_predict(self, feat):
        pred = self.ap.fit_predict(feat)
        return pred

class DensityPeakModel(ClusteringModel):

    def __init__(self, n_clusters=10):
        super().__init__()
        self.n_clusters= n_clusters

    def fit_predict(self, feat):
        import pydpc

        class MyDensityPeak(pydpc.Cluster):

            def __init__(self, feat, n_clusters):
                super().__init__(feat, fraction=0.02, autoplot=False)
                self.MY_n_clusters = n_clusters

            def _get_cluster_indices(self):
                data_size = self.density.shape[0]
                density_idx = np.argsort(-self.density)
                delta_idx = np.argsort(-self.delta)
                ideal_n_clusters = self.MY_n_clusters
                idx = ideal_n_clusters
                while True:
                    tmp_density_idx = density_idx[:idx]
                    tmp_delta_idx = delta_idx[:idx]
                    n = len(set(tmp_density_idx).intersection(set(tmp_delta_idx)))
                    if n >= ideal_n_clusters:
                        break
                    idx += 1
                    if idx >= data_size / 20 or idx >= 1.5 * ideal_n_clusters:
                        break
                min_density = self.density[density_idx[idx]]
                min_delta = self.delta[delta_idx[idx]]

                self.clusters = np.intersect1d(
                    np.where(self.density > min_density)[0],
                    np.where(self.delta > min_delta)[0], assume_unique=True).astype(np.intc)
                self.nclusters = self.clusters.shape[0]

        clu = MyDensityPeak(feat, n_clusters=self.n_clusters)
        clu.assign(20, 1.5)
        pred = clu.membership
        return pred

class LouvainModel(ClusteringModel):

    def __init__(self, n_clusters=0, n_neighbors=10):
        super().__init__()
        self.n_neighbors = n_neighbors

    def fit_predict(self, feat):
        nne = NearestNeighbors(n_neighbors=self.n_neighbors + 1)
        nne.fit(feat)
        kmatrix = nne.kneighbors_graph(feat) - scipy.sparse.identity(feat.shape[0])
        G = networkx.from_scipy_sparse_matrix(kmatrix)
        partition = community.best_partition(G)
        pred = []
        for i in range(feat.shape[0]):
            pred.append(partition[i])
        return np.array(pred)

if __name__ == '__main__':
    # INFO:root:Addition of 10 and 20 produces 30
    logging.basicConfig(filename='log.txt',
                        level=logging.DEBUG,
                        format='%(asctime)s %(message)s')

    def clustering_pipeline(sc_dataset, DR_methods, Cluster_methods, feature_dim_lst, dumpdir):
        # the list n_neighbors is used to lovain community detection algorithm
        n_neighbors_lst = [20, 50, 60, 70, 80, 100, 120, 150, 200, 500, 1000]
        assert isinstance(sc_dataset, SingleCellData)
        n_clusters = sc_dataset.label_num
        gene_num = sc_dataset.gene_num
        for dr_method in DR_methods.keys():
            for n_component in feature_dim_lst:
                if dr_method == 'identity':
                    dr_results_dir = os.path.join(dumpdir, dr_method)
                    logging.info('Begin to run {}'.format(dr_method))
                    n_component = sc_data.gene_num
                else:
                    dr_results_dir = os.path.join(dumpdir, '{}_{}'.format(dr_method, n_component))
                    # gene_num must be less than n_component
                    if gene_num < n_component:
                        continue
                    logging.info('Begin to run {}_{}'.format(dr_method, n_component))
                if not os.path.exists(dr_results_dir):
                    os.makedirs(dr_results_dir)
                dr_model = DR_methods[dr_method](n_components=n_component)
                dr_model.set_feature(sc_dataset.expression_matrix)
                dr_model.set_labels(sc_dataset.labels)
                logging.info('Dump tsne feat and distance matrix for {}_{}'.format(dr_method, n_component))
                dr_model.dump_results(dr_results_dir)
                for cluster_method in Cluster_methods.keys():
                    logging.info('Begin to run {} clustering for {}_{}'.format(cluster_method, dr_method, n_component))
                    for n_neighbor in n_neighbors_lst:
                        #
                        if cluster_method != 'lovain':
                            cluster_results_dir = os.path.join(dr_results_dir, cluster_method)
                        else:
                            cluster_results_dir = os.path.join(dr_results_dir, '{}_{}'.format(cluster_method, n_neighbor))
                        if not os.path.exists(cluster_results_dir):
                            os.makedirs(cluster_results_dir)
                        #
                        if cluster_method != 'lovain':
                            cluster_model = Cluster_methods[cluster_method](n_clusters=n_clusters)
                        else:
                            cluster_model = Cluster_methods[cluster_method](n_clusters=n_clusters, n_neighbors=n_neighbor)
                        cluster_model.set_feature(dr_model.transformed_feat)
                        cluster_model.set_labels(sc_dataset.labels)
                        logging.info('Dump {} clustering for {}_{}\' results'.format(cluster_method, dr_method, n_component))
                        cluster_model.dump_results(cluster_results_dir)
                        if cluster_method != 'lovain':
                            break
                if dr_method == 'identity':
                    break

    gse_series = 'GSE60361'
    result_dir = os.path.join('results', gse_series)
    if not os.path.exists(result_dir):
        os.makedirs(result_dir)
    logging.info('Loading dataset {}'.format(gse_series))
    cortex_data = SingleCellData(gse_series)
    import json
    with open(os.path.join(result_dir, 'label2id.json'), 'w') as f:
        f.write(json.dumps(cortex_data.label2id))

    DR_methods = {
        'identity': IdentityTransform,
        'pca': PCAFeatureTransform,
        'ica': ICAFeatureTransform,
        'nmf': NMFFeatureTransform
    }
    Cluster_methods = {
        #'dp': DensityPeakModel,
        'lovain': LouvainModel,
        'kmeans': KMeansModel,
        'hierachical': HierarchicalModel,
    }
    gene_num_lst = [500, 1000, 2000, 3000, -1]
    feature_dim_lst = [20, 30, 50, 100, 200, 300, 400, 500, 1000]

    logging.info('Algorithm begin to run')
    import pymp
    with pymp.Parallel(len(gene_num_lst)) as p:
        for i in p.range(len(gene_num_lst)):
            gene_num = gene_num_lst[i]
            if gene_num == -1:
                sc_data = cortex_data
                gene_num = sc_data.gene_num
            else:
                logging.info('Getting a shrunk dataset with gene num {}'.format(gene_num))
                sc_data = SingleCellData.get_shrunk_data_by_std(cortex_data, gene_num=gene_num)
            dumpdir = os.path.join(result_dir, str(gene_num))
            if not os.path.exists(dumpdir):
                os.makedirs(dumpdir)
            clustering_pipeline(sc_data, DR_methods, Cluster_methods, feature_dim_lst, dumpdir)


import math
from math import*
try:
    A1 = input("INPUT THE 1ST X COORDINATE: ")
    B1 = input("INPUT THE 1ST Y COORDINATE: ")
    A2 = input("INPUT THE 2ND X COORDINATE: ")
    B2 = input("INPUT THE 2ND Y COORDINATE: ")

    Coordinate_1 = (float(A1), float(B1))
    Coordinate_2 = (float(A2), float(B2))
    a1_float = float(A1)
    b1_float = float(B1)
    a2_float = float(A2)
    b2_float = float(B2)

    Formula = input("Enter:\n1 for distance formula \n2 for midpoint formula:")

    if Formula == "1":
        distance = str(math.sqrt(math.pow(a1_float-a2_float, 2) + math.pow(b1_float-b2_float, 2)))
        print("The distance between", Coordinate_1, "and", Coordinate_2,"is", distance)

    if Formula == "2":
        midpoint = [((a1_float + a2_float)/2), ((b1_float + b2_float)/2)]
        print("The midpoint of", Coordinate_1, "and", Coordinate_2, "is", midpoint)
except:
    print("Invalid input")


from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences

def tokenize(x):
    """
    Tokenize x
    :param x: List of sentences/strings to be tokenized
    :return: Tuple of (tokenized x data, tokenizer used to tokenize x)
    """
    keras_tokenizer = Tokenizer()
    keras_tokenizer.fit_on_texts(x)
    return keras_tokenizer.texts_to_sequences(x), keras_tokenizer

def pad(x, length=None):
    """
    Pad x
    :param x: List of sequences.
    :param length: Length to pad the sequence to.  If None, use length of longest sequence in x.
    :return: Padded numpy array of sequences
    """
    padded_x = pad_sequences(x, maxlen=length, padding='post')
    return padded_x

def preprocess(x, y):
    """
    Preprocess x and y
    :param x: Feature List of sentences
    :param y: Label List of sentences
    :return: Tuple of (Preprocessed x, Preprocessed y, x tokenizer, y tokenizer)
    """
    preprocess_x, x_tk = tokenize(x)
    preprocess_y, y_tk = tokenize(y)

    preprocess_x = pad(preprocess_x)
    preprocess_y = pad(preprocess_y)

    # Keras's sparse_categorical_crossentropy function requires the labels to be in 3 dimensions
    preprocess_y = preprocess_y.reshape(*preprocess_y.shape, 1)

    return preprocess_x, preprocess_y, x_tk, y_tk


def ladderLength(beginWord, endWord, wordList):
    minLength=float('inf')
    if(beginWord==endWord):
        return 1
    if(wordList is None):
        return 0
    for word in wordList:
        count=0
        for char1, char2 in zip(word,beginWord):
            if(char1!=char2):
                count+=1
        if(count==1):
            modWordList=wordList.copy()
            modWordList.remove(word)
            result= ladderLength(word, endWord, modWordList)
            if(result>0 and result<minLength):
                minLength=result+1
    if(minLength!=float('inf')):
        return minLength
    else:
        return 0

if __name__ == "__main__":

    beginWord="qa"
    endWord="sq"
    wordList=["si","go","se","cm","so","ph","mt","db","mb","sb","kr","ln","tm","le","av","sm","ar","ci","ca","br","ti","ba","to","ra","fa","yo","ow","sn","ya","cr","po","fe","ho","ma","re","or","rn","au","ur","rh","sr","tc","lt","lo","as","fr","nb","yb","if","pb","ge","th","pm","rb","sh","co","ga","li","ha","hz","no","bi","di","hi","qa","pi","os","uh","wm","an","me","mo","na","la","st","er","sc","ne","mn","mi","am","ex","pt","io","be","fm","ta","tb","ni","mr","pa","he","lr","sq","ye"]
    print(ladderLength(beginWord, endWord, wordList))


from gensim.models import word2vec
import pandas as pd

#data = word2vec.Text8Corpus('raw_bible.txt')
#model = word2vec.Word2Vec(data, size=100, window = 5, min_count=5 , workers=2)
## 保存する
#model.save('bible.model')
#
# Word2Vecのインスタンス作成
# sentences : 対象となる分かち書きされているテキスト
# size      : 出力するベクトルの次元数
# min_count : この数値よりも登場回数が少ない単語は無視する
# window    : 一つの単語に対してこの数値分だけ前後をチェックする
filenames = [ 'bhagvadgita','bible','quaran']
for filename in filenames:
    data = word2vec.Text8Corpus('raw_'+filename+'.txt')
    model = word2vec.Word2Vec(data, size=100, window = 5, min_count=10 , workers=2) #学習に使う前後の単語数
# 保存する
    model.save(filename+'.model')

# A more selective model
#model = word2vec.Word2Vec(corpus, size=100, window=20, min_count=500, workers=4)
#tsne_plot(model)


import os
from ppaxe import core
from ppaxe import report
import re
import smtplib
import base64
import datetime
from ppaxe import PubMedQueryError
from flask import Flask, request, session, g, redirect, url_for, abort, \
     render_template, flash
from xhtml2pdf import pisa
from StringIO import StringIO
from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.MIMEBase import MIMEBase
from email.utils import COMMASPACE, formatdate
from email import Encoders
from pycorenlp import StanfordCoreNLP
from flask import jsonify
import requests
from xml.dom import minidom

class ReverseProxied(object):
    '''Wrap the application in this middleware and configure the
    front-end server to add these headers, to let you quietly bind
    this to a URL other than / and to an HTTP scheme that is
    different than what is used locally.

    In nginx:
    location /myprefix {
        proxy_pass http://192.168.0.1:5001;
        proxy_set_header Host $host;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Scheme $scheme;
        proxy_set_header X-Script-Name /myprefix;
        }

    :param app: the WSGI application
    '''
    def __init__(self, app):
        self.app = app

    def __call__(self, environ, start_response):
        script_name = environ.get('HTTP_X_SCRIPT_NAME', '')
        if script_name:
            environ['SCRIPT_NAME'] = script_name
            path_info = environ['PATH_INFO']
            if path_info.startswith(script_name):
                environ['PATH_INFO'] = path_info[len(script_name):]

        scheme = environ.get('HTTP_X_SCHEME', '')
        if scheme:
            environ['wsgi.url_scheme'] = scheme
        return self.app(environ, start_response)

class PrefixMiddleware(object):

    def __init__(self, app, prefix=''):
        self.app = app
        self.prefix = prefix

    def __call__(self, environ, start_response):

        if environ['PATH_INFO'].startswith(self.prefix):
            environ['PATH_INFO'] = environ['PATH_INFO'][len(self.prefix):]
            environ['SCRIPT_NAME'] = self.prefix
            return self.app(environ, start_response)
        else:
            start_response('404', [('Content-Type', 'text/plain')])
            return ["This url does not belong to the app.".encode()]

core.NLP = StanfordCoreNLP(os.environ['PPAXE_CORENLP'])
app = Flask(__name__) # create the application instance
app.wsgi_app = ReverseProxied(app.wsgi_app)
# app.wsgi_app = PrefixMiddleware(app.wsgi_app, prefix=environ.get('SCRIPT_NAME', ''))

# FUNCTIONS
# -----------------------------------------------------------------------
def create_pdf(pdf_data):
    '''
    Creates pdf file
    '''
    pdf = StringIO()
    pisa.CreatePDF(StringIO(pdf_data), pdf)
    return pdf

# -----------------
def send_mail(send_from, send_to, subject, attachment=None):
    '''
    Returns message to be sent to ppaxe user
    '''
    msg = MIMEMultipart()
    msg['From'] = send_from
    msg['To'] = send_to
    msg['Date'] = formatdate(localtime=True)
    msg['Subject'] = subject
    part = MIMEBase('application', "octet-stream")
    # Remove header
    part.set_payload(attachment.getvalue())
    #Encoders.encode_base64(part)
    part.add_header('Content-Disposition', 'attachment; filename="ppaxe-report.pdf"')
    msg.attach(part)
    return msg

# VIEWS
# -----------------------------------------------------------------------
@app.route('/', methods=['GET', 'POST'])
def home_form():
    '''
    Home of the ppaxe web application containing the form
    '''
    identifiers = str()
    database    = str()
    response    = dict()
    response['search'] = False

    if 'identifiers' in request.form:
        # Get Form parameters
        response['search'] = True
        identifiers = request.form['identifiers']
        database = request.form['database']
        email = request.form['email']
        identifiers = re.split(",|\n|\r", identifiers)
        identifiers = [ident for ident in identifiers if ident]

        # Get articles from Pubmed or PMC
        source = str()
        try:
            query = core.PMQuery(ids=identifiers, database=database)
            query.get_articles()
        except PubMedQueryError:
            response['error'] = "PMQuery"

        # Retrieve interactions from 'source'
        if database == "PUBMED":
            source = "abstract"
        else:
            source = "fulltext"
        for article in query.articles:
            article.predict_interactions(source)

        summary = report.ReportSummary(query)
        summary.graphsummary.makesummary()
        summary.protsummary.makesummary()
        response['nints']  = summary.graphsummary.numinteractions
        response['nprots'] = summary.protsummary.totalprots

        if response['nprots'] > 0:
            response['prot_table'] = summary.protsummary.table_to_html()
        else:
            response['nprots'] = 0

        if response['nints'] > 0:
            # Tables
            response['int_table'] = summary.graphsummary.table_to_html()
            response['sum_table'] = summary.summary_table()
            # JSON graph
            response['graph'] = summary.graphsummary.graph_to_json()
            # Plots
            response['plots'] = dict()
            response['plots']['j_int_plot'], response['plots']['j_prot_plot'], response['plots']['a_year_plot'] = summary.journal_plots()
            response['plots']['j_prot_plot'] = response['plots']['j_prot_plot'].getvalue().encode("base64").strip()
            response['plots']['j_int_plot']  = response['plots']['j_int_plot'].getvalue().encode("base64").strip()
            response['plots']['a_year_plot'] = response['plots']['a_year_plot'].getvalue().encode("base64").strip()
            response['today'] = datetime.date.today()
            response['database'] = database
            response['pdf-plain'] = create_pdf(render_template('pdf.html', identifiers=identifiers, response=response))
            response['pdf'] = "data:application/pdf;base64," + base64.b64encode(response['pdf-plain'].getvalue())

            if email:
                server = smtplib.SMTP('smtp.gmail.com', 587)
                server.starttls()
                server.login(os.environ['PPAXE_EUSER'], os.environ['PPAXE_EPASSW'])
                msg = send_mail(os.environ['PPAXE_EMAIL'], email, 'PPaxe results', response['pdf-plain'])
                server.sendmail(os.environ['PPAXE_EMAIL'], email, msg.as_string())

    return render_template('home.html', identifiers=identifiers, response=response)

@app.route('/querypubmed', methods=['GET'])
def querypubmed():
    '''
    Queries PubMed and returns the pubmed identifiers
    '''
    response    = dict()
    response['error'] = True
    if request.is_xhr:
        query = request.args.get('query')
        req = requests.get("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&rettype=count&term=%s" % (query) )
        if req.status_code == 200:
            md = minidom.parseString(req.text)
            count = md.getElementsByTagName('Count')
            count = count[0].firstChild.nodeValue
            req = requests.get("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&retmax=%s&term=%s" % (count, query) )
            if req.status_code == 200:
                md = minidom.parseString(req.text)
                theids = md.getElementsByTagName('Id')
                try:
                    theids = [ iden.firstChild.nodeValue for iden in theids ]
                except Exception:
                    response['error'] = True
                if theids:
                    response['identifiers'] = theids
                    response['error'] = False
        return jsonify(response)

# -----------------
@app.route('/tutorial', methods=['GET'])
def tutorial():
    '''
    Tutorial page
    '''
    return render_template('tutorial.html')

@app.route('/download', methods=['GET'])
def download():
    '''
    download page
    '''
    return render_template('download.html')

# -----------------
@app.route('/about', methods=['GET'])
def about():
    '''
    About page
    '''
    return render_template('about.html')


from numpy import array, exp
from random import random
import numpy as np
from functools import reduce
from activation_functions import sigmoid, relu, softmax, normal

class Rede:

    dic_functions = {
        'sigmoid': sigmoid,
        'relu': relu,
        'softmax': softmax,
        None: normal
    }

    def __init__(self, shape_in):
        self.shape_in = shape_in
        self.bias = []
        self.camadas = []
        self.activation_functions = []

    def add_camada(self, neuronios: int, actvation_function=None):
        if len(self.camadas) == 0:
            camada = [[random() for i in range(neuronios)]
                      for j in range(self.shape_in)]
        else:
            camada = [[random() for i in range(neuronios)]
                      for j in range(self.camadas[-1].shape[1])]
        camada = array(camada)

        self.bias.append(array([random() for i in range(neuronios)]))

        self.camadas.append(camada)
        self.activation_functions.append(
            Rede.dic_functions[actvation_function])

    def predict(self, x):
        assert self.camadas is not None, "Deve inserir camadas primeiro"
        assert len(
            x) == self.shape_in, 'Erro na quantidade de elementos inseridos'
        x = array(x)

        dot = np.dot(x, self.camadas[0] + self.bias[0])
        dot = self.activation_functions[0](dot)

        if len(self.camadas) > 1:
            for funcao, camada, bias in zip(self.activation_functions[1:],
                                            self.camadas[1:],
                                            self.bias[1:]):
                dot = funcao(np.dot(dot, camada) + bias)
        return dot

    def get_shapes(self):
        return [camada.shape for camada in self.camadas]

    def get_prod_shapes(self):
        return [np.prod(shape) for shape in self.shapes]

    def get_sum_shapes(self):
        return sum(self.prod_shapes)

    prod_shapes = property(fget=get_prod_shapes)
    sum_shapes = property(fget=get_sum_shapes)
    shapes = property(fget=get_shapes)

    def __getitem__(self, i):
        return self.camadas[i]

    def __len__(self):
        return len(self.camadas)


#!/usr/bin/env python

import jinja2
from server import model

with open("base.fcfg.jinja") as f:
    base_fcfg = "\n".join(f.readlines())

template = jinja2.Template(base_fcfg)

categories = model.Category.query.all()
category_names = []
for cat in categories:
    name = cat.title
    category_names.append(name.lower())

    if '/' in name:
        for x in name.split('/'):
            category_names.append(x.lower())

    if ' ' in name:
        for x in name.split(' '):
            category_names.append(x.lower())

category_names.remove("")
category_names.remove("and")

knowledges = model.Knowledge.query.all()

with open("grammar.fcfg", "w+") as f:
    f.write(template.render(categories=category_names, knowledges=knowledges))


# -*- coding: utf-8 -*-
"""
Author: Shreyas Krishnamurthy
"""
import os
import datetime as dt
import unicodedata
import pymongo
import re

## Function to check if string is integer
def is_number(s):
    try:
        int(s)
        return int(s)
    except ValueError:
        pass

    try:
        unicodedata.numeric(s)
        return True
    except (TypeError, ValueError):
        pass
    return False

def read_log(fileName,fileDir):
    # The header variable stores the fields in the log files
    header = []
    # a list to save the dictionary for each line in the IIS log file
    l = []
    print('Processing the file '+fileName)
    ## try..except block to ensure that we do not encounter encoding errors
    try:
        file_data = open(fileDir+fileName,encoding="utf8").readlines()
    except:
        file_data = open(fileDir+fileName).readlines()

    ## reading and parsing the lines in the log file
    for line in file_data:
        if not line.startswith('#'):
            fields = line.split()
            date_idx = -1
            time_idx = -1
            for idx,val in enumerate(fields):
                if re.match('20[0-9][0-9]-[0-1][0-9]-[0-3][0-9]', val):
                    date_idx = idx
                elif re.match('[0-2][0-9]:[0-5][0-9]:[0-5][0-9]', val):
                    time_idx = idx
                elif is_number(val):
                    fields[idx] = is_number(val)
            fields[date_idx]=dt.datetime.strptime(fields[date_idx]+' '+fields[time_idx],'%Y-%m-%d %H:%M:%S')
            fields.pop(time_idx)
            #pprint (fields) # debug
            d = dict(zip(header, fields)) # create a <dict> based on <headers> & <split> log lines
            l.append(d)
        elif line.split()[0] == '#Fields:':
            header = line.split()
            header.remove('#Fields:')
            header.remove('time')
    return l

## function to insert the log data into the the database collection
## Modify this function based on your requirements
def insertMongoDB():
    ## connection to mongodb
    myclient = pymongo.MongoClient("mongodb://localhost:27017/")
    ## creating the database
    mydb = myclient["database_name"]
    ## creating a collection in the db
    mylogs = mydb["collection_name"]

    ## dir with all the log files
    log_dir = "/Data/"

    # ## reading all the log files in the dir, which have to be parsed and inserted into the DB
    # allFiles = os.listdir(log_dir)
    # for file_name in allFiles:
    #     l = read_log(file_name,log_dir)
    #     mylogs.insert_many(l)

    ## only for one log file
    l = read_log('iislog_sample.log',log_dir)
    mylogs.insert_many(l)

    print(myclient.list_database_names())
    print(mydb.list_collection_names())


from pyspark.sql import SparkSession
from pyspark.sql.types import DateType
from pyspark.sql.types import IntegerType
from pyspark.sql.types import StringType
from pyspark.sql.functions import asc
from pyspark.sql.functions import desc
from pyspark.sql.functions import sum
from pyspark.sql.functions import date_format
from pyspark.sql.functions import avg
from pyspark.sql.functions import col
from pyspark.sql.functions import max
from pyspark.sql.functions import min
from pyspark.sql.functions import count
from pyspark.sql.functions import to_timestamp
from pyspark.sql.functions import udf
from pyspark.sql.functions import round
from pyspark.sql.functions import lit
from pyspark.sql.functions import when
import math

interval = 60.0

def range_interval(duration):
	temp = interval
	while(True):
		if duration < temp:
			return '<' + str(temp)
		temp+=interval

def divide(cart, purchase):
	return int(math.ceil(cart/purchase))

udf_divide = udf(divide, IntegerType())
udf_intervals = udf(range_interval, StringType())

spark = SparkSession \
	.builder \
	.appName("PySpark example") \
	.getOrCreate()

#IMPORTANT: TENS DE FAZER DOWNLOAD DO CSV
# GOAL: Calcular a duração de cada sessão arredondando aos minutos e calcular nessa sessão o ratio de purchase/cart, de seguida agrupar por intervalos da duração de sessão(por ex de 10 em 10 min) e calcular para cada grupo a média do ratio anteriormente calculado
df = spark \
	.read \
	.option("header", "false") \
	.csv("database/smallerLargeFile_3.csv")

cart = df \
		.select(col("_c1").alias("event_type"), col("_c8").alias("sessions")) \
		.filter(col("event_type").isNotNull() & col("sessions").isNotNull()) \
		.filter(col("event_type").contains("cart")) \
		.groupby(col("event_type"), col("sessions")) \
		.count().withColumnRenamed("count", "#cart") \

purchases = df \
		.select(col("_c1").alias("event_type"), col("_c8").alias("session")) \
		.filter(col("event_type").isNotNull() & col("session").isNotNull()) \
		.filter(col("event_type").contains("purchase")) \
		.groupby(col("event_type"), col("session")) \
		.count().withColumnRenamed("count", "#purchases")

ratio = cart \
		.join(purchases, cart.sessions == purchases.session) \
		.withColumn("ratio", udf_divide("#cart", "#purchases")) \
		.drop("session", "event_type", "#cart", "#purchases")

session = df \
		  .select(to_timestamp(col("_c0"), 'yyyy-MM-dd HH:mm:ss').alias("time"), col("_c8").alias("session")) \
		  .filter(col("session").isNotNull()) \
		  .groupby(col("session")) \
		  .agg(max(col("time")).alias("end_time"), min(col("time")).alias("start_time")) \
		  .withColumn("duration", round(((col("end_time").cast("long") - col("start_time").cast("long"))/60), 1)) \
		  .drop("start_time", "end_time") \
		  .filter("duration > 0.0 AND duration < 600.0") \
		  .withColumn("intervals", udf_intervals("duration"))

result = ratio \
		 .join(session, session.session == ratio.sessions)

temp = spark.createDataFrame([('<0.0', 0)], ['interval', 'ratio'])

for i in range(int(interval), int(interval)*10+1, int(interval)):
	i = float(i)

	value = result.filter(col("intervals").contains(('<' + str(i)))).agg(avg(col("ratio"))).collect()[0]["avg(ratio)"]
	if value == None:
		value = 0

	temp = temp.union(spark.createDataFrame([(('<' + str(i)), int(math.ceil(value)))], ['interval', 'ratio']))

result = result \
			.groupby("intervals").count() \
			.orderBy(desc("intervals")) \
			.join(temp, temp.interval == result.intervals) \
			.drop("interval")

print(result.show())

spark.stop()


'''
31. 動詞
動詞の表層形をすべて抽出せよ．

argv[1]: neko.txt.mecab
'''
from no30 import read_morphemes
import sys

sentences = read_morphemes(sys.argv[1])
for sentence in sentences:
    for morpheme in sentence:
        if morpheme["pos"] == "動詞":
            print(morpheme["surface"])


from api.local.datahelper.tableloader.tableloader import TableLoader

class EntInsuranceLoader(TableLoader):
    """
    返回的是xzbz，一个公司的参保种类，是一个(1, 9)的01矩阵
    以及对应的entname
    """

    def __init__(self):
        super().__init__()
        self.load_name = 'enterprise_insurance'

    def load(self, table):

        xzbz = table['xzbz']
        cbztmc = table['cbztmc']
        company = table['entname']

        data = {}

        for i, name in enumerate(company):
            index = ord(xzbz[i]) - ord('A')
            if name in data:
                flag = (1 if cbztmc[i] == '参保缴费' else 0)
                data[name][index] = flag
            else:
                data[name] = [0 for _ in range(9)]  # 这里应该是只有A~I 9 种

        xzbz = []
        company = []
        for name in data:
            company.append(name)
            xzbz.append(data[name])

        result = {'xzbz': xzbz, 'entname': company, 'key': table['key']}
        # table['xzbz'] = xzbz
        # table['entname'] = company
        return result

    def describe(self, table):

        sorted(table.keys())
        company = table['entname']
        xzbz = table['xzbz']
        data = {}

        for i, name in enumerate(company):
            data[name] = xzbz[i]

        self.segment_name[table['key']] = ['xzbz'+str(i) for i in range(9)]

        return data

    def solve_unaccept_value(self, value, key):

        if key.startswith('xzbz'):
            value = 0
        else:
            value = super().solve_unaccept_value(value, key)

        return value

    def can_solve_the_unaccept_value(self, segment, value):
        if segment.startswith('xzbz'):
            return True
        return False