import os
import os.path
import shutil
import tempfile
import types
from typing import Dict, Optional, Set, MutableSequence
import zipfile

import tensorflow as tf

from tensorflow_federated.proto.v0 import computation_pb2 as pb
from tensorflow_federated.python.common_libs import py_typecheck
from tensorflow_federated.python.common_libs import serialization_utils
from tensorflow_federated.python.core.api import computation_types
from tensorflow_federated.python.core.impl import tf_computation_context
from tensorflow_federated.python.core.impl.context_stack import context_stack_base
from tensorflow_federated.python.core.impl.types import type_conversions
from tensorflow_federated.python.core.impl.types import type_serialization
from tensorflow_federated.python.core.impl.utils import function_utils
from tensorflow_federated.python.core.impl.utils import tensorflow_utils
from tensorflow_federated.python.tensorflow_libs import graph_keys

class SerializationError(Exception):
  """Error raised during value serialization or deserialization."""
  pass

def finalize_binding(binding, tensor_info_map):
  """Mutates binding by filling in actual tensor names.

  Args:
    binding: A `pb.Binding` or one of its submessages.
    tensor_info_map: A dict mapping the placeholder `tensor_name`s found in
      `binding` to final tensor names.
  """
  if not binding:
    if tensor_info_map:
      raise ValueError('Empty binding, but non-empty tensor_info_map {}'.format(
          tensor_info_map))
    return
  if isinstance(binding, pb.TensorFlow.Binding):
    sub_binding = getattr(binding, binding.WhichOneof('binding'))
    finalize_binding(sub_binding, tensor_info_map)

  elif isinstance(binding, pb.TensorFlow.TensorBinding):
    name = binding.tensor_name
    if name not in tensor_info_map:
      raise ValueError(
          'Did not find tensor_name {} in provided tensor_info_map with keys {}'
          .format(name, list(tensor_info_map.keys())))
    binding.tensor_name = tensor_info_map[name].name
  elif isinstance(binding, pb.TensorFlow.NamedTupleBinding):
    for sub_binding in binding.element:
      finalize_binding(sub_binding, tensor_info_map)
  else:
    raise ValueError('Unsupported binding type {}'.format(
        py_typecheck.type_string(type(binding))))

def serialize_tf2_as_tf_computation(target, parameter_type, unpack=None):
  """Serializes the 'target' as a TF computation with a given parameter type.

  Args:
    target: The entity to convert into and serialize as a TF computation. This
      can currently only be a Python function or `tf.function`, with arguments
      matching the 'parameter_type'.
    parameter_type: The parameter type specification if the target accepts a
      parameter, or `None` if the target doesn't declare any parameters. Either
      an instance of `types.Type`, or something that's convertible to it by
      `types.to_type()`.
    unpack: Whether to always unpack the parameter_type. Necessary for support
      of polymorphic tf2_computations.

  Returns:
    The constructed `pb.Computation` instance with the `pb.TensorFlow` variant
      set.

  Raises:
    TypeError: If the arguments are of the wrong types.
    ValueError: If the signature of the target is not compatible with the given
      parameter type.
  """
  py_typecheck.check_callable(target)
  parameter_type = computation_types.to_type(parameter_type)
  signature = function_utils.get_signature(target)
  if signature.parameters and parameter_type is None:
    raise ValueError(
        'Expected the target to declare no parameters, found {!r}.'.format(
            signature.parameters))

  # In the codepath for TF V1 based serialization (tff.tf_computation),
  # we get the "wrapped" function to serialize. Here, target is the
  # raw function to be wrapped; however, we still need to know if
  # the parameter_type should be unpacked into multiple args and kwargs
  # in order to construct the TensorSpecs to be passed in the call
  # to get_concrete_fn below.
  unpack = function_utils.infer_unpack_needed(target, parameter_type, unpack)
  arg_typespecs, kwarg_typespecs, parameter_binding = (
      tensorflow_utils.get_tf_typespec_and_binding(
          parameter_type,
          arg_names=list(signature.parameters.keys()),
          unpack=unpack))

  # Pseudo-global to be appended to once when target_poly below is traced.
  type_and_binding_slot = []

  # N.B. To serialize a tf.function or eager python code,
  # the return type must be a flat list, tuple, or dict. However, the
  # tff.tf_computation must be able to handle structured inputs and outputs.
  # Thus, we intercept the result of calling the original target fn, introspect
  # its structure to create a result_type and bindings, and then return a
  # flat dict output. It is this new "unpacked" tf.function that we will
  # serialize using tf.saved_model.save.
  #
  # TODO(b/117428091): The return type limitation is primarily a limitation of
  # SignatureDefs  and therefore of the signatures argument to
  # tf.saved_model.save. tf.functions attached to objects and loaded back with
  # tf.saved_model.load can take/return nests; this might offer a better
  # approach to the one taken here.

  @tf.function
  def target_poly(*args, **kwargs):
    result = target(*args, **kwargs)
    result_dict, result_type, result_binding = (
        tensorflow_utils.get_tf2_result_dict_and_binding(result))
    assert not type_and_binding_slot
    # A "side channel" python output.
    type_and_binding_slot.append((result_type, result_binding))
    return result_dict

  # Triggers tracing so that type_and_binding_slot is filled.
  cc_fn = target_poly.get_concrete_function(*arg_typespecs, **kwarg_typespecs)
  assert len(type_and_binding_slot) == 1
  result_type, result_binding = type_and_binding_slot[0]

  # N.B. Note that cc_fn does *not* accept the same args and kwargs as the
  # Python target_poly; instead, it must be called with **kwargs based on the
  # unique names embedded in the TensorSpecs inside arg_typespecs and
  # kwarg_typespecs. The (preliminary) parameter_binding tracks the mapping
  # between these tensor names and the components of the (possibly nested) TFF
  # input type. When cc_fn is serialized, concrete tensors for each input are
  # introduced, and the call finalize_binding(parameter_binding,
  # sigs['serving_default'].inputs) updates the bindings to reference these
  # concrete tensors.

  # Associate vars with unique names and explicitly attach to the Checkpoint:
  var_dict = {
      'var{:02d}'.format(i): v for i, v in enumerate(cc_fn.graph.variables)
  }
  saveable = tf.train.Checkpoint(fn=target_poly, **var_dict)

  try:
    # TODO(b/122081673): All we really need is the  meta graph def, we could
    # probably just load that directly, e.g., using parse_saved_model from
    # tensorflow/python/saved_model/loader_impl.py, but I'm not sure we want to
    # depend on that presumably non-public symbol. Perhaps TF can expose a way
    # to just get the MetaGraphDef directly without saving to a tempfile? This
    # looks like a small change to v2.saved_model.save().
    outdir = tempfile.mkdtemp('savedmodel')
    tf.saved_model.save(saveable, outdir, signatures=cc_fn)

    graph = tf.Graph()
    with tf.compat.v1.Session(graph=graph) as sess:
      mgd = tf.compat.v1.saved_model.load(
          sess, tags=[tf.saved_model.SERVING], export_dir=outdir)
  finally:
    shutil.rmtree(outdir)
  sigs = mgd.signature_def

  # TODO(b/123102455): Figure out how to support the init_op. The meta graph def
  # contains sigs['__saved_model_init_op'].outputs['__saved_model_init_op']. It
  # probably won't do what we want, because it will want to read from
  # Checkpoints, not just run Variable initializerse (?). The right solution may
  # be to grab the target_poly.get_initialization_function(), and save a sig for
  # that.

  # Now, traverse the signature from the MetaGraphDef to find
  # find the actual tensor names and write them into the bindings.
  finalize_binding(parameter_binding, sigs['serving_default'].inputs)
  finalize_binding(result_binding, sigs['serving_default'].outputs)

  annotated_type = computation_types.FunctionType(parameter_type, result_type)

  return pb.Computation(
      type=pb.Type(
          function=pb.FunctionType(
              parameter=type_serialization.serialize_type(parameter_type),
              result=type_serialization.serialize_type(result_type))),
      tensorflow=pb.TensorFlow(
          graph_def=serialization_utils.pack_graph_def(mgd.graph_def),
          parameter=parameter_binding,
          result=result_binding)), annotated_type

def serialize_py_fn_as_tf_computation(target, parameter_type, context_stack):
  """Serializes the 'target' as a TF computation with a given parameter type.

  See also `serialize_tf2_as_tf_computation` for TensorFlow 2
  serialization.

  Args:
    target: The entity to convert into and serialize as a TF computation. This
      can currently only be a Python function. In the future, we will add here
      support for serializing the various kinds of non-eager and eager
      functions, and eventually aim at full support for and compliance with TF
      2.0. This function is currently required to declare either zero parameters
      if `parameter_type` is `None`, or exactly one parameter if it's not
      `None`.  The nested structure of this parameter must correspond to the
      structure of the 'parameter_type'. In the future, we may support targets
      with multiple args/keyword args (to be documented in the API and
      referenced from here).
    parameter_type: The parameter type specification if the target accepts a
      parameter, or `None` if the target doesn't declare any parameters. Either
      an instance of `types.Type`, or something that's convertible to it by
      `types.to_type()`.
    context_stack: The context stack to use.

  Returns:
    A tuple of (`pb.Computation`, `tff.Type`), where the computation contains
    the instance with the `pb.TensorFlow` variant set, and the type is an
    instance of `tff.Type`, potentially including Python container annotations,
    for use by TensorFlow computation wrappers.

  Raises:
    TypeError: If the arguments are of the wrong types.
    ValueError: If the signature of the target is not compatible with the given
      parameter type.
  """
  # TODO(b/113112108): Support a greater variety of target type signatures,
  # with keyword args or multiple args corresponding to elements of a tuple.
  # Document all accepted forms with examples in the API, and point to there
  # from here.

  py_typecheck.check_type(target, types.FunctionType)
  py_typecheck.check_type(context_stack, context_stack_base.ContextStack)
  parameter_type = computation_types.to_type(parameter_type)
  signature = function_utils.get_signature(target)

  with tf.Graph().as_default() as graph:
    args = []
    if parameter_type is not None:
      if len(signature.parameters) != 1:
        raise ValueError(
            'Expected the target to declare exactly one parameter, found {!r}.'
            .format(signature.parameters))
      parameter_name = next(iter(signature.parameters))
      parameter_value, parameter_binding = tensorflow_utils.stamp_parameter_in_graph(
          parameter_name, parameter_type, graph)
      args.append(parameter_value)
    else:
      if signature.parameters:
        raise ValueError(
            'Expected the target to declare no parameters, found {!r}.'.format(
                signature.parameters))
      parameter_binding = None
    context = tf_computation_context.TensorFlowComputationContext(graph)
    with context_stack.install(context):
      result = target(*args)

      # TODO(b/122081673): This needs to change for TF 2.0. We may also
      # want to allow the person creating a tff.tf_computation to specify
      # a different initializer; e.g., if it is known that certain
      # variables will be assigned immediately to arguments of the function,
      # then it is wasteful to initialize them before this.
      #
      # The following is a bit of a work around: the collections below may
      # contain variables more than once, hence we throw into a set. TFF needs
      # to ensure all variables are initialized, but not all variables are
      # always in the collections we expect. tff.learning._KerasModel tries to
      # pull Keras variables (that may or may not be in GLOBAL_VARIABLES) into
      # VARS_FOR_TFF_TO_INITIALIZE for now.
      all_variables = set(tf.compat.v1.global_variables() +
                          tf.compat.v1.local_variables() +
                          tf.compat.v1.get_collection(
                              graph_keys.GraphKeys.VARS_FOR_TFF_TO_INITIALIZE))
      if all_variables:
        # Use a readable but not-too-long name for the init_op.
        name = 'init_op_for_' + '_'.join(
            [v.name.replace(':0', '') for v in all_variables])
        if len(name) > 50:
          name = 'init_op_for_{}_variables'.format(len(all_variables))
        with tf.control_dependencies(context.init_ops):
          # Before running the main new init op, run any initializers for sub-
          # computations from context.init_ops. Variables from import_graph_def
          # will not make it into the global collections, and so will not be
          # initialized without this code path.
          init_op_name = tf.group(
              tf.compat.v1.initializers.variables(all_variables, name=name),
              *tf.compat.v1.get_collection(
                  tf.compat.v1.GraphKeys.TABLE_INITIALIZERS)).name
      elif context.init_ops:
        init_op_name = tf.group(
            *context.init_ops, name='subcomputation_init_ops').name
      else:
        init_op_name = None

    result_type, result_binding = tensorflow_utils.capture_result_from_graph(
        result, graph)

  annotated_type = computation_types.FunctionType(parameter_type, result_type)

  # WARNING: we do not really want to be modifying the graph here if we can
  # avoid it. This is purely to work around performance issues uncovered with
  # the non-standard usage of Tensorflow and have been discussed with the
  # Tensorflow core team before being added.
  clean_graph_def = _clean_graph_def(graph.as_graph_def())

  return pb.Computation(
      type=pb.Type(
          function=pb.FunctionType(
              parameter=type_serialization.serialize_type(parameter_type),
              result=type_serialization.serialize_type(result_type))),
      tensorflow=pb.TensorFlow(
          graph_def=serialization_utils.pack_graph_def(clean_graph_def),
          parameter=parameter_binding,
          result=result_binding,
          initialize_op=init_op_name)), annotated_type

def _clean_graph_def(graph_def: tf.compat.v1.GraphDef) -> tf.compat.v1.GraphDef:
  """Edit the GraphDef proto to make it more performant for TFF.

  WARNING: This method must _NOT_ make any semantic changes (those that would
  change the results of the computation). TFF does not really want to be
  modifying the graph here if we can avoid it. This is purely to work around
  performance issues uncovered with the non-standard usage of Tensorflow and
  have been discussed with the Tensorflow core team before being added.

  Args:
    graph_def: the proto message to modify.

  Returns:
    A GraphDef that has been altered for performance, with no semantic
    modifications.
  """
  # TODO(b/153565654): remove this workaround once there is a way to prevent
  # the OptimizeDataset ops from being added when serializing FunctionDef that
  # received a tf.data.Dataset argument.
  _remove_optimize_dataset_ops(graph_def)
  return graph_def

# TODO(b/153565654): cleanup this workaround method when no longer needed.
def _remove_optimize_dataset_ops(graph_def: tf.compat.v1.GraphDef):
  """Removes `OptimizeDataset` and `ModelDataset` ops from the graph.

  TensorFlow Federated creates and tears down datasets frequently (one for each
  user); which is contrary to the TF expected usage of a Dataset where it is
  setup once and used throughout a long training process.

  In the TFF execution stack this leads to performance degradation where a lot
  of time is spent optimizing a dataset that will soon be thrown away. The time
  spent optimizing can even be as long as it takes to train on the dataset. For
  that reason TFF turns off this optimization.

  Luckily, it appears that generally `ModelDataset` and `OptimizeDataset` have a
  fairly straightforward pattern in graphs (though a fragile assumption); they
  have so far always appeared as:

    dataset -> OptimizeDataset -> ModelDataset -> (DatasetReduce ...)

  Each node is the first input to the next node. The following function simply
  cuts out the middle nodes and connect the first input into OptimizeDataset
  as the input to replace ModelDataset into the last op in the chain.

  Args:
    graph_def: the proto message to mutate in place.
  """
  ops_to_remove = ['OptimizeDataset', 'ModelDataset']

  def is_control_dep(tensor_name: str) -> bool:
    return tensor_name.startswith('^')

  def normalized_tensor_name(tensor_name: str) -> str:
    if is_control_dep(tensor_name):
      return tensor_name[1:]
    return tensor_name.split(':', maxsplit=2)[0]

  def clean_input_tensor(
      tensor_name: str,
      names_to_nodes: Dict[str, tf.compat.v1.NodeDef],
      input_args: Set[str],
  ) -> Optional[str]:
    """Rewire an input tensor that is output by a removed node."""
    node_name = normalized_tensor_name(tensor_name)
    if is_control_dep(tensor_name):
      # Simply delete control deps on removed nodes, otherwise pass through.
      input_node = names_to_nodes[node_name]
      if input_node.op in ops_to_remove:
        return None
      return tensor_name
    node = names_to_nodes.get(node_name)
    if node is None:
      if tensor_name in input_args:
        return node_name
      else:
        raise ValueError('cannot handle input {n} ({nn})'.format(
            n=tensor_name, nn=node_name))
    if node.op not in ops_to_remove:
      return tensor_name
    if node.op == 'OptimizeDataset':
      # The dataset is the first input to OptimizeDataset, so return to replace
      # the dependency on OptimizeDataset.
      return node.input[0]
    elif node.op == 'ModelDataset':
      # ModelDataset's first input is expected to be OptimizeDataset, we can
      # walk up input chain and find the input to the OptimizeDataset and return
      # that instead.
      input_node_name = normalized_tensor_name(node.input[0])
      input_node = names_to_nodes.get(input_node_name)
      if input_node is None or input_node.op != 'OptimizeDataset':
        raise ValueError('Input to ModelDataset node was not OptimizeDataset, '
                         'unknown graph structure, aborting.')
      return input_node.input[0]
    else:
      raise ValueError('Encoutered node [{n}] which is an op to remove, but '
                       'is not handled properly.'.format(n=node))

  def filter_nodes(node_defs: MutableSequence[tf.compat.v1.NodeDef], args):
    nodes_to_keep = []
    names_to_nodes = {}
    for node in node_defs:
      names_to_nodes[node.name] = node
      if node.op not in ops_to_remove:
        nodes_to_keep.append(node)
    func_arg_names = {arg.name for arg in args}
    for node in nodes_to_keep:
      clean_inputs = []
      for input_name in node.input:
        clean_input = clean_input_tensor(input_name, names_to_nodes,
                                         func_arg_names)
        if clean_input is not None:
          clean_inputs.append(clean_input)
      del node.input[:]
      node.input.extend(clean_inputs)
    del node_defs[:]
    node_defs.extend(nodes_to_keep)

  filter_nodes(graph_def.node, args=[])
  for function in graph_def.library.function:
    filter_nodes(function.node_def, args=function.signature.input_arg)

# The maximum size allowed for serialized sequence values. Sequence that
# serialize to values larger than this will result in errors being raised.  This
# likely occurs when the sequence is dependent on, and thus pulling in, many of
# variables from the graph.
DEFAULT_MAX_SERIALIZED_SEQUENCE_SIZE_BYTES = 20 * (1024**2)  # 20 MB

# TODO(b/137880330): there is likely opportunity here to share implementation
# with the serialization happening in
# `tensorflow_serialization.serialize_tf2_as_tf_computation()`. It would be good
# to sync with TF team about options for ensuring graph-only (variable-less)
# serializations.
def serialize_dataset(
    dataset,
    max_serialized_size_bytes=DEFAULT_MAX_SERIALIZED_SEQUENCE_SIZE_BYTES):
  """Serializes a `tf.data.Dataset` value into a `bytes` object.

  Args:
    dataset: A `tf.data.Dataset`.
    max_serialized_size_bytes: An `int` size in bytes designating the threshold
      on when to raise an error if the resulting serialization is too big.

  Returns:
    A `bytes` object that can be sent to
  `tensorflow_serialization.deserialize_dataset` to recover the original
  `tf.data.Dataset`.

  Raises:
    SerializationError: if there was an error in TensorFlow during
      serialization.
  """
  py_typecheck.check_type(dataset,
                          type_conversions.TF_DATASET_REPRESENTATION_TYPES)
  module = tf.Module()
  module.dataset = dataset
  module.dataset_fn = tf.function(lambda: module.dataset, input_signature=())

  temp_dir = tempfile.mkdtemp('dataset')
  fd, temp_zip = tempfile.mkstemp('zip')
  os.close(fd)
  try:
    tf.saved_model.save(module, temp_dir, signatures={})
    with zipfile.ZipFile(temp_zip, 'w') as z:
      for topdir, _, filenames in tf.io.gfile.walk(temp_dir):
        dest_dir = topdir[len(temp_dir):]
        for filename in filenames:
          z.write(
              os.path.join(topdir, filename), os.path.join(dest_dir, filename))
    with open(temp_zip, 'rb') as z:
      zip_bytes = z.read()
  except Exception as e:  # pylint: disable=broad-except
    raise SerializationError(
        'Error serializing tff.Sequence value. Inner error: {!s}'.format(
            e)) from e
  finally:
    tf.io.gfile.rmtree(temp_dir)
    tf.io.gfile.remove(temp_zip)

  if len(zip_bytes) > max_serialized_size_bytes:
    raise ValueError('Serialized size of Dataset ({:d} bytes) exceeds maximum '
                     'allowed ({:d} bytes)'.format(
                         len(zip_bytes), max_serialized_size_bytes))
  return zip_bytes

def deserialize_dataset(serialized_bytes):
  """Deserializes a `bytes` object to a `tf.data.Dataset`.

  Args:
    serialized_bytes: `bytes` object produced by
      `tensorflow_serialization.serialize_dataset`

  Returns:
    A `tf.data.Dataset` instance.

  Raises:
    SerializationError: if there was an error in TensorFlow during
      serialization.
  """
  py_typecheck.check_type(serialized_bytes, bytes)
  temp_dir = tempfile.mkdtemp('dataset')
  fd, temp_zip = tempfile.mkstemp('zip')
  os.close(fd)
  try:
    with open(temp_zip, 'wb') as f:
      f.write(serialized_bytes)
    with zipfile.ZipFile(temp_zip, 'r') as z:
      z.extractall(path=temp_dir)
    loaded = tf.saved_model.load(temp_dir)
    # TODO(b/156302055): Follow up here when bug is resolved, either remove
    # if this function call stops failing by default, or leave if this is
    # working as intended.
    with tf.device('cpu'):
      ds = loaded.dataset_fn()
  except Exception as e:  # pylint: disable=broad-except
    raise SerializationError(
        'Error deserializing tff.Sequence value. Inner error: {!s}'.format(
            e)) from e
  finally:
    tf.io.gfile.rmtree(temp_dir)
    tf.io.gfile.remove(temp_zip)
  return ds


# Introdução às Generator functions em Python
# generator = (n for n in range(1000000))

def generator(value=0, maximo=10):
    while True:
        yield value
        value += 1

        if value > maximo:
            return

gen = generator(6, 7)
print(next(gen))
for num in gen:
    print(num)


import logging
import datetime
import uuid

from eventlog.lib.events import Fields, InvalidField, MissingEventIDException
from eventlog.lib.feeds import Feed, MissingFeedIDException
from eventlog.lib.loader import load
from eventlog.lib.util import local_datetime_to_utc

from .pagination import ByTimeRangeCursor
from .eventquery import EventQuery
from .eventset import EventSetByQuery
from .search import Index
from .query import Query
from .pool import Pool

_LOG = logging.getLogger(__name__)

class Store:

    def __init__(self):
        self._pool = None
        self._config = None
        self._index = None

    def init_app(self, app):

        default_config = {
            'DB_USER': 'eventlog',
            'DB_PASS': 'eventlog',
            'DB_NAME': 'eventlog',
            'DB_POOL_MIN_CONN': 10,
            'DB_POOL_MAX_CONN': 20,
            'INDEX_DIR': None,
            'MEDIA_DIR': None,
            'THUMBNAIL_SUBDIR': 'thumbs',
            'THUMBNAIL_WIDTH': 200,
            'THUMBNAIL_HEIGHT': 200,
            'ORIGINAL_SUBDIR': 'originals',
            'ARCHIVE_SUBDIR': 'archives',
            'TIME_ZONE': 'UTC'
        }

        default_config.update(app.config.get('STORE', {}))

        self._config = default_config

        self._pool = Pool(
            self._config['DB_POOL_MIN_CONN'],
            self._config['DB_POOL_MAX_CONN'],
            self._config['DB_NAME'],
            self._config['DB_USER'],
            self._config['DB_PASS']
        )

        self._index = self._init_index()

    def _init_index(self):
        if self._config['INDEX_DIR'] is None:
            _LOG.warning(
                'Indexing disabled, please specify INDEX_DIR in config.'
            )
            return None
        else:
            index = Index(self._config['INDEX_DIR'])
            return index

    def exists(self, field, value):

        # verify field is valid
        if not isinstance(field, Fields):
            raise InvalidField

        query = Query("select {events}.* from events {events}")
        query = query.add_clause("{events}." + str(field) + "=%s", (value, ))

        with self._pool.connect() as cur:
            cur.execute(query.format(), query.params)

            res = cur.fetchone()

        return True if res is not None else False

    def update_feeds(self, feeds, dry=False):
        with self._pool.connect(
            dry=dry,
            error_message="rolled back update feed changes"
        ) as cur:

            for f in feeds:

                _LOG.info("updating %s", str(f))

                cur.execute(
                    """
                    update feeds
                    set (full_name, short_name, favicon, color, module, config,
                         is_public, is_updating, is_searchable) =
                        (%s, %s, %s, %s, %s, %s, %s, %s, %s)
                    where id = %s
                    """,
                    f.tuple()[1:] + (f.id, )
                )

                if not cur.rowcount:
                    raise MissingFeedIDException(
                        "Feed with ID '%s' does not exist" % (f.id)
                    )

    def add_events(self, events, dry=False):

        with self._pool.connect(
            dry=dry,
            error_message="rolled back new event changes"
        ) as cur:

            for e in events:

                cur.execute(
                    """
                    insert into events
                    values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                    on conflict do nothing
                    """,
                    e.tuple()
                )

                if not cur.rowcount:
                    _LOG.warning('skipping existing event id=%s', e.id)

                if e.related is not None:
                    for c in e.related:

                        # this is needed for .tuple call below to
                        # succeed for events fetched from the store since their
                        # related events have no feed data
                        if c.feed is None:
                            c.feed = e.feed

                        # if the event already exists need is_related to be set
                        # properly
                        cur.execute(
                            """
                            insert into events
                            values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                            on conflict (id) do update
                            set is_related = excluded.is_related
                            """,
                            c.tuple(is_related=True)
                        )

                        cur.execute(
                            """
                            insert into related_events (parent, child)
                            values (%s, %s)
                            on conflict do nothing
                            """,
                            (e.id, c.id)
                        )

                _LOG.info("saved %s", str(e))

        # index new events
        self._index.index(events, dry=dry)

    def update_events(self, events, dry=False):
        with self._pool.connect(
            dry=dry,
            error_message="rolled back update event changes"
        ) as cur:

            for e in events:

                cur.execute(
                    """
                    update events
                    set (title, text, link, occurred,
                         raw, thumbnail, original, archived) =
                        (%s, %s, %s, %s, %s, %s, %s, %s)
                    where id = %s
                    """,
                    e.tuple()[2:-1] + (e.id, )
                )

                if not cur.rowcount:
                    raise MissingEventIDException(
                        "Event with ID '%s' does not exist" % (e.id)
                    )

                _LOG.info("updated %s", str(e))

        # re-index events
        self._index.index(events, dry=dry)

    def remove_events(self, events=None, feed=None, dry=False):

        if events is None and feed is None:
            return

        with self._pool.connect(
            dry=dry,
            error_message="rolled back remove event changes"
        ) as cur:

            if events is not None:  # delete specified events
                for e in events:
                    if e.related is not None:
                        cur.execute(
                            "delete from related_events where parent=%s",
                            (e.id,)
                        )

                        for c in e.related:
                            cur.execute(
                                "delete from events where id=%s", (c.id,)
                            )

                    cur.execute("delete from events where id=%s", (e.id,))

                    _LOG.info("removed %s", str(e))

            elif feed is not None:
                cur.execute(
                    """
                    delete from events
                    where feed_id in (
                        select id from feeds where short_name = %s
                    )
                    """,
                    (feed, )
                )

                _LOG.info("removed all events for feed %s", feed)

        # remove index values here
        self._index.remove(events=events, feed=feed, dry=dry)

    def get_feeds(self, include_admin=False, **kwargs):
        flags = ['is_public', 'is_updating', 'is_searchable']

        config = {}

        # prepare query
        query = Query("select * from feeds")

        for flag, value in kwargs.items():
            if flag in flags:
                query = query.add_clause(flag + "=%s", (value,))

        # grab feeds from database
        with self._pool.connect(dict_cursor=True) as cur:

            cur.execute(query.format(), query.params)

            for row in cur:
                module = row['module']

                overrides = row['config']
                del row['config']

                config[module] = {
                    key: row[key] for key in row if not key.startswith('is_')
                }

                config[module]['flags'] = {
                    flag: row[flag] for flag in row if flag.startswith('is_')
                }

                config[module]['overrides'] = overrides

                config[module]['default'] = {
                    key.lower(): self._config[key]
                    for key in self._config if not key.startswith('DB_')
                }

        # load feeds
        feeds = load(Feed, config, store=self)

        return {feed.short_name: feed for feed in feeds}

    def get_events_by_ids(self, ids, pagesize=10, timezone=None,
                          embed_feeds=True, embed_related=True):

        basequery = Query("select {events}.* from events {events}")

        eq = EventQuery(
            basequery,
            embed_feeds=embed_feeds,
            embed_related=embed_related
        )

        # validate provided IDs as UUIDs
        validated = []

        for i in ids:
            try:
                uuid.UUID(i)

                validated.append(i)
            except ValueError:
                pass

        if validated:
            eq.add_clause("{events}.id in %s", (tuple(set(validated)), ))
        else:
            # want an empty result set if no valid IDs were provided
            eq.add_clause("(\"id\" != \"id\")")

        es = EventSetByQuery(self._pool, eq, pagesize, timezone=timezone)

        return es

    def get_events_by_latest(self, feed=None, timezone=None, pagesize=10,
                             embed_related=True):

        basequery = Query("""
            select {events}.*
            from events {events}
            inner join (
                select distinct on (feed_id) id
                from events where is_related=false
                order by feed_id, occurred desc
            ) latest on latest.id = {events}.id
        """)

        if feed is not None:
            basequery += (
                "inner join feeds {feeds} on {events}.feed_id = {feeds}.id"
            )

        eq = EventQuery(
            basequery,
            embed_feeds=True,
            embed_related=embed_related
        )

        if feed is not None:
            eq.add_clause("{feeds}.short_name = %s", (feed,))

        es = EventSetByQuery(self._pool, eq, pagesize, timezone=timezone)

        events = list(es)

        if feed is None:
            return {e.feed['short_name']: e for e in events}
        elif events:
            return events[0]
        else:
            return None

    def get_events_by_date(self, d, feeds=None, **kwargs):

        # start of day
        dt = datetime.datetime(d.year, d.month, d.day, 0, 0, 0, 0)

        after = dt - datetime.timedelta(microseconds=1)
        before = dt + datetime.timedelta(days=1)

        return self.get_events_by_timerange(
            before=before,
            after=after,
            feeds=feeds,
            **kwargs
        )

    def get_events_by_timerange(self, before=None, after=None, pagesize=10,
                                feeds=None, flattened=False, timezone=None,
                                embed_related=True):

        basequery = Query("select {events}.* from events {events}")

        if feeds is not None:
            basequery += ", feeds {feeds} where {events}.feed_id={feeds}.id"

        if flattened:
            embed_related = False

        eq = EventQuery(
            basequery,
            embed_feeds=True,
            embed_related=embed_related
        )

        if before is not None:
            eq.add_clause(
                "{events}.occurred < %s",
                (local_datetime_to_utc(before, timezone),)
            )

        if after is not None:
            eq.add_clause(
                "{events}.occurred > %s",
                (local_datetime_to_utc(after, timezone),)
            )

        if feeds is not None:
            eq.add_clause("{feeds}.short_name in %s", (tuple(feeds),))

        if not flattened:
            eq.add_clause("{events}.is_related=false")

        return EventSetByQuery(self._pool, eq, pagesize, timezone=timezone)

    def get_events_by_search(self, query, pagesize=10, **kwargs):

        basequery = Query(
            "select {events}.* from events {events} where {events}.id in %s"
        )

        eq = EventQuery(basequery, embed_feeds=True, embed_related=False)

        return self._index.search(query, eq, self._pool, pagesize, **kwargs)


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from statsmodels.tsa.stattools import coint, adfuller
from scipy.signal import savgol_filter
from scipy.stats import wasserstein_distance
from scipy.signal import welch
import datetime
from scipy.fftpack import fft, fftshift
from scipy.spatial.distance import euclidean
import itertools

make_plots = False
model = "All" # Train/Test/All
# Read in the data
df = pd.read_csv(r"C:\Users\60848\Desktop\opt\final_data_10s.csv")
if model == "Train":
    df = df.iloc[:400000, :]
if model == "Test":
    df = df.iloc[400000:,:]
if model == "All":
    df = df

# Convert Time column to Date Time object
df["Time"] = pd.to_datetime(df["Time"])
df["X_Spread"] = df["X_BID"] - df["X_ASK"]
df["Y_Spread"] = df["Y_BID"] - df["Y_ASK"]

if make_plots:
    # Plot the two time series
    plt.plot(df["Time"], df["X_BID"])
    plt.plot(df["Time"], df["Y_BID"])
    plt.xlabel("Time")
    plt.ylabel("Bid Price")
    plt.savefig("Opt_Bid_TS_"+model)
    plt.show()

    # Plot the two time series
    plt.plot(df["Time"], df["X_ASK"])
    plt.plot(df["Time"], df["Y_ASK"])
    plt.xlabel("Time")
    plt.ylabel("Ask Price")
    plt.savefig("Opt_Ask_TS_"+model)
    plt.show()

    # Plot the two time series
    plt.plot(df["Time"], df["X_Spread"], label="X_liquidity")
    plt.plot(df["Time"], df["Y_Spread"], label="Y_liquidity")
    plt.xlabel("Time")
    plt.ylabel("Bid-Ask: X & Y")
    plt.savefig("Opt_XY_Liquidity_"+model)
    plt.show()

# Compute Log returns of X & Y
df['X_BID_log_returns'] = np.log(df["X_BID"]) - np.log(df["X_BID"].shift(1))
df['Y_BID_log_returns'] = np.log(df["Y_BID"]) - np.log(df["Y_BID"].shift(1))
df['X_ASK_log_returns'] = np.log(df["X_ASK"]) - np.log(df["X_ASK"].shift(1))
df['Y_ASK_log_returns'] = np.log(df["Y_ASK"]) - np.log(df["Y_ASK"].shift(1))

# Study power spectrum
x_returns = np.nan_to_num(df["X_BID_log_returns"])
y_returns = np.nan_to_num(df["Y_BID_log_returns"])
x_returns_demean = x_returns - np.mean(x_returns)
y_returns_demean = y_returns - np.mean(y_returns)

# Plot power spectrum
log_periodogram_x = np.log(np.abs((fft(fftshift(x_returns_demean)))**2))[1:len(y_returns_demean)//2]
log_periodogram_y = np.log(np.abs((fft(fftshift(y_returns_demean)))**2))[1:len(y_returns_demean)//2]

# Parameters
window_size = (6 * 60 * 8) + 1 # Current smoothing corresponds to an 8 hour window (approximately a one-day trading window)
polynomial_order = 2
X_spectrum_smooth = np.nan_to_num(savgol_filter(log_periodogram_x, window_size, polynomial_order))
Y_spectrum_smooth = np.nan_to_num(savgol_filter(log_periodogram_y, window_size, polynomial_order))

if make_plots:
    # Plot power spectrum - X
    plt.plot(np.linspace(0, 0.5, (len(log_periodogram_x))), X_spectrum_smooth, label="X-log-periodogram", alpha=0.25)
    plt.legend()
    plt.xlabel("Frequency")
    plt.ylabel("Log-PSD")
    plt.savefig("opt_securities_power_spectrum_X_"+model)
    plt.show()

    # Plot power spectrum - Y
    plt.plot(np.linspace(0, 0.5, (len(log_periodogram_y))), Y_spectrum_smooth, label="Y-log-periodogram", alpha=0.25)
    plt.legend()
    plt.xlabel("Frequency")
    plt.ylabel("Log-PSD")
    plt.savefig("opt_securities_power_spectrum_Y_"+model)
    plt.show()

    # Plot element-wise difference - is there any trend here?
    plt.plot(np.linspace(0, 0.5, len(X_spectrum_smooth)), X_spectrum_smooth - Y_spectrum_smooth, label="Element-wise-first-difference")
    plt.plot(np.linspace(0, 0.5, len(X_spectrum_smooth)), savgol_filter((X_spectrum_smooth - Y_spectrum_smooth), 15001, 2), label="Smoothed-difference-in-XY-spectra")
    plt.xlabel("Frequency")
    plt.ylabel("L1 difference in Log-PSD")
    plt.savefig("Opt_securities_power_spectrum_difference_"+model)
    plt.show()

# Although there is not an obvious difference in periodic behaviour, the L1 difference plot indicates that there may be some trend in the differences
# Of course, we have to be aware of excessive smoothing
# It's possible that we see some medium-term reversal behaviour

# There is no notable difference in the Log power spectrum of the securities. Same peaks (implies limited intra-day, periodic phenomenon)
x_mean = np.mean(df["X_BID_log_returns"])
x_std = np.var(df["X_BID_log_returns"])
y_mean = np.mean(df["Y_BID_log_returns"])
y_std = np.var(df["Y_BID_log_returns"])

# Compute static correlations
correlation_bid = df["X_BID_log_returns"].corr(df["Y_BID_log_returns"])
correlation_ask = df["X_ASK_log_returns"].corr(df["Y_ASK_log_returns"])

# Add additional features for minute, hour, day, week, month, year
df["Date"] = df["Time"].dt.date
df["Second"] = df["Time"].dt.second
df["Minute"] = df["Time"].dt.minute
df["Hour"] = df["Time"].dt.hour
df["Day"] = df["Time"].dt.day
df["Week"] = df["Time"].dt.week
df["Month"] = df["Time"].dt.month
df["Year"] = df["Time"].dt.year

# Write a function to generate a table over the space which may identify dislocations
def dislocation_function(df, grid_periodicity):
    # Loop over all grid components and assess divergence in correlation
    grid = np.linspace(np.min(df[grid_periodicity]), np.max(df[grid_periodicity]), len(np.unique(df[grid_periodicity])))
    metrics_periodicity_list = []
    for i in range(len(grid)):
        slice = df.loc[df[grid_periodicity] == grid[i]]
        bid_correlation = slice["X_BID_log_returns"].corr(slice["Y_BID_log_returns"])
        ask_correlation = slice["X_ASK_log_returns"].corr(slice["Y_ASK_log_returns"])
        x_bid_liquidity = slice["X_BID_VOL"].mean()
        x_ask_liquidity = slice["X_ASK_VOL"].mean()
        y_bid_liquidity = slice["X_BID_VOL"].mean()
        y_ask_liquidity = slice["X_ASK_VOL"].mean()

        X_spread = slice["X_Spread"].mean()
        Y_spread = slice["Y_Spread"].mean()
        x_returns_mean = slice["X_BID_log_returns"].mean()
        x_returns_sd = slice["X_BID_log_returns"].std()
        y_returns_mean = slice["Y_BID_log_returns"].mean()
        y_returns_sd = slice["Y_BID_log_returns"].std()
        returns_mean_diff = x_returns_mean - y_returns_mean
        metrics_periodicity_list.append([grid[i], bid_correlation, ask_correlation, x_bid_liquidity, x_ask_liquidity,
                                         y_bid_liquidity, y_ask_liquidity, X_spread, Y_spread, x_returns_mean,
                                         y_returns_mean, x_returns_sd, y_returns_sd, returns_mean_diff])

    # Correlation scores minute array
    metrics_periodicity_df = pd.DataFrame(metrics_periodicity_list)
    metrics_periodicity_df.columns = ["Periodicity", "Bid_correlation", "Ask_correlation", "X_Bid_liquidity", "X_Ask_liquidity",
                                      "Y_Bid_liquidity", "Y_Ask_liquidity", "X_Spread", "Y_Spread", "X_log_returns",
                                     "Y_log_returns", "X_standard_deviation", "Y_standard_deviation", "Returns_mean_difference"]
    return metrics_periodicity_df

def stationarity_test(X, cutoff=0.01):
    # H_0 in adfuller is unit root exists (non-stationary)
    # We must observe significant p-value to convince ourselves that the series is stationary
    pvalue = adfuller(X)[1]
    if pvalue < cutoff:
        print('p-value = ' + str(pvalue) + ' The series is likely stationary.')
    else:
        print('p-value = ' + str(pvalue) + ' The series is likely non-stationary.')

### MODEL TRAINING ###
# Loop over all minutes
# and assess divergence in correlation
dislocation_minute_grid = dislocation_function(df, "Minute")
dislocation_hour_grid = dislocation_function(df, "Hour")
dislocation_day_grid = dislocation_function(df, "Day")
dislocation_week_grid = dislocation_function(df, "Week")
dislocation_month_grid = dislocation_function(df, "Month")
dislocation_year_grid = dislocation_function(df, "Year")

# Study evolution of correlation throughout the 10am time series
# Loop over unique days within the 10am hour
date_unique = df["Date"].unique()
hour_unique = df["Hour"].unique()
minute_unique = df["Minute"].unique()

# Hour/minute grid
hour_spacing = np.linspace(np.min(hour_unique), np.max(hour_unique), len(hour_unique))
minute_spacing = np.linspace(np.min(minute_unique), np.max(minute_unique), len(minute_unique))
second_spacing = np.arange(0,60,10)

# Grouped data by hour and minute
grouped_hour_minute = df.groupby(["Hour", "Minute"]).agg({k: np.mean for k in ['X_BID_log_returns', 'Y_BID_log_returns', 'X_Spread', 'Y_Spread']})

# Isolate first 10 seconds
unique_minute_hour_slice = df.loc[(df["Minute"]==0) & (df["Hour"]==8) & (df["Second"]==0)]
cum_sum_x = unique_minute_hour_slice["X_BID_log_returns"].fillna(0).cumsum()
cum_sum_y = unique_minute_hour_slice["Y_BID_log_returns"].fillna(0).cumsum()

# Plot cumulative returns of first 10 seconds
plt.plot(np.linspace(0,len(cum_sum_x),len(cum_sum_x)), cum_sum_x, label="Cumulative_returns_X")
plt.plot(np.linspace(0,len(cum_sum_y),len(cum_sum_y)), cum_sum_y, label = "Cumulative_returns_Y")
plt.xlabel("First 10 second instances")
plt.ylabel("Cumulative_returns")
plt.savefig("Cumulative_returns_X_Y_8_00_10")
plt.show()

# Loop over unique dates
up_start_days_x = []
up_start_days_y = []
down_start_days_x = []
down_start_days_y = []
for d in range(len(date_unique)):
    unique_date = df.loc[(df["Date"] == date_unique[d])]
    if unique_date["X_BID_log_returns"].iloc[0:6].mean() > unique_date["Y_BID_log_returns"].iloc[0:6].mean():
        up_start_days_x.append([list(unique_date["X_ASK_log_returns"].fillna(0))])
        up_start_days_y.append([list(unique_date["Y_ASK_log_returns"].fillna(0))])
    if unique_date["X_BID_log_returns"].iloc[0:6].mean() < unique_date["Y_BID_log_returns"].iloc[0:6].mean():
        down_start_days_x.append([list(unique_date["X_ASK_log_returns"].fillna(0))])
        down_start_days_y.append([list(unique_date["Y_ASK_log_returns"].fillna(0))])

# Flat data for histogram
up_start_x_array = np.array(list(itertools.chain.from_iterable(up_start_days_x)))
up_start_y_array = np.array(list(itertools.chain.from_iterable(up_start_days_y)))
down_start_x_array = np.array(list(itertools.chain.from_iterable(down_start_days_x)))
down_start_y_array = np.array(list(itertools.chain.from_iterable(down_start_days_y)))

# Determine consistency of argmin and argmax of elementwise difference curves
window_size = 31
polynomial_order = 2
argmin_up_list = []
argmax_up_list = []
argmin_down_list = []
argmax_down_list = []
for i in range(len(up_start_x_array)):
    print("Iteration ", i)
    # Up difference curve
    up_start_x_cumsum = np.array(up_start_x_array[i]).cumsum()
    up_start_y_cumsum = np.array(up_start_y_array[i]).cumsum()
    up_xy_diffs = up_start_x_cumsum - up_start_y_cumsum
    smoothed_diffs_up = np.nan_to_num(savgol_filter(up_xy_diffs, window_size, polynomial_order))
    argmin_up = np.argmin(smoothed_diffs_up)
    argmax_up = np.argmax(smoothed_diffs_up)
    argmin_up_list.append(argmin_up)
    argmax_up_list.append(argmax_up)

    # Plot smoothed difference curve and identify argmin/argmax
    plt.plot(up_xy_diffs, alpha = 0.4)
    plt.plot(smoothed_diffs_up)
plt.ylabel("Cumulative_difference_XY")
plt.xlabel("10_second_increments")
plt.savefig("opt_smoothed_diffs_up_days")
plt.show()

for j in range(len(down_start_x_array)):
    print("Iteration ", j)
    # Down difference curve
    down_start_x_cumsum = np.array(down_start_x_array[j]).cumsum()
    down_start_y_cumsum = np.array(down_start_y_array[j]).cumsum()
    down_xy_diffs = down_start_x_cumsum - down_start_y_cumsum
    smoothed_diffs_down = np.nan_to_num(savgol_filter(down_xy_diffs, window_size, polynomial_order))
    argmin_down = np.argmin(smoothed_diffs_down)
    argmax_down = np.argmax(smoothed_diffs_down)
    argmin_down_list.append(argmin_down)
    argmax_down_list.append(argmax_down)

    # Plot smoothed difference curve and identify argmin/argmax
    plt.plot(down_xy_diffs, alpha=0.4)
    plt.plot(smoothed_diffs_down)
plt.ylabel("Cumulative_difference_XY")
plt.xlabel("10_second_increments")
plt.savefig("opt_smoothed_diffs_down_days")
plt.show()

# Up days
plt.hist(argmin_up_list, label="argmin-up", alpha = 0.4, color='blue')
plt.hist(argmax_up_list, label="argmax-up", alpha = 0.4, color='red')
plt.legend()
plt.savefig("opt_min_max_divergence_up")
plt.show()

# Down days
plt.hist(argmin_down_list, label="argmin-down", alpha = 0.4, color='blue')
plt.hist(argmax_down_list, label="argmax-down", alpha = 0.4, color='red')
plt.legend()
plt.savefig("opt_min_max_divergence_down")
plt.show()

# Data in each minute/hour increment
minute_hour_second_returns_list = []
for i in range(len(hour_spacing)):
    for j in range(len(minute_spacing)):
        for k in range(len(second_spacing)):
            unique_minute_hour = df.loc[(df["Minute"]==minute_spacing[j]) & (df["Hour"]==hour_spacing[i]) & (df["Second"]==second_spacing[k])]
            correlation_bid_xy = np.nan_to_num(unique_minute_hour["X_BID_log_returns"].corr(unique_minute_hour["Y_BID_log_returns"]))
            x_spread = unique_minute_hour["X_Spread"].mean()
            y_spread = unique_minute_hour["Y_Spread"].mean()
            x_avg_returns = unique_minute_hour["X_BID_log_returns"].mean()
            y_avg_returns = unique_minute_hour["Y_BID_log_returns"].mean()
            x_y_avg_mispricing = x_avg_returns - y_avg_returns
            x_bid_liquidity = unique_minute_hour["X_BID_VOL"].mean()
            x_ask_liquidity = unique_minute_hour["X_ASK_VOL"].mean()
            y_bid_liquidity = unique_minute_hour["Y_BID_VOL"].mean()
            y_ask_liquidity = unique_minute_hour["Y_ASK_VOL"].mean()
            minute_hour_second_returns_list.append([hour_spacing[i], minute_spacing[j], len(unique_minute_hour), correlation_bid_xy,
                                             x_spread, y_spread, x_avg_returns, y_avg_returns,x_y_avg_mispricing,
                                             x_bid_liquidity, x_ask_liquidity, y_bid_liquidity, y_ask_liquidity])
# Convert to Dataframe
minute_hour_second_df = pd.DataFrame(minute_hour_second_returns_list)
minute_hour_second_df.columns = ["Hour", "Minute", "Samples", "Correlation", "X_avg_spread", "Y_avg_spread", "X_avg_returns", "Y_avg_returns", "X_Y_avg_mispricing",
                          "x_bid_liquidity", "x_ask_liquidity", "y_bid_liquidity", "y_ask_liquidity"]

#todo Fit local trend model to determine whether to go long X or long Y
# and subsequently close out

# On average, throughout the day - how do these stocks move?
x_bid = df["X_BID"]
y_bid = df["Y_BID"]
x_cleaned_bid = x_bid.fillna(method='bfill')
y_cleaned_bid = y_bid.fillna(method='bfill')
x_returns = pd.DataFrame(minute_hour_second_df["X_avg_returns"])
y_returns = pd.DataFrame(minute_hour_second_df["Y_avg_returns"])
x_cleaned_returns = x_returns.fillna(0)
y_cleaned_returns = y_returns.fillna(0)

if make_plots:
    # Plot Ratio of one time series to the other (this is meant to be stationary)
    plt.plot(np.nan_to_num(df["X_BID"]/df["Y_BID"]))
    plt.ylabel("X/Y ratio")
    plt.xlabel("Time")
    plt.savefig("X_Y_Ratio_"+model)
    plt.show()

    # Liquidity Throughout the day (bid-ask spread)
    plt.plot(minute_hour_second_df["X_avg_spread"], label="X-liquidity-spread")
    plt.plot(minute_hour_second_df["Y_avg_spread"], label="Y-liquidity-spread")
    plt.xlabel("Expected_minute_per_day")
    plt.ylabel("Average_spread")
    plt.savefig("Opt_Liquidity_over_time_"+model)
    plt.show()

    # Liquidity Throughout the day (bid-ask spread)
    plt.plot(minute_hour_second_df["x_bid_liquidity"], label = "X-liquidity-volume-bid")
    plt.plot(minute_hour_second_df["x_ask_liquidity"], label = "X-liquidity-volume-ask")
    plt.plot(minute_hour_second_df["y_bid_liquidity"], label = "Y-liquidity-volume-bid")
    plt.plot(minute_hour_second_df["y_ask_liquidity"], label = "Y-liquidity-volume-ask")
    plt.xlabel("Expected_minute_per_day")
    plt.ylabel("Avg_volume")
    plt.legend()
    plt.savefig("Opt_liquidity_volume_"+model)
    plt.show()

# # Spot Divergence throughout the average day (subject to liquidity)
# # Test for correlation and cointegration between the two series
# print('Correlation: ' + str(np.corrcoef(np.array(df["X_BID"]), np.array(df["Y_BID"]))))
# score, pvalue, _ = coint(x_cleaned_bid, y_cleaned_bid)
# print('Cointegration test p-value: ' + str(pvalue))

# Moving average function
def moving_average(returns, window):
    moving_avg = []
    for i in range(window, len(returns)):
        moving_avg.append(np.mean(returns[i - window:i]))
    return moving_avg

# The series appear to be both correlated and cointegrated
# Compute Cumulative returns from X & Y
x_avg_cum_returns = x_cleaned_returns.cumsum()
y_avg_cum_returns = y_cleaned_returns.cumsum()

# Smooth out returns with X & Y (20 minute blocks)
x_avg_cum_returns_smoothed = moving_average(x_avg_cum_returns, 20)
y_avg_cum_returns_smoothed = moving_average(y_avg_cum_returns, 20)
difference_smoothed_ts = np.array(x_avg_cum_returns_smoothed) - np.array(y_avg_cum_returns_smoothed)

if make_plots:
    plt.plot(x_cleaned_returns, label="X_avg time series")
    plt.plot(y_cleaned_returns, label="Y_avg time series")
    plt.xlabel("Minutes throughout the day")
    plt.ylabel("Avg cumulative returns")
    plt.legend()
    plt.savefig("Opt_avg_returns_"+model)
    plt.show()

    # Plot of cumulative log returns for X vs Y
    plt.plot(x_avg_cum_returns, label='X-avg cumulative time series', alpha = 0.2)
    plt.plot(y_avg_cum_returns, label='Y-avg cumulative time series', alpha = 0.2)
    plt.plot(x_avg_cum_returns_smoothed, label='X-avg cumulative time series smoothed')
    plt.plot(y_avg_cum_returns_smoothed, label='Y-avg cumulative time series smoothed')
    plt.xlabel("Minutes throughout the day")
    plt.ylabel("Avg cumulative returns")
    plt.legend()
    plt.savefig("Opt_avg_cumulative_returns_"+model)
    plt.show()

    # Plot the difference in smoothed Time series vs Time
    plt.plot(np.linspace(0,len(difference_smoothed_ts),len(difference_smoothed_ts)), difference_smoothed_ts)
    plt.xlabel("Minutes in day")
    plt.ylabel("Difference in smoothed TS")
    plt.title("Evolutionary_difference")
    plt.savefig("Opt_Evolutionary_difference_direction_"+model)
    plt.show()

    # Plot the difference in smoothed Time series vs Time
    plt.plot(np.linspace(0,len(difference_smoothed_ts),len(difference_smoothed_ts)), np.abs(difference_smoothed_ts))
    plt.xlabel("Minutes in day")
    plt.ylabel("Difference in smoothed TS")
    plt.title("Evolutionary_difference")
    plt.savefig("Opt_Evolutionary_difference_modulus_"+model)
    plt.show()

# Identify locations of buy/sell potential
argmax_100 = np.argmax(difference_smoothed_ts[0:100])
argmin_100 = np.argmin(difference_smoothed_ts[0:100])
argmax_200_400 = 200 + np.argmax(difference_smoothed_ts[200:])

data = pd.read_csv(r"C:\Users\60848\Desktop\opt\final_data_10s.csv", index_col='Time', parse_dates=True)
data.index #see index is datetime format
data.loc['2020-08-01 08'] #Filter data based on one day, hour etc

price_names=['X_BID','X_ASK','Y_BID','Y_ASK']
vol_names=['X_BID_VOL','X_ASK_VOL','Y_BID_VOL','Y_ASK_VOL']
price_df=data[price_names];vol_df=data[vol_names]
data['X_spread']=data.X_ASK - data.X_BID; data['Y_spread']=data.Y_ASK - data.Y_BID
spread_df=data[['X_spread','Y_spread']]

# Intra-day trading strategy
# When we refer to LONG, we are referring to the ratio of X/Y
x_spread_ub = np.arange(10,50,10)
y_spread_ub = np.arange(10,50,10)
open_minute = np.linspace(5, 19, 15)
open_second = np.arange(0, 20, 10)
close_minute = np.linspace(0, 10, 11)
close_second = np.arange(20, 60, 10)

# We are going to take the 30 second point within each minute (however this could also be optimised)
intra_day_results = []
profit_curve = []
for o in range(len(open_second)):
    for c in range(len(close_second)):

            # OPEN
            open_time = datetime.time(8, np.int(0), open_second[o])
            open_slice = data.loc[(data.index.time==open_time) & (spread_df.X_spread<20) & (spread_df.Y_spread<20)]
            open_slice.index=open_slice.index.date

            # CLOSE
            close_time = datetime.time(13, 55, close_second[c])
            close_slice=data.loc[data.index.time==close_time]
            close_slice.index=close_slice.index.date

            #At Open (around 8:10), go long X and short Y, and close later in the day (around 155).
            strategy = close_slice.X_BID + open_slice.Y_BID - open_slice.X_ASK - close_slice.Y_ASK

            # At open go long Y and short X, and seconds later
            # strategy = close_slice.Y_BID + open_slice.X_BID - open_slice.Y_ASK - close_slice.X_ASK
            strategy_clean = strategy.fillna(0)
            cumulative_profit = np.cumsum(strategy_clean)
            intra_day_results.append([cumulative_profit[-1], open_second[o], close_second[c]])
            profit_curve.append(cumulative_profit)

    print("Iteration ", open_second[o])

# Make intraday results a dataframe
intra_day_results_df = pd.DataFrame(intra_day_results)
intra_day_results_df.columns = ["Profit", "Open_second", "Close_second"]
intra_day_results_df.sort_values(by="Profit")

# Trend model with evolutionary mispricing
strategy_profit_list = []
regression_prediction = []
smoothing_window = 50
df["XY_mispricing"] = df["X_BID_log_returns"] - df["Y_BID_log_returns"]
from sklearn.linear_model import LinearRegression

# Evolutionary mispricing 8am
evolutionary_mispricing_8am = []
x_spread_8am = []
y_spread_8am = []
for i in range(len(date_unique)):
    unique_minute_hour_i = df.loc[(df["Minute"] == 0) & (df["Hour"] == 8) & (df["Second"] == 10)].iloc[i]
    mispricing = unique_minute_hour_i["X_BID_log_returns"] - unique_minute_hour_i["Y_BID_log_returns"]
    x_spread_8am.append(unique_minute_hour_i["X_Spread"])
    y_spread_8am.append(unique_minute_hour_i["Y_Spread"])
    evolutionary_mispricing_8am.append(np.nan_to_num(mispricing))

# Evolutionary mispricing at 8am
plt.plot(date_unique, evolutionary_mispricing_8am)
plt.title("Evolutionary mispricing SOD")
plt.savefig("Evolutionary_mispricing_SOD")
plt.show()

# Combined spread
merged_spread = np.array(x_spread_8am) + np.array(y_spread_8am)

plt.plot(date_unique, merged_spread)
plt.ylabel("Spread")
plt.xlabel("Days")
plt.savefig("Spread_8am_combined")
plt.show()

# Plot merged spread and profit on same plot and compare relationship
fig, ax1 = plt.subplots()
color = 'tab:red'
ax1.set_xlabel('Days')
ax1.set_ylabel('merged_spread', color=color)
ax1.plot(date_unique, merged_spread, color=color)
ax1.tick_params(axis='y', labelcolor=color)
ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis
color = 'tab:blue'
ax2.set_ylabel('intra_day_profit', color=color)  # we already handled the x-label with ax1
ax2.plot(date_unique, profit_curve[0], color=color)
ax2.tick_params(axis='y', labelcolor=color)
fig.tight_layout()  # otherwise the right y-label is slightly clipped
plt.savefig("opt_8am_spread_vs_profitability")
plt.show()

# Do we have a trend in the evolutionary 8am mispricing?
cumulative_mispricing = np.array(evolutionary_mispricing_8am).cumsum()
plt.plot(np.array(evolutionary_mispricing_8am).cumsum())
plt.savefig("Evolutionary_cumulative_SOD_mispricing")
plt.show()

for i in range(50, len(date_unique)-1):
    # Run rolling linear regression
    unique_date_i = df.loc[(df["Date"] == date_unique[i])]
    trailing_data_open = df.loc[(df["Hour"]==8) & (df["Minute"] == 0) & (df["Second"] == 0)]["XY_mispricing"].fillna(0).iloc[(i-smoothing_window):i]
    trailing_data_open_cumulative = evolutionary_mispricing_8am[(i-smoothing_window):i]
    grid = np.linspace(1,smoothing_window, smoothing_window)

    # Fit linear regression on mispricing at 8am and predict the next value being above/below 0
    reg = LinearRegression().fit(grid.reshape(-1,1), np.array(trailing_data_open_cumulative).reshape(-1,1))
    next_value_prediction = reg.predict(np.array(51).reshape(-1, 1))[0][0]
    regression_prediction.append(next_value_prediction)

    # Slice open and close times intra-day
    current_open = unique_date_i.loc[(unique_date_i["Hour"] == 8) & (unique_date_i["Minute"] == 0) & (unique_date_i["Second"] == 0)]
    current_close = unique_date_i.loc[(unique_date_i["Hour"] == 13) & (unique_date_i["Minute"] == 55) & (unique_date_i["Second"] == 0)]

    # The idea here is that if X-Y mispricing is HIGH, we short X and Long Y
    # This means X-Y is HIGH and therefore X is OVERPRICED
    # We would SHORT X and LONG Y
    if next_value_prediction > 0: # Additional conditions eg. the magnitude of change(np.abs(next_value_prediction) > .02)
        strategy_profit = current_close["Y_BID"].iloc[0] + current_open["X_BID"].iloc[0] - current_open["Y_ASK"].iloc[0] - current_close["X_ASK"].iloc[0]
        strategy_profit_list.append(strategy_profit)
    else:
    # Else perform trade other way
        strategy_profit = current_close["X_BID"].iloc[0] + current_open["Y_BID"].iloc[0] - current_open["X_ASK"].iloc[0] - current_close["Y_ASK"].iloc[0]
        strategy_profit_list.append(strategy_profit)
    print("Iteration ", i)

# Plot regression function
plt.plot(np.linspace(50,409,359), regression_prediction, label="Prediction")
plt.plot(np.linspace(1,len(np.array(evolutionary_mispricing_8am).cumsum()), len(np.array(evolutionary_mispricing_8am).cumsum())), np.array(evolutionary_mispricing_8am).cumsum(),
         label="Cumulative_8am_mispricing_X-Y")
plt.legend()
plt.savefig("opt_trend_model_mispricing")
plt.show()

# Print Strategy profit
print(np.sum(strategy_profit_list))

plt.plot(np.array(strategy_profit_list).cumsum())
plt.show()

x=1
y=2


"""
This module describe data model for "menu_item" table
"""

from sqlalchemy import (
    Column,
    Integer,
    Text,
    ForeignKey,
    Float,
    Numeric
)
from sqlalchemy.orm import relationship
from sqlalchemy.exc import IntegrityError
from pyramid.httpexceptions import HTTPForbidden

from .meta import Base

class MenuItem(Base):
    """
    The data model of "menu_items" table
    Defines data structure of "menu_items" table
    Relationship:
        menu_items -> category (One to One)
    """
    __tablename__ = 'menu_items'
    id = Column(Integer, primary_key=True)
    name = Column(Text)
    description = Column(Text)
    ingredients = Column(Text)
    img = Column(Text)
    price = Column(Numeric(precision=10, scale=2), default=0)
    amount = Column(Numeric())
    menu_id = Column(Integer, ForeignKey('menus.id'))
    category_id = Column(Integer, ForeignKey('categories.id'))

    category = relationship("Category")
    orders = relationship('OrderAssoc',  cascade="delete")
    menu = relationship('Menu')

    @staticmethod
    def create_item(session, form_data, menu_id):
        name = form_data["name"]
        description = form_data["description"]
        ingredients = form_data["ingredients"]
        img = form_data["img"]
        price = form_data["price"]
        amount = form_data["amount"]
        menu_id = menu_id
        category_id = form_data["category_id"]

        item = MenuItem(name=name, description=description,
                        ingredients=ingredients, img=img, price=price,
                        amount=amount, menu_id=menu_id, category_id=category_id)
        session.add(item)

        try:
            session.flush()
        except IntegrityError:
            session.rollback()
            raise HTTPForbidden("Can't Create Item")

        return item

    def update_item(self, session, form_data):
        name = form_data["name"]
        description = form_data["description"]
        ingredients = form_data["ingredients"]
        img = form_data["image"]
        price = float(form_data["price"]) * 100
        amount = form_data["value"]
        category_id = form_data["category"]

        self.name = name
        self.description = description
        self.ingredients = ingredients
        self.img = img
        self.price = price
        self.amount = amount
        self.category_id = category_id

        return self


import os

from google.cloud import storage
from xray.params import BUCKET_NAME, GCP_STORAGE_LOCATION

def storage_upload(
    model_name,
    base_model,
    version,
    rm=False,
):
    client = storage.Client().bucket(BUCKET_NAME)

    local_model_name = f"{model_name}"
    storage_location = f"models/{base_model}/{version}/{local_model_name}"
    blob = client.blob(storage_location)
    blob.upload_from_filename(local_model_name)
    print(
        f"=> {local_model_name} uploaded to bucket {BUCKET_NAME} inside {storage_location}",
        "green",
    )
    if rm:
        os.remove(local_model_name)


a = input().split()
b = int(input())
a = list(map(int, a))
a.append(b)
a.sort()
a = a[::-1]
for i in range(len(a) - 1, -1, -1):
    if a[i] == b:
        print(i + 1)
        break

pythontutor/Занятие 7. Списки/Задача «Шеренга».py

Задача «Шеренга».py


# Autores: Camilo Henrique Martins dos Santos (202004940027)
#          Eude Monteiro da Hora (202004940004)

#Imports
from grid import Grid
import numpy as np
import math

# Inicializa a grid
grid = Grid(extent=10, size=500)

def render_cells(selected_cells, rendered_cells, parameters):
    for cell in selected_cells:
        grid.render_cell(cell)

def translate(selected_cells, rendered_cells, parameters):

    x_offset = int(parameters['X'])
    y_offset = int(parameters['Y'])

    grid._clear_all()

    # Renderiza as novas células decorrentes da translação
    for cell in rendered_cells:
        new_cell = (cell[0] + x_offset, cell[1] + y_offset)
        grid.render_cell(new_cell)

def bresenham(selected_cells, rendered_cells=None, parameters=None):
    if len(selected_cells) != 2:
        print("O algoritmo requer exatamente duas células selecionadas.")
        return

    cell1, cell2 = selected_cells[0], selected_cells[1]

    x1, y1 = cell1
    x2, y2 = cell2
    dx = abs(x2 - x1)
    dy = abs(y2 - y1)
    sx = -1 if x1 > x2 else 1
    sy = -1 if y1 > y2 else 1
    err = dx - dy

    line_coordinates = []

    while True:
        line_coordinates.append((x1, y1))

        if x1 == x2 and y1 == y2:
            break

        e2 = 2 * err
        if e2 > -dy:
            err -= dy
            x1 += sx
        if e2 < dx:
            err += dx
            y1 += sy

    for cell in line_coordinates:
        grid.render_cell(cell)

    grid.vertices = selected_cells

def draw_circle(selected_cells, rendered_cells, parameters):
    radius = int(parameters['Raio'])
    if len(selected_cells) != 0:
        center_x,center_y = selected_cells[0][0], selected_cells[0][1]
    else:
        center_x = int(parameters['X do centro'])
        center_y = int(parameters['Y do centro'])

    x = radius
    y = 0
    decision = 1 - radius
    grid._clear_all()

    def plot_circle(center_x, center_y, x, y):
        """
        Realiza o plot do círculo utilizando as
        coordenadas do centro e coordenadas relativas
        """

        # Reflexão de pontos em octantes (simetria)
        points = [(x, y), (-x, y), (x, -y), (-x, -y), (y, x), (-y, x), (y, -x), (-y, -x)]
        for point in points:
            grid.render_cell((center_x + point[0], center_y + point[1]))

    while x >= y:
        plot_circle(center_x, center_y, x, y)

        y += 1
        if decision <= 0:
            decision += 2 * y + 1
        else:
            x -= 1
            decision += 2 * (y - x) + 1

    # Redesenha a grid após renderização
    grid._redraw()

def recursive_fill(selected_cells, rendered_cells, parameters):

    filled_cells = []

    def flood_fill(x, y, edge_pixels, rendered_pixels):
        point = (x,y)

        if ((abs(x) > 10 or abs(y) > 10) or
           (point in edge_pixels) or
           (point in rendered_pixels)):
            return

        rendered_pixels.append(point)

        flood_fill(x+1, y, edge_pixels, rendered_pixels)
        flood_fill(x-1, y, edge_pixels, rendered_pixels)
        flood_fill(x, y+1, edge_pixels, rendered_pixels)
        flood_fill(x, y-1, edge_pixels, rendered_pixels)

        return rendered_pixels

    result = flood_fill(selected_cells[0][0],
                        selected_cells[0][1],
                        rendered_cells,
                        filled_cells)

    for point in result:
        grid.raster.fill_cell((point))

def scanline_fill(selected_cells, rendered_cells, parameters):
    # Altura máxima e mínima da varredura
    min_y = min(point[1] for point in rendered_cells)
    max_y = max(point[1] for point in rendered_cells)

    # Varredura vertical
    for y in range(min_y, max_y + 1):
        # Intersecção entre as células coloridas
        # e arestas do polígono

        intersections = []
        for i, cell in enumerate(rendered_cells):
            x1, y1 = cell
            x2, y2 = rendered_cells[(i+1) % len(rendered_cells)]

            if (y1 <= y < y2) or (y2 <= y < y1):
                # Cálculo das intersecções
                x_intersect = int(x1 + (float(y - y1) / (y2 - y1)) * (x2 - x1))
                intersections.append(x_intersect)

        # Ordenação crescente  das coordenadas x
        intersections.sort()

        # Preenchimento dos pixels entre intersecções subsequentes
        for i in range(0, len(intersections), 2):
            x_start = intersections[i]
            x_end = intersections[i+1] if i+1 < len(intersections) else x_start

            # Preenchimento dos pixels entre os pontos de intersecção
            for x in range(x_start, x_end + 1):
                grid.raster.fill_cell((x, y))

def polyline(selected_cells, rendered_cells, parameters):

    if len(selected_cells) == 2:
        bresenham(selected_cells)
        return

    elif len(selected_cells) > 2:
        selected = selected_cells.copy()

    else:
        return

    length = len(selected)

    for i in range(length):
        if i+1 < length:
            draw = (selected[i],selected[i+1])
        else:
            draw = (selected[0],selected[i])

        bresenham(draw)
        grid.vertices = selected_cells

def scale(selected_cells, rendered_cells, parameters):

    x_factor = int(parameters['X'])
    y_factor = int(parameters['Y'])
    fixed_point = (int(parameters['FX']), int(parameters['FY']))

    points = grid.vertices.copy()
    new_points = []

    for i in range(len(points)):
        point = points[i]
        x = fixed_point[0] + (point[0] - fixed_point[0]) * x_factor
        y = fixed_point[1] + (point[1] - fixed_point[1]) * y_factor

        new_points.append((x,y))

    grid._clear_all()
    polyline(new_points,rendered_cells,parameters)

def cut(selected_cells, rendered_cells, parameters):

    x_list = [selected_cells[0][0],selected_cells[1][0]]
    y_list = [selected_cells[0][1],selected_cells[1][1]]
    x_max = max(x_list)
    x_min = min(x_list)
    y_max = max(y_list)
    y_min = min(y_list)
    for cell in rendered_cells:
        if ((x_min <= cell[0]) and (cell[0] <= x_max)) and ((y_min <= cell[1]) and (cell[1] <= y_max)):
            continue
        grid.clear_cell(cell)

def bezier_curve(selected_cells, rendered_cells, parameters):
    # Calcula o grau da curva
    degree = len(selected_cells) - 1

    # Verifica se há pontos de controle suficientes
    if degree + 1 < 3:
        return

    # Número de pontos a gerar na curva
    num_points = 100

    # Gera valores de t de 0 a 1
    t_values = [t / num_points for t in range(num_points + 1)]

    # Calcula os pontos na curva de Bezier
    for t in t_values:
        x = y = 0

        # Calcula as coordenadas x e y para o valor de t atual
        for i in range(degree + 1):
            coefficient = math.comb(degree, i)
            b = coefficient * (1 - t) ** (degree - i) * t ** i
            x += selected_cells[i][0] * b
            y += selected_cells[i][1] * b

        point = (int(x), int(y))

        rendered_cells.append(point)
        bresenham(rendered_cells[-2:], rendered_cells, parameters)

def rotation(selected_cells, rendered_cells, parameters):
    grid._clear_all()

    ang = np.deg2rad(float(parameters['Ângulo']))
    x_pivot = int(parameters['X do pivô'])
    y_pivot = int(parameters['Y do pivô'])

    cells = rendered_cells.copy()

    for i in range(len(cells)):
        x = cells[i][0] - x_pivot
        y = cells[i][1] - y_pivot

        x_rot = int(round(x*np.cos(ang) - y * np.sin(ang)))
        y_rot = int(round(x*np.sin(ang)+y*np.cos(ang)))

        x_rot += x_pivot
        y_rot += y_pivot

        cells[i] = (x_rot,y_rot)

    for cell in cells:
        grid.render_cell(cell)

#Adição dos algoritmos à grid
grid.add_algorithm(name='Renderizar', parameters=None, algorithm=render_cells)

grid.add_algorithm(name='Bresenham', parameters=None, algorithm=bresenham)

grid.add_algorithm(name='Translação', parameters=['X', 'Y'], algorithm=translate)

grid.add_algorithm(name='Rotação', parameters=['Ângulo', 'X do pivô', 'Y do pivô'], algorithm=rotation)

grid.add_algorithm(name='Escala', parameters=['X','Y','FX','FY'], algorithm=scale)

grid.add_algorithm(name='Polilinha', parameters=None, algorithm=polyline)

grid.add_algorithm(name='Círculo', parameters=['Raio', 'X do centro', 'Y do centro'], algorithm=draw_circle)

grid.add_algorithm(name='Preenchimento (Recursivo)', parameters=None, algorithm=recursive_fill)

grid.add_algorithm(name='Preenchimento (Varredura)', parameters=None, algorithm=scanline_fill)

grid.add_algorithm(name='Curva de Bezier', parameters=None, algorithm=bezier_curve)

grid.add_algorithm(name='Recorte', parameters=None, algorithm=cut)

grid.show()


class Solution:
    def nextGreaterElement(self, nums1: List[int], nums2: List[int]) -> List[int]:
        len1 = len(nums1)
        result = [-1]*len1
        for i in range(len1):
            index = nums2.index(nums1[i])
            for j in range(index + 1, len(nums2)):
                if nums2[j] > nums1[i]:
                    result[i] = nums2[j]
                    break

        return result


import pandas as pd
import numpy as np
from datetime import datetime as dt
from . import utilities as ut
from django.templatetags.static import static
from babel.numbers import format_currency

locale='en_IN'
currency = 'INR'

path_windows = "C:/Users/591705/django/plate-charts-NewUI/plate/static/data/"
path_mac="/Users/rags/Documents/ScopeAnalytica/plate-charts-NewUI/plate/static/data/"

'''
The below mentiond cleanData() function needs to be moved as a
module to onboarding or data upload application.This is a temp function just to read.
While on boarding properties needs to be set, while uploading those props needs to be used and final sheet needs to
be produced. Refer jupyter notebook "charts"
'''
def readData():
	df = pd.read_csv(path_windows+'input/data_f.csv' , parse_dates=["S2BillDate"])

	return df

def returnBasicDf(df):

	basic_df = df.groupby(['S2PName','S2PName-Category'], sort=False)['S2PGTotal'].agg([('totSale','sum'), ('count','size')]).reset_index()

	return basic_df

def filter_df_daterange(df,start_date,end_date):

	filter_by_date = df['S2BillDate'].between(start_date,end_date)
	df = df[filter_by_date]
	return df

def filter_df_franchise(df,franchise):
	filter_by_franchise = df['Franchise']==franchise
	df = df[filter_by_franchise]
	return df

'''function name says category trend, but it basically groups by category
and date, hence can be reused for overall sales performance but grouping category needs to be dealth with,
or else can create problems'''
def categoryTrend(df,flag):

	df = df.groupby(['S2PName-Category','background_color','border_color',df['S2BillDate'].dt.to_period(flag)], sort=False)['S2PGTotal'].agg([('totSale','sum'),('count','size')]).reset_index()
	df = df.sort_values('S2BillDate').reset_index(drop=True)
	df['S2BillDate'] = df['S2BillDate'].astype('str')

	return df

def franchiseTrend(df,flag):
	df = df.groupby([df['S2BillDate'].dt.to_period(flag)], sort=False)['S2PGTotal'].agg([('totSale','sum')]).reset_index()
	df = df.sort_values('S2BillDate').reset_index(drop=True)
	df['S2BillDate'] = df['S2BillDate'].astype('str')

	return df

def categoryChart(date_range):

	df = categoryTrend(df,start_date,end_date,flag)

	return df

def itemPerf(df,category,count):
	df = df.groupby(['S2PName','S2PName-Category','background_color','border_color'], sort=False)['S2PGTotal'].agg([('totSale','sum'), ('count','size')]).reset_index()
	filter_category = df['S2PName-Category'] == category
	bstPerf_df = df[filter_category].sort_values('totSale',ascending=False)
	not_bstPerf_df = df[filter_category].sort_values('totSale')
	return bstPerf_df.head(count),not_bstPerf_df.head(count)

def getDataForItemsTrend(df,start_date,end_date,flag):

	df = df.groupby(['S2PName',df['S2BillDate'].dt.to_period(flag)], sort=False)['S2PGTotal'].agg([('totSale','sum'),('count','size'),('Avg','mean')]).reset_index()
	#print(df.head(20))
	#df.sort_values('S2BillDate',inplace=True) - commented
	df = df.sort_values("S2BillDate").reset_index(drop=True)

	df['S2BillDate'] = df['S2BillDate'].astype('str')
	if(flag=='w'):
		df["S2BillDate"] = df["S2BillDate"].str.split("/").str[1].astype("datetime64[ns]")

	#print(df.head())
	return df

def getRollingPercentage(df,cardStartDate,endDate):

	df_d = df.groupby([df['S2BillDate'].dt.to_period('d')], sort=False)['S2PGTotal'].agg([('totSale','sum')]).reset_index()
	df_d['DoD'] = (df_d['totSale'].pct_change()*100).round(1)
	df_m = df.groupby([df['S2BillDate'].dt.to_period('m')], sort=False)['S2PGTotal'].agg([('totSale','sum')]).reset_index()
	df_m['MoM'] = (df_m['totSale'].pct_change()*100).round(1)

	cardStartMonth = ut.subtractDays(endDate,31)
	#daySales_1 means it referes to startdate oda overall sales
	if df_d[df_d['S2BillDate']==cardStartDate.to_period('d')].empty:
		daySales_1 = 'nan'
	else:
		daySales_1 = format_currency(df_d[df_d['S2BillDate']==cardStartDate.to_period('d')]['totSale'].iloc[0].round(1),currency,locale=locale)

	if df_d[df_d['S2BillDate']==endDate.to_period('d')].empty:
		daySales_2 = 'nan'
	else:
		daySales_2 = format_currency(df_d[df_d['S2BillDate']==endDate.to_period('d')]['totSale'].iloc[0].round(1),currency,locale=locale)
		daySales_percentage = df_d[df_d['S2BillDate']==endDate.to_period('d')]['DoD'].iloc[0]

	#print(df_m)
	if df_m[df_m['S2BillDate']==cardStartMonth.to_period('m')].empty:
		monthSales_1 = 'nan'
	else:
		monthSales_1 = format_currency(df_m[df_m['S2BillDate']==cardStartMonth.to_period('m')]['totSale'].iloc[0].round(1),currency, locale=locale)

	if df_m[df_m['S2BillDate']==endDate.to_period('m')].empty:
		monthSales_2='nan'
	else:
		monthSales_2 = format_currency(df_m[df_m['S2BillDate']==endDate.to_period('m')]['totSale'].iloc[0].round(1),currency,locale=locale)
		monthSales_percentage = df_m[df_m['S2BillDate']==endDate.to_period('m')]['MoM'].iloc[0]

	cardsData = {'daySales_1':daySales_1,
				 'daySales_2' : daySales_2,
				 'daySales_percentage':daySales_percentage,
				 'monthSales_1':monthSales_1,
				 'monthSales_2':monthSales_2,
				 'monthSales_percentage':monthSales_percentage

	}

	return cardsData


import urllib.request
import json
import sys
import time

name = input("Enter channel name : ")
ask_api = input('Do you have api? \n Type Y if you have API key : ')

if ask_api == 'y' or 'Y':
    pass
else:
    print('How to get your API : \n https://developers.google.com/youtube/v3/getting-started')
    sys.exit()

api = input('enter your api key : ')
t = input('Interval of Output \n [DEFAULT 1] :')
if t == '':
    t = int('1')
else:
    t = int(t)

while True:
    data = urllib.request.urlopen('https://www.googleapis.com/youtube/v3/channels?part=statistics&key=' + (api) + '&forUsername=' + (name)).read()
    subs = json.loads(data)["items"][0]["statistics"]["subscriberCount"]
    num = (name + " has " + "{:,d}".format(int(subs)) + " subscribers!")
    sys.stdout.write(num + '\n')
    time.sleep(t)


# Import modules
from math import sqrt
import pandas as pd
import numpy as np
import csv

# Define landmarks and normalized landmarks
landmark = [6, 168, 8, 193, 222, 224, 223, 113, 226, 25, 24, 22, 112, 243, 155, 153, 144, 23, 161, 160, 159, 157, 173, 246, 247, 27, 30, 56, 122, 231, 230, 229, 52, 53,
            55, 413, 464, 414, 463, 398, 382, 341, 453, 441, 285, 442, 385, 380, 252, 451, 443, 257, 386, 374, 253, 254, 173, 387, 445, 467, 263, 255, 359, 446, 342, 261, 348]

normalizer_lm = [(151, 175), (396, 337), (108, 171)]

# Read CSV file
df = pd.read_csv('initial_dataset.csv')

# Calculate normalization distance for each image based on normalizer_lm

df_original = df
large_empty_list = []
for element in normalizer_lm:

    normalizer_lm_dist = np.sqrt((
        df_original[str(element[0])+'x']-df_original[str(element[1])+'x'])**2+(df_original[str(element[0])+'y']-df_original[str(element[1])+'y'])**2)

    large_empty_list.append(list(normalizer_lm_dist))

norm_dist = np.transpose(np.mean(np.array(large_empty_list), axis=0))

# Generate tuple of (x,y) coordinate  point  for the "landmark" vector

x_coordinate = df[[str(i)+'x' for i in landmark]]
y_coordinate = df[[str(i)+'y' for i in landmark]]

df = pd.DataFrame(np.rec.fromarrays((x_coordinate.values, y_coordinate.values)
                                    ).tolist(), columns=x_coordinate.columns, index=x_coordinate.index)

# Generate pair-wise tuple and calculate the distance

input_dataset = []

def calc_distance(p1, p2):
    return sqrt((p1[0]-p2[0])**2+(p1[1]-p2[1])**2)

for row in df.to_numpy():
    test_list = row
    result_list = [(a, b) for idx, a in enumerate(test_list)
                   for b in test_list[idx + 1:]]
    distance_list = [calc_distance(pair[0], pair[1]) for pair in result_list]
    input_dataset.append(distance_list)

input_dataset = np.array(input_dataset)
# print(input_dataset.shape)  # Shape of the dataset

# Normalisng row-wise using python broadcasting

final_dataset = input_dataset/(norm_dist[:, np.newaxis])

final_dataset_df = pd.DataFrame(final_dataset)

# Slicing (from original data) and attaching output coloumn

test = df_original[['output']]
final = final_dataset_df.join(test)

print(final.shape)


import os

from data.conll03 import parse_raw_conll
from data.data_utils import build_vocab
from data.data_utils import iob2iobes
from global_vars import DATA_DIR, EMBEDDINGS_DIR
from utils.embeddings import trim_embeddings
from utils.evaluation import correct_iob

if __name__ == "__main__":

    source_path = DATA_DIR + "conll03/source/"
    original_path = DATA_DIR + "conll03/original/"

    if not os.path.exists(source_path):
        os.makedirs(source_path)
    if not os.path.exists(original_path):
        os.makedirs(original_path)

    ### Reformat Source data
    for split, file in zip(["train", "dev", "test"],
                           ["eng.train", "eng.testa", "eng.testb"]):
        words, tags, _ = parse_raw_conll(source_path + file, sep="\t")

        ### Correct IOB tags in case eng.train ..etc.. have only I- tags
        tags = correct_iob(tags)

        with open(original_path + "{}.words.txt".format(split), "w", encoding="utf8") as file:
            for sent in words:
                file.write("{}\n".format(" ".join(sent)))

        with open(original_path + "{}.iob.txt".format(split), "w", encoding="utf8") as file:
            for sent in tags:
                file.write("{}\n".format(" ".join(sent)))

        with open(original_path + "{}.iobes.txt".format(split), "w", encoding="utf8") as file:
            for sent in tags:
                file.write("{}\n".format(" ".join(iob2iobes(sent))))

    ### Compute vocabulary
    build_vocab(original_path)

    ### Trim glove embeddings
    word_vocab_path = original_path + "vocab.words.txt"
    embedding_path = EMBEDDINGS_DIR + "glove.840B/glove.840B.300d.txt"
    saving_path = original_path + "glove.840B.300d"
    trim_embeddings(word_vocab_path, embedding_path, saving_path)


import sys

import pytest

from data_generator.cli_parser import convert_args, parse_inputs, verify

@pytest.fixture(
    scope="module",
    params=[
        [
            # "python3",
            # "-m",
            "data_generator",
            "-sa",
            "xlsx",
            "-f",
            "output/test",
            "data",
            "column1:str:0:50",
            "column2:str:101:101",
            "column3:int:10:10",
            "column4:int:0:1000",
            "column5:float:0.0:1000.0",
            "1000",
        ],
        [
            # "python3",
            # "-m",
            "data_generator",
            "toml",
            "data_config_example01.toml",
            "data_config_example02.toml",
        ],
    ],
)
def valid_cli_input(request):
    return request.param

class TestFuncParseInputs:
    def test_with_valid_inputs(self, valid_cli_input):
        sys.argv = valid_cli_input
        parsed_inputs = parse_inputs()

        # data_parser
        if hasattr(parsed_inputs, "specify"):
            assert vars(parsed_inputs) == dict(
                folder="output/test",
                save_as="xlsx",
                specify=[
                    "column1:str:0:50",
                    "column2:str:101:101",
                    "column3:int:10:10",
                    "column4:int:0:1000",
                    "column5:float:0.0:1000.0",
                ],
                rows=1000,
            )

        # TOML parser
        if hasattr(parsed_inputs, "toml"):
            assert vars(parsed_inputs) == dict(
                # defaults in CLI parser!!!
                folder="output",
                save_as="csv",
                toml=["data_config_example01.toml", "data_config_example02.toml",],
            )

class TestFuncVerify:
    def test_with_correctly_parsed_inputs(self, valid_cli_input):
        sys.argv = valid_cli_input
        parsed_inputs = parse_inputs()

        assert verify(parsed_inputs) is None

class TestFuncConvertArgs:
    def test_with_correctly_parsed_inputs(self, valid_cli_input):
        sys.argv = valid_cli_input
        parsed_inputs = parse_inputs()
        converted_args = convert_args(parsed_inputs)

        if hasattr(parsed_inputs, "specify"):
            assert converted_args == dict(
                folder="output/test",
                save_as="xlsx",
                specify=[
                    dict(
                        data_type="str",
                        column_name="column1",
                        lower_bound=0,
                        upper_bound=50,
                    ),
                    dict(
                        data_type="str",
                        column_name="column2",
                        lower_bound=101,
                        upper_bound=101,
                    ),
                    dict(
                        data_type="int",
                        column_name="column3",
                        lower_bound=10,
                        upper_bound=10,
                    ),
                    dict(
                        data_type="int",
                        column_name="column4",
                        lower_bound=0,
                        upper_bound=1000,
                    ),
                    dict(
                        data_type="float",
                        column_name="column5",
                        lower_bound=0.0,
                        upper_bound=1000.0,
                    ),
                ],
                rows=1000,
            )

        if hasattr(parsed_inputs, "toml"):
            assert converted_args == dict(
                # defaults in CLI parser!!!
                folder="output",
                save_as="csv",
                toml=["data_config_example01.toml", "data_config_example02.toml"],
            )


# add reading id inside datapoint

import json

def set_filter_config(configuration):
    config = json.loads(configuration['config'])
    return True

# process one or more readings
def set_id(readings):
    for elem in list(readings):
        id = elem['id']
        reading = elem['reading']
        reading[b'id_datapoint'] = id
    return readings