from functions.interactions import find_command,send_msg, read_first_contacts,end_bot
from chatterbot import ChatBot
from chatterbot.trainers import ListTrainer
import time
import json
#creating bot intelligent and responses training

botleandor = ChatBot('Leandor')
trainer = ListTrainer(botleandor)

# read responses
with open ('docs\\botresponses.json',  encoding='utf-8') as f:
        read_j= f.read()

with open ('docs\\responsechatterbot.json', encoding='utf-8') as j:
    read_response = j.read()

def main ():

    dic_resp = json.loads(read_j)
    cond = ''
    while cond != 'finalizar' :

        for rotate in range(5,11):

            time.sleep(0.5)
            read_first_contacts(rotate)

            com = find_command()
            resp = dic_resp.get(com)

            if resp != None:
                send_msg(resp)

            if resp == 'bot finalizado':
                time.sleep(2)
                cond = 'finalizar'
                break
    end_bot()

def test ():

    dic_resp = json.loads(read_response).get('dialog')
    trainer.train(dic_resp)

    cond = ''
    while cond != 'finalizar' :

        for rotate in range(8,11):

            time.sleep(0.5)
            read_first_contacts(rotate)

            com = find_command()
            resp = botleandor.get_response(com)

            if float(resp.confidence) > 1:
              #corrigir a resposta para enviar ao whatsapp
                send_msg(str(resp))

            if com == 'finalizar':
                time.sleep(2)
                cond = 'finalizar'
                break

            else:
                cond = ''
    end_bot()

if __name__ == '__main__':
    main()


import typing

from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor as tensor_lib
from tensorflow.python.framework import tensor_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import array_ops_stack
from tensorflow.python.ops import cond
from tensorflow.python.ops import gen_string_ops
from tensorflow.python.ops import map_fn as map_fn_lib
from tensorflow.python.ops import string_ops
from tensorflow.python.ops.ragged import ragged_array_ops
from tensorflow.python.ops.ragged import ragged_functional_ops
from tensorflow.python.ops.ragged import ragged_math_ops
from tensorflow.python.ops.ragged import ragged_tensor
from tensorflow.python.util import compat as util_compat
from tensorflow.python.util import deprecation
from tensorflow.python.util import dispatch
from tensorflow.python.util.tf_export import tf_export

@tf_export("strings.bytes_split")
@dispatch.add_dispatch_support
def string_bytes_split(input, name=None):  # pylint: disable=redefined-builtin
  """Split string elements of `input` into bytes.

  Examples:

  >>> tf.strings.bytes_split('hello').numpy()
  array([b'h', b'e', b'l', b'l', b'o'], dtype=object)
  >>> tf.strings.bytes_split(['hello', '123'])
  <tf.RaggedTensor [[b'h', b'e', b'l', b'l', b'o'], [b'1', b'2', b'3']]>

  Note that this op splits strings into bytes, not unicode characters.  To
  split strings into unicode characters, use `tf.strings.unicode_split`.

  See also: `tf.io.decode_raw`, `tf.strings.split`, `tf.strings.unicode_split`.

  Args:
    input: A string `Tensor` or `RaggedTensor`: the strings to split.  Must
      have a statically known rank (`N`).
    name: A name for the operation (optional).

  Returns:
    A `RaggedTensor` of rank `N+1`: the bytes that make up the source strings.
  """
  with ops.name_scope(name, "StringsByteSplit", [input]):
    input = ragged_tensor.convert_to_tensor_or_ragged_tensor(input,
                                                             name="input")
    if isinstance(input, ragged_tensor.RaggedTensor):
      return input.with_flat_values(string_bytes_split(input.flat_values))

    rank = input.shape.ndims
    if rank is None:
      raise ValueError("input must have a statically-known rank.")

    if rank == 0:
      return string_bytes_split(array_ops_stack.stack([input]))[0]
    elif rank == 1:
      indices, values, shape = gen_string_ops.string_split(
          input, delimiter="", skip_empty=False)
      return ragged_tensor.RaggedTensor.from_value_rowids(
          values=values, value_rowids=indices[:, 0], nrows=shape[0],
          validate=False)
    else:
      return string_bytes_split(ragged_tensor.RaggedTensor.from_tensor(input))

# pylint: disable=redefined-builtin
@tf_export("strings.unicode_encode")
@dispatch.add_dispatch_support
def unicode_encode(input,
                   output_encoding,
                   errors="replace",
                   replacement_char=65533,
                   name=None):
  r"""Encodes each sequence of Unicode code points in `input` into a string.

  `result[i1...iN]` is the string formed by concatenating the Unicode
  codepoints `input[1...iN, :]`, encoded using `output_encoding`.

  Args:
    input: An `N+1` dimensional potentially ragged integer tensor with shape
      `[D1...DN, num_chars]`.
    output_encoding: Unicode encoding that should be used to encode each
      codepoint sequence.  Can be `"UTF-8"`, `"UTF-16-BE"`, or `"UTF-32-BE"`.
    errors: Specifies the response when an invalid codepoint is encountered
      (optional). One of:
            * `'replace'`: Replace invalid codepoint with the
              `replacement_char`. (default)
            * `'ignore'`: Skip invalid codepoints.
            * `'strict'`: Raise an exception for any invalid codepoint.
    replacement_char: The replacement character codepoint to be used in place of
      any invalid input when `errors='replace'`. Any valid unicode codepoint may
      be used. The default value is the default unicode replacement character
      which is 0xFFFD (U+65533).
    name: A name for the operation (optional).

  Returns:
    A `N` dimensional `string` tensor with shape `[D1...DN]`.

  #### Example:

  >>> input = tf.ragged.constant(
  ...     [[71, 246, 246, 100, 110, 105, 103, 104, 116], [128522]])
  >>> print(unicode_encode(input, 'UTF-8'))
  tf.Tensor([b'G\xc3\xb6\xc3\xb6dnight' b'\xf0\x9f\x98\x8a'],
            shape=(2,), dtype=string)
  """
  with ops.name_scope(name, "UnicodeEncode", [input]):
    input_tensor = ragged_tensor.convert_to_tensor_or_ragged_tensor(input)
    if input_tensor.shape.ndims is None:
      raise ValueError("Rank of input_tensor must be statically known.")
    if ragged_tensor.is_ragged(input_tensor):
      if input_tensor.flat_values.shape.ndims > 1:
        # If the flat_values of our ragged tensor is multi-dimensional, we can
        # process it separately and our output will have the same nested splits
        # as our input.
        return input_tensor.with_flat_values(
            unicode_encode(input_tensor.flat_values, output_encoding, errors,
                           replacement_char))
      elif input_tensor.ragged_rank > 1:
        # Recursively process the values of the ragged tensor.
        return input_tensor.with_values(
            unicode_encode(input_tensor.values, output_encoding, errors,
                           replacement_char))
      else:
        # Our ragged tensor is of the correct shape (rank 1 flat_values tensor
        # with ragged_rank of 1) so we can process it as normal.
        return gen_string_ops.unicode_encode(
            input_values=input_tensor.values,
            input_splits=input_tensor.row_splits,
            output_encoding=output_encoding,
            errors=errors,
            replacement_char=replacement_char)
    else:
      if input_tensor.shape.ndims == 2:
        # The input tensor is of the correct 2-D shape, it's just not ragged.
        return unicode_encode(
            ragged_tensor.RaggedTensor.from_tensor(input_tensor),
            output_encoding, errors, replacement_char)
      elif input_tensor.shape.ndims > 2:
        # We need to initially flatten the input tensor to 2-D, and then can
        # reshape the output of our processed flattened tensor.
        flat_input_tensor = array_ops.reshape(
            input_tensor,
            array_ops_stack.stack([-1, array_ops.shape(input_tensor)[-1]]))
        flat_output_tensor = unicode_encode(flat_input_tensor, output_encoding,
                                            errors, replacement_char)
        return array_ops.reshape(flat_output_tensor, input_tensor.shape[:-1])
      elif input_tensor.shape.ndims == 0:
        raise ValueError("input_tensor's rank must be at least 1.")
      else:
        # Our input tensor is rank 1, so we create a ragged tensor with an added
        # dimension to create the correct input shape & type, and then remove
        # the additional dimension from the output and return the string scalar.
        ragged_input_tensor = ragged_tensor.RaggedTensor.from_row_splits(
            input_tensor,
            array_ops_stack.stack(
                [0, array_ops.shape(input_tensor, out_type=dtypes.int32)[0]]),
            validate=False)
        output_tensor = unicode_encode(ragged_input_tensor, output_encoding,
                                       errors, replacement_char)
        return array_ops.reshape(output_tensor, [])

# pylint: disable=redefined-builtin
@tf_export("strings.unicode_decode")
@dispatch.add_dispatch_support
def unicode_decode(input,
                   input_encoding,
                   errors="replace",
                   replacement_char=0xFFFD,
                   replace_control_characters=False,
                   name=None):
  r"""Decodes each string in `input` into a sequence of Unicode code points.

  `result[i1...iN, j]` is the Unicode codepoint for the `j`th character in
  `input[i1...iN]`, when decoded using `input_encoding`.

  Args:
    input: An `N` dimensional potentially ragged `string` tensor with shape
      `[D1...DN]`.  `N` must be statically known.
    input_encoding: String name for the unicode encoding that should be used to
      decode each string.
    errors: Specifies the response when an input string can't be converted
      using the indicated encoding. One of:
      * `'strict'`: Raise an exception for any illegal substrings.
      * `'replace'`: Replace illegal substrings with `replacement_char`.
      * `'ignore'`: Skip illegal substrings.
    replacement_char: The replacement codepoint to be used in place of invalid
      substrings in `input` when `errors='replace'`; and in place of C0 control
      characters in `input` when `replace_control_characters=True`.
    replace_control_characters: Whether to replace the C0 control characters
      `(U+0000 - U+001F)` with the `replacement_char`.
    name: A name for the operation (optional).

  Returns:
    A `N+1` dimensional `int32` tensor with shape `[D1...DN, (num_chars)]`.
    The returned tensor is a `tf.Tensor` if `input` is a scalar, or a
    `tf.RaggedTensor` otherwise.

  #### Example:

  >>> input = [s.encode('utf8') for s in (u'G\xf6\xf6dnight', u'\U0001f60a')]
  >>> tf.strings.unicode_decode(input, 'UTF-8').to_list()
  [[71, 246, 246, 100, 110, 105, 103, 104, 116], [128522]]
  """
  with ops.name_scope(name, "UnicodeDecode", [input]):
    return _unicode_decode(input, input_encoding, errors, replacement_char,
                           replace_control_characters, with_offsets=False)

@tf_export("strings.unicode_decode_with_offsets")
@dispatch.add_dispatch_support
def unicode_decode_with_offsets(input,
                                input_encoding,
                                errors="replace",
                                replacement_char=0xFFFD,
                                replace_control_characters=False,
                                name=None):
  r"""Decodes each string into a sequence of code points with start offsets.

  This op is similar to `tf.strings.decode(...)`, but it also returns the
  start offset for each character in its respective string.  This information
  can be used to align the characters with the original byte sequence.

  Returns a tuple `(codepoints, start_offsets)` where:

  * `codepoints[i1...iN, j]` is the Unicode codepoint for the `j`th character
    in `input[i1...iN]`, when decoded using `input_encoding`.
  * `start_offsets[i1...iN, j]` is the start byte offset for the `j`th
    character in `input[i1...iN]`, when decoded using `input_encoding`.

  Args:
    input: An `N` dimensional potentially ragged `string` tensor with shape
      `[D1...DN]`.  `N` must be statically known.
    input_encoding: String name for the unicode encoding that should be used to
      decode each string.
    errors: Specifies the response when an input string can't be converted
      using the indicated encoding. One of:
      * `'strict'`: Raise an exception for any illegal substrings.
      * `'replace'`: Replace illegal substrings with `replacement_char`.
      * `'ignore'`: Skip illegal substrings.
    replacement_char: The replacement codepoint to be used in place of invalid
      substrings in `input` when `errors='replace'`; and in place of C0 control
      characters in `input` when `replace_control_characters=True`.
    replace_control_characters: Whether to replace the C0 control characters
      `(U+0000 - U+001F)` with the `replacement_char`.
    name: A name for the operation (optional).

  Returns:
    A tuple of `N+1` dimensional tensors `(codepoints, start_offsets)`.

    * `codepoints` is an `int32` tensor with shape `[D1...DN, (num_chars)]`.
    * `offsets` is an `int64` tensor with shape `[D1...DN, (num_chars)]`.

    The returned tensors are `tf.Tensor`s if `input` is a scalar, or
    `tf.RaggedTensor`s otherwise.

  #### Example:

  >>> input = [s.encode('utf8') for s in (u'G\xf6\xf6dnight', u'\U0001f60a')]
  >>> result = tf.strings.unicode_decode_with_offsets(input, 'UTF-8')
  >>> result[0].to_list()  # codepoints
  [[71, 246, 246, 100, 110, 105, 103, 104, 116], [128522]]
  >>> result[1].to_list()  # offsets
  [[0, 1, 3, 5, 6, 7, 8, 9, 10], [0]]

  """
  with ops.name_scope(name, "UnicodeDecodeWithOffsets", [input]):
    return _unicode_decode(input, input_encoding, errors, replacement_char,
                           replace_control_characters, with_offsets=True)

@tf_export("strings.unicode_split")
@dispatch.add_dispatch_support
def unicode_split(input,
                  input_encoding,
                  errors="replace",
                  replacement_char=0xFFFD,
                  name=None):
  r"""Splits each string in `input` into a sequence of Unicode code points.

  `result[i1...iN, j]` is the substring of `input[i1...iN]` that encodes its
  `j`th character, when decoded using `input_encoding`.

  Args:
    input: An `N` dimensional potentially ragged `string` tensor with shape
      `[D1...DN]`.  `N` must be statically known.
    input_encoding: String name for the unicode encoding that should be used to
      decode each string.
    errors: Specifies the response when an input string can't be converted
      using the indicated encoding. One of:
      * `'strict'`: Raise an exception for any illegal substrings.
      * `'replace'`: Replace illegal substrings with `replacement_char`.
      * `'ignore'`: Skip illegal substrings.
    replacement_char: The replacement codepoint to be used in place of invalid
      substrings in `input` when `errors='replace'`.
    name: A name for the operation (optional).

  Returns:
    A `N+1` dimensional `int32` tensor with shape `[D1...DN, (num_chars)]`.
    The returned tensor is a `tf.Tensor` if `input` is a scalar, or a
    `tf.RaggedTensor` otherwise.

  #### Example:

  >>> input = [s.encode('utf8') for s in (u'G\xf6\xf6dnight', u'\U0001f60a')]
  >>> tf.strings.unicode_split(input, 'UTF-8').to_list()
  [[b'G', b'\xc3\xb6', b'\xc3\xb6', b'd', b'n', b'i', b'g', b'h', b't'],
   [b'\xf0\x9f\x98\x8a']]
  """
  with ops.name_scope(name, "UnicodeSplit", [input]):
    codepoints = _unicode_decode(input, input_encoding, errors,
                                 replacement_char, False, with_offsets=False)
    return unicode_encode(
        ragged_array_ops.expand_dims(codepoints, -1),
        output_encoding=input_encoding,
        errors=errors,
        replacement_char=replacement_char)

@tf_export("strings.unicode_split_with_offsets")
@dispatch.add_dispatch_support
def unicode_split_with_offsets(input,
                               input_encoding,
                               errors="replace",
                               replacement_char=0xFFFD,
                               name=None):
  r"""Splits each string into a sequence of code points with start offsets.

  This op is similar to `tf.strings.decode(...)`, but it also returns the
  start offset for each character in its respective string.  This information
  can be used to align the characters with the original byte sequence.

  Returns a tuple `(chars, start_offsets)` where:

  * `chars[i1...iN, j]` is the substring of `input[i1...iN]` that encodes its
    `j`th character, when decoded using `input_encoding`.
  * `start_offsets[i1...iN, j]` is the start byte offset for the `j`th
    character in `input[i1...iN]`, when decoded using `input_encoding`.

  Args:
    input: An `N` dimensional potentially ragged `string` tensor with shape
      `[D1...DN]`.  `N` must be statically known.
    input_encoding: String name for the unicode encoding that should be used to
      decode each string.
    errors: Specifies the response when an input string can't be converted
      using the indicated encoding. One of:
      * `'strict'`: Raise an exception for any illegal substrings.
      * `'replace'`: Replace illegal substrings with `replacement_char`.
      * `'ignore'`: Skip illegal substrings.
    replacement_char: The replacement codepoint to be used in place of invalid
      substrings in `input` when `errors='replace'`.
    name: A name for the operation (optional).

  Returns:
    A tuple of `N+1` dimensional tensors `(codepoints, start_offsets)`.

    * `codepoints` is an `int32` tensor with shape `[D1...DN, (num_chars)]`.
    * `offsets` is an `int64` tensor with shape `[D1...DN, (num_chars)]`.

    The returned tensors are `tf.Tensor`s if `input` is a scalar, or
    `tf.RaggedTensor`s otherwise.

  #### Example:

  >>> input = [s.encode('utf8') for s in (u'G\xf6\xf6dnight', u'\U0001f60a')]
  >>> result = tf.strings.unicode_split_with_offsets(input, 'UTF-8')
  >>> result[0].to_list()  # character substrings
  [[b'G', b'\xc3\xb6', b'\xc3\xb6', b'd', b'n', b'i', b'g', b'h', b't'],
   [b'\xf0\x9f\x98\x8a']]
  >>> result[1].to_list()  # offsets
  [[0, 1, 3, 5, 6, 7, 8, 9, 10], [0]]

  """
  with ops.name_scope(name, "UnicodeSplitWithOffsets", [input]):
    codepoints, offsets = _unicode_decode(input, input_encoding, errors,
                                          replacement_char, False,
                                          with_offsets=True)
    chars = unicode_encode(
        ragged_array_ops.expand_dims(codepoints, -1),
        output_encoding=input_encoding,
        errors=errors,
        replacement_char=replacement_char)
    return chars, offsets

def _unicode_decode(input, input_encoding, errors, replacement_char,
                    replace_control_characters, with_offsets):
  """Decodes each string into a sequence of codepoints."""
  input = ragged_tensor.convert_to_tensor_or_ragged_tensor(input, name="input")
  input_ndims = input.shape.ndims
  if input_ndims is None:
    raise ValueError("Rank of `input` must be statically known.")

  if input_ndims > 1:
    # Convert to a ragged tensor with ragged_rank = input_ndims - 1.
    if not ragged_tensor.is_ragged(input):
      input = ragged_tensor.RaggedTensor.from_tensor(
          input, ragged_rank=input_ndims - 1)
    elif input.ragged_rank < input_ndims - 1:
      input = input.with_flat_values(
          ragged_tensor.RaggedTensor.from_tensor(
              input.flat_values,
              ragged_rank=input_ndims - input.ragged_rank - 1))

  # Reshape the input to a flat vector, and apply the gen_string_ops op.
  if ragged_tensor.is_ragged(input):
    flat_input = array_ops.reshape(input.flat_values, [-1])
  else:
    flat_input = array_ops.reshape(input, [-1])

  if with_offsets:
    decode_op = gen_string_ops.unicode_decode_with_offsets
  else:
    decode_op = gen_string_ops.unicode_decode
  flat_result = decode_op(
      input=flat_input,
      input_encoding=input_encoding,
      errors=errors,
      replacement_char=replacement_char,
      replace_control_characters=replace_control_characters)

  if input_ndims == 0:
    codepoints = flat_result.char_values
    if with_offsets:
      offsets = flat_result.char_to_byte_starts
  else:
    codepoints = ragged_tensor.RaggedTensor.from_row_splits(
        flat_result.char_values, flat_result.row_splits, validate=False)
    if input_ndims > 1:
      codepoints = input.with_flat_values(codepoints)
    if with_offsets:
      offsets = ragged_tensor.RaggedTensor.from_row_splits(
          flat_result.char_to_byte_starts, flat_result.row_splits,
          validate=False)
      if input_ndims > 1:
        offsets = input.with_flat_values(offsets)

  if with_offsets:
    return codepoints, offsets
  else:
    return codepoints

@tf_export("strings.split", v1=[])
@dispatch.add_dispatch_support
def string_split_v2(input, sep=None, maxsplit=-1, name=None):  # pylint: disable=redefined-builtin
  """Split elements of `input` based on `sep` into a `RaggedTensor`.

  Let N be the size of `input` (typically N will be the batch size). Split each
  element of `input` based on `sep` and return a `RaggedTensor` containing the
  split tokens. Empty tokens are ignored.

  Example:

  >>> tf.strings.split('hello world').numpy()
   array([b'hello', b'world'], dtype=object)
  >>> tf.strings.split(['hello world', 'a b c'])
  <tf.RaggedTensor [[b'hello', b'world'], [b'a', b'b', b'c']]>

  If `sep` is given, consecutive delimiters are not grouped together and are
  deemed to delimit empty strings. For example, `input` of `"1<>2<><>3"` and
  `sep` of `"<>"` returns `["1", "2", "", "3"]`. If `sep` is None or an empty
  string, consecutive whitespace are regarded as a single separator, and the
  result will contain no empty strings at the start or end if the string has
  leading or trailing whitespace.

  Note that the above mentioned behavior matches python's str.split.

  Args:
    input: A string `Tensor` of rank `N`, the strings to split.  If
      `rank(input)` is not known statically, then it is assumed to be `1`.
    sep: `0-D` string `Tensor`, the delimiter string.
    maxsplit: An `int`. If `maxsplit > 0`, limit of the split of the result.
    name: A name for the operation (optional).

  Raises:
    ValueError: If sep is not a string.

  Returns:
    A `RaggedTensor` of rank `N+1`, the strings split according to the
    delimiter.
  """
  with ops.name_scope(name, "StringSplit", [input]):
    input = ragged_tensor.convert_to_tensor_or_ragged_tensor(
        input, dtype=dtypes.string, name="input")
    if isinstance(input, ragged_tensor.RaggedTensor):
      return input.with_flat_values(
          string_split_v2(input.flat_values, sep, maxsplit))

    rank = input.shape.ndims
    if rank == 0:
      return string_split_v2(array_ops_stack.stack([input]), sep, maxsplit)[0]
    elif rank == 1 or rank is None:
      sparse_result = string_ops.string_split_v2(
          input, sep=sep, maxsplit=maxsplit)
      return ragged_tensor.RaggedTensor.from_value_rowids(
          values=sparse_result.values,
          value_rowids=sparse_result.indices[:, 0],
          nrows=sparse_result.dense_shape[0],
          validate=False)
    else:
      return string_split_v2(
          ragged_tensor.RaggedTensor.from_tensor(input), sep, maxsplit)

@tf_export(v1=["string_split"])
@dispatch.add_dispatch_support
@deprecation.deprecated_args(None,
                             "delimiter is deprecated, please use sep instead.",
                             "delimiter")
def string_split(source, sep=None, skip_empty=True, delimiter=None,
                 result_type="SparseTensor", name=None):  # pylint: disable=invalid-name
  """Split elements of `source` based on `delimiter`.

  Let N be the size of `source` (typically N will be the batch size). Split each
  element of `source` based on `delimiter` and return a `SparseTensor`
  or `RaggedTensor` containing the split tokens. Empty tokens are ignored.

  If `sep` is an empty string, each element of the `source` is split
  into individual strings, each containing one byte. (This includes splitting
  multibyte sequences of UTF-8.) If delimiter contains multiple bytes, it is
  treated as a set of delimiters with each considered a potential split point.

  Examples:

  >>> print(tf.compat.v1.string_split(['hello world', 'a b c']))
  SparseTensor(indices=tf.Tensor( [[0 0] [0 1] [1 0] [1 1] [1 2]], ...),
               values=tf.Tensor([b'hello' b'world' b'a' b'b' b'c'], ...),
               dense_shape=tf.Tensor([2 3], shape=(2,), dtype=int64))

  >>> print(tf.compat.v1.string_split(['hello world', 'a b c'],
  ...     result_type="RaggedTensor"))
  <tf.RaggedTensor [[b'hello', b'world'], [b'a', b'b', b'c']]>

  Args:
    source: `1-D` string `Tensor`, the strings to split.
    sep: `0-D` string `Tensor`, the delimiter character, the string should
      be length 0 or 1. Default is ' '.
    skip_empty: A `bool`. If `True`, skip the empty strings from the result.
    delimiter: deprecated alias for `sep`.
    result_type: The tensor type for the result: one of `"RaggedTensor"` or
      `"SparseTensor"`.
    name: A name for the operation (optional).

  Raises:
    ValueError: If delimiter is not a string.

  Returns:
    A `SparseTensor` or `RaggedTensor` of rank `2`, the strings split according
    to the delimiter.  The first column of the indices corresponds to the row
    in `source` and the second column corresponds to the index of the split
    component in this row.
  """
  with ops.name_scope(name, "StringSplit", [source]):
    sparse_result = string_ops.string_split(
        source, sep=sep, skip_empty=skip_empty, delimiter=delimiter)
    if result_type == "SparseTensor":
      return sparse_result
    elif result_type == "RaggedTensor":
      return ragged_tensor.RaggedTensor.from_value_rowids(
          values=sparse_result.values,
          value_rowids=sparse_result.indices[:, 0],
          nrows=sparse_result.dense_shape[0],
          validate=False)
    else:
      raise ValueError("result_type must be 'RaggedTensor' or 'SparseTensor'.")

# In TensorFlow 1.x, "tf.strings.split" uses the new signature (with maxsplit),
# but we need to add the result_type argument.
@tf_export(v1=["strings.split"])
@dispatch.add_dispatch_support
def strings_split_v1(input=None, sep=None, maxsplit=-1,  # pylint: disable=redefined-builtin
                     result_type="SparseTensor", source=None, name=None):
  """Split elements of `input` based on `sep`.

  Let N be the size of `input` (typically N will be the batch size). Split each
  element of `input` based on `sep` and return a `SparseTensor` or
  `RaggedTensor` containing the split tokens. Empty tokens are ignored.

  Examples:

  >>> print(tf.compat.v1.strings.split(['hello world', 'a b c']))
  SparseTensor(indices=tf.Tensor( [[0 0] [0 1] [1 0] [1 1] [1 2]], ...),
               values=tf.Tensor([b'hello' b'world' b'a' b'b' b'c'], ...),
               dense_shape=tf.Tensor([2 3], shape=(2,), dtype=int64))

  >>> print(tf.compat.v1.strings.split(['hello world', 'a b c'],
  ...     result_type="RaggedTensor"))
  <tf.RaggedTensor [[b'hello', b'world'], [b'a', b'b', b'c']]>

  If `sep` is given, consecutive delimiters are not grouped together and are
  deemed to delimit empty strings. For example, `input` of `"1<>2<><>3"` and
  `sep` of `"<>"` returns `["1", "2", "", "3"]`. If `sep` is None or an empty
  string, consecutive whitespace are regarded as a single separator, and the
  result will contain no empty strings at the start or end if the string has
  leading or trailing whitespace.

  Note that the above mentioned behavior matches python's str.split.

  Args:
    input: A string `Tensor` of rank `N`, the strings to split.  If
      `rank(input)` is not known statically, then it is assumed to be `1`.
    sep: `0-D` string `Tensor`, the delimiter character.
    maxsplit: An `int`. If `maxsplit > 0`, limit of the split of the result.
    result_type: The tensor type for the result: one of `"RaggedTensor"` or
      `"SparseTensor"`.
    source: alias for "input" argument.
    name: A name for the operation (optional).

  Raises:
    ValueError: If sep is not a string.

  Returns:
    A `SparseTensor` or `RaggedTensor` of rank `N+1`, the strings split
    according to the delimiter.
  """
  input = deprecation.deprecated_argument_lookup(
      "input", input, "source", source)
  with ops.name_scope(name, "StringSplit", [input]):
    input = ragged_tensor.convert_to_tensor_or_ragged_tensor(
        input, dtype=dtypes.string, name="input")

    if input.shape.rank == 0:
      input = array_ops.expand_dims(input, 0)

    if result_type == "SparseTensor":
      if input.shape.rank == 1:
        return string_ops.string_split_v2(input, sep=sep, maxsplit=maxsplit)
      else:
        return string_split_v2(input, sep=sep, maxsplit=maxsplit).to_sparse()
    elif result_type == "RaggedTensor":
      return string_split_v2(input, sep=sep, maxsplit=maxsplit)
    else:
      raise ValueError("result_type must be 'RaggedTensor' or 'SparseTensor'.")

@dispatch.dispatch_for_api(string_ops.reduce_join_v2)
def reduce_join(inputs: ragged_tensor.Ragged,
                axis=None,
                keepdims=None,
                separator="",
                name=None):
  """For docs, see: _RAGGED_REDUCE_DOCSTRING."""
  return ragged_math_ops.ragged_reduce_aggregate(
      string_ops.reduce_join, string_ops.unsorted_segment_join, inputs, axis,
      keepdims, separator, name or "RaggedSegmentJoin")

@tf_export("strings.ngrams")
@dispatch.add_dispatch_support
def ngrams(data,
           ngram_width,
           separator=" ",
           pad_values=None,
           padding_width=None,
           preserve_short_sequences=False,
           name=None):
  """Create a tensor of n-grams based on `data`.

  Creates a tensor of n-grams based on `data`. The n-grams are created by
  joining windows of `width` adjacent strings from the inner axis of `data`
  using `separator`.

  The input data can be padded on both the start and end of the sequence, if
  desired, using the `pad_values` argument. If set, `pad_values` should contain
  either a tuple of strings or a single string; the 0th element of the tuple
  will be used to pad the left side of the sequence and the 1st element of the
  tuple will be used to pad the right side of the sequence. The `padding_width`
  arg controls how many padding values are added to each side; it defaults to
  `ngram_width-1`.

  If this op is configured to not have padding, or if it is configured to add
  padding with `padding_width` set to less than ngram_width-1, it is possible
  that a sequence, or a sequence plus padding, is smaller than the ngram
  width. In that case, no ngrams will be generated for that sequence. This can
  be prevented by setting `preserve_short_sequences`, which will cause the op
  to always generate at least one ngram per non-empty sequence.

  Examples:

  >>> tf.strings.ngrams(["A", "B", "C", "D"], 2).numpy()
  array([b'A B', b'B C', b'C D'], dtype=object)
  >>> tf.strings.ngrams(["TF", "and", "keras"], 1).numpy()
  array([b'TF', b'and', b'keras'], dtype=object)

  Args:
    data: A Tensor or RaggedTensor containing the source data for the ngrams.
    ngram_width: The width(s) of the ngrams to create. If this is a list or
      tuple, the op will return ngrams of all specified arities in list order.
      Values must be non-Tensor integers greater than 0.
    separator: The separator string used between ngram elements. Must be a
      string constant, not a Tensor.
    pad_values: A tuple of (left_pad_value, right_pad_value), a single string,
      or None. If None, no padding will be added; if a single string, then that
      string will be used for both left and right padding. Values must be Python
      strings.
    padding_width: If set, `padding_width` pad values will be added to both
      sides of each sequence. Defaults to `ngram_width`-1. Must be greater than
      0. (Note that 1-grams are never padded, regardless of this value.)
    preserve_short_sequences: If true, then ensure that at least one ngram is
      generated for each input sequence.  In particular, if an input sequence is
      shorter than `min(ngram_width) + 2*pad_width`, then generate a single
      ngram containing the entire sequence.  If false, then no ngrams are
      generated for these short input sequences.
    name: The op name.

  Returns:
    A RaggedTensor of ngrams. If `data.shape=[D1...DN, S]`, then
    `output.shape=[D1...DN, NUM_NGRAMS]`, where
    `NUM_NGRAMS=S-ngram_width+1+2*padding_width`.

  Raises:
    TypeError: if `pad_values` is set to an invalid type.
    ValueError: if `pad_values`, `padding_width`, or `ngram_width` is set to an
      invalid value.
  """

  with ops.name_scope(name, "StringNGrams", [data]):
    if pad_values is None:
      left_pad = ""
      right_pad = ""
    elif isinstance(pad_values, (list, tuple)):
      if (not isinstance(pad_values[0], util_compat.bytes_or_text_types) or
          not isinstance(pad_values[1], util_compat.bytes_or_text_types)):
        raise TypeError(
            "pad_values must be a string, tuple of strings, or None.")
      left_pad = pad_values[0]
      right_pad = pad_values[1]
    else:
      if not isinstance(pad_values, util_compat.bytes_or_text_types):
        raise TypeError(
            "pad_values must be a string, tuple of strings, or None.")
      left_pad = pad_values
      right_pad = pad_values

    if padding_width is not None and padding_width < 1:
      raise ValueError("padding_width must be greater than 0.")

    if padding_width is not None and pad_values is None:
      raise ValueError("pad_values must be provided if padding_width is set.")

    data = ragged_tensor.convert_to_tensor_or_ragged_tensor(
        data, name="data", dtype=dtypes.string)

    # preserve the shape of the data if it is a tensor
    to_tensor = False
    if isinstance(data, tensor_lib.Tensor):
      dense_shape = array_ops.concat([array_ops.shape(data)[:-1], [-1]], axis=0)
      to_tensor = True

    if not isinstance(data, ragged_tensor.RaggedTensor):
      if data.shape.ndims is None:
        raise ValueError("Rank of data must be known.")
      elif data.shape.ndims == 0:
        raise ValueError("Data must have rank>0")
      elif data.shape.ndims == 1:
        rt = ragged_tensor.RaggedTensor.from_row_starts(
            data, [0], validate=False)
        return ngrams(rt, ngram_width, separator, pad_values, padding_width,
                      preserve_short_sequences, name)[0]
      else:
        data = ragged_tensor.RaggedTensor.from_tensor(
            data, ragged_rank=data.shape.ndims - 1)

    if data.ragged_rank > 1:
      output = data.with_values(
          ngrams(data.values, ngram_width, separator, pad_values, padding_width,
                 preserve_short_sequences, name))
      return array_ops.reshape(output.flat_values,
                               dense_shape) if to_tensor else output

    if pad_values is None:
      padding_width = 0

    if pad_values is not None and padding_width is None:
      padding_width = -1

    if not isinstance(ngram_width, (list, tuple)):
      ngram_widths = [ngram_width]
    else:
      ngram_widths = ngram_width
    for width in ngram_widths:
      if width < 1:
        raise ValueError("All ngram_widths must be greater than 0. Got %s" %
                         ngram_width)

    output, output_splits = gen_string_ops.string_n_grams(
        data=data.flat_values,
        data_splits=data.row_splits,
        separator=separator,
        ngram_widths=ngram_widths,
        left_pad=left_pad,
        right_pad=right_pad,
        pad_width=padding_width,
        preserve_short_sequences=preserve_short_sequences)

    # if the input is Dense tensor, the output should also be a dense tensor
    output = ragged_tensor.RaggedTensor.from_row_splits(
        values=output, row_splits=output_splits, validate=False)
    return array_ops.reshape(output.flat_values,
                             dense_shape) if to_tensor else output

@dispatch.dispatch_for_api(string_ops.string_format)
def string_format(
    template: str,
    inputs: typing.Union[ragged_tensor.Ragged,
                         typing.List[ragged_tensor.RaggedOrDense]],
    placeholder="{}",
    summarize=3,
    name=None):
  """Version of tf.strings.format that handles RaggedTensors."""
  if tensor_util.is_tf_type(inputs) or ragged_tensor.is_ragged(inputs):
    inputs = [inputs]

  split_template = template.split(placeholder)
  if len(inputs) != len(split_template) - 1:
    raise ValueError("num placeholders in template and num inputs must match"
                     ": {} vs {}".format(len(split_template) - 1, len(inputs)))

  with ops.name_scope(name, "StringFormat", [inputs]):
    output_pieces = [constant_op.constant(split_template[0])]
    for i, input in enumerate(inputs):
      if ragged_tensor.is_ragged(input):
        output_pieces.append(ragged_tensor_to_string(input, summarize))
      else:
        output_pieces.append(string_ops.string_format(
            "{}", [input], summarize=summarize))
      output_pieces.append(constant_op.constant(split_template[i + 1]))
    if len(output_pieces) == 1:
      return output_pieces[0]
    else:
      return string_ops.reduce_join(output_pieces)

def ragged_tensor_to_string(rt, summarize=None):
  """Returns a scalar string tensor with the contents of a RaggedTensor.

  Requires that `rt.shape.rank` is not `None`.

  Note: this converts the entire `RaggedTensor` into a single string scalar.
  If you want to convert individual elements, use `tf.strings.as_string(rt)`.

  >>> rt1 = tf.ragged.constant([[1, 2, 3], [4, 5]])
  >>> ragged_tensor_to_string(rt1).numpy()
  b'[[1, 2, 3], [4, 5]]'

  >>> rt2 = tf.ragged.constant([[['a'], ['b', 'c']], [['d', 'e', 'f'], []]])
  >>> ragged_tensor_to_string(rt2).numpy()
  b"[[['a'], ['b', 'c']], [['d', 'e', 'f'], []]]"

  >>> rt3 = tf.ragged.constant([[1], [2, 3, 4, 5, 6], [], [], [7], [8, 9]])
  >>> ragged_tensor_to_string(rt3, summarize=2).numpy()
  b'[[1], [2, 3, ..., 5, 6], ..., [7], [8, 9]]'

  Args:
    rt: The RaggedTensor that should be converted to a string.
    summarize: If specified, then only the first and last `summarize` elements
      within each dimension are included in the string. If `-1` or `None`, then
      all elements are included.
  """
  if (summarize is not None and summarize != -1 and
      not (isinstance(summarize, int) and summarize > 0)):
    raise ValueError("Expected summarize to be -1 or a positive int, got %r" %
                     summarize)
  with ops.name_scope(None, "AsString", [rt]):
    rt = ragged_tensor.convert_to_tensor_or_ragged_tensor(rt)
    if rt.shape.rank is None:
      raise ValueError("RaggedTensor to_string requires that rt.shape.rank "
                       "is not None.")
    # Convert all elements of `rt` to strings.
    if rt.dtype == dtypes.string:
      escaped = string_ops.regex_replace(rt.flat_values, r"(['\\])", r"\\\1")
      str_t = rt.with_flat_values("'" + escaped + "'")
    else:
      str_t = rt.with_flat_values(string_ops.as_string(rt.flat_values))

    return _ragged_tensor_to_string(str_t, summarize)

def _ragged_tensor_to_string(string_tensor, summarize):
  """Returns a scalar string tensor with the contents of `string_tensor`.

  Args:
    string_tensor: A potentially ragged tensor with dtype=string.
    summarize: Include only the first and last `summarize` elements of each
      dimension.  If `-1` or `None`, then include all elements.

  Returns:
    A scalar string Tensor.
  """
  if string_tensor.shape.rank == 1:
    pieces = string_tensor
  else:
    pieces = map_fn_lib.map_fn(
        lambda s: _ragged_tensor_to_string(s, summarize),
        string_tensor,
        fn_output_signature=tensor_lib.TensorSpec(None, dtypes.string))
  if summarize not in (-1, None):
    pieces = cond.cond(
        _nrows(string_tensor) <= 2 * summarize,
        lambda: pieces,
        lambda: array_ops.concat(  # pylint: disable=g-long-lambda
            [pieces[:summarize], ["..."], pieces[-summarize:]],
            axis=0))
  return "[" + string_ops.reduce_join(pieces, separator=", ") + "]"

def _nrows(tensor, out_type=dtypes.int32):
  if isinstance(tensor, ragged_tensor.RaggedTensor):
    return tensor.nrows(out_type=out_type)
  else:
    return array_ops.shape(tensor, out_type=out_type)[0]

@dispatch.dispatch_for_api(string_ops.string_join)
def string_join(inputs: typing.List[ragged_tensor.RaggedOrDense],
                separator="",
                name=None):
  """RaggedTensor implementation for tf.strings.join."""
  if len(inputs) < 0:
    raise ValueError("tf.strings.join: expected at least one input.")
  with ops.name_scope(name, "RaggedStringJoin", inputs):
    return ragged_functional_ops.map_flat_values(string_ops.string_join, inputs,
                                                 separator)


class Solution:
    def corpFlightBookings(self, bookings: List[List[int]], n: int) -> List[int]:
        answer = [0] * n
        for first, last, seat in bookings:
            answer[first - 1] += seat
            if last < n:
                answer[last] -= seat
        for i in range(1, n):
            answer[i] += answer[i - 1]
        return answer


import datetime
import pytz
from tensorflow.python.client import timeline
import numpy as np
import os
import tensorflow as tf
os.environ['TF_CPP_MIN_LOG_LEVEL']="2"
#os.environ["CUDA_VISIBLE_DEVICES"]="0,1,2,3,4,5"
os.environ["CUDA_VISIBLE_DEVICES"]=""
import input_data_video
from tf_flags import FLAGS
import json
import pprint

class TimeLiner:
    _timeline_dict = None

    def update_timeline(self, chrome_trace):
        # convert crome trace to python dict
        chrome_trace_dict = json.loads(chrome_trace)
        # for first run store full trace
        if self._timeline_dict is None:
            self._timeline_dict = chrome_trace_dict
        # for other - update only time consumption, not definitions
        else:
            for event in chrome_trace_dict['traceEvents']:
                # events time consumption started with 'ts' prefix
                if 'ts' in event:
                    self._timeline_dict['traceEvents'].append(event)

    def save(self, f_name):
        with open(f_name, 'w') as f:
            json.dump(self._timeline_dict, f)

pp = pprint.PrettyPrinter()
pp.pprint(FLAGS.__flags)

filenames = []
for path, subdirs, files in os.walk("/hdd/wuzhenyu_sjtu/SRx3/train/"):
    for name in files:
        filenames.append(os.path.join(path, name))
#print(filenames)

dir = "SRx3/train/part0"
#dir = "/hdd/wuzhenyu_sjtu/SRx3/train/part0"

test_files = [os.path.join(dir, f) for f in
                      os.listdir(dir) if f.endswith('.tfrecords')]
#print(test_files)

videos_LR, videos_HR, labels = input_data_video.inputs(filenames=test_files,
                                                       batch_size=FLAGS.gpu_num * FLAGS.batch_size,
                                                       num_epochs=1,
                                                       num_threads=40,
                                                       vshape=[FLAGS.depth, FLAGS.height, FLAGS.width,
                                                               FLAGS.nchannel],
                                                       num_examples_per_epoch=1000,
                                                       shuffle=False
                                                       )

#config = tf.ConfigProto(device_count = {'GPU': 0})
with tf.Session() as sess:
    run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
    run_metadata = tf.RunMetadata()
    init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
    sess.run(init_op, options=run_options, run_metadata=run_metadata)
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(coord=coord, sess=sess)
    start_time = datetime.datetime.now(pytz.timezone('America/Chicago'))
    step = 1
    speed_lst = []
    many_runs_timeline = TimeLiner()
    try:
        while not coord.should_stop():
            _, _, _ = sess.run([videos_LR, videos_HR, labels],options=run_options, run_metadata=run_metadata)
            fetched_timeline = timeline.Timeline(run_metadata.step_stats)
            chrome_trace = fetched_timeline.generate_chrome_trace_format()
            many_runs_timeline.update_timeline(chrome_trace)
            step +=1
            end_time = datetime.datetime.now(pytz.timezone('America/Chicago'))
            sec = (end_time - start_time).total_seconds()
            speed = int((FLAGS.gpu_num * FLAGS.batch_size)/sec)
            print("[{}] time[{:6.2f}] step[{:10d}] speed[{:6d}]".format(
                    str(end_time).split(".")[0], sec, step,
                    speed
                    ))
            speed_lst.append(speed)
            start_time = end_time

    except tf.errors.OutOfRangeError:
        print("Done training after reading all data")
    finally:
        coord.request_stop()
        print("coord stopped")

    coord.join(threads)

    many_runs_timeline.save('timeline_merged_%d_steps.json' % step)

    speed_arr = np.asarray(speed_lst, dtype=np.float32)
    print("avg speed: {:6.6f}".format(np.mean(speed_arr)))
    print("stddev speed: {:6.6f}".format(np.std(speed_arr)))
    print("all done")


"""SaveComputeRepeat"""
def main():
    """days, hours, minutes, seconds, milliseconds"""
    start = 492137954293754252786
    millisecond = start
    seconds = millisecond//1000
    millisecond = millisecond%1000
    minute = seconds//60
    seconds = seconds%60
    hours = minute//60
    minute = minute%60
    days = hours//24
    hours = hours%24
    print(days, hours, minute, seconds, millisecond)
main()


"""
.. module: license.tests.models.test_plan
    :synopsis: Plan model tests.
"""

from __future__ import unicode_literals
import unittest
from license.models.plan import Plan
from license.exceptions.plan import (
    PlanTypeError,
    PlanUpgradeError,
    PlanDowngradeError,
)

class TestPlanModel(unittest.TestCase):
    """Test Plan Model."""

    def test__init(self):
        """Test __init__."""
        for plan_type in Plan.PLANS:
            plan = Plan(plan_type)
            self.assertEqual(plan.plan_type, plan_type)
            self.assertEqual(plan.allowance(), Plan.ALLOWANCE[plan_type])
            self.assertEqual(plan.price(), Plan.PRICES[plan_type])

    def test_invalid_plan_type(self):
        """Test invalid plan type."""
        with self.assertRaises(PlanTypeError):
            Plan('not valid')

    def test_upgrade(self):
        """Test plan upgrade."""
        plan = Plan(Plan.PLANS[0])
        for plan_type in Plan.PLANS[1:]:
            plan.upgrade(plan_type)
            self.assertEqual(plan.plan_type, plan_type)

    def test_downgrade(self):
        """Test plan downgrade."""
        plans = Plan.PLANS[:]
        plans.reverse()
        plan = Plan(plans[0])
        for plan_type in plans[1:]:
            plan.downgrade(plan_type)
            self.assertEqual(plan.plan_type, plan_type)

    def test_upgrade_error(self):
        """Test upgrade error."""
        plan = Plan(Plan.PLANS[-1])
        for plan_type in Plan.PLANS[:-1]:
            with self.assertRaises(PlanUpgradeError):
                plan.upgrade(plan_type)
                self.assertEqual(plan.plan_type, Plan.PLANS[-1])

    def test_downgrade_error(self):
        """Test downgrade error."""
        plan = Plan(Plan.PLANS[0])
        for plan_type in Plan.PLANS[1:]:
            with self.assertRaises(PlanDowngradeError):
                plan.downgrade(plan_type)
                self.assertEqual(plan.plan_type, Plan.PLANS[0])

    def test_upgrade_invalid_error(self):
        """Test upgrade to an invalid plan."""
        plan = Plan(Plan.PLANS[0])
        with self.assertRaises(PlanUpgradeError):
            plan.upgrade('not valid')

    def test_downgrade_invalid_error(self):
        """Test downgrade to an invalid plan."""
        plan = Plan(Plan.PLANS[-1])
        with self.assertRaises(PlanDowngradeError):
            plan.downgrade('not valid')


import logging
import uuid

import requests

from codecov_cli.types import RequestError, RequestResult

logger = logging.getLogger("codecovcli")

def send_post_request(
    url: str,
    data: dict = None,
    headers: dict = None,
):
    resp = requests.post(
        url=url,
        data=data,
        headers=headers,
    )
    return request_result(resp)

def get_token_header_or_fail(token: uuid.UUID) -> dict:
    if token is None:
        raise Exception("Codecov token not found.")
    if not isinstance(token, uuid.UUID):
        raise Exception(f"Token must be UUID. Received {type(token)}")
    return {"Authorization": f"token {token.hex}"}

def send_put_request(
    url: str,
    data: dict = None,
    headers: dict = None,
):
    resp = requests.put(url=url, data=data, headers=headers)
    return request_result(resp)

def request_result(resp):
    if resp.status_code >= 400:
        return RequestResult(
            status_code=resp.status_code,
            error=RequestError(
                code=f"HTTP Error {resp.status_code}",
                description=resp.text,
                params={},
            ),
            warnings=[],
            text=resp.text,
        )

    return RequestResult(
        status_code=resp.status_code, error=None, warnings=[], text=resp.text
    )

def log_warnings_and_errors_if_any(
    sending_result: RequestResult, process_desc: str, fail_on_error: bool = False
):
    logger.debug(
        f"Process {process_desc} complete.",
        extra=dict(extra_log_attributes=dict(result=sending_result)),
    )
    if sending_result.warnings:
        number_warnings = len(sending_result.warnings)
        pluralization = "s" if number_warnings > 1 else ""
        logger.info(
            f"{process_desc} process had {number_warnings} warning{pluralization}",
        )
        for ind, w in enumerate(sending_result.warnings):
            logger.warning(f"Warning {ind + 1}: {w.message}")
    if sending_result.error is not None:
        logger.error(f"{process_desc} failed: {sending_result.error.description}")
        if fail_on_error:
            exit(1)


#! /usr/bin/env python
# -*- coding: utf-8 -*-
# Author: AxiaCore S.A.S. http://axiacore.com
from django.conf import settings
from django.utils.safestring import mark_safe
from string import Template

TRACKING_CODE = """
<script>
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};
ga.l=+new Date;
ga('create', '$ga_code', 'auto');
ga('send', 'pageview');
</script>
<script async src='https://www.google-analytics.com/analytics.js'></script>
"""

def logo_company(request):
    return {'LOGO_COMPANY': settings.LOGO_COMPANY}

def analytics(request):
    """
    Enable analytics script if debug is False
    """
    script = ''
    if not settings.DEBUG:
        template = Template(TRACKING_CODE)
        script = mark_safe(
            template.substitute(
                ga_code=settings.GOOGLE_ANALYTICS_CODE,
            )
        )

    return {'ANALYTICS': script}


import itertools

import pygame as pg

from consts import *
import matrix

class MatrixSurface(pg.Surface):
    """Класс для отрисовки игрового поля"""

    def __init__(self):
        super().__init__((MATRIX_WIDTH * BLOCK_SIZE,
                          MATRIX_HEIGHT * BLOCK_SIZE))

        # экземпляр игрового поля
        self.m = matrix.Matrix(MATRIX_WIDTH, MATRIX_HEIGHT)

    def update(self):
        """Рисует ячейки"""

        # отрисовываем стакан
        for i, j in itertools.product(range(self.m.height),
                                      range(self.m.width)):
            # если ячейка в стакане не пустая, отрисовываем цветом
            # иначе фоновым
            color = BG
            if self.m.grid[i][j]:
                color = self.m.get_block_color(i, j)

            # прямоугольник для отрисовки
            _rect = (j * BLOCK_SIZE, i * BLOCK_SIZE,
                     BLOCK_SIZE, BLOCK_SIZE)

            pg.draw.rect(self, color, _rect, border_radius=BORDER_RADIUS)

        # отрисовываем фигуру
        color = self.m.p_figure.get_fig_color()
        for i, j in itertools.product(range(self.m.p_figure.height),
                                      range(self.m.p_figure.width)):
            if not (0 <= self.m.figure_top + i < MATRIX_HEIGHT
                    and 0 <= self.m.figure_left + j < MATRIX_WIDTH):
                continue

            # если ячейка пустая, не отрисовываем
            if not self.m.p_figure.grid[i][j]:
                continue

            _rect = ((self.m.figure_left + j) * BLOCK_SIZE, (self.m.figure_top + i) * BLOCK_SIZE,
                     BLOCK_SIZE, BLOCK_SIZE)

            pg.draw.rect(self, color, _rect, border_radius=BORDER_RADIUS)


from openpiv.lib import replace_nans
import numpy as np
from scipy.signal import convolve

"""The openpiv.filters module contains some filtering/smoothing routines."""

__licence_ = """
Copyright (C) 2011  www.openpiv.net

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

def _gaussian_kernel(half_width=1):
    """A normalized 2D Gaussian kernel array

    Parameters
    ----------
    half_width : int
        the half width of the kernel. Kernel
        has shape 2*half_width + 1 (default half_width = 1, i.e.
        a Gaussian of 3 x 3 kernel)

    Examples
    --------

    >>> from openpiv.filters import _gaussian_kernel
    >>> _gaussian_kernel(1)
    array([[ 0.04491922,  0.12210311,  0.04491922],
       [ 0.12210311,  0.33191066,  0.12210311],
       [ 0.04491922,  0.12210311,  0.04491922]])

    """
    # size = int(half_width)
    x, y = np.mgrid[-half_width:half_width + 1, -half_width:half_width + 1]
    g = np.exp(-(x ** 2 / float(half_width) + y ** 2 / float(half_width)))
    return g / g.sum()

def gaussian_kernel(sigma, truncate=4.0):
    """
    Return Gaussian that truncates at the given number of standard deviations.
    """

    sigma = float(sigma)
    radius = int(truncate * sigma + 0.5)

    x, y = np.mgrid[-radius:radius + 1, -radius:radius + 1]
    sigma = sigma ** 2

    k = 2 * np.exp(-0.5 * (x ** 2 + y ** 2) / sigma)
    k = k / np.sum(k)

    return k

def gaussian(u, v, half_width=1):
    """Smooths the velocity field with a Gaussian kernel.

    Parameters
    ----------
    u : 2d np.ndarray
        the u velocity component field

    v : 2d np.ndarray
        the v velocity component field

    half_width : int
        the half width of the kernel. Kernel
        has shape 2*half_width+1, default = 1

    Returns
    -------
    uf : 2d np.ndarray
        the smoothed u velocity component field

    vf : 2d np.ndarray
        the smoothed v velocity component field

    """
    g = _gaussian_kernel(half_width=half_width)
    uf = convolve(u, g, mode="same")
    vf = convolve(v, g, mode="same")
    return uf, vf

def replace_outliers(u, v, w=None, method="localmean",
                     max_iter=5, tol=1e-3, kernel_size=1):
    """Replace invalid vectors in an velocity field using an iterative image
        inpainting algorithm.

    The algorithm is the following:

    1) For each element in the arrays of the ``u`` and ``v`` components,
       replace it by a weighted average
       of the neighbouring elements which are not invalid themselves. The
       weights depends of the method type. If ``method=localmean`` weight
       are equal to 1/( (2*kernel_size+1)**2 -1 )

    2) Several iterations are needed if there are adjacent invalid elements.
       If this is the case, inforation is "spread" from the edges of the
       missing regions iteratively, until the variation is below a certain
       threshold.

    Parameters
    ----------

    u : 2d or 3d np.ndarray
        the u velocity component field

    v : 2d or 3d  np.ndarray
        the v velocity component field

    w : 2d or 3d  np.ndarray
        the w velocity component field

    max_iter : int
        the number of iterations

    kernel_size : int
        the size of the kernel, default is 1

    method : str
        the type of kernel used for repairing missing vectors

    Returns
    -------
    uf : 2d or 3d np.ndarray
        the smoothed u velocity component field, where invalid vectors have
        been replaced

    vf : 2d or 3d np.ndarray
        the smoothed v velocity component field, where invalid vectors have
        been replaced

    wf : 2d or 3d np.ndarray
        the smoothed w velocity component field, where invalid vectors have
        been replaced

    """
    uf = replace_nans(
        u, method=method, max_iter=max_iter, tol=tol,
        kernel_size=kernel_size
    )
    vf = replace_nans(
        v, method=method, max_iter=max_iter, tol=tol,
        kernel_size=kernel_size
    )

    if isinstance(w, np.ndarray):
        wf = replace_nans(
            w, method=method, max_iter=max_iter, tol=tol,
            kernel_size=kernel_size
        )
        return uf, vf, wf

    return uf, vf


from _Globals import *
from _DeviceClass import *
from _MessageClass import *

class Control(Device):
    """ Controls to be used to turn on relays """

    def __init__(self, Name):

        super(Control, self).__init__(Name)
        self.mqttc = MClient('/' + Globals._PlantName + '/Controls/' + Name, Globals._MQTTLogLevel)
        self.mqttc.Connect()

    def ValidateDevice(self):
        """method to make sure the Control has all that is needed to operate.  Interface to hardware, logging, etc"""
        isvalid=True
        if not self.Props.has_key('_id'):
            isvalid=False
        if self.interface == None:
            isvalid=False

        return isvalid

    def Subscribe(self,callback):
        """Setup subscription to MQTT channel with callback when message received"""
        self.mqttc.Receive(callback)

    def WriteRelay(self,onoff='off'):
        """sets relay to on or off."""
        action=0
        if isinstance(onoff,str):
            if onoff == 'on' or onoff=='1' or onoff=='yes' or onoff=='Yes':
                action=1
        elif isinstance(onoff,bool):
            if onoff:
                action=1
        elif isinstance(onoff,int):
            if onoff > 0:
                action=1
        self.WriteInterface({'type':'GPIO','action':action})

    def WritePWM(self,duty=50,freq=1000,polarity=1):
        """Starts the output with duty, freq and polarity"""
        self.WriteInterface({'duty':duty,'freq':freq,'polarity':polarity})

    def LogEntry(self,entry):
        """
            Incoming data can be either Errors or data.
            Typically takes raw data from a ReadInterface callback and puts it into a Loggable/MQTT format
        """
        if entry.has_key('Error'):
            logentry = {'Description': 'Error:'+entry['Error']}
            mqtt = 'Error:'+str(entry['Error'])
        else:
            logentry = {'Description': 'Data Written','details':[{'data':entry['data']}]}
            mqtt = str(entry['data'])

        super(Control, self).LogEntry(logentry)
        self.mqttc.Send(mqtt)

    def StopWrite(self):
        """Tears down connection to MQTT broker then calls the hardware (Device) StopWrite to tear down hardware"""
        self.mqttc.Disconnect()
        super(Control, self).StopWrite()


import unittest
from ds2.mapping import (Mapping,
                        ListMappingSimple,
                        ListMapping,
                        ListMapping_notDRY,
                        HashMappingSimple,
                        HashMapping_notDRY,
                        HashMapping,
                        )
from ds2.orderedmapping import (BSTMapping,
                                BalancedBST,
                                WBTree,
                                AVLTree,
                                SplayTree
                                )

class MappingTests:
    def Mapping(self):
        raise NotImplementedError

    def testput(self):
        M = self.Mapping()
        M.put(3, "three")
        M.put(1, "one")
        M.put(2, "two")

    def testgetandput(self):
        M = self.Mapping()
        M.put(3, "three")
        M.put(1, "one")
        self.assertEqual(M.get(1), "one")
        self.assertEqual(M.get(3), "three")
        self.assertEqual(M.get(1), "one")
        M.put(2, "two")
        self.assertEqual(M.get(2), "two")
        self.assertEqual(M.get(1), "one")

    def testgetraisesKeyError(self):
        M = self.Mapping()
        M.put(3, "three")
        M.put(1, "one")
        with self.assertRaises(KeyError):
            M.get(2)
        with self.assertRaises(KeyError):
            M.get(4)

    def testputoverwrites(self):
        M = self.Mapping()
        M.put(1, 1000)
        M.put(2, 'two')
        M.put(1, 'one')
        self.assertEqual(M.get(1), 'one')

class ExtendedMappingTests:
    def testlen(self):
        M = self.Mapping()
        for i in range(30):
            M.put(i, 2*i)
        self.assertEqual(len(M), 30)
        for i in range(40):
            M.put(i, i * 3)
        self.assertEqual(len(M), 40)

    def testcontains(self):
        M = self.Mapping()
        for i in range(10):
            M.put(i * 2, None)
        self.assertTrue(2 in M)
        self.assertTrue(4 in M)
        self.assertTrue(9 not in M)
        self.assertTrue(13 not in M)

    def testgetsetitem(self):
        M = self.Mapping()
        M[0] = 1000
        M[1000] = 10
        self.assertEqual(M[0], 1000)
        self.assertEqual(M[1000], 10)

    def testiter(self):
        M = self.Mapping()
        keys = {'one', 'two', 'three', '1', '2', '3'}
        for k in keys:
            M[k] = 100
        self.assertEqual(set(M), keys)

    def testvalues(self):
        M = self.Mapping()
        for i in range(10):
            M[2*i] = i + 30
        self.assertEqual(set(M.values()), set(range(30,40)))

    def testitems(self):
        M = self.Mapping()
        items = {(1, 'one'), (2,'two'), (3,'three')}
        for k, v in items:
            M[k] = v
        self.assertEqual(set(M.items()), items)

    def testmanyitems(self):
        M = self.Mapping()
        for i in range(900):
            M[i] = 1
        self.assertEqual(len(M), 900)

    def teststr(self):
        M = self.Mapping()
        M[1] = 2
        self.assertEqual(str(M), '{1 : 2}')

def _test(mapping, extended=True):
    """ Produce a TestCase class that uses the given implementation.
    """
    if extended:
        class MappingTestCase(unittest.TestCase, MappingTests, ExtendedMappingTests):
            Mapping = mapping
    else:
        class MappingTestCase(unittest.TestCase, MappingTests):
            Mapping = mapping
    return MappingTestCase

TestListMappingSimple = _test(ListMappingSimple, extended = False)
TestHashMappingSimple = _test(HashMappingSimple, extended = False)
TestListMapping = _test(ListMapping)
TestListMapping_notDRY = _test(ListMapping_notDRY)
TestHashMapping = _test(HashMapping)
TestHashMapping_notDRY = _test(HashMapping_notDRY)
TestBSTMapping = _test(BSTMapping)
TestBalancedBST = _test(BalancedBST)
TestWBTree = _test(WBTree)
TestAVLTree = _test(AVLTree)
TestSplayTree = _test(SplayTree)

class TestAbstractMapping(unittest.TestCase):
    """ These tests just check (and document) the methods that must
    be implemented by any class that extends `Mapping`.
    They are mostly here to exercise these code paths.
    """
    def testmustimplementget(self):
        M = Mapping()
        with self.assertRaises(NotImplementedError):
            M.get(1)

    def testmustimplementput(self):
        M = Mapping()
        with self.assertRaises(NotImplementedError):
            M.put(1, 2)

    def testmustimplementlen(self):
        M = Mapping()
        with self.assertRaises(NotImplementedError):
            len(M)

    def testmustimplement_entryiter(self):
        M = Mapping()
        with self.assertRaises(NotImplementedError):
            M._entryiter()

if __name__ == '__main__':
    unittest.main()


import logging
from datetime import datetime
import warnings
from tqdm import tqdm
try:
    from apex import amp
except ImportError:
    amp = None

import json
from pathlib import Path
import argparse
from typing import Union
import csv
from prettytable import PrettyTable
from typing import Dict

import torch
from torch.optim import AdamW
from torch import nn

# gpt_composer sub-package
from musicaiz.models.transformer_composers.configs import (
    GPTConfigs,
    TrainConfigs,
)
from musicaiz.models.transformer_composers.dataset import (
    build_torch_loaders,
    get_vocabulary
)
from musicaiz.models.transformer_composers.transformers import (
    GPT2,
)

def initialize_model(
    model_name: str,
    configs: Dict[str, Union[str, int]],
    device: str
):
    confs = configs["model_configs"]
    if model_name == "gpt":
        model = GPT2(
            vocab_size=confs["VOCAB_SIZE"],
            embedding_dim=confs["EMBED_DIM"],
            n_decoders=confs["N_DECODERS"],
            sequence_len=confs["SEQ_LEN"],
            n_heads=confs["N_HEADS"],
            device=device
        )
    return model

def train(
    dataset_path: Union[str, Path],
    sequence_length: int = GPTConfigs.SEQ_LEN,
    batch_size: int = TrainConfigs.BATCH_SIZE,
    train_split: float = TrainConfigs.TRAIN_SPLIT,
    is_splitted: bool = False,
    epochs: int = TrainConfigs.EPOCHS,
    lr: float = TrainConfigs.LR,
    adam_epsilon: float = TrainConfigs.ADAM_EPSILON,
    gradient_accumulation_steps: float = TrainConfigs.GRAD_ACUM_STEPS,
    fp16: bool = TrainConfigs.FP16,
    fp16_opt_level: str = TrainConfigs.FP16_OPT_LEVEL,
    log_steps: int = TrainConfigs.LOG_STEPS,
    ckpt_steps: int = TrainConfigs.CKPT_STEPS,
    checkpoint_path: Union[str, Path] = TrainConfigs.CHECKPOINT_PATH,
    log_dir: Union[str, Path] = TrainConfigs.LOG_DIR,
    model_name: str = TrainConfigs.MODEL_NAME
):

    """

    Parameters
    ----------

    dataset_path: Union[str, Path]

    sequence_length: int
        Default is :func:`~musicaiz.models.gpt_composer.GPT2Configs.SEQ_LEN`

    batch_size: int
        Default is :func:`~musicaiz.models.gpt_composer.GPT2Configs.BATCH_SIZE`

    embedding_dim: int
        Default is :func:`~musicaiz.models.gpt_composer.GPT2Configs.EMBED_DIM`

    n_decoders: int
        Default is :func:`~musicaiz.models.gpt_composer.GPT2Configs.N_DECODERS`

    n_heads: int
        Default is :func:`~musicaiz.models.gpt_composer.GPT2Configs.N_HEADS`

    dropout: float
        Default is :func:`~musicaiz.models.gpt_composer.GPT2Configs.DROPOUT`

    train_split: float
        Default is :func:`~musicaiz.models.gpt_composer.GPT2TrainConfigs.TRAIN_SPLIT`

    epochs: int
        Default is :func:`~musicaiz.models.gpt_composer.GPT2TrainConfigs.EPOCHS`

    lr: float
        Default is :func:`~musicaiz.models.gpt_composer.GPT2TrainConfigs.LR`

    adam_epsilon: float
        Default is :func:`~musicaiz.models.gpt_composer.GPT2TrainConfigs.ADAM_EPSILON`

    gradient_accumulation_steps: float
        Default is :func:`~musicaiz.models.gpt_composer.GPT2TrainConfigs.GRAD_ACUM_STEPS`

    fp16: bool
        Default is :func:`~musicaiz.models.gpt_composer.GPT2TrainConfigs.FP16`

    fp16_opt_level: str
        Default is :func:`~musicaiz.models.gpt_composer.GPT2TrainConfigs.FP16_OPT_LEVEL`

    log_steps: int
        Default is :func:`~musicaiz.models.gpt_composer.GPT2TrainConfigs.LOG_STEPS`

    ckpt_steps: int
        Default is :func:`~musicaiz.models.gpt_composer.GPT2TrainConfigs.CKPT_STEPS`

    checkpoint_path: Union[str, Path]
        Default is :func:`~musicaiz.models.gpt_composer.GPT2TrainConfigs.CHECKPOINT_PATH`

    log_dir: Union[str, Path]
        Default is :func:`~musicaiz.models.gpt_composer.GPT2TrainConfigs.LOG_DIR`

    model_name: str
        Default is :func:`~musicaiz.models.gpt_composer.GPT2TrainConfigs.MODEL_NAME`
    """

    if isinstance(log_dir, str):
        log_dir = Path(log_dir)
    log_dir.mkdir(parents=True, exist_ok=True)
    if isinstance(checkpoint_path, str):
        checkpoint_path = Path(checkpoint_path)
    checkpoint_path.mkdir(parents=True, exist_ok=True)

    logging.getLogger("Train")

    logging.info("Creating dataloaders...")

    train_dataloader, val_dataloader = build_torch_loaders(
        dataset_path=dataset_path,
        sequence_length=sequence_length,
        batch_size=batch_size,
        train_split=train_split,
        is_splitted=is_splitted
    )
    logging.info(f"Train samples {len(train_dataloader.dataset)} | Validation samples {len(val_dataloader.dataset)}")
    logging.info(
        f"Train tokens {len(train_dataloader.dataset) * sequence_length} \
            | Validation samples {len(val_dataloader.dataset) * sequence_length}"
    )

    # sample
    sample_idx = next(iter(train_dataloader))
    logging.info(f"Sample len {len(sample_idx)} data {sample_idx}")
    logging.info(f"input shape: {sample_idx.shape}")

    tokens = get_vocabulary(dataset_path)
    vocab_size = len(tokens)
    logging.info(f"Vocabulary size is {vocab_size}")

    # TODO: This needs to go into a separated method
    # Save hyperparams to json
    if model_name == "gpt":
        model_configs = {
            key: value for key, value in GPTConfigs.__dict__.items() if "__" not in key
        }
    model_configs.update({"VOCAB_SIZE": vocab_size})
    train_configs = {
        key: value for key, value in TrainConfigs.__dict__.items() if "__" not in key and not isinstance(value, Path)
    }
    configs_dict = {
        "model_configs": model_configs,
        "train_configs": train_configs,
    }
    dir_configs = Path(str(log_dir), model_name + "_configs.json")
    with open(dir_configs, 'w') as outfile:
        json.dump(configs_dict, outfile)
    logging.info(f"Saved configs.json in {dir_configs}")

    # Initialize model
    device = "cuda:0" if torch.cuda.is_available() else "cpu"
    model = initialize_model(model_name, configs_dict, device)
    model.to(device)

    def count_parameters(model):
        table = PrettyTable(["Modules", "Parameters"])
        total_params = 0
        for name, parameter in model.named_parameters():
            if not parameter.requires_grad: continue
            params = parameter.numel()
            table.add_row([name, params])
            total_params+=params
        return table, total_params
    table, total_params = count_parameters(model)
    print(table)
    logging.info(f"Total trainable params: {total_params}")

    # write model params in txt
    dir_params = Path(str(log_dir), model_name + "_model.txt")
    with open(dir_params, 'a+') as results_file:
        results_file.write(f"{table} \n Total trainable params: {total_params}")
        results_file.close()

    param_optimizer = list(model.named_parameters())
    no_decay = ['bias', 'LayerNorm.weight']
    optimizer_grouped_parameters = [
    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
        'weight_decay': 0.01},
    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
        'weight_decay': 0.0}
    ]

    loss_fn = nn.CrossEntropyLoss(ignore_index=-1)

    optimizer = AdamW(
        optimizer_grouped_parameters,
        lr=lr,
        eps=adam_epsilon
    )

    scheduler = torch.optim.lr_scheduler.OneCycleLR(
        optimizer,
        max_lr=lr,
        steps_per_epoch=len(train_dataloader) // gradient_accumulation_steps,
        epochs=epochs,
        pct_start=0.05,
        anneal_strategy='linear'
    )

    model, optimizer = amp.initialize(model, optimizer, opt_level=fp16_opt_level)

    losses = {}
    global_steps = 0
    local_steps = 0
    step_loss = 0.0
    train_loss = 0.0
    train_perplexity = 0.0
    start_epoch = 0
    start_step = 0
    step_perplexity = 0.0

    model.train()

    logging.info(f'{datetime.now()} | Moved model to: {device}')
    logging.info(f'{datetime.now()} | train_batch_size: {batch_size} | eval_batch_size: {batch_size}')
    logging.info(f'{datetime.now()} | Epochs: {epochs} | log_steps: {log_steps} | ckpt_steps: {ckpt_steps}')
    logging.info(f'{datetime.now()} | gradient_accumulation_steps: {gradient_accumulation_steps}')

    model.zero_grad()  # Reset gradients tensors
    optimizer.zero_grad()
    for epoch in range(start_epoch, epochs):  # tqdm(range(epochs), desc='Epochs', position=0):
        logging.info(f"{datetime.now()} | Epoch: {epoch} \n")
        pb = tqdm(
            enumerate(train_dataloader),
            total=len(train_dataloader),
            bar_format="{l_bar}{bar:10}{r_bar}"
        )
        for step, batch in pb:
            if step < start_step:
                continue
            inputs = batch  # _ is input_mask
            inputs = inputs.to(device)
            lm_logits = model(inputs)
            shift_logits = lm_logits[..., :-1, :].contiguous()

            # if we don't use softmax at the end of last layer
            loss = loss_fn(shift_logits.view(-1, vocab_size), inputs[:, 1:].contiguous().view(-1).type(torch.LongTensor).to(device))

            step_perplexity = torch.exp(loss)
            origin_loss = loss.item()

            loss = loss / gradient_accumulation_steps  # divide loss into gradient accumulation step
            if fp16:
                with amp.scale_loss(loss, optimizer) as scaled_loss:
                    scaled_loss.backward()
            else:
                loss.backward()

            step_loss = origin_loss
            losses[global_steps] = origin_loss

            train_loss += loss.item()
            train_perplexity += step_perplexity.item()

            local_steps += 1
            global_steps += 1

            if global_steps % gradient_accumulation_steps == 0:
                if fp16:
                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), max_norm=1.0)
                else:
                    torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)

                scheduler.step()
                optimizer.step()
                model.zero_grad()

            total_train_loss = train_loss / local_steps
            total_train_perplexity = train_perplexity / local_steps

            step_loss = 0.0
            local_steps = 0

            dir_train = Path(str(log_dir), model_name + "_train_results.txt")
            with open(dir_train, 'a+') as results_file:
                json.dump(losses, results_file)
            results_file.close()

            # Evaluate every epoch
            model.eval()

            eval_loss = 0.0
            perplexity = 0.0
            eval_steps = 0

            for step, batch in tqdm(
                enumerate(val_dataloader),
                total=len(val_dataloader),
                bar_format='{l_bar}{bar:10}{r_bar}'
            ):

                inputs = batch  # _ is input_mask
                inputs = inputs.to(device)
                labels = inputs
                labels = labels.to(device)
                lm_logits = model(inputs)

                with torch.no_grad():
                    lm_logits = model(inputs)

                shift_logits = lm_logits[..., :-1, :].contiguous()

                # if we don't use spftmax at the end of last layer
                loss = loss_fn(shift_logits.view(-1, vocab_size), inputs[:, 1:].contiguous().view(-1).type(torch.LongTensor).to(device))

                tmp_eval_loss = loss
                tmp_perplexity = torch.exp(tmp_eval_loss)

                eval_loss += tmp_eval_loss.item()
                perplexity += tmp_perplexity.item()
                eval_steps += 1

                total_eval_loss = eval_loss / eval_steps
                total_perplexity = perplexity / eval_steps

                dir_eval = Path(str(log_dir), model_name + "_eval_results.txt")
                with open(dir_eval, 'a+') as results_file:
                    results_file.write(
                        f'{datetime.now()} | Step: {step} | Eval Loss: {total_eval_loss} | Perplexity: {total_perplexity}\n'
                    )
                    results_file.close()

            # We finished 1 global step
            # write csv with results
            csv_eval = Path(str(log_dir), model_name + "_results.csv")
            with open(csv_eval, 'a+') as results_file:
                writer = csv.writer(results_file)
                if global_steps == 1:
                    writer.writerow(["Step", "Train PPL", "Eval PPL", "Train Loss", "Eval Loss"])
                writer.writerow([global_steps, total_train_loss, total_perplexity, total_train_perplexity, total_eval_loss])
                results_file.close()

            pb.set_postfix_str(
                f"{datetime.now()} | Epoch {epoch} | Global Step: {global_steps} | Train Loss: {origin_loss} | Train PPL: {step_perplexity} \n" \
            )
            pb.set_postfix_str(
                f"{datetime.now()} | Epoch {epoch} | Global Step: {global_steps} | Eval Loss: {total_eval_loss} | Perplexity: {total_perplexity} \n"
            )

            model.train()
            start_step = 0

            if global_steps % 100 == 0:
                torch.save({
                    'epoch': epoch,
                    'model_state_dict': model.state_dict(),
                    'optimizer_state_dict': optimizer.state_dict(),
                    'scheduler_state_dict': scheduler.state_dict(),
                    'losses': losses,
                    'train_step': global_steps,
                    'amp': amp.state_dict()
                    }, f'{checkpoint_path}/ep{epoch}_step{global_steps}_{model_name}.pth'
                )
                logging.info(f'\n {datetime.now()} | Saved checkpoint to: {checkpoint_path} \n')

        if epoch % 10 == 0:
            torch.save({
                'epoch': epoch,
                'model_state_dict': model.state_dict(),
                'optimizer_state_dict': optimizer.state_dict(),
                'scheduler_state_dict': scheduler.state_dict(),
                'losses': losses,
                'train_step': global_steps,
                'amp': amp.state_dict()
                }, f'{checkpoint_path}/ep{epoch}_{model_name}.pth'
            )
            logging.info(f'\n {datetime.now()} | Saved checkpoint to: {checkpoint_path} \n')

    torch.save({
        'epoch': epoch,
        'model_state_dict': model.state_dict(),
        'optimizer_state_dict': optimizer.state_dict(),
        'scheduler_state_dict': scheduler.state_dict(),
        'losses': losses,
        'train_step': global_steps,
        'amp': amp.state_dict()
        }, f'{checkpoint_path}/ep{epoch}_{model_name}.pth'
    )
    logging.info(f'\n {datetime.now()} | Saved checkpoint to: {checkpoint_path} \n')

def parse_args():
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "--dataset_path",
        type=str,
        help="",
        required=True,
    )
    parser.add_argument(
        "--sequence_length",
        type=int,
        help="",
        required=False,
        default=GPTConfigs.SEQ_LEN,
    )
    parser.add_argument(
        "--batch_size",
        type=int,
        help="",
        required=False,
        default=TrainConfigs.BATCH_SIZE,
    )
    parser.add_argument(
        "--train_split",
        type=float,
        help="",
        required=False,
        default=TrainConfigs.TRAIN_SPLIT,
    )
    parser.add_argument(
        "--is_splitted",
        type=bool,
        help="",
        required=False,
        default=TrainConfigs.IS_SPLITTED,
    )
    parser.add_argument(
        "--epochs",
        type=int,
        help="",
        required=False,
        default=TrainConfigs.EPOCHS,
    )
    parser.add_argument(
        "--lr",
        type=float,
        help="",
        required=False,
        default=TrainConfigs.LR,
    )
    parser.add_argument(
        "--adam_epsilon",
        type=float,
        help="",
        required=False,
        default=TrainConfigs.ADAM_EPSILON,
    )
    parser.add_argument(
        "--gradient_accumulation_steps",
        type=float,
        help="",
        required=False,
        default=TrainConfigs.GRAD_ACUM_STEPS,
    )
    parser.add_argument(
        "--fp16",
        type=bool,
        help="",
        required=False,
        default=TrainConfigs.FP16,
    )
    parser.add_argument(
        "--fp16_opt_level",
        type=str,
        help="",
        required=False,
        default=TrainConfigs.FP16_OPT_LEVEL,
    )
    parser.add_argument(
        "--log_steps",
        type=int,
        help="",
        required=False,
        default=TrainConfigs.LOG_STEPS,
    )
    parser.add_argument(
        "--ckpt_steps",
        type=int,
        help="",
        required=False,
        default=TrainConfigs.CKPT_STEPS,
    )
    parser.add_argument(
        "--checkpoint_path",
        type=str,
        help="",
        required=False,
        default=TrainConfigs.CHECKPOINT_PATH,
    )
    parser.add_argument(
        "--log_dir",
        type=str,
        help="",
        required=False,
        default=TrainConfigs.LOG_DIR,
    )
    parser.add_argument(
        "--model_name",
        type=str,
        help="",
        required=False,
        default=TrainConfigs.MODEL_NAME,
    )
    return parser.parse_args()

if __name__ == "__main__":
    args = parse_args()
    train(
        dataset_path=args.dataset_path,
        train_split=args.train_split,
        is_splitted=args.is_splitted,
        epochs=args.epochs,
        lr=args.lr,
        adam_epsilon=args.adam_epsilon,
        gradient_accumulation_steps=args.gradient_accumulation_steps,
        fp16=args.fp16,
        fp16_opt_level=args.fp16_opt_level,
        log_steps=args.log_steps,
        ckpt_steps=args.ckpt_steps,
        checkpoint_path=args.checkpoint_path,
        log_dir=args.log_dir,
        model_name=args.model_name,
    )


#exec() compiles and evaluates whatever you pass through it.
#NO SECURITY
#its a compiler in compiler

exec("print('so this works')")

list_str = "[4,5657,3,43345,234]"
list_str = exec(list_str)
print(list_str)

exec("list_str2 = [423,235,567]")
print(list_str2)

exec("def test(): print('Oh snip snap if this works')")
exec("test()")

exec("""
def test1():
    print('See whether multiline works.')

test1()
"""
   )


from setuptools import setup, find_packages

version = '3.0.0'

setup(name='multisearch',
      version=version,
      description='HI LGS web price scraper for MTG',
      url='https://github.com/kylemccreery/multisearch',
      author='Kyle McCreery',
      author_email='mccreery.kyle@gmail.com',
      packages=find_packages(),
      install_requires=['BeautifulSoup4','bs4','requests'],
      keywords=['multisearch', 'MTG', 'Hawaii', 'LGS'],
      license='MIT',
      classifiers=[
          # Indicate who your project is intended for
          'Intended Audience :: Developers',
          # Pick your license as you wish (should match "license" above)
          'License :: OSI Approved :: MIT License',

          # Specify the Python versions you support here. In particular, ensure
          # that you indicate whether you support Python 2, Python 3 or both.
          'Programming Language :: Python :: 3'
      ])


"""
API for creating tree scatter subnet
"""

import hou
from . import classNodeNetwork as cnn
from . import helper

def findNodeInList(nodeList, nodeType):
    """
    Simple function to return the first node that matches the node type
    :param nodeList: List of nodes
    :param nodeType: String of the node type
    :return: hou.node object
    """
    for node in nodeList:
        if node.type().name() == nodeType:
            break
        else:
            node = None
    return node

def createTreeScatterSubnet(treeSubnet, hfGeoNode, matchsize=True):
    """ Create Subnet used for scattering in hf_scatter SOP"""
    treeSubnetNet = cnn.MyNetwork(treeSubnet)
    hfGeoNodeNet = cnn.MyNetwork(hfGeoNode)

    treeScatterSubnetName = treeSubnetNet.name + "_scatter"

    # Create tree scatter subnet
    oldScatterSubnetList = hfGeoNodeNet.findNodes(treeScatterSubnetName, method="name")
    if oldScatterSubnetList:
        oldScatterSubnet = oldScatterSubnetList[0]
        # Store old position and name
        oldScatterSubnetPos = oldScatterSubnet.position()
        oldScatterSubnetName = oldScatterSubnet.name()
        # Store wired merge node if exists
        if oldScatterSubnet.outputs():
            scatterMergeNodes = oldScatterSubnet.outputs()
            scatterConnections = oldScatterSubnet.outputConnections()
            scatterConnectionIndicies = [scatterConnection.inputIndex() for scatterConnection in scatterConnections]
        else:
            scatterMergeNodes = ()
            scatterConnectionIndicies = []
        # Find network box that the node is in
        currentNetworkBox = helper.getNetworkBox(oldScatterSubnet, hfGeoNode)

        # Copy old scatter subnet and delete contents
        scatterSubnet = oldScatterSubnet.copyTo(oldScatterSubnet.parent())
        oldScatterSubnet.destroy()
        for child in scatterSubnet.children():
            child.destroy()
        scatterSubnet.setPosition(oldScatterSubnetPos)
        scatterSubnet.setName(oldScatterSubnetName)
        # Hide default parameters
        scatterSubnet = helper.hideParms(scatterSubnet, ["label1", "label2", "label3", "label4"])
        # Put in network box
        if currentNetworkBox:
            currentNetworkBox.addNode(scatterSubnet)

        # Rewire scatterSubnet
        if scatterMergeNodes:
            for i in range(len(scatterMergeNodes)):
                inputIndex = scatterConnectionIndicies[i]
                scatterMergeNodes[i].insertInput(inputIndex, scatterSubnet)

        # Action message
        action = "Updated"
    else:
        # Create new tree scatter subnet
        scatterSubnet = hfGeoNode.createNode("subnet", treeScatterSubnetName)
        # Hide default parameters
        scatterSubnet = helper.hideParms(scatterSubnet, ["label1", "label2", "label3", "label4"])
        # Create new parameters
        group = scatterSubnet.parmTemplateGroup()
        labelTemplate = hou.LabelParmTemplate("treeScatterSubnet", "Tree Scatter Subnet")
        group.append(labelTemplate)
        treeScaleTemplate = hou.FloatParmTemplate("treeScale", ("Tree Scale"), 1, default_value=([1]), min=0, max=6)
        group.append(treeScaleTemplate)
        weightTemplate = hou.FloatParmTemplate("weight", ("Weight"), 1, default_value=([1]), min=0, max=1)
        group.append(weightTemplate)
        scatterSubnet.setParmTemplateGroup(group)
        # Action message
        action = "Created"

    # ScatterSubnet set creator state
    scatterSubnet.setCreatorState("SpeedTree Asset Generator by Daniel")

    scatterSubnetNet = cnn.MyNetwork(scatterSubnet)

    # Create merge, xform, attribcreate, and output
    nodePrefix1 = treeSubnetNet.name + "_"  # Can be string or None
    nodePrefix1 = str(nodePrefix1 or "")
    newNodesGroup1 = scatterSubnetNet.addNodes("merge", "xform", "attribcreate::2.0", "output", prefix=nodePrefix1)
    mergeNode = findNodeInList(newNodesGroup1, "merge")
    attribCreateNode = findNodeInList(newNodesGroup1, "attribcreate::2.0")
    attribCreateNode.setParms({"name1": "weight", "class1": 1, "value1v1": 1})
    outputNode =findNodeInList(newNodesGroup1, "output")
    # Wires new nodes group 1 in the order of the list
    scatterSubnetNet.wireNodes(newNodesGroup1)

    # Create object_merge and matchsize
    i = 0
    for child in treeSubnetNet.children:
        # Bypass material networks
        if child.type().name() == "matnet" or child.type().name() == "shopnet":
            continue

        nodePrefix2 = child.name() + "_"
        newNodesGroup2Temp = scatterSubnetNet.addNodes("object_merge", "matchsize", prefix=nodePrefix2)
        # Wires new nodes group 2 temp in the order of the list
        scatterSubnetNet.wireNodes(newNodesGroup2Temp)

        # Object merge node
        objMergeNode = findNodeInList(newNodesGroup2Temp, "object_merge")
        objMergeNode.setParms({"objpath1": "/obj/{TREESUBNETNAME}/{CHILDNAME}".format(TREESUBNETNAME=treeSubnetNet.name,
                                                                                CHILDNAME=child.name())})
        # Match size node
        matchsizeNode = findNodeInList(newNodesGroup2Temp, "matchsize")
        mergeNode.setInput(i, matchsizeNode)
        if matchsize:
            matchsizeNode.setParms({"justify_y": 1})
            matchsizeNode.setParms({"doscale": 1})
            matchsizeNode.setParms({"uniformscale": 1})
            matchsizeNode.setParms({"scale_axis": 0})
        else:
            newNodesGroup2Temp.remove(matchsizeNode)
            matchsizeNode.destroy()

        # Wires new nodes group 2 temp in the order of the list
        scatterSubnetNet.wireNodes(newNodesGroup2Temp)
        i += 1

    # Set relative references
    xformNode = scatterSubnetNet.findNodes("xform", method="type")[0]
    xformScale = xformNode.parm("scale")
    xformScale.setExpression("ch(\"../treeScale\")")
    attrWeightNode = scatterSubnetNet.findNodes("attribcreate::2.0", method="type")[0]
    attrWeightParm = attrWeightNode.parm("value1v1")
    attrWeightParm.setExpression("ch(\"../weight\")")

    # Set flags
    outputNode.setRenderFlag(True)
    outputNode.setDisplayFlag(True)

    # Layout Children
    scatterSubnet.layoutChildren()

    # Action Message
    actionMessage = "{ACTION} Scatter Subnet: {SCATTERSUBNETNAME}".format(ACTION=action,
                                                                          SCATTERSUBNETNAME=treeScatterSubnetName)

    return scatterSubnet, actionMessage