from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import abc
import collections

import numpy as np

from tensorflow.python.feature_column import lookup_ops
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib
from tensorflow.python.framework import tensor_shape
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import check_ops
from tensorflow.python.ops import embedding_ops
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import nn_ops
from tensorflow.python.ops import parsing_ops
from tensorflow.python.ops import sparse_ops
from tensorflow.python.ops import string_ops
from tensorflow.python.ops import variable_scope
from tensorflow.python.ops import variables
from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.util import nest

def make_input_layer(features,
                     feature_columns,
                     weight_collections=None,
                     trainable=True):
  """Returns a dense `Tensor` as input layer based on given `feature_columns`.

  Generally a single example in training data is described with FeatureColumns.
  At the first layer of the model, this column oriented data should be converted
  to a single `Tensor`.

  Example:

  ```python
  price = numeric_column('price')
  keywords_embedded = embedding_column(
      categorical_column_with_hash_bucket("keywords", 10K), dimensions=16)
  all_feature_columns = [price, keywords_embedded, ...]
  dense_tensor = make_input_layer(features, all_feature_columns)
  for units in [128, 64, 32]:
    dense_tensor = tf.layers.dense(dense_tensor, units, tf.nn.relu)
  prediction = tf.layers.dense(dense_tensor, 1)
  ```

  Args:
    features: A mapping from key to tensors. `FeatureColumn`s look up via these
      keys. For example `numeric_column('price') will look at 'price' key in
      this dict. Values can be a `SparseTensor` or a `Tensor` depends on
      corresponding `FeatureColumn`.
    feature_columns: An iterable containing all the `FeatureColumn`s. All items
      should be instances of classes derived from `_DenseColumn` such as
      `numeric_column`, `embedding_column`, `bucketized_column`,
      `indicator_column`. If you have categorical features, you can wrap them
      with with an `embedding_column` or `indicator_column`.
    weight_collections: A list of collection names to which the Variable will be
      added. Note that, variables will also be added to collections
      `tf.GraphKeys.GLOBAL_VARIABLES` and `ops.GraphKeys.MODEL_VARIABLES`.
    trainable: If `True` also add the variable to the graph collection
      `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).

  Returns:
    A `Tensor` which represents input layer of a model. Its shape
    is (batch_size, first_layer_dimension) and its dtype is `float32`.
    first_layer_dimension is determined based on given `feature_columns`.

  Raises:
    ValueError: if an item in `feature_columns` is not a `_DenseColumn`.
  """
  _check_feature_columns(feature_columns)
  for column in feature_columns:
    if not isinstance(column, _DenseColumn):
      raise ValueError(
          'Items of feature_columns must be a _DenseColumn. '
          'You can wrap a categorical column with an '
          'embedding_column or indicator_column. Given: {}'.format(column))
  weight_collections = list(weight_collections or [])
  if ops.GraphKeys.GLOBAL_VARIABLES not in weight_collections:
    weight_collections.append(ops.GraphKeys.GLOBAL_VARIABLES)
  if ops.GraphKeys.MODEL_VARIABLES not in weight_collections:
    weight_collections.append(ops.GraphKeys.MODEL_VARIABLES)
  with variable_scope.variable_scope(
      None, default_name='make_input_layer', values=features.values()):
    builder = _LazyBuilder(features)
    output_tensors = []
    for column in sorted(feature_columns, key=lambda x: x.name):
      with variable_scope.variable_scope(None, default_name=column.name):
        tensor = column._get_dense_tensor(  # pylint: disable=protected-access
            builder,
            weight_collections=weight_collections,
            trainable=trainable)
        num_elements = column._variable_shape.num_elements()  # pylint: disable=protected-access
        batch_size = array_ops.shape(tensor)[0]
        tensor = array_ops.reshape(tensor, shape=(batch_size, num_elements))
        output_tensors.append(tensor)
    return array_ops.concat(output_tensors, 1)

def make_linear_model(features,
                      feature_columns,
                      units=1,
                      sparse_combiner='sum',
                      weight_collections=None,
                      trainable=True):
  """Returns a linear prediction `Tensor` based on given `feature_columns`.

  This function generates a weighted sum for each unitss`. Weighted sum
  refers to logits in classification problems. It refers to the prediction
  itself for linear regression problems.

  Main difference of `make_linear_model` and `make_input_layer` is handling of
  categorical columns. `make_linear_model` treats them as `indicator_column`s
  while `make_input_layer` explicitly requires wrapping each of them with an
  `embedding_column` or an `indicator_column`.

  Example:

  ```python
  price = numeric_column('price')
  price_buckets = bucketized_column(price, boundaries=[0., 10., 100., 1000.])
  keywords = categorical_column_with_hash_bucket("keywords", 10K)
  all_feature_columns = [price_buckets, keywords, ...]
  prediction = make_linear_model(features, all_feature_columns)
  ```

  Args:
    features: A mapping from key to tensors. `FeatureColumn`s look up via these
      keys. For example `numeric_column('price')` will look at 'price' key in
      this dict. Values are `Tensor` or `SparseTensor` depending on
      corresponding `FeatureColumn`.
    feature_columns: An iterable containing all the FeatureColumns. All items
      should be instances of classes derived from FeatureColumn.
    units: units: An integer, dimensionality of the output space. Default
      value is 1.
    sparse_combiner: A string specifying how to reduce if a sparse column is
      multivalent. Currently "mean", "sqrtn" and "sum" are supported, with "sum"
      the default. "sqrtn" often achieves good accuracy, in particular with
      bag-of-words columns. It combines each sparse columns independently.
        * "sum": do not normalize features in the column
        * "mean": do l1 normalization on features in the column
        * "sqrtn": do l2 normalization on features in the column
    weight_collections: A list of collection names to which the Variable will be
      added. Note that, variables will also be added to collections
      `tf.GraphKeys.GLOBAL_VARIABLES` and `ops.GraphKeys.MODEL_VARIABLES`.
    trainable: If `True` also add the variable to the graph collection
      `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).

  Returns:
    A `Tensor` which represents predictions/logits of a linear model. Its shape
    is (batch_size, units) and its dtype is `float32`.

  Raises:
    ValueError: if an item in `feature_columns` is neither a `_DenseColumn`
      nor `_CategoricalColumn`.
  """
  _check_feature_columns(feature_columns)
  for column in feature_columns:
    if not isinstance(column, (_DenseColumn, _CategoricalColumn)):
      raise ValueError('Items of feature_columns must be either a _DenseColumn '
                       'or _CategoricalColumn. Given: {}'.format(column))
  weight_collections = list(weight_collections or [])
  if ops.GraphKeys.GLOBAL_VARIABLES not in weight_collections:
    weight_collections.append(ops.GraphKeys.GLOBAL_VARIABLES)
  if ops.GraphKeys.MODEL_VARIABLES not in weight_collections:
    weight_collections.append(ops.GraphKeys.MODEL_VARIABLES)
  with variable_scope.variable_scope(
      None, default_name='make_linear_model', values=features.values()):
    weigthed_sums = []
    builder = _LazyBuilder(features)
    for column in sorted(feature_columns, key=lambda x: x.name):
      with variable_scope.variable_scope(None, default_name=column.name):
        if isinstance(column, _CategoricalColumn):
          weigthed_sums.append(_create_categorical_column_weighted_sum(
              column, builder, units, sparse_combiner, weight_collections,
              trainable))
        else:
          weigthed_sums.append(_create_dense_column_weighted_sum(
              column, builder, units, weight_collections, trainable))
    predictions_no_bias = math_ops.add_n(
        weigthed_sums, name='weighted_sum_no_bias')
    bias = variable_scope.get_variable(
        'bias_weight',
        shape=[units],
        initializer=init_ops.zeros_initializer(),
        trainable=trainable,
        collections=weight_collections)
    predictions = nn_ops.bias_add(
        predictions_no_bias, bias, name='weighted_sum')

    return predictions

def numeric_column(key,
                   shape=(1,),
                   default_value=None,
                   dtype=dtypes.float32,
                   normalizer_fn=None):
  """Represents real valued or numerical features.

  Example:

  ```python
  price = numeric_column('price')
  all_feature_columns = [price, ...]
  dense_tensor = make_input_layer(features, all_feature_columns)

  # or
  bucketized_price = bucketized_column(price, boundaries=[...])
  all_feature_columns = [bucketized_price, ...]
  linear_prediction = make_linear_model(features, all_feature_columns)

  ```

  Args:
    key: A unique string identifying the input feature. It is used as the
      column name and the dictionary key for feature parsing configs, feature
      `Tensor` objects, and feature columns.
    shape: An iterable of integers specifies the shape of the `Tensor`. An
      integer can be given which means a single dimension `Tensor` with given
      width. The `Tensor` representing the column will have the shape of
      [batch_size] + `shape`.
    default_value: A single value compatible with `dtype` or an iterable of
      values compatible with `dtype` which the column takes on during
      `tf.Example` parsing if data is missing. A default value of `None` will
      cause `tf.parse_example` to fail if an example does not contain this
      column. If a single value is provided, the same value will be applied as
      the default value for every item. If an iterable of values is provided,
      the shape of the `default_value` should be equal to the given `shape`.
    dtype: defines the type of values. Default value is `tf.float32`. Must be a
      non-quantized, real integer or floating point type.
    normalizer_fn: If not `None`, a function that can be used to normalize the
      value of the tensor after `default_value` is applied for parsing.
      Normalizer function takes the input `Tensor` as its argument, and returns
      the output `Tensor`. (e.g. lambda x: (x - 3.0) / 4.2). Please note that
      even though most common use case of this function is normalization, it can
      be used for any kind of Tensorflow transformations.

  Returns:
    A _NumericColumn.

  Raises:
    TypeError: if any dimension in shape is not an int
    ValueError: if any dimension in shape is not a positive integer
    TypeError: if `default_value` is an iterable but not compatible with `shape`
    TypeError: if `default_value` is not compatible with `dtype`.
    ValueError: if `dtype` is not convertible to `tf.float32`.
  """
  shape = _check_shape(shape, key)
  if not (dtype.is_integer or dtype.is_floating):
    raise ValueError('dtype must be convertible to float. '
                     'dtype: {}, key: {}'.format(dtype, key))
  default_value = _check_default_value(shape, default_value, dtype, key)

  if normalizer_fn is not None and not callable(normalizer_fn):
    raise TypeError(
        'normalizer_fn must be a callable. Given: {}'.format(normalizer_fn))

  return _NumericColumn(
      key,
      shape=shape,
      default_value=default_value,
      dtype=dtype,
      normalizer_fn=normalizer_fn)

def bucketized_column(source_column, boundaries):
  """Represents discretized dense input.

  Buckets include the left boundary, and exclude the right boundary. Namely,
  `boundaries=[0., 1., 2.]` generates buckets `(-inf, 0.)`, `[0., 1.)`,
  `[1., 2.)`, and `[2., +inf)`.

  For example, if the inputs are
    `boundaries` = [0, 10, 100]
    input tensor = [[-5, 10000]
                    [150,   10]
                    [5,    100]]

  then the output will be
    output = [[0, 3]
              [3, 2]
              [1, 3]]

  Example:

  ```python
  price = numeric_column('price')
  bucketized_price = bucketized_column(price, boundaries=[...])
  all_feature_columns = [bucketized_price, ...]
  linear_prediction = make_linear_model(features, all_feature_columns)

  # or
  all_feature_columns = [bucketized_price, ...]
  dense_tensor = make_input_layer(features, all_feature_columns)
  ```

  Args:
    source_column: A one-dimensional dense column which is generated with
      `numeric_column`.
    boundaries: A sorted list or tuple of floats specifying the boundaries.

  Returns:
    A `_BucketizedColumn`.

  Raises:
    ValueError: If `source_column` is not a numeric column, or if it is not
      one-dimensional.
    ValueError: If `boundaries` is not a sorted list or tuple.
  """
  if not isinstance(source_column, _NumericColumn):
    raise ValueError(
        'source_column must be a column generated with numeric_column(). '
        'Given: {}'.format(source_column))
  if len(source_column.shape) > 1:
    raise ValueError(
        'source_column must be one-dimensional column. '
        'Given: {}'.format(source_column))
  if (not boundaries or
      not (isinstance(boundaries, list) or isinstance(boundaries, tuple))):
    raise ValueError('boundaries must be a sorted list.')
  for i in range(len(boundaries) - 1):
    if boundaries[i] >= boundaries[i + 1]:
      raise ValueError('boundaries must be a sorted list.')
  return _BucketizedColumn(source_column, tuple(boundaries))

def _assert_string_or_int(dtype, prefix):
  if (dtype != dtypes.string) and (not dtype.is_integer):
    raise ValueError(
        '{} dtype must be string or integer. dtype: {}.'.format(prefix, dtype))

def categorical_column_with_hash_bucket(key,
                                        hash_bucket_size,
                                        dtype=dtypes.string):
  """Represents sparse feature where ids are set by hashing.

  Use this when your sparse features are in string or integer format where you
  want to distribute your inputs into a finite number of buckets by hashing.
  output_id = Hash(input_feature_string) % bucket_size

  Example:

  ```python
  keywords = categorical_column_with_hash_bucket("keywords", 10K)
  linear_prediction = make_linear_model(features, [keywords, ...])

  # or
  keywords_embedded = embedding_column(keywords, 16)
  dense_tensor = make_input_layer(features, [keywords_embedded, ...])
  ```

  Args:
    key: A unique string identifying the input feature. It is used as the
      column name and the dictionary key for feature parsing configs, feature
      `Tensor` objects, and feature columns.
    hash_bucket_size: An int > 1. The number of buckets.
    dtype: The type of features. Only string and integer types are supported.

  Returns:
    A `_HashedCategoricalColumn`.

  Raises:
    ValueError: `hash_bucket_size` is not greater than 1.
    ValueError: `dtype` is neither string nor integer.
  """
  if hash_bucket_size is None:
    raise ValueError('hash_bucket_size must be set. ' 'key: {}'.format(key))

  if hash_bucket_size < 1:
    raise ValueError('hash_bucket_size must be at least 1. '
                     'hash_bucket_size: {}, key: {}'.format(
                         hash_bucket_size, key))

  _assert_string_or_int(dtype, prefix='column_name: {}'.format(key))

  return _HashedCategoricalColumn(key, hash_bucket_size, dtype)

def categorical_column_with_vocabulary_file(
    key, vocabulary_file, vocabulary_size, num_oov_buckets=0,
    default_value=None, dtype=dtypes.string):
  """A `_CategoricalColumn` with a vocabulary file.

  Use this when your inputs are in string or integer format, and you have a
  vocabulary file that maps each value to an integer ID. By default,
  out-of-vocabulary values are ignored. Use either (but not both) of
  `num_oov_buckets` and `default_value` to specify how to include
  out-of-vocabulary values.

  Inputs can be either `Tensor` or `SparseTensor`. If `Tensor`, missing values
  can be represented by `-1` for int and `''` for string. Note that these values
  are independent of the `default_value` argument.

  Example with `num_oov_buckets`:
  File '/us/states.txt' contains 50 lines, each with a 2-character U.S. state
  abbreviation. All inputs with values in that file are assigned an ID 0-49,
  corresponding to its line number. All other values are hashed and assigned an
  ID 50-54.
  ```python
  states = categorical_column_with_vocabulary_file(
      key='states', vocabulary_file='/us/states.txt', vocabulary_size=50,
      num_oov_buckets=5)
  linear_prediction = make_linear_model(features, [states, ...])
  ```

  Example with `default_value`:
  File '/us/states.txt' contains 51 lines - the first line is 'XX', and the
  other 50 each have a 2-character U.S. state abbreviation. Both a literal 'XX'
  in input, and other values missing from the file, will be assigned ID 0. All
  others are assigned the corresponding line number 1-50.
  ```python
  states = categorical_column_with_vocabulary_file(
      key='states', vocabulary_file='/us/states.txt', vocabulary_size=51,
      default_value=0)
  linear_prediction, _, _ = make_linear_model(features, [states, ...])

  And to make an embedding with either:
  ```python
  dense_tensor = make_input_layer(features, [embedding_column(states, 3),...])
  ```

  Args:
    key: A unique string identifying the input feature. It is used as the
      column name and the dictionary key for feature parsing configs, feature
      `Tensor` objects, and feature columns.
    vocabulary_file: The vocabulary file name.
    vocabulary_size: Number of the elements in the vocabulary. This must be no
      greater than length of `vocabulary_file`, if less than length, later
      values are ignored.
    num_oov_buckets: Non-negative integer, the number of out-of-vocabulary
      buckets. All out-of-vocabulary inputs will be assigned IDs in the range
      `[vocabulary_size, vocabulary_size+num_oov_buckets)` based on a hash of
      the input value. A positive `num_oov_buckets` can not be specified with
      `default_value`.
    default_value: The integer ID value to return for out-of-vocabulary feature
      values, defaults to -1. This can not be specified with a positive
      `num_oov_buckets`.
    dtype: The type of features. Only string and integer types are supported.

  Returns:
    A `_CategoricalColumn` with a vocabulary file.

  Raises:
    ValueError: `vocabulary_file` is missing.
    ValueError: `vocabulary_size` is missing or < 1.
    ValueError: `num_oov_buckets` is not a non-negative integer.
    ValueError: `dtype` is neither string nor integer.
  """
  if not vocabulary_file:
    raise ValueError('Missing vocabulary_file in {}.'.format(key))
  # `vocabulary_size` isn't required for lookup, but it is for `_num_buckets`.
  # TODO(ptucker): Should we fail for vocabulary_size==1?
  if (vocabulary_size is None) or (vocabulary_size < 1):
    raise ValueError('Invalid vocabulary_size in {}.'.format(key))
  if num_oov_buckets:
    if default_value is not None:
      raise ValueError(
          'Can\'t specify both num_oov_buckets and default_value in {}.'.format(
              key))
    if num_oov_buckets < 0:
      raise ValueError('Invalid num_oov_buckets {} in {}.'.format(
          num_oov_buckets, key))
  _assert_string_or_int(dtype, prefix='column_name: {}'.format(key))
  return _VocabularyFileCategoricalColumn(
      key=key,
      vocabulary_file=vocabulary_file,
      vocabulary_size=vocabulary_size,
      num_oov_buckets=0 if num_oov_buckets is None else num_oov_buckets,
      default_value=-1 if default_value is None else default_value,
      dtype=dtype)

def categorical_column_with_vocabulary_list(
    key, vocabulary_list, dtype=None, default_value=-1):
  """A `_CategoricalColumn` with in-memory vocabulary.

  Logic for feature f is:
  id = f in vocabulary_list ? vocabulary_list.index(f) : default_value

  Use this when your inputs are in string or integer format, and you have an
  in-memory vocabulary mapping each value to an integer ID. By default,
  out-of-vocabulary values are ignored. Use `default_value` to specify how to
  include out-of-vocabulary values.

  Inputs can be either `Tensor` or `SparseTensor`. If `Tensor`, missing values
  can be represented by `-1` for int and `''` for string. Note that these values
  are independent of the `default_value` argument.

  In the following examples, each input in `vocabulary_list` is assigned an ID
  0-4 corresponding to its index (e.g., input 'B' produces output 2). All other
  inputs are assigned `default_value` 0.

  Linear model:
  ```python
  colors = categorical_column_with_vocabulary_list(
      key='colors', vocabulary_list=('X', 'R', 'G', 'B', 'Y'), default_value=0)
  linear_prediction, _, _ = make_linear_model(features, [colors, ...])
  ```

  Embedding for a DNN model:
  ```python
  dense_tensor = make_input_layer(features, [embedding_column(colors, 3),...])
  ```

  Args:
    key: A unique string identifying the input feature. It is used as the
      column name and the dictionary key for feature parsing configs, feature
      `Tensor` objects, and feature columns.
    vocabulary_list: An ordered iterable defining the vocabulary. Each feature
      is mapped to the index of its value (if present) in `vocabulary_list`.
      Must be castable to `dtype`.
    dtype: The type of features. Only string and integer types are supported.
      If `None`, it will be inferred from `vocabulary_list`.
    default_value: The value to use for values not in `vocabulary_list`.

  Returns:
    A `_CategoricalColumn` with in-memory vocabulary.

  Raises:
    ValueError: if `vocabulary_list` is empty, or contains duplicate keys.
    ValueError: if `dtype` is not integer or string.
  """
  if (vocabulary_list is None) or (len(vocabulary_list) < 1):
    raise ValueError(
        'vocabulary_list {} must be non-empty, column_name: {}'.format(
            vocabulary_list, key))
  if len(set(vocabulary_list)) != len(vocabulary_list):
    raise ValueError(
        'Duplicate keys in vocabulary_list {}, column_name: {}'.format(
            vocabulary_list, key))
  vocabulary_dtype = dtypes.as_dtype(np.array(vocabulary_list).dtype)
  _assert_string_or_int(
      vocabulary_dtype, prefix='column_name: {} vocabulary'.format(key))
  if dtype is None:
    dtype = vocabulary_dtype
  elif dtype.is_integer != vocabulary_dtype.is_integer:
    raise ValueError(
        'dtype {} and vocabulary dtype {} do not match, column_name: {}'.format(
            dtype, vocabulary_dtype, key))
  _assert_string_or_int(dtype, prefix='column_name: {}'.format(key))

  return _VocabularyListCategoricalColumn(
      key=key, vocabulary_list=tuple(vocabulary_list), dtype=dtype,
      default_value=default_value)

def categorical_column_with_identity(key, num_buckets, default_value=None):
  """A `_CategoricalColumn` that returns identity values.

  Use this when your inputs are integers in the range `[0, num_buckets)`. Values
  outside this range will result in `default_value` if specified, otherwise it
  will fail.

  Inputs can be either `Tensor` or `SparseTensor`.
  ```

  Args:
    key: A unique string identifying the input feature. It is used as the
      column name and the dictionary key for feature parsing configs, feature
      `Tensor` objects, and feature columns.
    num_buckets: Range of inputs and outputs is `[0, num_buckets)`.
    default_value: If `None`, this column's graph operations will fail for
      out-of-range inputs. Otherwise, this value must be in the range
      `[0, num_buckets)`, and will replace inputs in that range.

  Returns:
    A `_CategoricalColumn` that returns identity values.

  Raises:
    ValueError: if `num_buckets` is less than one.
    ValueError: if `default_value` is not in range `[0, num_buckets)`.
  """
  if num_buckets < 1:
    raise ValueError(
        'num_buckets {} < 1, column_name {}'.format(num_buckets, key))
  if (default_value is not None) and (
      (default_value < 0) or (default_value >= num_buckets)):
    raise ValueError(
        'default_value {} not in range [0, {}), column_name {}'.format(
            default_value, num_buckets, key))
  return _IdentityCategoricalColumn(
      key=key, num_buckets=num_buckets, default_value=default_value)

class _FeatureColumn(object):
  """Represents a feature column abstraction.

  WARNING: Do not subclass this layer unless you know what you are doing:
  the API is subject to future changes.

  To distinguish the concept of a feature family and a specific binary feature
  within a family, we refer to a feature family like "country" as a feature
  column. Following is an example feature in a `tf.Example` format:
    {key: "country",  value: [ "US" ]}
  In this example the value of feature is "US" and "country" refers to the
  column of the feature.

  This class is an abstract class. User should not create instances of this.
  """
  __metaclass__ = abc.ABCMeta

  @abc.abstractproperty
  def name(self):
    """Returns string. used for variable_scope and naming."""
    pass

  @abc.abstractmethod
  def _transform_feature(self, inputs):
    """Returns transformed `Tensor`, uses `inputs` to access input tensors.

    It uses `inputs` to get either raw feature or transformation of other
    FeatureColumns.

    Example input access:
    Let's say a Feature column depends on raw feature ('raw') and another
    `_FeatureColumn` (input_fc). To access corresponding Tensors, inputs will
    be used as follows:

    ```python
    raw_tensor = inputs.get('raw')
    fc_tensor = inputs.get(input_fc)
    ```

    Args:
      inputs: A `_LazyBuilder` object to access inputs.

    Returns:
      Transformed feature `Tensor`.
    """
    pass

  @abc.abstractproperty
  def _parse_example_config(self):
    """Returns a `tf.Example` parsing spec as dict.

    It is used for get_parsing_spec for `tf.parse_example`. Returned spec is a
    dict from keys ('string') to `VarLenFeature`, `FixedLenFeature`, and other
    supported objects. Please check documentation of ${tf.parse_example} for all
    supported spec objects.

    Let's say a Feature column depends on raw feature ('raw') and another
    `_FeatureColumn` (input_fc). One possible implementation of
    _parse_example_config is as follows:

    ```python
    spec = {'raw': tf.FixedLenFeature(...)}
    spec.update(input_fc._parse_example_config)
    return spec
    ```
    """
    pass

class _DenseColumn(_FeatureColumn):
  """Represents a column which can be represented as `Tensor`.

  WARNING: Do not subclass this layer unless you know what you are doing:
  the API is subject to future changes.

  Some examples of this type are: numeric_column, embedding_column,
  indicator_column.
  """

  __metaclass__ = abc.ABCMeta

  @abc.abstractproperty
  def _variable_shape(self):
    """Returns a `TensorShape` of variable compatible with _get_dense_tensor."""
    pass

  @abc.abstractmethod
  def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None):
    """Returns a `Tensor`.

    The output of this function will be used by model-buildier-functions. For
    example the pseudo code of `make_input_layer` will be like that:

    ```python
    def make_input_layer(features, feature_columns, ...):
      outputs = [fc._get_dense_tensor(...) for fc in feature_columns]
      return tf.concat(outputs)
    ```

    Args:
      inputs: A `_LazyBuilder` object to access inputs.
      weight_collections: List of graph collections to which Variables (if any
        will be created) are added.
      trainable: If `True` also add variables to the graph collection
        `GraphKeys.TRAINABLE_VARIABLES` (see ${tf.Variable}).
    """
    pass

def _create_dense_column_weighted_sum(
    column, builder, units, weight_collections, trainable):
  """Create a weighted sum of a dense column for make_linear_model."""
  tensor = column._get_dense_tensor(  # pylint: disable=protected-access
      builder,
      weight_collections=weight_collections,
      trainable=trainable)
  num_elements = column._variable_shape.num_elements()  # pylint: disable=protected-access
  batch_size = array_ops.shape(tensor)[0]
  tensor = array_ops.reshape(tensor, shape=(batch_size, num_elements))
  weight = variable_scope.get_variable(
      name='weight',
      shape=[num_elements, units],
      initializer=init_ops.zeros_initializer(),
      trainable=trainable,
      collections=weight_collections)
  return math_ops.matmul(tensor, weight, name='weighted_sum')

class _CategoricalColumn(_FeatureColumn):
  """Represents a categorical feautre.

  WARNING: Do not subclass this layer unless you know what you are doing:
  the API is subject to future changes.

  A categorical feature typically handled with a ${tf.SparseTensor} of IDs.
  """
  __metaclass__ = abc.ABCMeta

  IdWeightPair = collections.namedtuple(  # pylint: disable=invalid-name
      'IdWeightPair', ['id_tensor', 'weight_tensor'])

  @abc.abstractproperty
  def _num_buckets(self):
    """Returns number of buckets in this sparse feature."""
    pass

  @abc.abstractmethod
  def _get_sparse_tensors(self,
                          inputs,
                          weight_collections=None,
                          trainable=None):
    """Returns an IdWeightPair.

    `IdWeightPair` is a pair of `SparseTensor`s which represents ids and
    weights.

    `IdWeightPair.id_tensor` is typically a `batch_size` x `num_buckets`
    `SparseTensor` of `int64`. `IdWeightPair.weight_tensor` is either a
    `SparseTensor` of `float` or `None` to indicate all weights should be
    taken to be 1. If specified, `weight_tensor` must have exactly the same
    shape and indices as `sp_ids`. Expected `SparseTensor` is same as parsing
    output of a `VarLenFeature` which is a ragged matrix.

    Args:
      inputs: A `LazyBuilder` as a cache to get input tensors required to
        create `IdWeightPair`.
      weight_collections: List of graph collections to which variables (if any
        will be created) are added.
      trainable: If `True` also add variables to the graph collection
        `GraphKeys.TRAINABLE_VARIABLES` (see ${tf.get_variable}).
    """
    pass

def _create_categorical_column_weighted_sum(
    column, builder, units, sparse_combiner, weight_collections, trainable):
  """Create a weighted sum of a categorical column for make_linear_model."""
  sparse_tensors = column._get_sparse_tensors(  # pylint: disable=protected-access
      builder,
      weight_collections=weight_collections,
      trainable=trainable)
  weight = variable_scope.get_variable(
      name='weight',
      shape=[column._num_buckets, units],  # pylint: disable=protected-access
      initializer=init_ops.zeros_initializer(),
      trainable=trainable,
      collections=weight_collections)
  return _safe_embedding_lookup_sparse(
      weight,
      sparse_tensors.id_tensor,
      sparse_weights=sparse_tensors.weight_tensor,
      combiner=sparse_combiner,
      name='weighted_sum')

class _LazyBuilder(object):
  """Handles caching of transformations while building the model.

  `FeatureColumn` specifies how to digest an input column to the network. Some
  feature columns require data transformations. This class caches those
  transformations.

  Some features may be used in more than one place. For example, one can use a
  bucketized feature by itself and a cross with it. In that case we
  should create only one bucketization op instead of creating ops for each
  feature column separately. To handle re-use of transformed columns,
  `_LazyBuilder` caches all previously transformed columns.

  Example:
  We're trying to use the following `FeatureColumns`:

  ```python
    bucketized_age = fc.bucketized_column(fc.numeric_column("age"), ...)
    keywords = fc.categorical_column_with_hash_buckets("keywords", ...)
    age_X_keywords = fc.crossed_column([bucketized_age, keywords])
    ... = make_linear_model(features,
                            [bucketized_age, keywords, age_X_keywords]
  ```

  If we transform each column independently, then we'll get duplication of
  bucketization (one for cross, one for bucketization itself).
  The `_LazyBuilder` eliminates this duplication.
  """

  def __init__(self, features):
    """Creates a `_LazyBuilder`.

    Args:
      features: A mapping from feature column to objects that are `Tensor` or
        `SparseTensor`, or can be converted to same via
        `sparse_tensor.convert_to_tensor_or_sparse_tensor`. A `string` key
        signifies a base feature (not-transformed). A `FeatureColumn` key
        means that this `Tensor` is the output of an existing `FeatureColumn`
        which can be reused.
    """
    self._features = features.copy()
    self._feature_tensors = {}

  def get(self, key):
    """Returns a `Tensor` for the given key.

    A `str` key is used to access a base feature (not-transformed). When a
    `_FeatureColumn` is passed, the transformed feature is returned if it
    already exists, otherwise the given `_FeatureColumn` is asked to provide its
    transformed output, which is then cached.

    Args:
      key: a `str` or a `_FeatureColumn`.

    Returns:
      The transformed `Tensor` corresponding to the `key`.

    Raises:
      ValueError: if key is not found or a transformed `Tensor` cannot be
        computed.
    """
    if key in self._feature_tensors:
      # FeatureColumn is already transformed or converted.
      return self._feature_tensors[key]

    if key in self._features:
      # FeatureColumn is a raw feature.
      feature_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(
          self._features[key])
      self._feature_tensors[key] = feature_tensor
      return feature_tensor

    if not isinstance(key, (str, _FeatureColumn)):
      raise TypeError('"key" must be either a "str" or "_FeatureColumn". '
                      'Provided: {}'.format(key))

    if not isinstance(key, _FeatureColumn):
      raise ValueError('Feature {} is not in features dictionary.'.format(key))

    column = key
    logging.debug('Transforming feature_column %s.', column)
    # pylint: disable=protected-access
    transformed = column._transform_feature(self)
    # pylint: enable=protected-access
    if transformed is None:
      raise ValueError('Column {} is not supported.'.format(column.name))
    self._feature_tensors[column] = transformed
    return transformed

# TODO(ptucker): Move to third_party/tensorflow/python/ops/sparse_ops.py
def _shape_offsets(shape):
  """Returns moving offset for each dimension given shape."""
  offsets = []
  for dim in reversed(shape):
    if offsets:
      offsets.append(dim * offsets[-1])
    else:
      offsets.append(dim)
  offsets.reverse()
  return offsets

# TODO(ptucker): Move to third_party/tensorflow/python/ops/sparse_ops.py
def _to_sparse_input(input_tensor, ignore_value=None):
  """Converts a `Tensor` to a `SparseTensor`, dropping ignore_value cells.

  If `input_tensor` is already a `SparseTensor`, just return it.

  Args:
    input_tensor: A string or integer `Tensor`.
    ignore_value: Entries in `dense_tensor` equal to this value will be
      absent from the resulting `SparseTensor`. If `None`, default value of
      `dense_tensor`'s dtype will be used ('' for `str`, -1 for `int`).

  Returns:
    A `SparseTensor` with the same shape as `input_tensor`.

  Raises:
    ValueError: when `input_tensor`'s rank is `None`.
  """
  input_tensor = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(
      input_tensor)
  if isinstance(input_tensor, sparse_tensor_lib.SparseTensor):
    return input_tensor
  with ops.name_scope(None, 'to_sparse_input', (input_tensor, ignore_value,)):
    input_rank = input_tensor.get_shape().ndims
    if input_rank is None:
      # TODO(b/32318825): Implement dense_to_sparse_tensor for undefined rank.
      raise ValueError('Undefined input_tensor shape.')
    if ignore_value is None:
      ignore_value = '' if input_tensor.dtype == dtypes.string else -1
    dense_shape = math_ops.cast(array_ops.shape(input_tensor), dtypes.int64)
    indices = array_ops.where(math_ops.not_equal(
        input_tensor, math_ops.cast(ignore_value, input_tensor.dtype)))
    # Flattens the tensor and indices for use with gather.
    flat_tensor = array_ops.reshape(input_tensor, [-1])
    flat_indices = indices[:, input_rank - 1]
    # Computes the correct flattened indices for 2d (or higher) tensors.
    if input_rank > 1:
      higher_dims = indices[:, :input_rank - 1]
      shape_offsets = array_ops.stack(
          _shape_offsets(array_ops.unstack(dense_shape)[1:]))
      offsets = math_ops.reduce_sum(
          math_ops.multiply(higher_dims, shape_offsets),
          reduction_indices=[1])
      flat_indices = math_ops.add(flat_indices, offsets)
    values = array_ops.gather(flat_tensor, flat_indices)
    return sparse_tensor_lib.SparseTensor(indices, values, dense_shape)

def _check_feature_columns(feature_columns):
  if isinstance(feature_columns, dict):
    raise ValueError('Expected feature_columns to be iterable, found dict.')
  for column in feature_columns:
    if not isinstance(column, _FeatureColumn):
      raise ValueError('Items of feature_columns must be a _FeatureColumn.')
  name_to_column = dict()
  for column in feature_columns:
    if column.name in name_to_column:
      raise ValueError('Duplicate feature column name found for columns: {} '
                       'and {}. This usually means that these columns refer to '
                       'same base feature. Either one must be discarded or a '
                       'duplicated but renamed item must be inserted in '
                       'features dict.'.format(column,
                                               name_to_column[column.name]))
    name_to_column[column.name] = column

class _NumericColumn(_DenseColumn,
                     collections.namedtuple('_NumericColumn', [
                         'key', 'shape', 'default_value', 'dtype',
                         'normalizer_fn'
                     ])):
  """see `numeric_column`."""

  @property
  def name(self):
    return self.key

  @property
  def _parse_example_config(self):
    return {
        self.key:
            parsing_ops.FixedLenFeature(self.shape, self.dtype,
                                        self.default_value)
    }

  def _transform_feature(self, inputs):
    input_tensor = inputs.get(self.key)
    if isinstance(input_tensor, sparse_tensor_lib.SparseTensor):
      raise ValueError(
          'The corresponding Tensor of numerical column must be a Tensor. '
          'SparseTensor is not supported. key: {}'.format(self.key))
    if self.normalizer_fn is not None:
      input_tensor = self.normalizer_fn(input_tensor)
    return math_ops.to_float(input_tensor)

  @property
  def _variable_shape(self):
    return tensor_shape.TensorShape(self.shape)

  def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None):
    del weight_collections
    del trainable
    return inputs.get(self)

class _BucketizedColumn(_DenseColumn, _CategoricalColumn,
                        collections.namedtuple('_BucketizedColumn', [
                            'source_column', 'boundaries'])):
  """See `bucketized_column`."""

  @property
  def name(self):
    return '{}_bucketized'.format(self.source_column.name)

  @property
  def _parse_example_config(self):
    return self.source_column._parse_example_config  # pylint: disable=protected-access

  def _transform_feature(self, inputs):
    source_tensor = inputs.get(self.source_column)
    return math_ops._bucketize(  # pylint: disable=protected-access
        source_tensor,
        boundaries=self.boundaries)

  @property
  def _variable_shape(self):
    return tensor_shape.TensorShape(
        tuple(self.source_column.shape) + (len(self.boundaries) + 1,))

  def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None):
    del weight_collections
    del trainable
    input_tensor = inputs.get(self)
    return array_ops.one_hot(
        indices=math_ops.to_int64(input_tensor),
        depth=len(self.boundaries) + 1,
        on_value=1.,
        off_value=0.)

  @property
  def _num_buckets(self):
    # By construction, source_column is always one-dimensional.
    return (len(self.boundaries) + 1) * self.source_column.shape[0]

  def _get_sparse_tensors(self, inputs, weight_collections=None,
                          trainable=None):
    input_tensor = inputs.get(self)
    batch_size = array_ops.shape(input_tensor)[0]
    # By construction, source_column is always one-dimensional.
    source_dimension = self.source_column.shape[0]

    i1 = array_ops.reshape(
        array_ops.tile(
            array_ops.expand_dims(math_ops.range(0, batch_size), 1),
            [1, source_dimension]),
        (-1,))
    i2 = array_ops.tile(math_ops.range(0, source_dimension), [batch_size])
    # Flatten the bucket indices and unique them across dimensions
    # E.g. 2nd dimension indices will range from k to 2*k-1 with k buckets
    bucket_indices = (
        array_ops.reshape(input_tensor, (-1,)) +
        (len(self.boundaries) + 1) * i2)

    indices = math_ops.to_int64(array_ops.transpose(array_ops.stack((i1, i2))))
    dense_shape = math_ops.to_int64(array_ops.stack(
        [batch_size, source_dimension]))
    sparse_tensor = sparse_tensor_lib.SparseTensor(
        indices=indices,
        values=bucket_indices,
        dense_shape=dense_shape)
    return _CategoricalColumn.IdWeightPair(sparse_tensor, None)

def _create_tuple(shape, value):
  """Returns a tuple with given shape and filled with value."""
  if shape:
    return tuple([_create_tuple(shape[1:], value) for _ in range(shape[0])])
  return value

def _as_tuple(value):
  if not nest.is_sequence(value):
    return value
  return tuple([_as_tuple(v) for v in value])

def _check_shape(shape, key):
  """Returns shape if it's valid, raises error otherwise."""
  assert shape is not None
  if not nest.is_sequence(shape):
    shape = [shape]
  shape = tuple(shape)
  for dimension in shape:
    if not isinstance(dimension, int):
      raise TypeError('shape dimensions must be integer. '
                      'shape: {}, key: {}'.format(shape, key))
    if dimension < 1:
      raise ValueError('shape dimensions must be greater than 0. '
                       'shape: {}, key: {}'.format(shape, key))
  return shape

def _is_shape_and_default_value_compatible(default_value, shape):
  """Verifies compatibility of shape and default_value."""
  # Invalid condition:
  #  * if default_value is not a scalar and shape is empty
  #  * or if default_value is an iterable and shape is not empty
  if nest.is_sequence(default_value) != bool(shape):
    return False
  if not shape:
    return True
  if len(default_value) != shape[0]:
    return False
  for i in range(shape[0]):
    if not _is_shape_and_default_value_compatible(default_value[i], shape[1:]):
      return False
  return True

def _check_default_value(shape, default_value, dtype, key):
  """Returns default value as tuple if it's valid, otherwise raises errors.

  This function verifies that `default_value` is compatible with both `shape`
  and `dtype`. If it is not compatible, it raises an error. If it is compatible,
  it casts default_value to a tuple and returns it. `key` is used only
  for error message.

  Args:
    shape: An iterable of integers specifies the shape of the `Tensor`.
    default_value: If a single value is provided, the same value will be applied
      as the default value for every item. If an iterable of values is
      provided, the shape of the `default_value` should be equal to the given
      `shape`.
    dtype: defines the type of values. Default value is `tf.float32`. Must be a
      non-quantized, real integer or floating point type.
    key: Column name, used only for error messages.

  Returns:
    A tuple which will be used as default value.

  Raises:
    TypeError: if `default_value` is an iterable but not compatible with `shape`
    TypeError: if `default_value` is not compatible with `dtype`.
    ValueError: if `dtype` is not convertible to `tf.float32`.
  """
  if default_value is None:
    return None

  if isinstance(default_value, int):
    return _create_tuple(shape, default_value)

  if isinstance(default_value, float) and dtype.is_floating:
    return _create_tuple(shape, default_value)

  if callable(getattr(default_value, 'tolist', None)):  # Handles numpy arrays
    default_value = default_value.tolist()

  if nest.is_sequence(default_value):
    if not _is_shape_and_default_value_compatible(default_value, shape):
      raise ValueError(
          'The shape of default_value must be equal to given shape. '
          'default_value: {}, shape: {}, key: {}'.format(
              default_value, shape, key))
    # Check if the values in the list are all integers or are convertible to
    # floats.
    is_list_all_int = all(
        isinstance(v, int) for v in nest.flatten(default_value))
    is_list_has_float = any(
        isinstance(v, float) for v in nest.flatten(default_value))
    if is_list_all_int:
      return _as_tuple(default_value)
    if is_list_has_float and dtype.is_floating:
      return _as_tuple(default_value)
  raise TypeError('default_value must be compatible with dtype. '
                  'default_value: {}, dtype: {}, key: {}'.format(
                      default_value, dtype, key))

class _HashedCategoricalColumn(
    _CategoricalColumn,
    collections.namedtuple('_HashedCategoricalColumn',
                           ['key', 'hash_bucket_size', 'dtype'])):
  """see `categorical_column_with_hash_bucket`."""

  @property
  def name(self):
    return self.key

  @property
  def _parse_example_config(self):
    return {self.key: parsing_ops.VarLenFeature(self.dtype)}

  def _transform_feature(self, inputs):
    input_tensor = _to_sparse_input(inputs.get(self.key))
    if not isinstance(input_tensor, sparse_tensor_lib.SparseTensor):
      raise ValueError('SparseColumn input must be a SparseTensor.')

    _assert_string_or_int(
        input_tensor.dtype,
        prefix='column_name: {} input_tensor'.format(self.key))

    if self.dtype.is_integer != input_tensor.dtype.is_integer:
      raise ValueError(
          'Column dtype and SparseTensors dtype must be compatible. '
          'key: {}, column dtype: {}, tensor dtype: {}'.format(
              self.key, self.dtype, input_tensor.dtype))

    if self.dtype == dtypes.string:
      sparse_values = input_tensor.values
    else:
      sparse_values = string_ops.as_string(input_tensor.values)

    sparse_id_values = string_ops.string_to_hash_bucket_fast(
        sparse_values, self.hash_bucket_size, name='lookup')
    return sparse_tensor_lib.SparseTensor(
        input_tensor.indices, sparse_id_values, input_tensor.dense_shape)

  @property
  def _num_buckets(self):
    """Returns number of buckets in this sparse feature."""
    return self.hash_bucket_size

  def _get_sparse_tensors(self, inputs, weight_collections=None,
                          trainable=None):
    return _CategoricalColumn.IdWeightPair(inputs.get(self), None)

class _VocabularyFileCategoricalColumn(
    _CategoricalColumn,
    collections.namedtuple('_VocabularyFileCategoricalColumn', (
        'key', 'vocabulary_file', 'vocabulary_size', 'num_oov_buckets', 'dtype',
        'default_value'
    ))):
  """See `categorical_column_with_vocabulary_file`."""

  @property
  def name(self):
    return self.key

  @property
  def _parse_example_config(self):
    return {self.key: parsing_ops.VarLenFeature(self.dtype)}

  def _transform_feature(self, inputs):
    input_tensor = _to_sparse_input(inputs.get(self.key))

    if self.dtype.is_integer != input_tensor.dtype.is_integer:
      raise ValueError(
          'Column dtype and SparseTensors dtype must be compatible. '
          'key: {}, column dtype: {}, tensor dtype: {}'.format(
              self.key, self.dtype, input_tensor.dtype))

    _assert_string_or_int(
        input_tensor.dtype,
        prefix='column_name: {} input_tensor'.format(self.key))

    key_dtype = self.dtype
    if input_tensor.dtype.is_integer:
      # `index_table_from_file` requires 64-bit integer keys.
      key_dtype = dtypes.int64
      input_tensor = math_ops.to_int64(input_tensor)

    return lookup_ops.index_table_from_file(
        vocabulary_file=self.vocabulary_file,
        num_oov_buckets=self.num_oov_buckets,
        vocab_size=self.vocabulary_size,
        default_value=self.default_value,
        key_dtype=key_dtype,
        name='{}_lookup'.format(self.key)).lookup(input_tensor)

  @property
  def _num_buckets(self):
    """Returns number of buckets in this sparse feature."""
    return self.vocabulary_size + self.num_oov_buckets

  def _get_sparse_tensors(
      self, inputs, weight_collections=None, trainable=None):
    return _CategoricalColumn.IdWeightPair(inputs.get(self), None)

class _VocabularyListCategoricalColumn(
    _CategoricalColumn,
    collections.namedtuple('_VocabularyListCategoricalColumn', (
        'key', 'vocabulary_list', 'dtype', 'default_value'
    ))):
  """See `categorical_column_with_vocabulary_list`."""

  @property
  def name(self):
    return self.key

  @property
  def _parse_example_config(self):
    return {self.key: parsing_ops.VarLenFeature(self.dtype)}

  def _transform_feature(self, inputs):
    input_tensor = _to_sparse_input(inputs.get(self.key))

    if self.dtype.is_integer != input_tensor.dtype.is_integer:
      raise ValueError(
          'Column dtype and SparseTensors dtype must be compatible. '
          'key: {}, column dtype: {}, tensor dtype: {}'.format(
              self.key, self.dtype, input_tensor.dtype))

    _assert_string_or_int(
        input_tensor.dtype,
        prefix='column_name: {} input_tensor'.format(self.key))

    key_dtype = self.dtype
    if input_tensor.dtype.is_integer:
      # `index_table_from_tensor` requires 64-bit integer keys.
      key_dtype = dtypes.int64
      input_tensor = math_ops.to_int64(input_tensor)

    return lookup_ops.index_table_from_tensor(
        mapping=tuple(self.vocabulary_list),
        default_value=self.default_value,
        dtype=key_dtype,
        name='{}_lookup'.format(self.key)).lookup(input_tensor)

  @property
  def _num_buckets(self):
    """Returns number of buckets in this sparse feature."""
    return len(self.vocabulary_list)

  def _get_sparse_tensors(
      self, inputs, weight_collections=None, trainable=None):
    return _CategoricalColumn.IdWeightPair(inputs.get(self), None)

class _IdentityCategoricalColumn(
    _CategoricalColumn,
    collections.namedtuple('_IdentityCategoricalColumn', (
        'key', 'num_buckets', 'default_value'
    ))):

  """See `categorical_column_with_identity`."""

  @property
  def name(self):
    return self.key

  @property
  def _parse_example_config(self):
    return {self.key: parsing_ops.VarLenFeature(dtypes.int64)}

  def _transform_feature(self, inputs):
    input_tensor = _to_sparse_input(inputs.get(self.key))

    if not input_tensor.dtype.is_integer:
      raise ValueError(
          'Invalid input, not integer. key: {} dtype: {}'.format(
              self.key, input_tensor.dtype))

    values = math_ops.to_int64(input_tensor.values, name='values')
    num_buckets = math_ops.to_int64(self.num_buckets, name='num_buckets')
    zero = math_ops.to_int64(0, name='zero')
    if self.default_value is None:
      # Fail if values are out-of-range.
      assert_less = check_ops.assert_less(
          values, num_buckets, data=(values, num_buckets),
          name='assert_less_than_num_buckets')
      assert_greater = check_ops.assert_greater_equal(
          values, zero, data=(values,),
          name='assert_greater_or_equal_0')
      with ops.control_dependencies((assert_less, assert_greater)):
        values = array_ops.identity(values)
    else:
      # Assign default for out-of-range values.
      values = array_ops.where(
          math_ops.logical_or(
              values < zero, values >= num_buckets, name='out_of_range'),
          array_ops.fill(
              dims=array_ops.shape(values),
              value=math_ops.to_int64(self.default_value),
              name='default_values'),
          values)

    return sparse_tensor_lib.SparseTensor(
        indices=input_tensor.indices,
        values=values,
        dense_shape=input_tensor.dense_shape)

  @property
  def _num_buckets(self):
    """Returns number of buckets in this sparse feature."""
    return self.num_buckets

  def _get_sparse_tensors(
      self, inputs, weight_collections=None, trainable=None):
    return _CategoricalColumn.IdWeightPair(inputs.get(self), None)

# TODO(zakaria): Move this to embedding_ops and make it public.
def _safe_embedding_lookup_sparse(embedding_weights,
                                  sparse_ids,
                                  sparse_weights=None,
                                  combiner=None,
                                  default_id=None,
                                  name=None,
                                  partition_strategy='div',
                                  max_norm=None):
  """Lookup embedding results, accounting for invalid IDs and empty features.

  The partitioned embedding in `embedding_weights` must all be the same shape
  except for the first dimension. The first dimension is allowed to vary as the
  vocabulary size is not necessarily a multiple of `P`.  `embedding_weights`
  may be a `PartitionedVariable` as returned by using `tf.get_variable()` with a
  partitioner.

  Invalid IDs (< 0) are pruned from input IDs and weights, as well as any IDs
  with non-positive weight. For an entry with no features, the embedding vector
  for `default_id` is returned, or the 0-vector if `default_id` is not supplied.

  The ids and weights may be multi-dimensional. Embeddings are always aggregated
  along the last dimension.

  Args:
    embedding_weights:  A list of `P` float tensors or values representing
        partitioned embedding tensors.  Alternatively, a `PartitionedVariable`,
        created by partitioning along dimension 0.  The total unpartitioned
        shape should be `[e_0, e_1, ..., e_m]`, where `e_0` represents the
        vocab size and `e_1, ..., e_m` are the embedding dimensions.
    sparse_ids: `SparseTensor` of shape `[d_0, d_1, ..., d_n]` containing the
        ids. `d_0` is typically batch size.
    sparse_weights: `SparseTensor` of same shape as `sparse_ids`, containing
        float weights corresponding to `sparse_ids`, or `None` if all weights
        are be assumed to be 1.0.
    combiner: A string specifying how to combine embedding results for each
        entry. Currently "mean", "sqrtn" and "sum" are supported, with "mean"
        the default.
    default_id: The id to use for an entry with no features.
    name: A name for this operation (optional).
    partition_strategy: A string specifying the partitioning strategy.
        Currently `"div"` and `"mod"` are supported. Default is `"div"`.
    max_norm: If not None, all embeddings are l2-normalized to max_norm before
        combining.

  Returns:
    Dense tensor of shape `[d_0, d_1, ..., d_{n-1}, e_1, ..., e_m]`.

  Raises:
    ValueError: if `embedding_weights` is empty.
  """
  if combiner is None:
    logging.warn('The default value of combiner will change from \"mean\" '
                 'to \"sqrtn\" after 2016/11/01.')
    combiner = 'mean'
  if embedding_weights is None:
    raise ValueError('Missing embedding_weights %s.' % embedding_weights)
  if isinstance(embedding_weights, variables.PartitionedVariable):
    embedding_weights = list(embedding_weights)  # get underlying Variables.
  if not isinstance(embedding_weights, list):
    embedding_weights = [embedding_weights]
  if len(embedding_weights) < 1:
    raise ValueError('Missing embedding_weights %s.' % embedding_weights)

  dtype = sparse_weights.dtype if sparse_weights is not None else None
  if isinstance(embedding_weights, variables.PartitionedVariable):
    embedding_weights = list(embedding_weights)
  embedding_weights = [
      ops.convert_to_tensor(w, dtype=dtype) for w in embedding_weights
  ]

  with ops.name_scope(name, 'embedding_lookup',
                      embedding_weights + [sparse_ids,
                                           sparse_weights]) as scope:
    # Reshape higher-rank sparse ids and weights to linear segment ids.
    original_shape = sparse_ids.dense_shape
    original_rank_dim = sparse_ids.dense_shape.get_shape()[0]
    original_rank = (
        array_ops.size(original_shape)
        if original_rank_dim.value is None
        else original_rank_dim.value)
    sparse_ids = sparse_ops.sparse_reshape(sparse_ids, [
        math_ops.reduce_prod(
            array_ops.slice(original_shape, [0], [original_rank - 1])),
        array_ops.gather(original_shape, original_rank - 1)])
    if sparse_weights is not None:
      sparse_weights = sparse_tensor_lib.SparseTensor(
          sparse_ids.indices,
          sparse_weights.values, sparse_ids.dense_shape)

    # Prune invalid ids and weights.
    sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights)

    # Fill in dummy values for empty features, if necessary.
    sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows(sparse_ids,
                                                                 default_id or
                                                                 0)
    if sparse_weights is not None:
      sparse_weights, _ = sparse_ops.sparse_fill_empty_rows(sparse_weights, 1.0)

    result = embedding_ops.embedding_lookup_sparse(
        embedding_weights,
        sparse_ids,
        sparse_weights,
        combiner=combiner,
        partition_strategy=partition_strategy,
        name=None if default_id is None else scope,
        max_norm=max_norm)

    if default_id is None:
      # Broadcast is_row_empty to the same shape as embedding_lookup_result,
      # for use in Select.
      is_row_empty = array_ops.tile(
          array_ops.reshape(is_row_empty, [-1, 1]),
          array_ops.stack([1, array_ops.shape(result)[1]]))

      result = array_ops.where(is_row_empty,
                               array_ops.zeros_like(result),
                               result,
                               name=scope)

    # Reshape back from linear ids back into higher-dimensional dense result.
    final_result = array_ops.reshape(
        result,
        array_ops.concat([
            array_ops.slice(
                math_ops.cast(original_shape, dtypes.int32), [0],
                [original_rank - 1]),
            array_ops.slice(array_ops.shape(result), [1], [-1])
        ], 0))
    final_result.set_shape(tensor_shape.unknown_shape(
        (original_rank_dim - 1).value).concatenate(result.get_shape()[1:]))
    return final_result

def _prune_invalid_ids(sparse_ids, sparse_weights):
  """Prune invalid IDs (< 0) from the input ids and weights."""
  is_id_valid = math_ops.greater_equal(sparse_ids.values, 0)
  if sparse_weights is not None:
    is_id_valid = math_ops.logical_and(
        is_id_valid, math_ops.greater(sparse_weights.values, 0))
  sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid)
  if sparse_weights is not None:
    sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid)
  return sparse_ids, sparse_weights


import numpy as np
import lda
import lda.datasets
from numpy import genfromtxt
import csv

my_data = genfromtxt('Restaurant_data_matrix.csv', delimiter=',', skip_header=1)
word_matrix=np.delete(my_data,0,1).astype(int)   #delete the 1st column, which is NAN; convert from float to int type

with open('Restaurant_data_matrix.csv', newline='') as f:
    reader = csv.reader(f)
    vocab = next(reader)[1:]

#X=lda.datasets.load_reuters()
#vocab=lda.datasets.load_reuters_vocab()
#titles=lda.datasets.load_reuters_titles()
model=lda.LDA(n_topics=20,n_iter=1500, random_state=1)
model.fit(word_matrix)
topic_word=model.topic_word_   # it's the beta in the model
n_top_words=10
for i, topic_dist in enumerate(topic_word):
	topic_words=np.array(vocab)[np.argsort(topic_dist)][:-(n_top_words+1):-1]
	print('Topic {0} : {1}'.format(i, ' '.join(topic_words)))

doc_topic = model.doc_topic_
for i in range(10):
     print("{} (top topic: {})".format(i, doc_topic[i].argmax()))


# coding: utf-8

from __future__ import absolute_import

import flask

import auth
import config
import model
import util

from main import app

microsoft_config = dict(
  access_token_method='POST',
  access_token_url='https://login.live.com/oauth20_token.srf',
  authorize_url='https://login.live.com/oauth20_authorize.srf',
  base_url='https://apis.live.net/v5.0/',
  consumer_key=config.CONFIG_DB.microsoft_client_id,
  consumer_secret=config.CONFIG_DB.microsoft_client_secret,
  request_token_params={'scope': 'wl.emails'},
)

microsoft = auth.create_oauth_app(microsoft_config, 'microsoft')

@app.route('/api/auth/callback/microsoft/')
def microsoft_authorized():
  response = microsoft.authorized_response()
  if response is None:
    flask.flash('You denied the request to sign in.')
    return flask.redirect(util.get_next_url())
  flask.session['oauth_token'] = (response['access_token'], '')
  me = microsoft.get('me')
  if me.data.get('error', {}):
    return 'Unknown error: error:%s error_description:%s' % (
      me['error']['code'],
      me['error']['message'],
    )
  user_db = retrieve_user_from_microsoft(me.data)
  return auth.signin_user_db(user_db)

@microsoft.tokengetter
def get_microsoft_oauth_token():
  return flask.session.get('oauth_token')

@app.route('/signin/microsoft/')
def signin_microsoft():
  return auth.signin_oauth(microsoft)

def retrieve_user_from_microsoft(response):
  auth_id = 'microsoft_%s' % response['id']
  user_db = model.User.get_by('auth_ids', auth_id)
  if user_db:
    return user_db
  email = response['emails']['preferred'] or response['emails']['account']
  return auth.create_user_db(
    auth_id=auth_id,
    name=response.get('name', ''),
    username=email,
    email=email,
    verified=bool(email),
  )


#!/usr/bin/env python
# coding: utf-8

# 예제 내용
# * 기본 위젯을 사용하여 기본 창을 생성
# * '창 이름'을 변경한다.

import sys

from PyQt5.QtWidgets import QWidget
from PyQt5.QtWidgets import QSplitter
from PyQt5.QtWidgets import QTextEdit
from PyQt5.QtWidgets import QVBoxLayout

from PyQt5.QtWidgets import QApplication
from PyQt5.QtCore import Qt

__author__ = "Deokyu Lim <hong18s@gmail.com>"

class Form(QWidget):
    """
    만들고자 하는 프로그램의 기본이 되는 창 또는 폼 위젯.
    본 위젯 위에 다른 위젯을 올려서 모양을 만든다.

    QWidget을 상속받아서 필요한 메소드를 작성.
    """

    def __init__(self):
        """
        보통 __init__ (생성자)에서 필요한 것들을 다를 위젯들을 선언해줘도 되지만 init_widget을 따로 만들어서 호출한다.
        """
        QWidget.__init__(self, flags=Qt.Widget)

        self.te_1 = QTextEdit()
        self.te_2 = QTextEdit()
        self.te_3 = QTextEdit()
        self.split_1 = QSplitter()
        self.split_2 = QSplitter()
        self.vbox = QVBoxLayout()
        self.container_vbox = QVBoxLayout()
        self.init_widget()

    def init_widget(self):
        """
        현재 위젯의 모양등을 초기화
        """
        self.setWindowTitle("Hello World")
        self.split_1.addWidget(self.te_1)
        self.split_1.addWidget(self.te_2)
        self.container_vbox.addWidget(self.split_1)

        self.split_2.setOrientation(Qt.Vertical)
        self.split_2.addWidget(self.split_1)
        self.split_2.addWidget(self.te_3)

        self.vbox.addWidget(self.split_2)
        self.setLayout(self.vbox)

if __name__ == "__main__":
    app = QApplication(sys.argv)
    form = Form()
    form.show()
    exit(app.exec_())


from django.utils.functional import lazy

from mock import patch

from django_browserid import helpers
from django_browserid.tests import JSON_STRING, TestCase

def _lazy_request_args():
    return {'siteName': 'asdf'}
lazy_request_args = lazy(_lazy_request_args, dict)

class BrowserIDInfoTests(TestCase):
    def setUp(self):
        patcher = patch('django_browserid.helpers.render_to_string')
        self.addCleanup(patcher.stop)
        self.render_to_string = patcher.start()

    def test_defaults(self):
        with self.settings(BROWSERID_REQUEST_ARGS={'foo': 'bar', 'baz': 1}):
            output = helpers.browserid_info()

        self.assertEqual(output, self.render_to_string.return_value)
        expected_info = JSON_STRING({
            'loginUrl': '/browserid/login/',
            'logoutUrl': '/browserid/logout/',
            'csrfUrl': '/browserid/csrf/',
            'requestArgs': {'foo': 'bar', 'baz': 1},
        })
        self.render_to_string.assert_called_with('browserid/info.html', {'info': expected_info})

    def test_lazy_request_args(self):
        with self.settings(BROWSERID_REQUEST_ARGS=lazy_request_args()):
            output = helpers.browserid_info()

        self.assertEqual(output, self.render_to_string.return_value)
        expected_info = JSON_STRING({
            'loginUrl': '/browserid/login/',
            'logoutUrl': '/browserid/logout/',
            'csrfUrl': '/browserid/csrf/',
            'requestArgs': {'siteName': 'asdf'},
        })
        self.render_to_string.assert_called_with('browserid/info.html', {'info': expected_info})

class BrowserIDJSTests(TestCase):
    def test_basic(self):
        output = helpers.browserid_js()
        self.assertHTMLEqual(output, """

        """)

    def test_no_shim(self):
        output = helpers.browserid_js(include_shim=False)
        self.assertHTMLEqual(output, """

        """)

    def test_custom_shim(self):
        with self.settings(BROWSERID_SHIM='http://example.com/test.js'):
            output = helpers.browserid_js()
        self.assertHTMLEqual(output, """

        """)

    def test_autologin_email(self):
        """
        If BROWSERID_AUTOLOGIN_ENABLED is True, do not include the shim
        and include the autologin mock script.
        """
        with self.settings(BROWSERID_AUTOLOGIN_ENABLED=True):
            output = helpers.browserid_js()
            self.assertHTMLEqual(output, """

            """)

class BrowserIDCSSTests(TestCase):
    def test_basic(self):
        output = helpers.browserid_css()
        self.assertHTMLEqual(output, """
            <link rel="stylesheet" href="static/browserid/persona-buttons.css" />
        """)

class BrowserIDButtonTests(TestCase):
    def test_basic(self):
        button = helpers.browserid_button(text='asdf', next='1234', link_class='fake-button',
                                          href="/test", attrs={'target': '_blank'})
        self.assertHTMLEqual(button, """

                asdf

        """)

    def test_json_attrs(self):
        button = helpers.browserid_button(text='qwer', next='5678', link_class='fake-button',
                                          attrs='{"target": "_blank"}')
        self.assertHTMLEqual(button, """

                qwer

        """)

class BrowserIDLoginTests(TestCase):
    def test_login_class(self):
        button = helpers.browserid_login(link_class='go button')
        self.assertHTMLEqual(button, """

                Sign in

        """)

    def test_default_class(self):
        """
        If no class is provided, it should default to
        'browserid-login persona-button'.
        """
        button = helpers.browserid_login()
        self.assertHTMLEqual(button, """

                Sign in

        """)

    def test_color_class(self):
        button = helpers.browserid_login(color='dark')
        self.assertHTMLEqual(button, """

                Sign in

        """)

    def test_color_custom_class(self):
        """
        If using a color and a custom link class, persona-button should
        be added to the link class.
        """
        button = helpers.browserid_login(link_class='go button', color='dark')
        self.assertHTMLEqual(button, """

                Sign in

        """)

    def test_next(self):
        button = helpers.browserid_login(next='/foo/bar')
        self.assertHTMLEqual(button, """

                Sign in

        """)

class BrowserIDLogoutTests(TestCase):
    def test_logout_class(self):
        button = helpers.browserid_logout(link_class='go button')
        self.assertHTMLEqual(button, """

                Sign out

        """)

    def test_next(self):
        button = helpers.browserid_logout(next='/foo/bar')
        self.assertHTMLEqual(button, """

                Sign out

        """)


# advance max and min function in python

#exmple one
name = ['Mostaifz','limon','Toufiq']
# find_max = max(name,key= lambda items: len(items))

# print(find_max)

#exmple two
students_one ={
    'Mostafiz' : {'score':90,'age':20},
    'Toufiq' : {'score': 80,'age':21}
}

# print(max(students_one,key=lambda items: students_one[items]['age'] ))

# Exmple three

students_two = [
    {'name': 'Mostafiz','score' :90,'age': 20},
    {'name': 'Toufiq','score' :100,'age': 22},
    {'name': 'Limon','score' :80,'age': 21},
]

print(max(students_two,key=lambda items: items.get('age')).get('name'))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Sep 17 22:11:59 2020

@author: abhijithneilabraham
"""

def isValid(s):
    vals={i:s.count(i) for i in s}
    c=list(vals.values())
    counts={i:c.count(i) for i in c}
    print(counts)
    if len(counts)==1:
        return "YES"
    if len(counts)==2:
        keys=list(counts.keys())
        values=list(counts.values())
        if abs(keys[0]-keys[1])==1 or 1 in keys:
            if min(values)==1:
                return "YES"
    return "NO"


# Inputs for parameters
stepSize = float(input("What do you want as the step size? "))
steps = int(input("How many steps do you want to take? "))
initialX = float(input("What is the initial x value? "))
initialY = float(input("What is the initial y value? "))

def differentialEquation(x, y): # Input your function here
  dydx = x ** 2
  # Change this function to your function

  return dydx

yNew = [initialY]
xNew = [initialX]

# Running Euler's Method
for i in range(steps):
   delta = differentialEquation(xNew[-1], yNew[-1]) * stepSize
   newVal = yNew[-1] + delta
   newX = xNew[-1] + stepSize
   xNew.append(newX)
   yNew.append(newVal)

# Prints final value of the function
print(yNew[-1])


import os
from app import db
import services.bibtex_format as bibtex_format

def add_article(user, ref_key, author, title, journal, year, volume):
    try:
        sql = ('INSERT INTO articles (user_id, ref_key, author, title, journal, year, volume)'
               'VALUES (:user, :ref_key, :author, :title, :journal, :year, :volume)')
        db.session.execute(
            sql, {'user': user, 'ref_key': ref_key, 'author': author,
                  'title': title, 'journal': journal, 'year': year, 'volume': volume})
        db.session.commit()
        return True
    except:
        return False

def add_book(user, ref_key, author, title, publisher, year):
    try:
        sql = ('INSERT INTO books (user_id, ref_key, author, title, publisher, year)'
               ' VALUES (:user, :ref_key, :author, :title, :publisher, :year)')
        db.session.execute(
            sql, {'user': user, 'ref_key': ref_key, 'author': author,
                  'title': title, 'publisher': publisher, 'year': year})
        db.session.commit()
        return True
    except:
        return False

def add_inproceedings(user, ref_key, author, title, booktitle, year):
    try:
        sql = ('INSERT INTO inproceedings (user_id, ref_key, author, title, booktitle, year) '
               'VALUES (:user, :ref_key, :author, :title, :booktitle, :year)')
        db.session.execute(
            sql, {'user': user, 'ref_key': ref_key, 'author': author,
                  'title': title, 'booktitle': booktitle, 'year': year})
        db.session.commit()
        return True
    except:
        return False

def add_masterthesis(user, ref_key, author, title, school, year):
    try:
        sql = ('INSERT INTO masterthesis (user_id, ref_key, author, title, school, year) '
               'VALUES (:user, :ref_key, :author, :title, :school, :year)')
        db.session.execute(
            sql, {'user': user, 'ref_key': ref_key, 'author': author,
                  'title': title, 'school': school, 'year': year})
        db.session.commit()
        return True
    except:
        return False

def get_articles(user_id, ref_key, search):
    try:
        sql = ('SELECT * FROM articles '
               'WHERE user_id=:user_id '
               'AND ref_key LIKE :ref_key '
               'AND (title LIKE :search '
               'OR ref_key LIKE :search '
               'OR author LIKE :search '
               'OR journal LIKE :search)')
        articles = db.session.execute(
            sql, {'user_id': user_id, 'ref_key': '%' + ref_key + '%',
                  'search': '%' + search + '%'}).fetchall()
        return articles
    except:
        return False

def get_books(user_id, ref_key, search):
    try:
        sql = ('SELECT * FROM books '
               'WHERE user_id=:user_id '
               'AND ref_key LIKE :ref_key '
               'AND (title LIKE :search '
               'OR ref_key LIKE :search '
               'OR author LIKE :search '
               'OR publisher LIKE :search)')
        books = db.session.execute(
            sql, {'user_id': user_id, 'ref_key': '%' + ref_key + '%',
                  'search': '%' + search + '%'}).fetchall()
        return books
    except:
        return False

def get_inproceedings(user_id, ref_key, search):
    try:
        sql = ('SELECT * FROM inproceedings '
               'WHERE user_id=:user_id '
               'AND ref_key LIKE :ref_key '
               'AND (title LIKE :search '
               'OR ref_key LIKE :search '
               'OR author LIKE :search '
               'OR booktitle LIKE :search)')
        inproceedings = db.session.execute(
            sql, {'user_id': user_id, 'ref_key': '%' + ref_key + '%',
                  'search': '%' + search + '%'}).fetchall()
        return inproceedings
    except:
        return False

def get_master_thesis(user_id, ref_key, search):
    try:
        sql = ('SELECT * FROM masterthesis '
               'WHERE user_id=:user_id '
               'AND ref_key LIKE :ref_key '
               'AND (title LIKE :search '
               'OR ref_key LIKE :search '
               'OR author LIKE :search '
               'OR school LIKE :search)')
        masterthesis = db.session.execute(
            sql, {'user_id': user_id, 'ref_key': '%' + ref_key + '%',
                  'search': '%' + search + '%'}).fetchall()
        return masterthesis
    except:
        return False

def check_refkey(user_id, refkey, type):
    try:
        sql = f'SELECT * FROM {type} WHERE ref_key=:refkey AND user_id=:user_id'
        reference = db.session.execute(
            sql, {'refkey': refkey, 'user_id': user_id}).fetchall()
        return reference
    except:
        return False

def find_refkey(user_id, refkey):
    if check_refkey(user_id, refkey, 'books'):
        return True
    if check_refkey(user_id, refkey, 'articles'):
        return True
    if check_refkey(user_id, refkey, 'masterthesis'):
        return True
    if check_refkey(user_id, refkey, 'inproceedings'):
        return True
    return False

def delete_all():
    sql = 'TRUNCATE TABLE articles CASCADE'
    db.session.execute(sql)
    sql = 'TRUNCATE TABLE books CASCADE'
    db.session.execute(sql)
    sql = 'TRUNCATE TABLE inproceedings CASCADE'
    db.session.execute(sql)
    sql = 'TRUNCATE TABLE masterthesis CASCADE'
    db.session.execute(sql)
    db.session.commit()

def get_bibtex_forms(user_id, ref_keys):
    bibtex_list = []

    for reference in ref_keys:
        articles = get_articles(user_id, reference, '')
        for article in articles:
            bibtex_list.append(bibtex_format.article_to_bibtex(article))

        books = get_books(user_id, reference, '')
        for book in books:
            bibtex_list.append(bibtex_format.book_to_bibtex(book))

        inproceedings = get_inproceedings(user_id, reference, '')
        for inproceeding in inproceedings:
            bibtex_list.append(
                bibtex_format.inproceedings_to_bibtex(inproceeding))

        masterthesis = get_master_thesis(user_id, reference, '')
        for thesis in masterthesis:
            bibtex_list.append(bibtex_format.masterthesis_to_bibtex(thesis))

    return bibtex_list

def add_references_to_file(user_id, ref_keys):
    try:
        if os.path.exists('src/bibtex.bib'):
            os.remove('src/bibtex.bib')
        file = open('src/bibtex.bib', 'a')
        bibtex_form = get_bibtex_forms(user_id, ref_keys)
        for i in bibtex_form:
            for row in i:
                file.write(row)
                file.write("\n")
        return True
    except:
        return False


from tkinter import *

root = Tk()
root.title("Conversion Program")
root.iconbitmap("wizard.jpg")
root.configure(background="#D6EAF8")

# Define Frame 1

lengthframe1 = LabelFrame(root, text="Converting Length", padx=5, pady=5, font=("Futura Bold", "15"), bg="#FCF3CF")
lengthframe1.grid(row=0, column=0, padx=10, pady=10)

# Define Frame 2
weightframe2 = LabelFrame(root, text="Converting Weight", padx=5, pady=5, font=("Futura Bold", "15"), bg="#ABEBC6")
weightframe2.grid(row=2, column=0, padx=10, pady=10)

# Define Result Labels
lengthresultlbl = Label(lengthframe1)
weightresultlbl = Label(weightframe2)

# Define Lenth Function

def length():
    global lengthresultlbl
    lengthresultlbl.destroy()
    lengthchoice = lengthclicked.get()
    if lengthchoice == "inches to cm":
        lengthnum1 = float(lengthnum.get())
        lengthresult = round(lengthnum1 * 2.54, 2)
    elif lengthchoice == "feet to cm":
        lengthnum1 = float(lengthnum.get())
        lengthresult = round(lengthnum1 * 30.48, 2)
    elif lengthchoice == "miles to km":
        lengthnum1 = float(lengthnum.get())
        lengthresult = round(lengthnum1 * 1.60934, 2)
    else:
        lengthresult = "Error"

    lengthresultlbl = Label(lengthframe1, text=lengthresult)
    lengthresultlbl.grid(row=4, column=1, sticky="W")

# Define Weight Function

def weight():
    global weightresultlbl
    weightresultlbl.destroy()
    weightchoice = weightclicked.get()
    if weightchoice == "ounces to grams":
        weightnum1 = float(weightnum.get())
        weightresult = round(weightnum1 * 28.35, 2)
    elif weightchoice == "pounds to grams":
        weightnum1 = float(weightnum.get())
        weightresult = round(weightnum1 * 454.592, 2)
    elif weightchoice == "pounds to kilograms":
        weightnum1 = float(weightnum.get())
        weightresult = round(weightnum1 / 2.205, 2)
    elif weightchoice == "stones to kilograms":
        weightnum1 = float(weightnum.get())
        weightresult = round(weightnum1 * 6.35, 2)
    else:
        weightresult = "Error"

    weightresultlbl = Label(weightframe2, text=weightresult)
    weightresultlbl.grid(row=4, column=1, sticky="W")

# Length Menu Items
lengthoption = [
    "inches to cm",
    "feet to cm",
    "miles to km",
]
lengthclicked = StringVar()
lengthclicked.set(lengthoption[0])

lengthdrop = OptionMenu(lengthframe1, lengthclicked, *lengthoption)
lengthdrop.grid(row=0, columnspan=2)
lengthdrop.config(font=("Arial Black", "15"), bg="yellow")

# Length User Input box
lengthnum = Entry(lengthframe1)
lengthnum.grid(row=1, column=1)

lengthuslbl = Label(lengthframe1, text="Enter number to convert")
lengthuslbl.grid(row=1, column=0, padx=10, pady=10)

lengthconvbtn = Button(lengthframe1, text="Convert Length", command=length)
lengthconvbtn.grid(row=3, column=1)

lengthanswer = Label(lengthframe1, text="The result is....")
lengthanswer.grid(row=4, column=0)

# Weight Menu Items
weightoption = [
    "ounces to grams",
    "pounds to grams",
    "pounds to kilograms",
    "stones to kilograms"
]
weightclicked = StringVar()
weightclicked.set(weightoption[0])

weightdrop = OptionMenu(weightframe2, weightclicked, *weightoption)
weightdrop.grid(row=0, columnspan=2)
weightdrop.config(font=("Arial Black", "15"), bg="green")

# Weight User Input box
weightnum = Entry(weightframe2)
weightnum.grid(row=1, column=1)

weightuslbl = Label(weightframe2, text="Enter number to convert")
weightuslbl.grid(row=1, column=0, padx=10, pady=10)

weightconvbtn = Button(weightframe2, text="Convert Weight", command=weight)
weightconvbtn.grid(row=3, column=1)

weightanswer = Label(weightframe2, text="The result is....")
weightanswer.grid(row=4, column=0)

root.mainloop()


from Crypto.PublicKey import ECC
from Crypto.Signature import DSS
from Crypto.Hash import SHA256

def load_private_key(private_key_path):
    with open(private_key_path, 'rt') as f:
        private_key = ECC.import_key(f.read())
    return private_key

def generate_ecdsa_signature(private_key, file_path):
    with open(file_path, 'rb') as f:
        data = f.read()
    hash_obj = SHA256.new(data)
    signer = DSS.new(private_key, 'fips-186-3')
    signature = signer.sign(hash_obj)
    return signature

# Load the private key
private_key = load_private_key('keys/ecc_private_key.pem')

# Generate the ECDSA signature
digital_signature = generate_ecdsa_signature(private_key, 'text.txt')

# Save the digital signature
def save_signature(signature, file_path):
    with open(file_path, 'wb') as f:
        f.write(signature)

save_signature(digital_signature, 'keys/digital_signature.bin')

print('Digital Signature saved to keys/digital_signature.bin')

print('Digital Signature:', digital_signature)


# -*- coding: utf-8 -*-
"""
Created on Fri Mar  5 00:49:27 2021

@author: Tarandeep
"""

import PySimpleGUI as sg
import json

# create layout

def gui_method():

    bold_method = ('Arial', 12, 'bold')

    layout_gui = [
        [
            sg.Text(
                "This form requires files listed below to calculate model that returns the lowest distance.", font=bold_method)
        ],
        [
            sg.Text("File locations left blank will use default values.")
        ],
        [
            sg.Text("Select Excel Layout", size=(30, 1)),
            sg.In(size=(25, 1), enable_events=True, key="layout"),
            sg.FileBrowse(),
        ],
        [
            sg.Text("Select Specifications File", size=(30, 1)),
            sg.In(size=(25, 1), enable_events=True, key="specs"),
            sg.FileBrowse(),
        ],
        [
            sg.Text("Select Sales Data File", size=(30, 1)),
            sg.In(size=(25, 1), enable_events=True, key="salesData"),
            sg.FileBrowse(),
        ],
        [
            sg.Text("Select Store Orders", size=(30, 1)),
            sg.In(size=(25, 1), enable_events=True, key="storeOrder"),
            sg.FileBrowse(),
        ],
        [
            sg.Text("Select Order Lines Location", size=(30, 1)),
            sg.In(size=(25, 1), enable_events=True, key="orderLinesLocation"),
            sg.FolderBrowse(),
        ],
        [
            sg.Text("Export File Location", size=(30, 1)),
            sg.In(size=(25, 1), enable_events=True, key="exportLocation"),
            sg.FolderBrowse(),
        ],
        [
            sg.Text("What range of weeks would you like to look at?",
                    font=bold_method)
        ],
        [
            sg.Text("Starting Week", size=(30, 1)),
            sg.InputText(size=(5, 1), key="startWeek")
        ],
        [
            sg.Text("Ending Week", size=(30, 1)),
            sg.InputText(size=(5, 1), key="endWeek")
        ],
        [
            sg.Text("Select store order date to analyze:", font=bold_method)
        ],
        [
            sg.Text("Month", size=(10, 1)),
            sg.InputText(size=(5, 1), key="salesMonth"),
            sg.Text("Day", size=(10, 1)),
            sg.InputText(size=(5, 1), key="salesDay"),
            sg.Text("Year", size=(10, 1)),
            sg.InputText(size=(5, 1), key="salesYear")
        ],
        [
            sg.Checkbox("Would you like to compute Space Allocation Model? If not, previous results will be used.",
                        default=True, key="spaceAllocate"),
        ],
        [
            sg.Text("Space Allocation Parameters:", font=bold_method)
        ],
        [
            sg.Text("Max Spaces per SKU", size=(30, 1)),
            sg.InputText(size=(5, 1), key="maxSpaces")
        ],
        [
            sg.Text(
                "Aisle Declaration (ex. 1st row, 22nd row, 50th row) ", font=bold_method)
        ],
        [
            sg.Text("Top Aisle", size=(10, 1)),
            sg.InputText(size=(5, 1), key="aisleTop"),
            sg.Text("Cross Aisle", size=(10, 1)),
            sg.InputText(size=(5, 1), key="aisleMiddle"),
            sg.Text("Bottom Aisle", size=(10, 1)),
            sg.InputText(size=(5, 1), key="aisleBottom")
        ],
        [
            sg.Text("Determine SKU List to Calculate", font=bold_method)
        ],
        [
            sg.Checkbox("Random", default=True, key="random"),
            sg.Checkbox("COI", default=True, key="coi"),
            sg.Checkbox("Weight", default=True, key="weight"),
            sg.Checkbox("ABC (across)", default=True, key="across"),
            sg.Checkbox("ABC (vertical)", default=True, key="vertical")
        ],
        [
            sg.Submit(), sg.Cancel()
        ]
    ]

    window = sg.Window("Metro SKU Assignment Tool").Layout(layout_gui)
    while True:
        event, values = window.Read()  # Run the window until an "event" is triggered
        if event == "Submit":
            window.close()
            return values
        elif event is None or event == "Cancel":
            window.close()
            return None


import argparse
import cv2
import numpy as np
import random

def addConstant(image_file_name='data/test.png', length=50, num=5):
    image = cv2.imread(image_file_name, 1)
    modifiedImage = np.empty(
        [len(image),  len(image[0]), 3], dtype=np.uint8)
    for i in range(len(image)):
        for j in range(len(image[0])):
            for k in range(3):
                modifiedImage[i][j][k] = image[i][j][k]
    for i in range(num):
        startX = random.randint(0, len(image) - length)
        startY = random.randint(0, len(image) - length)
        addConstantOnce(modifiedImage, length, startX, startY)
    cv2.imwrite("saved/constant.png", (modifiedImage))

def addConstantOnce(modifiedImage, length, startX, startY):
    for i in range(startX, startX + length):
        for j in range(startY, startY + length):
            for k in range(3):
                modifiedImage[i][j][k] = 0

    return modifiedImage

addConstant(num=20)


import youtube_dl

print("Colocar o link de uma playlist ou o nome de uma musica do youtube")
url = input()

ydl_opts = {
        'format' : 'bestaudio/best',
        'keepvideo': False,
        'postprocessors': [{
            'key': 'FFmpegExtractAudio',
            'preferredcodec': 'mp3',
            'preferredquality': '192',
        }]
}
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
    ydl.download([url])


# -*- coding: utf-8 -*-

from odoo import api, fields, models
from datetime import datetime, timedelta
from PyPDF2 import PdfFileWriter, PdfFileReader
import base64, io
import logging
_logger= logging.getLogger(__name__)

class eCommerceShop(models.Model):
    _inherit = 'ecommerce.shop'

#    def order_tracking_no_push(self, ordersn, tracking_no):
#        self.env['sale.order'].search([('ecommerce_shop_id','=',self.id),('client_order_ref','=',ordersn)])[:1].picking_ids.write({
#            'carrier_tracking_ref': tracking_no,
#            })
#        return True

    def _get_orders_detail_lazada(self, lazada_orders):
        zip_orders = super(eCommerceShop, self)._get_orders_detail_lazada(lazada_orders)
        to_update_pairs = [(d, d['order_items'][0]['order_item_id']) for o, d in zip_orders if d['order_items'][0].get('tracking_code')]
        if not to_update_pairs:
            return zip_orders
        orders_detail, order_item_ids = zip(*to_update_pairs)
        resp = self._py_client_lazada_request('/order/document/get', 'GET', doc_type='shippingLabel', order_item_ids=str(list(order_item_ids)))
        if not resp.get('data'):
            return zip_orders
        raw = resp['data']['document']['file']
        Report = self.env['ir.actions.report']
        stream = io.BytesIO(Report._run_wkhtmltopdf([base64.b64decode(raw)]))
        streams = [stream]
        pdf = PdfFileReader(stream)
        for page in range(pdf.getNumPages()):
            writer = PdfFileWriter()
            writer.addPage(pdf.getPage(page))
            res_stream = io.BytesIO()
            streams.append(res_stream)
            writer.write(res_stream)
            orders_detail[page]['order_items'][0]['shippingLabel'] = base64.encodestring(res_stream.getvalue())
        for stream in streams:
            try:
                stream.close()
            except Exception:
                pass
        return zip_orders

    def _create_order_lazada(self, order, detail=False):
        detail = detail or self._py_client_lazada_request('/order/items/get','GET', order_id = order['order_id']).get('data')
        sale_order = super(eCommerceShop, self)._create_order_lazada(order, detail=detail)
        for line in sale_order.order_line:
            if order.get('warehouse_code') == 'dropshipping':
                line.route_id = self.env.ref('connector_lazada_stock.stock_location_route_lazada_transit')
            elif order.get('warehouse_code','').startswith('OMS-LAZADA'):
                line.route_id = self.env.ref('connector_lazada_stock.stock_location_route_lazada_fbl')
        return sale_order

    def _update_order_lazada(self, order, statuses=[], detail=False):
        report = self.env['ir.actions.report']._get_report_from_name('stock.report_deliveryslip')
        detail = detail or self._py_client_lazada_request('/order/items/get','GET', order_id = order.client_order_ref).get('data')
        order = super(eCommerceShop, self)._update_order_lazada(order, statuses=statuses, detail=detail)
        for status in statuses:
            if status in ['pending','ready_to_ship','shipped']:
                order.invoice_shipping_on_delivery = False
                #tracking code generated mean delivery info is ready
                if not detail[0].get('tracking_code'):
                    continue
                if not order.carrier_id:
                    order.carrier_id = self.env['ecommerce.carrier'].search([
                        ('name','=', detail[0].get('shipment_provider','').split('Delivery: ')[-1])
                    ])[:1].carrier_id
                    picks = order.picking_ids.filtered(lambda r: r.state not in ['done', 'cancel'])
                    picks.write({
                        'carrier_id': order.carrier_id.id,
                        'carrier_tracking_ref': detail[0].get('tracking_code','')
                    })
                start_picking = order.picking_ids.filtered(lambda r: r.state not in ['done', 'cancel'] and r.picking_type_id == order.warehouse_id.out_type_id)[:1]
                if not start_picking or start_picking.ecomm_delivery_slip_loaded:
                    continue
                report.render(start_picking.ids)
                attachment = report.retrieve_attachment(start_picking)
                if attachment:
                    streams = [io.BytesIO(base64.decodestring(attachment.datas)), io.BytesIO(base64.decodestring(detail[0]['shippingLabel']))]
                    writer = PdfFileWriter()
                    for stream in streams:
                        writer.appendPagesFromReader(PdfFileReader(stream))
                    if writer.getNumPages()%2 == 1:
                        writer.addBlankPage()
                    res_stream = io.BytesIO()
                    streams.append(res_stream)
                    writer.write(res_stream)
                    try:
                        attachment.write({
                            'datas': base64.encodestring(res_stream.getvalue()),
                            'datas_fname': attachment.name
                            })
                    except exceptions.AccessError:
                        _logger.info("Cannot save PDF report")
                    finally:
                        start_picking.ecomm_delivery_slip_loaded = True
                        for stream in streams:
                            try:
                                stream.close()
                            except Exception:
                                pass

            elif status in ['returned', 'failed', 'cancel']:
                pick_ids = order.picking_ids.filtered(lambda r: r.picking_type_id == self.env.ref('connector_lazada_stock.stock_picking_type_lazada_out')) + order.picking_ids.filtered(lambda r: r.picking_type_id == order.warehouse_id.out_type_id)
                for pick_id in pick_ids:
                    if pick_id.state not in ['done','cancel']: pick_id.action_cancel()
                    elif pick_id.state == 'done':
                        returns = order.picking_ids.filtered(lambda r: r.location_id == pick_id.location_dest_id and r.location_dest_id == pick_id.location_id and r.state != 'cancel')
                        if not returns:
                            wiz_model = self.env['stock.return.picking']
                            wiz = wiz_model.create(wiz_model.with_context(active_id = pick_id.id).default_get(wiz_model._fields.keys()))
                            wiz.product_return_moves.write({'to_refund': True})
                            new_picking_id, pick_type_id = wiz._create_returns()
                            return_pick = self.env['stock.picking'].browse(new_picking_id)
                            return_pick.write({
                                'carrier_id': pick_id.carrier_id.id,
                                'carrier_tracking_ref': pick_id.carrier_tracking_ref
                                })
                            for moves in zip(pick_id.move_lines, return_pick.move_lines):
                                l = len(moves[1].move_line_ids)
                                moves[1].write({
                                    'move_line_ids': [(1, moves[1].move_line_ids[i].id, {
                                        'lot_id': v.lot_id and v.lot_id.id
                                    }) for i,v in enumerate(moves[0].move_line_ids[:l])] + [(0, _, {
                                        'product_uom_id': moves[1].product_uom.id,
                                        'picking_id': moves[1].picking_id.id,
                                        'move_id': moves[1].id,
                                        'product_id': moves[1].product_id.id,
                                        'location_id': moves[1].location_id.id,
                                        'location_dest_id': moves[1].location_dest_id.id,
                                        'lot_id': v.lot_id and v.lot_id.id
                                    }) for i,v in enumerate(moves[0].move_line_ids[l:])]
                                })
                for line in order.order_line:
                    line.product_uom_qty = line.qty_delivered

            elif status == 'delivered':
                pick_ids = order.picking_ids.filtered(lambda r: r.state not in ['done', 'cancel'] and r.picking_type_id in [self.env.ref('connector_lazada_stock.stock_picking_type_lazada_out'), self.env.ref('connector_lazada_stock.stock_picking_type_lazada_in')])
                for pick_id in pick_ids:
                    for line in pick_id.move_line_ids:
                        if line.state not in ['done', 'cancel']: line.qty_done = line.product_uom_qty
                    self.env['stock.immediate.transfer'].create({'pick_ids': [(4, pick_id.id)]}).process()
        return order

    def _get_logistic_lazada(self):
        self.ensure_one()
        logistics = self._py_client_lazada_request('/shipment/providers/get','GET').get('data',{}).get('shipment_providers')
        for l in logistics:
            carrier = self.env['ecommerce.shop.carrier'].search([
                ('shop_id', '=', self.id),
                ('name','=',l.get('name')),
            ])
            if carrier:
                carrier[0].write({
                    'enable': l.get('enabled'),
                    'default': l.get('preferred'),
                    'cod': l.get('has_cod'),
                })
            else:
                self.env['ecommerce.shop.carrier'].create({
                    'ecomm_carrier_id': self.env['ecommerce.carrier'].search([
                        ('platform_id','=', self.platform_id.id),
                        ('name','=',l.get('name')),
                    ])[:1].id or self.env['ecommerce.carrier'].create({
                        'name': l.get('name'),
                        'platform_id': self.platform_id.id,
                        'carrier_id' : self.env['delivery.carrier'].create({
                            'name': l.get('name'),
                            'product_id': self.env.ref('delivery.product_product_delivery').id
                        }).id
                    }).id,
                    'shop_id': self.id,
                    'default': l.get('preferred'),
                    'cod': l.get('has_cod'),
                })


class Solution:
    def maxProfit(self, prices: List[int]) -> int:
        day1 = prices[0]
        profit = 0
        for i in range(1,len(prices)):
            if prices[i]>day1:
                profit = max(profit,prices[i]-day1)
            day1 = min(day1,prices[i])

        return profit

121 : Best Time to Buy and Sell Stock.py