#!/usr/bin/python
# -*- coding: utf-8 -*-

import pandas as pd

import numpy as np

from nltk.tokenize import word_tokenize

from utils import config_reader

from collections import Counter

from sklearn.model_selection import train_test_split

from keras.preprocessing.sequence import pad_sequences

import re

class Dataset(object):
    """
    Dataset class.

    Reads train, test, and validation data from zip or CSV and
    performs preprocessing.
    """

    def __init__(self, shuffle=True, seed=42, config=None):
        """Initialize dataset."""
        self.df = None
        self.x_train = []
        self.x_test = []
        self.y_train = []
        self.y_test = []
        self.x_val = []
        self.y_val = []

        self.header = []
        self.possible_labels = []
        self.data = []
        self.avrg_comment_len = 0

        self.shuffle = shuffle
        self.seed = seed

        self.pattern = re.compile(r"(.)\1{2,}")
        self.insulting_words = self.get_insulting_words()

        self.vocab = set()
        self.word2id = {}
        self.id2word = {}

        self.config = config_reader.read_json(config)

        self.filter_symbols = self.config["filter_symbols"]
        self.filter_repeated_chars = self.config["filter_repeated_chars"]
        self.filter_insulting_words = self.config["filter_insulting_words"]
        self.vocab_size = self.config["vocab_size"]
        self.max_seq_len = self.config["max_seq_len"]

    def get_insulting_words(self):
        """Collect insulting words in set."""
        hate_words = set()
        with open("data/hate_speech_lexicon.txt",
                  encoding="utf-8", mode="r") as hate_lex:
            for line in hate_lex.readlines():
                hate_words.add(line.strip().lower())

        return hate_words

    def prepare_data(self, path_to_data):
        """Load raw data and perform preprocessing."""
        # load data from zip file
        raw_train_data = self.load_data(path_to_data)

        # collect all possible labels
        self.get_possible_labels()

        # tokenize the data and collect the labels
        tokenized, labels = self.tokenize(raw_train_data)

        # split data
        x_train, y_train, x_test, y_test, x_val, y_val = self.split_data(
            tokenized,
            labels)

        # create vocabulary only for train data
        self.create_vocabulary(x_train)

        # preprocess all data splits
        self.x_train = self.preprocess_data(x_train)
        self.x_test = self.preprocess_data(x_test)
        self.x_val = self.preprocess_data(x_val)

        self.y_train = y_train
        self.y_test = y_test
        self.y_val = y_val

    def split_data(self, tokenized, labels):
        """Split and shuffle the data."""
        assert (self.config["data_split"]["test"] * 2 +
                self.config["data_split"]["train"]) == 1.0

        x_train, x_test, y_train, y_test = train_test_split(
            tokenized,
            labels,
            test_size=self.config["data_split"]["test"] * 2,
            shuffle=self.shuffle,
            random_state=self.seed)

        x_test, x_val, y_test, y_val = train_test_split(
            x_test,
            y_test,
            test_size=0.5,
            shuffle=self.shuffle,
            random_state=self.seed)

        return x_train, y_train, x_test, y_test, x_val, y_val

    def get_possible_labels(self):
        """Get list of possible labels."""
        for label in self.header:
            if label not in ("id", "comment_text"):
                self.possible_labels.append(label)

    def load_data(self, path_to_data):
        """Read in the data."""
        raw_data = []

        try:
            self.df = pd.read_csv(path_to_data, compression='zip', header=0,
                                  sep=',', encoding="utf-8")

            self.header = list(self.df)

            for index, row in self.df.iterrows():
                instance = {}

                for key, value in row.iteritems():
                    try:
                        instance[key] = int(value)

                    except ValueError:
                        instance[key] = value.strip()

                raw_data.append(instance)

        except FileNotFoundError:
            print("\nFile '{}' does not exist. Ignore data.\n".format(
                  path_to_data))

        # restrict dataset for debugging:
        if self.config["dataset_max_for_debug"]:
            raw_data = raw_data[:self.config["dataset_max_for_debug"]]

        return raw_data

    def tokenize(self, data):
        """Tokenize the data."""
        print("Tokenizing data ...")
        tokenized = []
        labels = []
        num_tokens = []

        symbol_list = ['``', "''", '#', "[", "]", "(", ")", '·']

        for instance in data:
            # for better tokenization
            comment = instance["comment_text"]
            if "/" in comment:
                comment = comment.replace("/", " / ")

            tokens = word_tokenize(comment.lower())

            # filter certain symbols
            if self.filter_symbols:
                tokens = [word for word in tokens if word not in symbol_list]

            num_tokens.append(len(tokens))

            tokenized.append(tokens)

            inst_labels = [instance[label] for label in self.possible_labels]
            labels.append(inst_labels)

        self.avrg_comment_len = np.mean(num_tokens)
        self.longest_comment_len = np.max(num_tokens)
        self.shortest_comment_len = np.min(num_tokens)

        return tokenized, np.array(labels)

    def create_vocabulary(self, tokenized):
        """Add most common words to vocabulary."""
        word_freq = Counter(word.lower() for sentence in tokenized for
                            word in sentence)

        self.vocab = set([word for word, freq in word_freq.most_common(
                          self.vocab_size - 5)])

        # start at index 1 (because of padding)
        for i, word in enumerate(self.vocab):
            self.id2word[i + 1] = word
            self.word2id[word] = i + 1

        # placeholder for padding
        self.id2word[0] = "@@@"
        self.word2id["@@@"] = 0
        self.vocab.add("@@@")

        # add UNKNOWN to vocabulary
        self.id2word[i + 2] = "UNKNOWN"
        self.word2id["UNKNOWN"] = i + 2
        self.vocab.add("UNKNOWN")

        # add punctuation replacement to vocabulary
        self.id2word[i + 3] = "PUNCT"
        self.word2id["PUNCT"] = i + 3
        self.vocab.add("PUNCT")

        # add XXX to vocabulary
        self.id2word[i + 4] = "XXX"
        self.word2id["XXX"] = i + 4
        self.vocab.add("XXX")

        # add INSULT to vocabulary
        self.id2word[i + 5] = "INSULT"
        self.word2id["INSULT"] = i + 5
        self.vocab.add("INSULT")

    def preprocess_data(self, tokenized_data):
        """Pre-process data.

        For each element in the given list, a preprocessing task is
        executed.
        """
        print("\nPreprocessing data ...")
        data = []
        instance_count = 0

        for i, instance in enumerate(tokenized_data):
            instance_count += 1

            # replace special occurrences like repeated punctuation
            instance = self.replace_special_occurrences(instance)

            # convert the list of words to a list of indices
            data.append(self.convert_words_to_ids(instance))

            if i % 10000 == 0:
                print("{}/{} done.\n".format(instance_count,
                                             len(tokenized_data)))

        # zero-pad sequences to same length
        data = pad_sequences(data, maxlen=self.max_seq_len, padding="post")

        return data

    def replace_special_occurrences(self, instance):
        """Replace infrequent words."""
        for i, word in enumerate(instance):

            if word in ("!", "?", ".", "-", ","):
                instance[i] = "PUNCT"

            if word not in self.vocab:
                instance[i] = "UNKNOWN"

            # replace words with repeated characters
            # (from 3 repetitions on)
            if self.filter_repeated_chars:
                if re.findall(self.pattern, word):
                    instance[i] = "XXX"

            if self.filter_insulting_words:
                if word in self.insulting_words:
                    instance[i] = "INSULT"

        return instance

    def convert_words_to_ids(self, list_of_words):
        """Replace words by their ids."""
        return [self.word2id[word] for word in list_of_words]

    def convert_ids_to_words(self, list_of_ids):
        """Replace words by their ids."""
        return [self.id2word[idx] for idx in list_of_ids]

if __name__ == "__main__":

    path_to_data = "data/train.csv.zip"
    config_file = "configs/config_test.json"

    dataset = Dataset(shuffle=True,
                      seed=42,
                      config=config_file)

    dataset.prepare_data(path_to_data)


#!/usr/bin/env python
#
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Provisions Android devices with settings required for bots.

Usage:
  ./provision_devices.py [-d <device serial number>]
"""

import argparse
import datetime
import json
import logging
import os
import posixpath
import re
import sys
import time

# Import _strptime before threaded code. datetime.datetime.strptime is
# threadsafe except for the initial import of the _strptime module.
# See crbug.com/584730 and https://bugs.python.org/issue7980.
import _strptime  # pylint: disable=unused-import

if __name__ == '__main__':
  sys.path.append(
      os.path.abspath(
          os.path.join(os.path.dirname(__file__), '..', '..', '..')))

from devil.android import battery_utils
from devil.android import device_denylist
from devil.android import device_errors
from devil.android import device_temp_file
from devil.android import device_utils
from devil.android import settings
from devil.android.sdk import adb_wrapper
from devil.android.sdk import intent
from devil.android.sdk import keyevent
from devil.android.sdk import shared_prefs
from devil.android.sdk import version_codes
from devil.android.tools import script_common
from devil.constants import exit_codes
from devil.utils import logging_common
from devil.utils import timeout_retry

logger = logging.getLogger(__name__)

_SYSTEM_APP_DIRECTORIES = ['/system/app/', '/system/priv-app/']
_SYSTEM_WEBVIEW_NAMES = ['webview', 'WebViewGoogle']
_CHROME_PACKAGE_REGEX = re.compile('.*chrom.*')
_TOMBSTONE_REGEX = re.compile('tombstone.*')
_STANDALONE_VR_DEVICES = [
    'vega',  # Lenovo Mirage Solo
]

class _DEFAULT_TIMEOUTS(object):
  # L can take a while to reboot after a wipe.
  LOLLIPOP = 600
  PRE_LOLLIPOP = 180

  HELP_TEXT = '{}s on L, {}s on pre-L'.format(LOLLIPOP, PRE_LOLLIPOP)

class ProvisionStep(object):
  def __init__(self, cmd, reboot=False):
    self.cmd = cmd
    self.reboot = reboot

def ProvisionDevices(devices,
                     denylist_file,
                     adb_key_files=None,
                     disable_location=False,
                     disable_mock_location=False,
                     disable_network=False,
                     disable_system_chrome=False,
                     emulators=False,
                     enable_java_debug=False,
                     max_battery_temp=None,
                     min_battery_level=None,
                     output_device_denylist=None,
                     reboot_timeout=None,
                     remove_system_webview=False,
                     system_app_remove_list=None,
                     system_package_remove_list=None,
                     wipe=True):
  denylist = (device_denylist.Denylist(denylist_file)
              if denylist_file else None)
  system_app_remove_list = system_app_remove_list or []
  system_package_remove_list = system_package_remove_list or []
  try:
    devices = script_common.GetDevices(devices, denylist)
  except device_errors.NoDevicesError:
    logging.error('No available devices to provision.')
    if denylist:
      logging.error('Local device denylist: %s', denylist.Read())
    raise
  devices = [d for d in devices if not emulators or d.adb.is_emulator]
  parallel_devices = device_utils.DeviceUtils.parallel(devices)

  steps = []
  if wipe:
    steps += [ProvisionStep(lambda d: Wipe(d, adb_key_files), reboot=True)]
  steps += [
      ProvisionStep(
          lambda d: SetProperties(d, enable_java_debug, disable_location,
                                  disable_mock_location),
          reboot=not emulators)
  ]

  if disable_network:
    steps.append(ProvisionStep(DisableNetwork))

  if disable_system_chrome:
    steps.append(ProvisionStep(DisableSystemChrome))

  if max_battery_temp:
    steps.append(
        ProvisionStep(lambda d: WaitForBatteryTemperature(d, max_battery_temp)))

  if min_battery_level:
    steps.append(ProvisionStep(lambda d: WaitForCharge(d, min_battery_level)))

  if remove_system_webview:
    system_app_remove_list.extend(_SYSTEM_WEBVIEW_NAMES)

  if system_app_remove_list or system_package_remove_list:
    steps.append(
        ProvisionStep(lambda d: RemoveSystemApps(d, system_app_remove_list,
                                                 system_package_remove_list)))

  steps.append(ProvisionStep(SetDate))
  steps.append(ProvisionStep(CheckExternalStorage))
  steps.append(ProvisionStep(StandaloneVrDeviceSetup))

  parallel_devices.pMap(ProvisionDevice, steps, denylist, reboot_timeout)

  denylisted_devices = denylist.Read() if denylist else []
  if output_device_denylist:
    with open(output_device_denylist, 'w') as f:
      json.dump(denylisted_devices, f)
  if all(d in denylisted_devices for d in devices):
    raise device_errors.NoDevicesError
  return 0

def ProvisionDevice(device, steps, denylist, reboot_timeout=None):
  try:
    if not reboot_timeout:
      if device.build_version_sdk >= version_codes.LOLLIPOP:
        reboot_timeout = _DEFAULT_TIMEOUTS.LOLLIPOP
      else:
        reboot_timeout = _DEFAULT_TIMEOUTS.PRE_LOLLIPOP

    for step in steps:
      try:
        device.WaitUntilFullyBooted(timeout=reboot_timeout, retries=0)
      except device_errors.CommandTimeoutError:
        logger.error('Device did not finish booting. Will try to reboot.')
        device.Reboot(timeout=reboot_timeout)
      step.cmd(device)
      if step.reboot:
        device.Reboot(False, retries=0)
        device.adb.WaitForDevice()

  except device_errors.CommandTimeoutError:
    logger.exception('Timed out waiting for device %s. Adding to denylist.',
                     str(device))
    if denylist:
      denylist.Extend([str(device)], reason='provision_timeout')

  except (device_errors.CommandFailedError,
          device_errors.DeviceUnreachableError):
    logger.exception('Failed to provision device %s. Adding to denylist.',
                     str(device))
    if denylist:
      denylist.Extend([str(device)], reason='provision_failure')

def Wipe(device, adb_key_files=None):
  if (device.IsUserBuild()
      or device.build_version_sdk >= version_codes.MARSHMALLOW):
    WipeChromeData(device)

    package = 'com.google.android.gms'
    version_name = device.GetApplicationVersion(package)
    if version_name:
      logger.info('Version name for %s is %s', package, version_name)
    else:
      logger.info('Package %s is not installed', package)
  else:
    WipeDevice(device, adb_key_files)

def WipeChromeData(device):
  """Wipes chrome specific data from device

  (1) uninstall any app whose name matches *chrom*, except
      com.android.chrome, which is the chrome stable package. Doing so also
      removes the corresponding dirs under /data/data/ and /data/app/
  (2) remove any dir under /data/app-lib/ whose name matches *chrom*
  (3) remove any files under /data/tombstones/ whose name matches "tombstone*"
  (4) remove /data/local.prop if there is any
  (5) remove /data/local/chrome-command-line if there is any
  (6) remove anything under /data/local/.config/ if the dir exists
      (this is telemetry related)
  (7) remove anything under /data/local/tmp/

  Arguments:
    device: the device to wipe
  """
  try:
    if device.IsUserBuild():
      _UninstallIfMatch(device, _CHROME_PACKAGE_REGEX)
      device.RunShellCommand(
          'rm -rf %s/*' % device.GetExternalStoragePath(),
          shell=True,
          check_return=True)
      device.RunShellCommand(
          'rm -rf /data/local/tmp/*', shell=True, check_return=True)
    else:
      device.EnableRoot()
      _UninstallIfMatch(device, _CHROME_PACKAGE_REGEX)
      _WipeUnderDirIfMatch(device, '/data/app-lib/', _CHROME_PACKAGE_REGEX)
      _WipeUnderDirIfMatch(device, '/data/tombstones/', _TOMBSTONE_REGEX)

      _WipeFileOrDir(device, '/data/local.prop')
      _WipeFileOrDir(device, '/data/local/chrome-command-line')
      _WipeFileOrDir(device, '/data/local/.config/')
      _WipeFileOrDir(device, '/data/local/tmp/')
      device.RunShellCommand(
          'rm -rf %s/*' % device.GetExternalStoragePath(),
          shell=True,
          check_return=True)
  except device_errors.CommandFailedError:
    logger.exception('Possible failure while wiping the device. '
                     'Attempting to continue.')

def _UninstallIfMatch(device, pattern):
  installed_packages = device.RunShellCommand(['pm', 'list', 'packages'],
                                              check_return=True)
  installed_system_packages = [
      pkg.split(':')[1]
      for pkg in device.RunShellCommand(['pm', 'list', 'packages', '-s'],
                                        check_return=True)
  ]
  for package_output in installed_packages:
    package = package_output.split(":")[1]
    if pattern.match(package) and package not in installed_system_packages:
      device.Uninstall(package)

def _WipeUnderDirIfMatch(device, path, pattern):
  for filename in device.ListDirectory(path):
    if pattern.match(filename):
      _WipeFileOrDir(device, posixpath.join(path, filename))

def _WipeFileOrDir(device, path):
  if device.PathExists(path):
    device.RunShellCommand(['rm', '-rf', path], check_return=True)

def WipeDevice(device, adb_key_files):
  """Wipes data from device, keeping only the adb_keys for authorization.

  After wiping data on a device that has been authorized, adb can still
  communicate with the device, but after reboot the device will need to be
  re-authorized because the adb keys file is stored in /data/misc/adb/.
  Thus, adb_keys file is rewritten so the device does not need to be
  re-authorized.

  Arguments:
    device: the device to wipe
  """
  try:
    device.EnableRoot()
    device_authorized = device.FileExists(adb_wrapper.ADB_KEYS_FILE)
    if device_authorized:
      adb_keys = device.ReadFile(
          adb_wrapper.ADB_KEYS_FILE, as_root=True).splitlines()
    device.RunShellCommand(['wipe', 'data'], as_root=True, check_return=True)
    device.adb.WaitForDevice()

    if device_authorized:
      adb_keys_set = set(adb_keys)
      for adb_key_file in adb_key_files or []:
        try:
          with open(adb_key_file, 'r') as f:
            adb_public_keys = f.readlines()
          adb_keys_set.update(adb_public_keys)
        except IOError:
          logger.warning('Unable to find adb keys file %s.', adb_key_file)
      _WriteAdbKeysFile(device, '\n'.join(adb_keys_set))
  except device_errors.CommandFailedError:
    logger.exception('Possible failure while wiping the device. '
                     'Attempting to continue.')

def _WriteAdbKeysFile(device, adb_keys_string):
  dir_path = posixpath.dirname(adb_wrapper.ADB_KEYS_FILE)
  device.RunShellCommand(['mkdir', '-p', dir_path],
                         as_root=True,
                         check_return=True)
  device.RunShellCommand(['restorecon', dir_path],
                         as_root=True,
                         check_return=True)
  device.WriteFile(adb_wrapper.ADB_KEYS_FILE, adb_keys_string, as_root=True)
  device.RunShellCommand(['restorecon', adb_wrapper.ADB_KEYS_FILE],
                         as_root=True,
                         check_return=True)

def SetProperties(device, enable_java_debug, disable_location,
                  disable_mock_location):
  try:
    device.EnableRoot()
  except device_errors.CommandFailedError as e:
    logger.warning(str(e))

  if not device.IsUserBuild():
    _ConfigureLocalProperties(device, enable_java_debug)
  else:
    logger.warning('Cannot configure properties in user builds.')
  settings.ConfigureContentSettings(device,
                                    settings.DETERMINISTIC_DEVICE_SETTINGS)
  if disable_location:
    settings.ConfigureContentSettings(device,
                                      settings.DISABLE_LOCATION_SETTINGS)
  else:
    settings.ConfigureContentSettings(device, settings.ENABLE_LOCATION_SETTINGS)

  if disable_mock_location:
    settings.ConfigureContentSettings(device,
                                      settings.DISABLE_MOCK_LOCATION_SETTINGS)
  else:
    settings.ConfigureContentSettings(device,
                                      settings.ENABLE_MOCK_LOCATION_SETTINGS)

  settings.SetLockScreenSettings(device)

  # Some device types can momentarily disappear after setting properties.
  device.adb.WaitForDevice()

def DisableNetwork(device):
  settings.ConfigureContentSettings(device, settings.NETWORK_DISABLED_SETTINGS)
  if device.build_version_sdk >= version_codes.MARSHMALLOW:
    # Ensure that NFC is also switched off.
    device.RunShellCommand(['svc', 'nfc', 'disable'],
                           as_root=True,
                           check_return=True)

def DisableSystemChrome(device):
  # The system chrome version on the device interferes with some tests.
  device.RunShellCommand(['pm', 'disable', 'com.android.chrome'],
                         as_root=True,
                         check_return=True)

def _FindSystemPackagePaths(device, system_package_list):
  found_paths = []
  for system_package in system_package_list:
    found_paths.extend(device.GetApplicationPaths(system_package))
  return [p for p in found_paths if p.startswith('/system/')]

def _FindSystemAppPaths(device, system_app_list):
  found_paths = []
  for system_app in system_app_list:
    for directory in _SYSTEM_APP_DIRECTORIES:
      path = os.path.join(directory, system_app)
      if device.PathExists(path):
        found_paths.append(path)
  return found_paths

def RemoveSystemApps(device, system_app_remove_list,
                     system_package_remove_list):
  """Attempts to remove the provided system apps from the given device.

  Arguments:
    device: The device to remove the system apps from.
    system_app_remove_list: A list of app names to remove, e.g.
        ['WebViewGoogle', 'GoogleVrCore']
    system_package_remove_list: A list of app packages to remove, e.g.
        ['com.google.android.webview']
  """
  device.EnableRoot()
  if device.HasRoot():
    system_app_paths = (
        _FindSystemAppPaths(device, system_app_remove_list) +
        _FindSystemPackagePaths(device, system_package_remove_list))
    if system_app_paths:
      # Disable Marshmallow's Verity security feature
      if device.build_version_sdk >= version_codes.MARSHMALLOW:
        logger.info('Disabling Verity on %s', device.serial)
        device.adb.DisableVerity()
        device.Reboot()
        device.WaitUntilFullyBooted()
        device.EnableRoot()

      device.adb.Remount()
      device.RunShellCommand(['stop'], check_return=True)
      device.RemovePath(system_app_paths, force=True, recursive=True)
      device.RunShellCommand(['start'], check_return=True)
  else:
    raise device_errors.CommandFailedError(
        'Failed to remove system apps from non-rooted device', str(device))

def _ConfigureLocalProperties(device, java_debug=True):
  """Set standard readonly testing device properties prior to reboot."""
  local_props = [
      'persist.sys.usb.config=adb',
      'ro.monkey=1',
      'ro.test_harness=1',
      'ro.audio.silent=1',
      'ro.setupwizard.mode=DISABLED',
  ]
  if java_debug:
    local_props.append('%s=all' % device_utils.DeviceUtils.JAVA_ASSERT_PROPERTY)
    local_props.append('debug.checkjni=1')
  try:
    device.WriteFile(
        device.LOCAL_PROPERTIES_PATH, '\n'.join(local_props), as_root=True)
    # Android will not respect the local props file if it is world writable.
    device.RunShellCommand(['chmod', '644', device.LOCAL_PROPERTIES_PATH],
                           as_root=True,
                           check_return=True)
  except device_errors.CommandFailedError:
    logger.exception('Failed to configure local properties.')

def FinishProvisioning(device):
  # The lockscreen can't be disabled on user builds, so send a keyevent
  # to unlock it.
  if device.IsUserBuild():
    device.SendKeyEvent(keyevent.KEYCODE_MENU)

def WaitForCharge(device, min_battery_level):
  battery = battery_utils.BatteryUtils(device)
  try:
    battery.ChargeDeviceToLevel(min_battery_level)
  except device_errors.DeviceChargingError:
    device.Reboot()
    battery.ChargeDeviceToLevel(min_battery_level)

def WaitForBatteryTemperature(device, max_battery_temp):
  try:
    battery = battery_utils.BatteryUtils(device)
    battery.LetBatteryCoolToTemperature(max_battery_temp)
  except device_errors.CommandFailedError:
    logger.exception('Unable to let battery cool to specified temperature.')

def SetDate(device):
  def _set_and_verify_date():
    if device.build_version_sdk >= version_codes.MARSHMALLOW:
      date_format = '%m%d%H%M%Y.%S'
      set_date_command = ['date', '-u']
      get_date_command = ['date', '-u']
    else:
      date_format = '%Y%m%d.%H%M%S'
      set_date_command = ['date', '-s']
      get_date_command = ['date']

    # TODO(jbudorick): This is wrong on pre-M devices -- get/set are
    # dealing in local time, but we're setting based on GMT.
    strgmtime = time.strftime(date_format, time.gmtime())
    set_date_command.append(strgmtime)
    device.RunShellCommand(set_date_command, as_root=True, check_return=True)

    get_date_command.append('+"%Y%m%d.%H%M%S"')
    device_time = device.RunShellCommand(
        get_date_command, check_return=True, as_root=True,
        single_line=True).replace('"', '')
    device_time = datetime.datetime.strptime(device_time, "%Y%m%d.%H%M%S")
    correct_time = datetime.datetime.strptime(strgmtime, date_format)
    tdelta = (correct_time - device_time).seconds
    if tdelta <= 1:
      logger.info('Date/time successfully set on %s', device)
      return True
    else:
      logger.error('Date mismatch. Device: %s Correct: %s',
                   device_time.isoformat(), correct_time.isoformat())
      return False

  # Sometimes the date is not set correctly on the devices. Retry on failure.
  if device.IsUserBuild():
    # TODO(bpastene): Figure out how to set the date & time on user builds.
    pass
  else:
    if not timeout_retry.WaitFor(
        _set_and_verify_date, wait_period=1, max_tries=2):
      raise device_errors.CommandFailedError(
          'Failed to set date & time.', device_serial=str(device))
    device.EnableRoot()
    # The following intent can take a bit to complete when ran shortly after
    # device boot-up.
    device.BroadcastIntent(
        intent.Intent(action='android.intent.action.TIME_SET'), timeout=180)

def LogDeviceProperties(device):
  props = device.RunShellCommand(['getprop'], check_return=True)
  for prop in props:
    logger.info('  %s', prop)

# TODO(jbudorick): Relocate this either to device_utils or a separate
# and more intentionally reusable layer on top of device_utils.
def CheckExternalStorage(device):
  """Checks that storage is writable and if not makes it writable.

  Arguments:
    device: The device to check.
  """
  try:
    with device_temp_file.DeviceTempFile(
        device.adb, suffix='.sh', dir=device.GetExternalStoragePath()) as f:
      device.WriteFile(f.name, 'test')
  except device_errors.CommandFailedError:
    logger.info('External storage not writable. Remounting / as RW')
    device.RunShellCommand(['mount', '-o', 'remount,rw', '/'],
                           check_return=True,
                           as_root=True)
    device.EnableRoot()
    with device_temp_file.DeviceTempFile(
        device.adb, suffix='.sh', dir=device.GetExternalStoragePath()) as f:
      device.WriteFile(f.name, 'test')

def StandaloneVrDeviceSetup(device):
  """Performs any additional setup necessary for standalone Android VR devices.

  Arguments:
    device: The device to check.
  """
  if device.product_name not in _STANDALONE_VR_DEVICES:
    return

  # Modify VrCore's settings so that any first time setup, etc. is skipped.
  shared_pref = shared_prefs.SharedPrefs(
      device,
      'com.google.vr.vrcore',
      'VrCoreSettings.xml',
      use_encrypted_path=True)
  shared_pref.Load()
  # Skip first time setup.
  shared_pref.SetBoolean('DaydreamSetupComplete', True)
  # Disable the automatic prompt that shows anytime the device detects that a
  # controller isn't connected.
  shared_pref.SetBoolean('gConfigFlags:controller_recovery_enabled', False)
  # Use an automated controller instead of a real one so we get past the
  # controller pairing screen that's shown on startup.
  shared_pref.SetBoolean('UseAutomatedController', True)
  shared_pref.Commit()

def main(raw_args):
  # Recommended options on perf bots:
  # --disable-network
  #     TODO(tonyg): We eventually want network on. However, currently radios
  #     can cause perfbots to drain faster than they charge.
  # --min-battery-level 95
  #     Some perf bots run benchmarks with USB charging disabled which leads
  #     to gradual draining of the battery. We must wait for a full charge
  #     before starting a run in order to keep the devices online.

  parser = argparse.ArgumentParser(
      description='Provision Android devices with settings required for bots.')
  logging_common.AddLoggingArguments(parser)
  script_common.AddDeviceArguments(parser)
  script_common.AddEnvironmentArguments(parser)
  parser.add_argument(
      '--adb-key-files',
      type=str,
      nargs='+',
      help='list of adb keys to push to device')
  parser.add_argument(
      '--disable-location',
      action='store_true',
      help='disable Google location services on devices')
  parser.add_argument(
      '--disable-mock-location',
      action='store_true',
      default=False,
      help='Set ALLOW_MOCK_LOCATION to false')
  parser.add_argument(
      '--disable-network',
      action='store_true',
      help='disable network access on devices')
  parser.add_argument(
      '--disable-java-debug',
      action='store_false',
      dest='enable_java_debug',
      default=True,
      help='disable Java property asserts and JNI checking')
  parser.add_argument(
      '--disable-system-chrome',
      action='store_true',
      help='DEPRECATED: use --remove-system-packages com.android.google '
      'Disable the system chrome from devices.')
  parser.add_argument(
      '--emulators',
      action='store_true',
      help='provision only emulators and ignore usb devices '
      '(this will not wipe emulators)')
  parser.add_argument(
      '--max-battery-temp',
      type=int,
      metavar='NUM',
      help='Wait for the battery to have this temp or lower.')
  parser.add_argument(
      '--min-battery-level',
      type=int,
      metavar='NUM',
      help='wait for the device to reach this minimum battery'
      ' level before trying to continue')
  parser.add_argument('--output-device-denylist',
                      help='Json file to output the device denylist.')
  parser.add_argument(
      '--reboot-timeout',
      metavar='SECS',
      type=int,
      help='when wiping the device, max number of seconds to'
      ' wait after each reboot '
      '(default: %s)' % _DEFAULT_TIMEOUTS.HELP_TEXT)
  parser.add_argument(
      '--remove-system-apps',
      nargs='*',
      dest='system_app_remove_list',
      help='DEPRECATED: use --remove-system-packages instead. '
      'The names of system apps to remove. ')
  parser.add_argument(
      '--remove-system-packages',
      nargs='*',
      dest='system_package_remove_list',
      help='The names of system packages to remove.')
  parser.add_argument(
      '--remove-system-webview',
      action='store_true',
      help='DEPRECATED: use --remove-system-packages '
      'com.google.android.webview com.android.webview '
      'Remove the system webview from devices.')
  parser.add_argument(
      '--skip-wipe',
      action='store_true',
      default=False,
      help='do not wipe device data during provisioning')

  # No-op arguments for compatibility with build/android/provision_devices.py.
  # TODO(jbudorick): Remove these once all callers have stopped using them.
  parser.add_argument(
      '--chrome-specific-wipe', action='store_true', help=argparse.SUPPRESS)
  parser.add_argument('--phase', action='append', help=argparse.SUPPRESS)
  parser.add_argument(
      '-r', '--auto-reconnect', action='store_true', help=argparse.SUPPRESS)
  parser.add_argument('-t', '--target', help=argparse.SUPPRESS)

  args = parser.parse_args(raw_args)

  logging_common.InitializeLogging(args)
  script_common.InitializeEnvironment(args)

  try:
    return ProvisionDevices(
        args.devices,
        args.denylist_file,
        adb_key_files=args.adb_key_files,
        disable_location=args.disable_location,
        disable_mock_location=args.disable_mock_location,
        disable_network=args.disable_network,
        disable_system_chrome=args.disable_system_chrome,
        emulators=args.emulators,
        enable_java_debug=args.enable_java_debug,
        max_battery_temp=args.max_battery_temp,
        min_battery_level=args.min_battery_level,
        output_device_denylist=args.output_device_denylist,
        reboot_timeout=args.reboot_timeout,
        remove_system_webview=args.remove_system_webview,
        system_app_remove_list=args.system_app_remove_list,
        system_package_remove_list=args.system_package_remove_list,
        wipe=not args.skip_wipe and not args.emulators)
  except (device_errors.DeviceUnreachableError, device_errors.NoDevicesError):
    logging.exception('Unable to provision local devices.')
    return exit_codes.INFRA

if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))


import re
from pyecharts import Pie
from collections import Counter
from wordcloud import WordCloud
import matplotlib.pyplot as plt

class FocusRatioChart:
    def __init__(self, category=None):
        self.pat = None
        self.category = category
        self.file_path = './lg_gov_title_1.txt'
        if self.category == 'dev':
            self.pat = r'更新|整治|利益统筹|拆除重建|治理|战略|先行|整备|建设|教育|医疗|文化|体育|文体|儿童|养老|老年|高等教育|创新|科技|产业|人口|制造业|服务业'
        else:
            self.pat = r'平湖|横岗|龙岗街道|龙城|坪地|布吉|南湾|坂田|吉华|圆山|宝龙|龙岗河|大运|龙园|龙西|五联|云创|甘坑|坂雪岗|华为|比亚迪|低碳城'

    def gene_pie_chart(self, width='1000', height='600', title="", output=None, **kwargs):
        with open(self.file_path, 'r', encoding='utf-8') as f:
            res = re.findall(self.pat, f.read())
            count_dict = Counter(res)
        if not title:
            title = '发展关注占比' if self.category == 'dev' else '地区关注占比'
        attr, value = count_dict.keys(), count_dict.values()
        pie = Pie(title=title,
                  width=width,
                  height=height,
                  subtitle='',
                  title_pos="center",
                  title_top="95%",
                  title_color="#000",
                  subtitle_color="#aaa",
                  title_text_size=18,
                  subtitle_text_size=12,
                  background_color="#fff",
                  is_grid=False
                  )

        pie.add("", attr, value, is_label_show=True, center=['50%', '55%'])
        # pie.show_config()
        pie.render(path=output or (title + '.html'))

    def gene_word_cloud(self, **kwargs):
        f = open(self.file_path, 'r', encoding='UTF-8').read()
        kws = re.findall(self.pat, f)
        counter = Counter(kws)

        word_cloud = WordCloud(
            font_path="C:/Windows/Fonts/simsun.ttc",
            **kwargs
            # background_color="white",
            # min_font_size=6,
            # width=1000,
            # height=880,
            # max_words=max_words
        )
        word_cloud.generate_from_frequencies(counter)
        plt.imshow(word_cloud, interpolation="bilinear")
        plt.axis("off")
        word_cloud.to_file('res_title_1.png')
        plt.show()

if __name__ == '__main__':
    frc = FocusRatioChart()
    # frc.gene_pie_chart()
    frc.gene_word_cloud(background_color="white")


#! /usr/local/bin/python27
import mrjob
from mrjob.job import MRJob
import json
import urllib2
import sys
import math
sys.path.append( '../' )
import time
import unicodedata

# if YIELD_ALL is true, we yield all densities, otherwise,
# we yield just the extremes
YIELD_ALL = True

def get_geonames(lat, lng, types):
    url = 'http://maps.googleapis.com/maps/api/geocode/json' + \
            '?latlng={0},{1}&sensor=false'.format(lat, lng)
    jsondata = json.load(urllib2.urlopen(url))
    address_comps = jsondata['results'][0]['address_components']

    #print json.dumps(jsondata, sort_keys=True, indent=4, separators=(',', ': '))

    filter_method = lambda x: len(set(x['types']).intersection(types))
    return filter(filter_method, address_comps)

def get_fips(lat, lng):
    url = 'http://data.fcc.gov/api/block/2010/find?latitude={0}&longitude={1}&format=json'.format(lat,lng)
    jsondata = json.load(urllib2.urlopen(url))
    return jsondata['County']['FIPS']

def get_zip(lat, lng):
    results = get_geonames(lat, lng, ['country', 'postal_code'])
    #print results
    retVal = None
    try:
        if results[0]["short_name"] == "US":
            retVal = results[1]["long_name"]
    except:
        pass
        #print json.dumps(results, sort_keys=True, indent=4, separators=(',', ': '))

    return retVal

#types = ['locality', 'postal_code']

class ArtistZip(MRJob):
    # Kills the stupid quotes around strings in the output.
    #DEFAULT_OUTPUT_PROTOCOL = mrjob.protocol.RawValueProtocol

    INPUT_PROTOCOL = mrjob.protocol.JSONProtocol
    #INTERNAL_PROTOCOL = mrjob.protocol.JSONProtocol
    OUTPUT_PROTOCOL = mrjob.protocol.RawValueProtocol

    def reducer(self, key, val):
        total = 0
        artists = {}
        terms = []
        for v in val:
            if not "rap" in v[3].lower() or "hip-hop" in v[3].lower():
                continue
            total = total + v[0]
            if isinstance(v[1], unicode):
                nice_artist = unicodedata.normalize('NFKD', v[1]).encode('ascii','ignore')
            else:
                nice_artist = v[1]
            if(nice_artist not in artists):
                artists[nice_artist] = v[2]
            if(v[3] not in terms):
                terms.append(v[3])

        sorted_artists = sorted(artists, key=artists.get, reverse=True)

        #print sorted_artists
        #final_artists = []
        #for k,v in sorted_artists.iteritems():
        #    final_artists.append(k)
        if total == 0:
            return
        yield key, '"' + str(key) + '":' + '{"count": ' + str(total) + ', "artists":'  + str(sorted_artists) +'},'

if __name__ == '__main__':
    ArtistZip.run()

#1                   bars [(1.54119, 0.267), (4.58833, 0.24), (7.6054, 0.183), (1
#2         end_of_fade_in 0.235
#3                 energy 0.0
#4              cover_art http://cdn.7static.com/static/img/sleeveart/00/002/420/
#5                  beats [(0.30104, 0.738), (0.91303, 0.232), (1.54119, 0.0), (2
#6     release_7digitalid 242014
#7                   year 2007
#8               duration 385.43628
#9            artist_mbid 494d1c4b-fb74-4a4f-b73e-408577a97370
#10       artist_longitude nan
#11       track_7digitalid 2639239
#12        similar_artists ['ARTKVEL1187FB548E0', 'ARW5R811187B98BC96', 'AR11Z1W1
#13               track_id TRAAWCO128F423A29B
#14                  title Cold Waste
#15               segments [{'confidence': 0.0, 'timbre': [0.0, 171.13, 9.469, -2
#16        artist_playmeid 161669
#17      artist_7digitalid 182218
#18           danceability 0.0
#19          artist_mbtags []
#20     artist_familiarity 0.543086782202
#21              artist_id ARL6NP61187B98C1FC
#22                preview http://previews.7digital.com/clips/34/2639239.clip.mp3
#23               sections [(0.0, 1.0), (17.85461, 0.822), (26.83628, 1.0), (44.8
#24      start_of_fade_out 366.393
#25                 tatums [(0.09973, 0.37), (0.30104, 0.353), (0.50133, 0.29), (
#26   analysis_sample_rate 22050
#27         time_signature (5, 1.0)
#28                  tempo 99.728
#29            artist_name The Foreshadowing
#30           artist_terms [('doom metal', 1.0, 1.0), ('heavy metal', 0.988538582
#31      artist_hotttnesss 0.411627769292
#32                    key (7, 0.249)
#33              audio_md5 cc05d52c09d6075e724936ae4bfd2bdc
#34        artist_location
#35                song_id SOABCEU12A8C132027
#36        artist_latitude nan
#37                   mode (1, 0.403)
#38        song_hotttnesss 0.596010764183
#39                release Days Of Nothing
#40               loudness -4.819


# -*- coding: utf-8 -*-

# (c) Copyright 2015 Hewlett Packard Enterprise Development LP
#
# GNU Zebra is free software; you can rediTestribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2, or (at your option) any
# later version.
#
# GNU Zebra is diTestributed in the hope that it will be useful, but
# WITHoutputputput ANY WARRANTY; withoutputputput even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with GNU Zebra; see the file COPYING.  If not, write to the Free
# Software Foundation, Inc., 59 Temple Place - Suite 330, BoTeston, MA
# 02111-1307, USA.

TOPOLOGY = """
# +-------+
# |  sw1  |
# +-------+

# Nodes
[type=openswitch name="Switch 1"] sw1
"""

def init_temp_sensor_table(sw1):
    output = sw1('list subsystem', shell='vsctl')
    lines = output.split('\n')
    for line in lines:
        if '_uuid' in line:
            _id = line.split(':')
            uuid = _id[1].strip()
            output = sw1('ovs-vsctl -- set Subsystem {} '
                         ' temp_sensors=@fan1 -- --id=@fan1 '
                         ' create Temp_sensor name=base-1 '
                         ' location=Faceplate_side_of_switch_chip_U16 '
                         ' status=normal fan-state=normal min=0 '
                         ' max=21000 temperature=20500'.format(uuid),
                         shell='bash')

def show_system_temperature(sw1, step):
    counter = 0
    step('Test to verify \'show system temperature\' command')
    output = sw1('show system temperature detail')
    lines = output.split('\n')
    for line in lines:
        if 'base-1' in line:
            counter += 1
        if 'Faceplate_side_of_switch_chip_U16' in line:
            counter += 1
        if 'normal' in line:
            counter += 1
    assert counter is 3

def test_tempd_ct_tempsensor(topology, step):
    sw1 = topology.get("sw1")
    assert sw1 is not None
    step("Initializing temperature sensor table with dummy data")
    init_temp_sensor_table(sw1)
    step('Test to verify \'show system temperature\' command')
    show_system_temperature(sw1, step)


from mcpi.minecraft import Minecraft
DD=Minecraft.create()

while True:
    hits=DD.events.pollBlockHits()
    if len(hits)>0:
        a=hits[0]
        x,y,z=a.pos.x,a.pos.y,a.pos.z
        bl=DD.getBlock(x,y,z)
        DD.postToChat("你打到"+str(bl))
        if bl==1:
            DD.setBlock(x,y,z,41)


from cms.plugin_base import CMSPluginBase
from cms.plugin_pool import plugin_pool
from cms.models.pluginmodel import CMSPlugin
from django.utils.translation import ugettext_lazy as _
from .models import Partner, TeamMember, Project

@plugin_pool.register_plugin
class PartnersPlugin(CMSPluginBase):
    model = CMSPlugin
    name = _("Partners Plugin")
    render_template = "partner_list.html"
    cache = False

    def render(self, context, instance, placeholder):
        context['partners'] = Partner.objects.filter(is_active=True).order_by('position')
        return context

@plugin_pool.register_plugin
class MembersPlugin(CMSPluginBase):
    model = CMSPlugin
    name = _("Members Plugin")
    render_template = "member_list.html"
    cache = False

    def render(self, context, instance, placeholder):
        context['active_members'] = TeamMember.objects.filter(is_active=True, is_lab_affiliate=False).order_by('position')
        context['lab_affiliate_members'] = TeamMember.objects.filter(is_active=True, is_lab_affiliate=True).order_by('position')
        context['inactive_members'] = TeamMember.objects.filter(is_active=False).order_by('position')
        return context

@plugin_pool.register_plugin
class SeparatorPlugin(CMSPluginBase):
    model = CMSPlugin
    name = _("Separator")
    render_template = "separator.html"
    cache = False

    def render(self, context, instance, placeholder):
        return context

@plugin_pool.register_plugin
class ProjectsActivePlugin(CMSPluginBase):
    model = CMSPlugin
    name = _("Current Projects")
    render_template = "project_active_list.html"
    cache = False

    def render(self, context, instance, placeholder):
        context['active_projects'] = Project.objects.order_by('position').filter(is_active=True)
        return context

@plugin_pool.register_plugin
class ProjectsInactivePlugin(CMSPluginBase):
    model = CMSPlugin
    name = _("Past Projects")
    render_template = "project_inactive_list.html"
    cache = False

    def render(self, context, instance, placeholder):
        context['inactive_projects'] = Project.objects.order_by('position').filter(is_active=False)
        return context


# 100
def f1():
    y = None
    for i, line in enumerate(field):
        if line[x:x + w] != ['.'] * w:
            y = i - 1
            break
    if y is None:
        y = len(field) - 1

    y -= (h - 1)
    for line in field[y:y + h]:
        line[x:x + w] = ['#'] * w

H, W, N = map(int, input().split())

field = [['.'] * W for _ in range(H)]

for n in range(N):
    h, w, x = map(int, input().split())
    f1()

for _line in field:
    print(''.join(_line))


from helper import get_processed_data, get_df_from_row, pre_process, get_5_min_data
import numpy as np
import datetime
import talib
import pandas as pd
import s3fs
weekly_expiry = []
all_df = get_processed_data()

st = datetime.date(2021, 1, 1)
en = datetime.date(2021, 7, 29)

all_df['expiry'] = all_df['expiry'].apply(lambda x: x.date())

futures_df = all_df[all_df['type'] == 'FUT']
futures_files = futures_df['file'].unique()
complete_df = pd.DataFrame()

for _file in futures_files:
    df = pd.read_csv(_file, index_col=0, parse_dates=True)
    df = df[(df['ticker'] == 'BANKNIFTY-I') | (df['ticker'] == 'BANKNIFTY-I.NFO')]
    df['ticker'] = 'BANKNIFTY-I'
    complete_df = pd.concat([complete_df, df])

bnf_5 = get_5_min_data(complete_df)

options_df = all_df[all_df['type'] == 'OPT']

options_df.to_csv('options.csv')

weekly_expiry_dates = options_df['expiry'].unique()
monthly_expiry_dates = futures_df['expiry'].unique()

for i, row in bnf_5.iterrows():
    if i.date() in weekly_expiry_dates:
        bnf_5.at[i, 'expiry'] = i.date()
expiry_bfill = bnf_5['expiry'].bfill()

bnf_5['expiry'] = np.where(
        bnf_5['expiry'].isnull(), expiry_bfill, bnf_5['expiry']
    )

bnf_5['RSI'] = talib.RSI(bnf_5['close'])

start = datetime.datetime(year=2019, month=1, day=1)

ohlc_dict = {
    'open': 'first',
    'high': 'max',
    'low': 'min',
    'close': 'last',
    'open_interest': 'last',
    'volume': 'sum',
}

exchange_start_time = datetime.time(minute=15, hour=9)

def get_option_from_strike(strike, option_type, expiry):
    opt = options_df[
        (options_df['strike'] == strike) &
        (options_df['expiry'] == expiry) &
        (options_df['option_type'] == option_type)
        ]
    return opt.iloc[0]

positions = dict()
transactions = []

def _transact(row, quantity, reason, param='close', price=0, price_given=False):
    temp = dict()
    temp['instrument_code'] = row['code']
    temp['time'] = row.name
    if price_given:
        temp['price'] = price
    else:
        temp['price'] = row[param]
    temp['quantity'] = quantity
    temp['reason'] = reason
    transactions.append(temp)

def sell(row, quantity=-1, reason=None, param='close', price=0, price_given=False):
    _transact(row, quantity, reason, param, price, price_given)

def buy(row, quantity=1, reason=None):
    _transact(row, quantity, reason)

def weekly_to_monthly(weekly_date):
    for expiry_day in monthly_expiry_dates:
        if expiry_day.month == weekly_date.month:
            return expiry_day

def vwap(df):
    current_date = datetime.date(2017, 1, 1)
    previous_date = datetime.date(2016, 12, 31)
    for i, row in df.iterrows():  # need to intialise yest_close, day_close, today_high
        current_date = i.date()
        if current_date == previous_date:  # inside the same day 9:16 to 3:30

            ##############Calculating Vwap###############
            typ_p = (row['open'] + row['high'] + row['close']) / 3
            cummulative_volume = cummulative_volume + row['volume']
            cummulative_sum = cummulative_sum + typ_p * row['volume']
            if cummulative_volume == 0:
                df.at[i, 'vwap'] = 0
            else:
                df.at[i, 'vwap'] = cummulative_sum / cummulative_volume
        else:  # at 9:15
            # Calculating Vwap
            typ_p = (row['open'] + row['high'] + row['close']) / 3
            cummulative_volume = row['volume']
            cummulative_sum = typ_p * cummulative_volume
            if cummulative_volume == 0:
                df.at[i, 'vwap'] = 0
            else:
                df.at[i, 'vwap'] = cummulative_sum / cummulative_volume
        previous_date = current_date
    return df

capital = 200000
risk = 5000
def conditions_check(df, time):
    if df.loc[time]['RSI'] > 60 and df.loc[time]['vwap'] > df.loc[time]['close']:
        if df.loc[time]['open_interest'] < df.loc[time]['previous_open_interest'] and df.loc[time]['volume'] > df.loc[time]['volume_sma']:
            return True

        else:
            return False
    else:
        return False

position = False
rr_reached = False
count = 1

in_trade_df = pd.DataFrame()
bnf_5.dropna(inplace=True)

bnf_5 = bnf_5[st:en]

for i, row in bnf_5.iterrows():
    try:
        if not position:
            if row['RSI'] > 20 and i.time() > datetime.time(9, 19):
                close_price = row['close']
                strike_price = (round(close_price/100))*100
                expiry = row['expiry']
                instrument_row = get_option_from_strike(strike_price, 'CE', expiry)
                instrument_df = get_df_from_row(instrument_row)
                instrument_df['RSI'] = talib.RSI(instrument_df['close'])

                instrument_df['volume_sma'] = talib.SMA(instrument_df['volume'], timeperiod=20)
                instrument_df['previous_open_interest'] = instrument_df['open_interest'].shift(1)
                instrument_df = vwap(instrument_df)

                if i in instrument_df.index:
                    if conditions_check(df=instrument_df, time=i) and not position:
                        print(i, 'Code is Running', 'CE')
                        buy_df = instrument_df
                        buy(buy_df.loc[i], quantity=50, reason='BUY CE Triggered')
                        buy_price = buy_df.loc[i]['close']
                        init_stop_loss = buy_df.loc[i]['vwap'] - 30
                        stop_loss = init_stop_loss
                        in_trade_df = in_trade_df.append(buy_df.loc[i])
                        in_trade_df.at[i, 'stop_loss'] = stop_loss
                        in_trade_df.at[i, 'trade_no'] = count
                        position = True
            if row['RSI'] < 80 and i.time() > datetime.time(9, 19):#########PE Buying
                close_price = row['close']
                strike_price = (round(close_price / 100)) * 100
                expiry = row['expiry']
                instrument_row = get_option_from_strike(strike_price, 'PE', expiry)
                instrument_df = get_df_from_row(instrument_row)
                instrument_df['RSI'] = talib.RSI(instrument_df['close'])

                instrument_df['volume_sma'] = talib.SMA(instrument_df['volume'], timeperiod=20)
                instrument_df = vwap(instrument_df)
                instrument_df['previous_open_interest'] = instrument_df['open_interest'].shift(1)
                if i in instrument_df.index:
                    if conditions_check(df=instrument_df, time=i) and not position:
                        print(i, 'Code is Running', 'PE')
                        buy_df = instrument_df
                        buy(buy_df.loc[i], quantity=50, reason='BUY CE Triggered')
                        buy_price = buy_df.loc[i]['close']
                        init_stop_loss = buy_df.loc[i]['vwap'] - 40
                        stop_loss = init_stop_loss
                        in_trade_df = in_trade_df.append(buy_df.loc[i])
                        in_trade_df.at[i, 'stop_loss'] = stop_loss
                        in_trade_df.at[i, 'trade_no'] = count
                        position = True

        elif position:
            in_trade_df = in_trade_df.append(buy_df.loc[i])
            in_trade_df.at[i, 'stop_loss'] = stop_loss
            in_trade_df.at[i, 'trade_no'] = count
            current_close = buy_df.loc[i]['close']
            if i.time() == datetime.time(15, 25):
                sell(buy_df.loc[i], quantity=-50, reason='Time Over')
                position = False
                count = count + 1
            if i.time() == datetime.time(15, 25) and rr_reached and position:
                sell(buy_df.loc[i], quantity=-25, reason='Time Over')
                position = False
                rr_reached = False
                count = count + 1
            if stop_loss > buy_df.loc[i]['low'] and not rr_reached and position:
                sell(buy_df.loc[i], quantity=-50, reason='First SL Hit', price=stop_loss, price_given=True)
                position = False
                count = count + 1
            if buy_df.loc[i]['close'] > (buy_price + buy_price-init_stop_loss) and not rr_reached and position:##1:1 RR
                stop_loss = buy_price
                sell(buy_df.loc[i], quantity=-25, reason='Half Profit Booked')
                rr_reached = True
            if stop_loss > buy_df.loc[i]['low'] and rr_reached and position:
                sell(buy_df.loc[i], quantity=-25, reason='Trail SL Hit', price=stop_loss, price_given=True)
                rr_reached = False
                previous_close = current_close
                position = False
                count = count + 1
            if rr_reached:
                stop_loss = buy_df.loc[i]['open']
    except:
        print('Error in', i)

in_trade_df.to_csv('in_trade_df.csv')
transactions = pd.DataFrame(transactions)

# del transactions['instrument_code']
transactions.to_csv('transactions.csv')
# print(transactions/)


default = '''# To restore the default configuration, delete this file and config.pyc

RESURRECT = False
QUICK_START = False
DEBUG_KEYS = False
DEBUG_ITEMS = False
'''

try:
	from config import *

except ImportError:
	f = open('config.py', 'w')
	f.write(default)
	f.close()

	from config import *


with open('input') as f:
    lines = [x.strip() for x in f]

asts = set()
for y in range(len(lines)):
    for x in range(len(lines[y])):
        if lines[y][x] == '#':
            asts.add((x,y))

def get_angle(dx,dy):
# Return angle in sortable (quadrant,gradient) integer format
# Avoid floats to avoid any equality comparison issues
    dy = -dy # Reverse y to match normal geometry
    scale = 1000000
    if dy == 0:
        angle = 0
        quad = 1 if dx > 0 else 3
    if (dx >= 0 and dy > 0):
        quad = 0
        angle = dx * scale // dy
    if (dx >  0 and dy < 0):
        quad = 1
        angle = -dy * scale // dx
    if (dx <= 0 and dy < 0):
        quad = 2
        angle = dx * scale // dy
    if (dx < 0 and dy > 0):
        quad = 3
        angle = -dy * scale // dx
    return (quad,angle)

def dist(a,b): return abs(a[0]-b[0]) + abs(a[1]-b[1])
def diff(a,b): return (a[0]-b[0], a[1]-b[1])

visible_count = []
for ast in asts:
    visible_set = {get_angle(*diff(ast2,ast)) for ast2 in asts - {ast}}
    visible_count.append((len(visible_set),ast))
visible_count.sort()
count, pos = visible_count[-1]
print(count, pos)

others = []
for ast in asts - {pos}:
    others.append((get_angle(*diff(ast,pos)), dist(ast,pos), ast))
others.sort() # Asteroids sorted by angle and distance from base

destroyed = 0
while others:
    angle = None
    temp = others.copy()
    for ast in temp:
        if ast[0] != angle:
            angle = ast[0]
            destroyed += 1
            print(destroyed,ast)
            others.remove(ast)


from flask import Flask, redirect, url_for, render_template, session

import filters, constants

import iso8601
import pytz
import requests
import json
import os
import datetime

app = Flask(__name__)
app.secret_key = os.urandom(24)

@app.route('/')
def index():
  return redirect(url_for('order'))

@app.route('/order/')
def order():
  with open('order.json') as f:
    order = json.load(f)
    session['order'] = order

  payload = {
    'pickup_address': order['pickup']['address'],
    'dropoff_address': order['dropoff']['address']
  }
  quote_response = requests.post(constants.QUOTE_ENDPOINT, data=payload, auth=(constants.API_KEY, ''))

  if quote_response.status_code == requests.codes.ok:
    quote = quote_response.json()
    session['quote_id'] = quote['id']
    return render_template('order.html', order=order, quote=quote)
  else:
    quote_response.raise_for_status()

@app.route('/delivery/')
def delivery():
  if 'quote_id' and 'order' in session:
    order = session['order']
    payload = {
      'dropoff_address': order['dropoff']['address'],
      'dropoff_name': order['dropoff']['name'],
      'dropoff_phone_number': order['dropoff']['phone_number'],
      'manifest':'fancy hot selling socks',
      'pickup_name': order['pickup']['name'],
      'pickup_address': order['pickup']['address'],
      'pickup_phone_number': order['pickup']['phone_number']
    }
    delivery_response = requests.post(constants.DELIVERY_ENDPOINT, data=payload, auth=(constants.API_KEY, ''))
    if delivery_response.status_code == requests.codes.ok:
      return render_template('delivery.html', delivery=delivery_response.json(), order=order)
    else:
      delivery_response.raise_for_status()
  else:
      return redirect(url_for('order'))

def parse_isodate(timestamp):
  tz = pytz.timezone('US/Pacific')
  return iso8601.parse_date(timestamp).astimezone(tz)

@app.template_filter('currency')
def currency(value):
  value = float(value)/100
  return "${:,.2f}".format(value)

@app.template_filter('format_datetime')
def format_datetime(value):
  return parse_isodate(value).strftime('%m/%d/%y @ %I:%M %p')

@app.context_processor
def utility_processor():
  def time_from_now(value):
    now = datetime.datetime.now()
    timedelta = parse_isodate(value).replace(tzinfo=None) - now
    return  '%s minutes from now' % int(timedelta.total_seconds() / 60)
  return dict(time_from_now=time_from_now)


import tensorflow as tf
from tensorflow.python.layers import base
import scipy.io as sio
import numpy as np
TF_DATA_TYPE = tf.float32
TF_WEIGHT_DECAY_LAMBDA = 1e-08
ksize = 3
out_channel = 31

class AddCoords(base.Layer):
    """Add coords to a tensor"""
    def __init__(self, x_dim=64, y_dim=64, with_r=False):
        super(AddCoords, self).__init__()
        self.x_dim = x_dim
        self.y_dim = y_dim
        self.with_r = with_r

    def call(self, input_tensor):
        """
        input_tensor: (batch, x_dim, y_dim, c)
        """
        batch_size_tensor = tf.shape(input_tensor)[0]
        xx_ones = tf.ones([batch_size_tensor, self.x_dim], dtype=tf.int32)
        xx_ones = tf.expand_dims(xx_ones, -1)

        xx_range = tf.tile(tf.expand_dims(tf.range(self.x_dim), 0), [batch_size_tensor, 1])
        xx_range = tf.expand_dims(xx_range, 1)

        xx_channel = tf.matmul(xx_ones, xx_range)
        xx_channel = tf.expand_dims(xx_channel, -1)

        yy_ones = tf.ones([batch_size_tensor, self.y_dim], dtype=tf.int32)
        yy_ones = tf.expand_dims(yy_ones, 1)

        yy_range = tf.tile(tf.expand_dims(tf.range(self.y_dim), 0), [batch_size_tensor, 1])
        yy_range = tf.expand_dims(yy_range, -1)

        yy_channel = tf.matmul(yy_range, yy_ones)
        yy_channel = tf.expand_dims(yy_channel, -1)
        xx_channel = tf.cast(xx_channel, 'float32') / (self.x_dim - 1)
        yy_channel = tf.cast(yy_channel, 'float32') / (self.y_dim - 1)
        xx_channel = xx_channel*2 - 1
        yy_channel = yy_channel*2 - 1

        ret = tf.concat([input_tensor, xx_channel, yy_channel], axis=-1)

        if self.with_r:
            rr = tf.sqrt( tf.square(xx_channel-0.5) + tf.square(yy_channel-0.5))
            ret = tf.concat([ret, rr], axis=-1)
        return ret

class CoordConv_2(base.Layer):
    """CoordConv layer as in the paper."""
    def __init__(self, x_dim, y_dim, with_r):
        super(CoordConv_2, self).__init__()
        self.addcoords = AddCoords(x_dim=x_dim, y_dim=y_dim, with_r=with_r)

    def call(self, input_tensor, filter, strides, padding):
        ret = self.addcoords(input_tensor)
        ret = tf.nn.conv2d(ret, filter, strides=strides, padding=padding)
        return ret

# MyConv = CoordConv_2(96, 96, False)
def conv_first_block(img, n_feature,layer_name, conv_weight_list):
	'''
	input : img[batch, h, w, chs]
	output: img[batch, h, w, chs * 2]
	'''
	strides = [1,1,1,1]
	_, h, w, chs = img.shape
	n_feature = chs * 10
	filter_1 = tf.get_variable("weight_1" + layer_name,
								shape=[ksize, ksize, chs + 2, n_feature],
								initializer=tf.contrib.layers.xavier_initializer_conv2d(),
								trainable=True)
	conv_bias_1 = tf.Variable(tf.zeros([n_feature], dtype=TF_DATA_TYPE),
							  name='bias_1' + layer_name,
							  trainable=True)
	filter_2 = tf.get_variable("weight_2" + layer_name,
								shape=[ksize, ksize, n_feature + 2, n_feature],
								initializer=tf.contrib.layers.xavier_initializer_conv2d(),
								trainable=True)
	conv_bias_2 = tf.Variable(tf.zeros([n_feature], dtype=TF_DATA_TYPE),
							  name='bias_2' + layer_name,
							  trainable=True)
	filter_3 = tf.get_variable("weight_3" + layer_name,
								shape=[ksize, ksize, n_feature + 2, n_feature],
								initializer=tf.contrib.layers.xavier_initializer_conv2d(),
								trainable=True)
	conv_bias_3 = tf.Variable(tf.zeros([n_feature], dtype=TF_DATA_TYPE),
							  name='bias_3' + layer_name,
							  trainable=True)

	MyConv = CoordConv_2(96, 96, False)
	img = MyConv(img, filter_1, strides, 'SAME')
	img = tf.nn.bias_add(img, conv_bias_1)
	img = tf.nn.relu(img)
	img = MyConv(img, filter_2, strides=strides, padding='SAME')
	img = tf.nn.bias_add(img, conv_bias_2)
	img = tf.nn.relu(img)
	img = MyConv(img, filter_3, strides = strides, padding='SAME')
	img = tf.nn.bias_add(img, conv_bias_3)
	img = tf.nn.relu(img)

	# result = tf.concat([origin, img],3)
	conv_weight_list.append(filter_1)
	conv_weight_list.append(filter_2)
	conv_weight_list.append(filter_3)
	return img

# input = tf.Variable(tf.random_normal([10,96,96,2]))
# result = Residual(input, 30, 'hahah')
# print(result.shape)

def conv_second_block(img, n_feature, layer_name, layer_seq, conv_weight_list):
	_, h, w, chs = img.shape
	n_feature = chs
	strides = [1,1,1,1]
	filter_1 = tf.get_variable("weight_1" + layer_name + layer_seq,
								shape=[ksize, ksize, n_feature + 2, n_feature],
								initializer=tf.contrib.layers.xavier_initializer_conv2d(),
								trainable=True)
	conv_bias_1 = tf.Variable(tf.zeros([n_feature], dtype=TF_DATA_TYPE),
							  name='bias_1' + layer_name + layer_seq,
							  trainable=True)
	filter_2 = tf.get_variable("weight_2" + layer_name + layer_seq,
								shape=[ksize, ksize, n_feature + 2, n_feature],
								initializer=tf.contrib.layers.xavier_initializer_conv2d(),
								trainable=True)
	conv_bias_2 = tf.Variable(tf.zeros([n_feature], dtype=TF_DATA_TYPE),
							  name='bias_2' + layer_name + layer_seq,
							  trainable=True)
	# batch_norm = tf.layers.batch_normalization(
	MyConv = CoordConv_2(96, 96, False)
	img = MyConv(img, filter_1, strides = strides, padding='SAME')
	img = tf.nn.bias_add(img, conv_bias_1)
	img = tf.nn.relu(img)
	img = MyConv(img, filter_2, strides = strides, padding='SAME')
	img = tf.nn.bias_add(img, conv_bias_2)
	img = tf.layers.batch_normalization(img, training=True)
	img = tf.nn.relu(img)

	conv_weight_list.append(filter_1)
	conv_weight_list.append(filter_2)

	return img

# input = tf.Variable(tf.random_normal([10,96,96,80]))
# result = conv_second_block(input, 20, 'hhh', '1')
# print(result.shape)

def conv_final_block(img, n_feature, layer_name, layer_seq, conv_weight_list):
	_, h, w, chs = img.shape
	n_feature = chs
	strides = [1,1,1,1]
	filter_1 = tf.get_variable("weight_1" + layer_name + layer_seq,
								shape=[ksize, ksize, chs + 2, n_feature],
								initializer=tf.contrib.layers.xavier_initializer_conv2d(),
								trainable=True)
	conv_bias_1 = tf.Variable(tf.zeros([n_feature], dtype=TF_DATA_TYPE),
							  name='bias_1' + layer_name + layer_seq,
							  trainable=True)
	filter_2 = tf.get_variable("weight_2" + layer_name + layer_seq,
								shape=[ksize, ksize, n_feature + 2, out_channel],
								initializer=tf.contrib.layers.xavier_initializer_conv2d(),
								trainable=True)
	conv_bias_2 = tf.Variable(tf.zeros([out_channel], dtype=TF_DATA_TYPE),
							  name='bias_2' + layer_name + layer_seq,
							  trainable=True)
	# batch_norm = tf.layers.batch_normalization(
	MyConv = CoordConv_2(96, 96, False)
	img = MyConv(img, filter_1, strides = strides, padding='SAME')
	img = tf.nn.bias_add(img, conv_bias_1)
	img = tf.nn.relu(img)
	img = MyConv(img, filter_2, strides = strides, padding='SAME')
	img = tf.nn.bias_add(img, conv_bias_2)

	conv_weight_list.append(filter_1)
	conv_weight_list.append(filter_2)

	return img

# input = tf.Variable(tf.random_normal([10,96,96,80]))
# result = conv_final_block(input, 20, 'hhh', '1')
# print(result.shape)
def recon(hidden_layer_num):
	placeholder_shape = [64,96,96,2]
	conv_weight_list = []

	img_origin_horizontal = tf.placeholder(TF_DATA_TYPE, name='data',shape=placeholder_shape)
	img_origin_left_diagonal = tf.placeholder(TF_DATA_TYPE, name='data',shape=placeholder_shape)
	img_origin_vertical = tf.placeholder(TF_DATA_TYPE, name='data',shape=placeholder_shape)
	img_origin_right_diagonal = tf.placeholder(TF_DATA_TYPE, name='data',shape=placeholder_shape)
	img_target = tf.placeholder(TF_DATA_TYPE, name='data')

	response_horizontal = conv_first_block(img_origin_horizontal, 20,'horizontal', conv_weight_list)
	response_left_diagonal = conv_first_block(img_origin_left_diagonal, 20,'left_diagonal', conv_weight_list)
	response_vertical = conv_first_block(img_origin_vertical, 20,'vertical', conv_weight_list)
	response_right_diagonal = conv_first_block(img_origin_right_diagonal, 20,'right_diagonal', conv_weight_list)

	layer_name = 'mix_responses'
	response = tf.concat([response_horizontal, response_left_diagonal, response_vertical, response_right_diagonal], 3)
	for i in range(hidden_layer_num):
		layer_seq = str(i)
		response = conv_second_block(response, 80, layer_name, layer_seq, conv_weight_list)
	response = conv_final_block(response, 80, 'final_responses', '0', conv_weight_list)

	x_data_predicted_node = response
	# return response
	data_loss = tf.reduce_mean(tf.square(x_data_predicted_node - img_target), name='training_error')
	training_error = data_loss

	# add weight loss
	weight_lambda = TF_WEIGHT_DECAY_LAMBDA
	weight_dacay_term\
		= weight_lambda * tf.add_n([tf.nn.l2_loss(v) for v in conv_weight_list])
	weight_dacay_term /= len(conv_weight_list)
	training_error += weight_dacay_term

	optimizer = tf.train.AdamOptimizer(learning_rate=1e-04).minimize(training_error)
	model = {'img_origin_horizontal':img_origin_horizontal,
			 'img_origin_left_diagonal':img_origin_left_diagonal,
			 'img_origin_vertical':img_origin_vertical,
			 'img_origin_right_diagonal':img_origin_right_diagonal,
			 'img_target':img_target,
			 'x_data_predicted_node':x_data_predicted_node,
			 'optimizer':optimizer,
			 'conv_weight_list':conv_weight_list,
			 'data_loss':data_loss,
			 }
	return model

# input = tf.Variable(tf.random_normal([10,96,96,20]))
# result = recon(7, input, input, input, input)
# print(result.shape)


from django.conf.urls.defaults import *
from django.conf import settings

# Uncomment the next two lines to enable the admin:
# from django.contrib import admin
# admin.autodiscover()

urlpatterns = patterns('',
	(r'^static_media/(?P<path>.*)$', 'django.views.static.serve',
        {'document_root': settings.STATIC_ROOT}),
	(r'shapes', 'main.views.shapes'),
	(r'landing', 'main.views.landing'),
	(r'trips', 'main.views.trips'),
	(r'shape', 'main.views.shape'),
	(r'', 'main.views.trips'),
    # Uncomment the admin/doc line below to enable admin documentation:
    # (r'^admin/doc/', include('django.contrib.admindocs.urls')),
    # Uncomment the next line to enable the admin:
    # (r'^admin/', include(admin.site.urls)),
)


import csv
import sys
import seaborn as sns
from sklearn.metrics import confusion_matrix, precision_score, recall_score, accuracy_score, f1_score

"""
Plots confusion matrix and prints some metrics computed on input report file (command line arg).
Report file is a csv file with named columns. Names are: ["link", "classificator", "moderator"], order is arbitrary.
link,classificator,moderator
http://habrahabr.ru/post/200376/,False,False
.....
"""

def to_bool(s):
    if s == "True":
        return True
    else:
        return False

input_path = sys.argv[1]
with open(input_path) as csvreport:
    reportreader = csv.DictReader(csvreport, delimiter=",")
    y_true = []
    y_pred = []
    for row in reportreader:
        y_true.append(to_bool(row["moderator"]))
        y_pred.append(to_bool(row["classificator"]))
print(len(y_true), sum(y_true))

conf = confusion_matrix(y_true, y_pred, labels=[True, False])
print("Precision:", precision_score(y_true, y_pred))
print("Recall:", recall_score(y_true, y_pred))
print("Accuracy:", accuracy_score(y_true, y_pred))
print("F1 score:", f1_score(y_true, y_pred))
sns.heatmap(conf, fmt="d", cbar=False, annot=True, xticklabels=["Predicted: True", "Predicted: False"], yticklabels=["Real: True", "Real: False"])
sns.plt.show()


from unittest import mock
from unittest.mock import sentinel

import pytest

from h.services.annotation_json_presentation import (
    annotation_json_presentation_service_factory,
)

class TestAnnotationJSONPresentationServiceFactory:
    def test_it(
        self,
        pyramid_request,
        AnnotationJSONPresentationService,
        flag_service,
        links_service,
        user_service,
    ):
        service = annotation_json_presentation_service_factory(
            sentinel.context, pyramid_request
        )

        assert service == AnnotationJSONPresentationService.return_value

        AnnotationJSONPresentationService.assert_called_once_with(
            session=pyramid_request.db,
            user=pyramid_request.user,
            has_permission=pyramid_request.has_permission,
            links_svc=links_service,
            flag_svc=flag_service,
            user_svc=user_service,
        )

    @pytest.fixture
    def AnnotationJSONPresentationService(self, patch):
        return patch(
            "h.services.annotation_json_presentation.factory.AnnotationJSONPresentationService"
        )

    @pytest.fixture
    def pyramid_request(self, pyramid_request):
        pyramid_request.user = mock.Mock()
        return pyramid_request


#
#   Copyright 2016 RIFT.IO Inc
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
import gi
gi.require_version('rwpython_util', '1.0')

from gi.repository import (
    GObject,
    Peas,
    rwpython_util,
    RwTypes)

class Plugin(GObject.Object, rwpython_util.Api):
    def __init__(self):
        GObject.Object.__init__(self)
        self._data_store = {}

    def do_python_run_string(self, python_util):
        status = RwTypes.RwStatus.SUCCESS
        try:
            exec(python_util, self._data_store)
        except Exception:
            status = RwTypes.RwStatus.FAILURE
        return status

    def do_python_eval_integer(self, python_variable):
        try:
            ret = eval(python_variable, self._data_store)
        except NameError:
            return (RwTypes.RwStatus.FAILURE, 0)
        return (RwTypes.RwStatus.SUCCESS, ret)

    def do_python_eval_string(self, python_variable):
        try:
            ret = eval(python_variable, self._data_store)
        except NameError:
            return (RwTypes.RwStatus.FAILURE, '')
        return (RwTypes.RwStatus.SUCCESS, ret)

# vim: et sw=4


#!/usr/bin/python3
"""This is the state class"""
from models.base_model import BaseModel, Base
from sqlalchemy import Column, Integer, String, DateTime, ForeignKey
from sqlalchemy.orm import relationship
from models.city import City
from os import getenv
import models

class State(BaseModel, Base):
    """This is the class for State
    Attributes:
    __tablename__: the name of the table created in the database
        name: input name
    """
    __tablename__ = 'states'

    if getenv('HBNB_TYPE_STORAGE') == 'db':
        name = Column(String(128), nullable=False)
        cities = relationship("City", cascade="all, delete-orphan",
                              backref="state")

    else:
        name = ""
    # for FileStorage

    @property
    def cities(self):
        """"
        Returns the list of City instances with state_id equal to the
        current State.id
        """
        ls = []
        for value in models.storage.all("City").values():
            if value.state_id == self.id:
                ls.append(value)
                return ls


import pygame
from Path import base_path

class Plant(object):

    # Constructor. Pass in the color of the block,
    # and its x and y position
    def __init__(self, position):
        # Call the parent class (Sprite) constructor
        pygame.sprite.Sprite.__init__(self)
        self.width = 50
        self.height = 50
        self.x_offset = 10
        self.y_offset = 0
        self.dry = pygame.transform.scale(pygame.image.load(base_path + "Assets/images/plant_dry.png"), (20, 30))
        self.wet = pygame.transform.scale(pygame.image.load(base_path + "Assets/images/plant_wet.png"), (20, 30))
        self.image = self.dry
        self.color = (0, 100, 0)
        # Create an image of the block, and fill it with a color.
        # This could also be an image loaded from the disk.
        self.watered = False
        self.position = position
        # Fetch the rectangle object that has the dimensions of the image
        # Update the position of this object by setting the values of rect.x and rect.y
        self.rect = pygame.Rect(position[0]-self.x_offset, position[1]+self.y_offset, self.width, self.height)
        #self.success_noise = pygame.mixer.Sound()

    def draw_rect(self, game_display):
        pygame.draw.rect(game_display, self.color, (self.position[0]-self.x_offset, self.position[1]+self.y_offset, self.width, self.height), 0 )
    def water(self):
        self.watered = True
        self.image = self.wet


'''
Created on 2017415

@author: zhou
'''

from threading import Thread
from flask import current_app, render_template
from flask_mail import Message
from . import mail
import sendgrid
from sendgrid.helpers.mail import *
import os

def send_async_email(app, msg):
    with app.app_context():
        mail.send(msg)

def send_email(to, subject, template, **kwargs):
    app = current_app._get_current_object()
    msg = Message(app.config['FLASKY_MAIL_SUBJECT_PREFIX'] + ' ' + subject,
                  sender=app.config['FLASKY_MAIL_SENDER'], recipients=[to])
    msg.body = render_template(template + '.txt', **kwargs)
    msg.html = render_template(template + '.html', **kwargs)
    thr = Thread(target=send_async_email, args=[app, msg])
    thr.start()
    return thr

def send_mail(to , subject, template, **kwargs):
    app = current_app._get_current_object()
    sg = sendgrid.SendGridAPIClient(apikey='Your sendgrid Key')
    msg = Message(app.config['FLASKY_MAIL_SUBJECT_PREFIX'] + ' ' + subject,
                  sender=app.config['FLASKY_MAIL_SENDER'], recipients=[to])
    from_email = Email(app.config['FLASKY_MAIL_SENDER'])
    subject = subject
    to_email = Email(to)
    content = Content("text/plain", render_template(template + '.txt', **kwargs))
    mail = Mail(from_email, subject, to_email, content)
    sg.client.mail.send.post(request_body=mail.get())


# Simple CNN model for CIFAR-10
# Scott McElfresh sme1d1 4/12/2021

import numpy
from keras.datasets import cifar10
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import Flatten
from keras.constraints import maxnorm
from keras.optimizers import SGD
from keras.layers.convolutional import Conv2D
from keras.layers.convolutional import MaxPooling2D
from keras.utils import np_utils
from keras import backend as K

import matplotlib.pyplot as plt
import numpy as np

K.set_image_data_format('channels_last')

# fix random seed for reproducibility
seed = 7
numpy.random.seed(seed)
# load data
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
# normalize inputs from 0-255 to 0.0-1.0
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train = X_train / 255.0
X_test = X_test / 255.0
# one hot encode outputs
y_train = np_utils.to_categorical(y_train)
y_test = np_utils.to_categorical(y_test)
num_classes = y_test.shape[1]
#print("Num Classes " + str(num_classes))
# Create the model
model = Sequential()
model.add(
    Conv2D(32, (3, 3), input_shape=X_train.shape[1:], padding='same', activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.2))

model.add(Conv2D(32, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3)))
model.add(MaxPooling2D(pool_size=(2, 2), data_format='channels_last'))

model.add(Conv2D(64, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.2))

model.add(Conv2D(64, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3)))
model.add(MaxPooling2D(pool_size=(2, 2), data_format='channels_last'))

model.add(Conv2D(128, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.2))

model.add(Conv2D(128, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3)))
model.add(MaxPooling2D(pool_size=(2, 2), data_format='channels_last'))

model.add(Flatten())
model.add(Dropout(0.2))

model.add(Dense(1024, activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.2))

model.add(Dense(512, activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.2))

model.add(Dense(num_classes, activation='softmax'))
# Compile model
epochs = 20
lrate = 0.01
decay = lrate / epochs
sgd = SGD(lr=lrate, momentum=0.9, decay=decay, nesterov=False)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
print(model.summary())
# Fit the model
history = model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=epochs, batch_size=32, )
model.save('cifar10_model.h5')
# Final evaluation of the model
scores = model.evaluate(X_test, y_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[1] * 100))

print("Categories: 0: airplane, 1: automobile, 2: bird, 3: cat, 4: deer, 5: dog, 6: frog, 7: horse, 8: ship, 9: truck")

result = np.argmax(model.predict(X_test), axis=-1)  # create array of predicted categories

labelsAndTrainingImages = list(zip(X_test, y_test))
# plot images and their labels
for index, (image, label) in enumerate(labelsAndTrainingImages[:6]):
    plt.subplot(2, 3, index + 1)
    plt.imshow(image, interpolation='nearest')
    plt.title("Label " + '%i' % result[index])

# Plot accuracy
plot2 = plt.figure(2)
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='lower right')

# "Loss Plot"
plot3 = plt.figure(3)
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='upper left')
plt.show()


import json

from django.conf.urls import re_path
from django.contrib.postgres.fields.jsonb import KeyTextTransform
from django.db.models import CharField, IntegerField, JSONField
from django.db.models.expressions import F
from django.db.models.functions import Cast
from django.urls import reverse
from pytimeparse.timeparse import timeparse
from tastypie import fields
from tastypie.exceptions import BadRequest
from tastypie.utils import trailing_slash

from clist.api.common import BaseModelResource as CommmonBaseModuelResource
from clist.api.paginator import EstimatedCountPaginator
from clist.models import Contest, Resource
from ranking.models import Account, Statistics
from true_coders.models import Coder, Filter

class BaseModelResource(CommmonBaseModuelResource):

    class Meta(CommmonBaseModuelResource.Meta):
        paginator_class = EstimatedCountPaginator

    def build_filters(self, filters=None, *args, **kwargs):
        filters = filters or {}
        filters.pop('total_count', None)
        return super().build_filters(filters, *args, **kwargs)

    def dehydrate(self, *args, **kwargs):
        bundle = super().dehydrate(*args, **kwargs)
        bundle.data.pop('total_count', None)
        return bundle

class ResourceResource(BaseModelResource):
    name = fields.CharField('host')
    icon = fields.CharField('icon')
    total_count = fields.BooleanField()

    class Meta(BaseModelResource.Meta):
        abstract = False
        queryset = Resource.objects.all()
        resource_name = 'resource'
        excludes = ('total_count', )
        filtering = {
            'total_count': ['exact'],
            'id': ['exact', 'in'],
            'name': ['exact', 'iregex', 'regex', 'in'],
        }
        ordering = ['id', 'name', ]

    def dehydrate(self, *args, **kwargs):
        bundle = super().dehydrate(*args, **kwargs)
        bundle.data['icon'] = '/imagefit/static_resize/64x64/' + bundle.data['icon']
        return bundle

class ContestResource(BaseModelResource):
    resource = fields.CharField('resource__host',
                                help_text='Unicode string data. Use comma to filter multiple resources')
    resource_id = fields.IntegerField('resource_id')
    host = fields.CharField('host')
    event = fields.CharField('title')
    start = fields.DateTimeField('start_time')
    end = fields.DateTimeField('end_time')
    duration = fields.DateTimeField('duration_in_secs', help_text='Time delta: Ex: "864000" or "10 days"')
    href = fields.CharField('url')
    filtered = fields.BooleanField(help_text='Use user filters')
    category = fields.CharField(help_text=f'Category to filter (default: api, allowed {Filter.CATEGORIES})')
    problems = fields.CharField('problems', null=True, help_text='Dict or List data')
    with_problems = fields.BooleanField()
    total_count = fields.BooleanField()

    class Meta(BaseModelResource.Meta):
        abstract = False
        queryset = Contest.visible.all()
        resource_name = 'contest'
        excludes = ('filtered', 'category', 'total_count', 'with_problems')
        filtering = {
            'total_count': ['exact'],
            'with_problems': ['exact'],
            'id': ['exact', 'in'],
            'resource_id': ['exact', 'in'],
            'resource': ['exact'],
            'host': ['exact', 'iregex', 'regex'],
            'event': ['exact', 'iregex', 'regex'],
            'start': ['exact', 'gt', 'lt', 'gte', 'lte', 'week_day'],
            'end': ['exact', 'gt', 'lt', 'gte', 'lte', 'week_day'],
            'duration': ['exact', 'gt', 'lt', 'gte', 'lte', 'week_day'],
            'filtered': ['exact'],
            'category': ['exact'],
        }
        ordering = ['id', 'event', 'start', 'end', 'resource_id', 'duration']

    def dehydrate(self, *args, **kwargs):
        bundle = super().dehydrate(*args, **kwargs)
        bundle.data.pop('filtered')
        bundle.data.pop('category')

        bundle.data.pop('with_problems', None)
        problems = bundle.data.pop('problems')
        if problems:
            problems = json.loads(problems)
        bundle.data['problems'] = problems

        return bundle

    def build_filters(self, filters=None, *args, **kwargs):
        filters = filters or {}
        filtered = filters.pop('filtered', None)
        category = filters.pop('category', 'api')
        resource = filters.pop('resource', None)
        with_problems = filters.pop('with_problems', None)
        filters = super().build_filters(filters, *args, **kwargs)
        if filtered is not None:
            filters['filtered'] = filtered[-1]
            filters['category'] = category[-1]
        if resource:
            filters['resource__host__in'] = ','.join(resource).split(',')
        if with_problems:
            filters['with_problems'] = with_problems
        return filters

    def apply_filters(self, request, applicable_filters):
        filtered = None
        category = 'api'
        for f in list(applicable_filters.keys()):
            if f.startswith('duration'):
                v = applicable_filters.pop(f)
                v = int(v) if v.isdigit() else timeparse(v)
                applicable_filters[f] = v
            elif f == 'filtered':
                filtered = applicable_filters.pop(f).lower() in ['true', 'yes', '1']
            elif f == 'category':
                category = applicable_filters.pop(f)
        with_problems = applicable_filters.pop('with_problems', None)

        qs = super().apply_filters(request, applicable_filters)
        qs = qs.select_related('resource')

        if filtered is not None and request.user:
            filter_ = request.user.coder.get_contest_filter([category])
            if not filtered:
                filter_ = ~filter_
            qs = qs.filter(filter_)

        if with_problems and with_problems[0].lower() in ['yes', 'true', '1']:
            qs = qs.annotate(problems=Cast(KeyTextTransform('problems', 'info'), CharField()))

        return qs

class StatisticsResource(BaseModelResource):
    account_id = fields.IntegerField('account_id')
    handle = fields.CharField('account__key')
    contest_id = fields.IntegerField('contest_id')
    event = fields.CharField('contest__title')
    date = fields.DateTimeField('contest__end_time')
    coder_id = fields.IntegerField()
    place = fields.IntegerField('place_as_int', null=True)
    score = fields.FloatField('solving')
    new_rating = fields.IntegerField('new_rating', null=True)
    old_rating = fields.IntegerField('old_rating', null=True)
    rating_change = fields.IntegerField('rating_change', null=True)
    problems = fields.DictField('problems', null=True)
    more_fields = fields.DictField('more_fields', null=True)
    with_problems = fields.BooleanField()
    with_more_fields = fields.BooleanField()
    total_count = fields.BooleanField()

    class Meta(BaseModelResource.Meta):
        abstract = False
        queryset = Statistics.objects.all()
        resource_name = 'statistics'
        excludes = ('total_count', 'with_problems', 'with_more_fields', 'coder_id')
        filtering = {
            'total_count': ['exact'],
            'with_problems': ['exact'],
            'with_more_fields': ['exact'],
            'contest_id': ['exact', 'in'],
            'account_id': ['exact', 'in'],
            'coder_id': ['exact'],
            'place': ['exact', 'isnull'],
            'new_rating': ['isnull'],
            'rating_change': ['isnull'],
        }
        ordering = ['score', 'place', 'new_rating', 'rating_change', 'date']
        detail_allowed_methods = []

    def build_filters(self, filters=None, *args, **kwargs):
        filters = filters or {}
        tmp = {}
        for k in 'new_rating__isnull', 'rating_change__isnull', 'coder_id', 'with_problems', 'with_more_fields':
            tmp[k] = filters.pop(k, None)
        filters = super().build_filters(filters, *args, **kwargs)
        filters.update(tmp)
        return filters

    def apply_filters(self, request, applicable_filters):
        one_of = ['contest_id__exact', 'contest_id__in', 'account_id__exact', 'account_id__in', 'coder_id']
        for k in one_of:
            if applicable_filters.get(f'{k}'):
                break
        else:
            raise BadRequest(f'One of {[k.split("__")[0] for k in one_of]} is required')

        rating_change_isnull = applicable_filters.pop('rating_change__isnull', None)
        new_rating_isnull = applicable_filters.pop('new_rating__isnull', None)
        coder_id = applicable_filters.pop('coder_id', None)
        with_problems = applicable_filters.pop('with_problems', None)
        with_more_fields = applicable_filters.pop('with_more_fields', None)

        qs = super().apply_filters(request, applicable_filters)
        qs = qs.select_related('account', 'contest')

        if rating_change_isnull:
            qs = qs.filter(addition__rating_change__isnull=rating_change_isnull[0] in ['true', '1', 'yes'])
        if new_rating_isnull:
            qs = qs.filter(addition__new_rating__isnull=new_rating_isnull[0] in ['true', '1', 'yes'])
        if coder_id:
            qs = qs.filter(account__coders=coder_id[0])

        qs = qs \
            .annotate(new_rating=Cast(KeyTextTransform('new_rating', 'addition'), IntegerField())) \
            .annotate(old_rating=Cast(KeyTextTransform('old_rating', 'addition'), IntegerField())) \
            .annotate(rating_change=Cast(KeyTextTransform('rating_change', 'addition'), IntegerField()))

        if with_problems and with_problems[0].lower() in ['yes', 'true', '1']:
            qs = qs.annotate(problems=Cast(KeyTextTransform('problems', 'addition'), JSONField()))

        if with_more_fields:
            qs = qs.annotate(more_fields=Cast(F('addition'), JSONField()))

        return qs

    def dehydrate(self, *args, **kwargs):
        bundle = super().dehydrate(*args, **kwargs)
        bundle.data.pop('coder_id', None)
        bundle.data.pop('with_problems', None)
        bundle.data.pop('with_more_fields', None)

        problems = bundle.data['problems']
        if problems:
            for problem in problems.values():
                for k in list(problem.keys()):
                    if k.startswith('_'):
                        problem.pop(k, None)
                for k in 'solution', 'external_solution':
                    problem.pop(k, None)

        more_fields = bundle.data['more_fields']
        if more_fields:
            for k in list(more_fields.keys()):
                if k.startswith('_') or k in bundle.data:
                    more_fields.pop(k, None)
            for k in 'problems', 'solved':
                more_fields.pop(k, None)
        return bundle

class AccountResource(BaseModelResource):
    resource = fields.CharField('resource__host')
    resource_id = fields.IntegerField('resource_id')
    handle = fields.CharField('key')
    name = fields.CharField('name', null=True)
    rating = fields.IntegerField('rating', null=True)
    n_contests = fields.IntegerField('n_contests')
    total_count = fields.BooleanField()

    class Meta(BaseModelResource.Meta):
        abstract = False
        queryset = Account.objects.all()
        resource_name = 'account'
        excludes = ('total_count', )
        filtering = {
            'total_count': ['exact'],
            'id': ['exact', 'in'],
            'resource_id': ['exact', 'in'],
            'resource': ['exact'],
            'handle': ['exact', 'iregex', 'regex'],
            'rating': ['exact', 'gt', 'lt', 'gte', 'lte', 'isnull'],
        }
        ordering = ['id', 'handle', 'rating', 'n_contests']

    def build_filters(self, filters=None, *args, **kwargs):
        filters = filters or {}
        resource = filters.pop('resource', None)
        filters = super().build_filters(filters, *args, **kwargs)
        if resource:
            filters['resource__host'] = resource[-1]
        return filters

    def apply_filters(self, request, applicable_filters):
        qs = super().apply_filters(request, applicable_filters)
        qs = qs.select_related('resource')
        return qs

def use_in_me_only(bundle, *args, **kwargs):
    url = reverse('clist:api:v2:api_dispatch_me', kwargs={'api_name': 'v2', 'resource_name': 'coder'})
    return url == bundle.request.path

def use_in_detail_only(bundle, *args, **kwargs):
    with_accounts = bundle.request.GET.get('with_accounts') in ['true', '1', 'yes']
    query_by_id = bool(bundle.request.GET.get('id') or bundle.request.GET.get('username'))
    url = reverse('clist:api:v2:api_dispatch_list', kwargs={'api_name': 'v2', 'resource_name': 'coder'})
    return with_accounts or query_by_id or not (url == bundle.request.path or use_in_me_only(bundle, *args, **kwargs))

class CoderResource(BaseModelResource):
    username = fields.CharField('username')
    first_name = fields.CharField('user__first_name')
    last_name = fields.CharField('user__last_name')
    country = fields.CharField('country')
    timezone = fields.CharField('timezone', use_in=use_in_me_only)
    email = fields.CharField('user__email', use_in=use_in_me_only)
    n_accounts = fields.IntegerField('n_accounts', use_in='list')
    accounts = fields.ManyToManyField(AccountResource, 'account_set', use_in=use_in_detail_only, full=True)
    with_accounts = fields.BooleanField()
    total_count = fields.BooleanField()

    class Meta(BaseModelResource.Meta):
        abstract = False
        object_class = Coder
        queryset = Coder.objects.all()
        resource_name = 'coder'
        excludes = ('total_count', 'with_accounts')
        filtering = {
            'total_count': ['exact'],
            'with_accounts': ['exact'],
            'country': ['exact'],
            'id': ['exact', 'in'],
            'username': ['exact', 'iregex', 'regex', 'in'],
        }
        extra_actions = [
            {
                'name': 'me',
                'summary': 'Retrieve your coder',
                'resource_type': 'list',
                'responseClass': 'coder',
            }
        ]
        ordering = ['n_accounts']

    def me(self, request, *args, **kwargs):
        kwargs['me'] = True
        return self.dispatch('detail', request, **kwargs)

    def prepend_urls(self):
        return [
            re_path(
                r'^(?P<resource_name>%s)/me%s$' % (self._meta.resource_name, trailing_slash),
                self.wrap_view('me'),
                name='api_dispatch_me'
            )
        ]

    def dehydrate(self, *args, **kwargs):
        bundle = super().dehydrate(*args, **kwargs)
        for k in self.fields.keys():
            if k in bundle.data and not bundle.data[k] and isinstance(bundle.data[k], str):
                bundle.data[k] = None
        bundle.data.pop('with_accounts', None)
        return bundle

    def build_filters(self, filters=None, *args, **kwargs):
        filters = filters or {}
        me = filters.pop('me', None)
        filters.pop('with_accounts', None)
        filters = super().build_filters(filters, *args, **kwargs)
        filters['me'] = me
        return filters

    def apply_filters(self, request, applicable_filters):
        me = applicable_filters.pop('me', None)
        qs = super().apply_filters(request, applicable_filters)
        if me:
            qs = qs.filter(pk=request.user.coder.pk)
        qs = qs.select_related('user')

        fake_bundle = type('obj', (object,), {'request': request})
        if use_in_detail_only(fake_bundle):
            qs = qs.prefetch_related('account_set__resource')
        return qs