# System imports
import pandas as pd
import numpy as np
from datetime import datetime, timedelta

# Third-party imports
from pathlib import Path
import yfinance as yf

# Local imports
from portfolio.transactions import Trades
import datehandler

class Portfolio:
    def __init__(self):
        '''View portfolio data and returns
        '''
        # Dates and file paths
        self.today = datetime.today()
        self.save_dir = Path(__file__).parents[1] / 'data'
        self.portfolio_csv_name = f'portfolio_holdings.csv'
        self.portfolio_csv_path = self.save_dir / self.portfolio_csv_name

        # Initialise trade transaction data
        self.t = Trades()
        self.trades = self.t.all

        # Create portfolio from trades
        self.holdings = self.build()

    def build(self):
        '''Builds portfolio holdings from transaction data

        Returns:
            Dataframe -- (Date, Ticker): [Volume, Close]
        '''

        # Construct range of dates over portfolio period
        p_inception = self.trades.index[0][0]
        p_dates = datehandler.date_list(p_inception, self.today)

        # Create base dataframe
        df_base_dates = pd.DataFrame(index=p_dates)
        df_base_dates.index.name = 'Date'

        df_pfolio = pd.DataFrame()  # Initialise the main portfolio df as empty

        # Build list of tickers seen throughout investment period
        tickers = list(sorted(set(self.trades.reset_index().Ticker.to_list())))

        # Use yfinance to get close prices
        lookup_tickers = [f'{ticker}.AX' for ticker in tickers]  # Only supports ASX stocks
        lookup_tickers = ' '.join(lookup_tickers)

        prices = yf.download(lookup_tickers, start=p_dates[0], end=p_dates[-1])
        close = prices['Adj Close'] # Using Adj Close instead of Close to account for stocksplits, dividends
        close.columns = tickers

        # Step through transactions dataframe, ticker by ticker
        for ticker in tickers:
            vol_df = self.t.by_ticker(ticker)['Volume']  # Get volumes

            # Merge with base_dates_df and cumulative sum the volumes
            ticker_df = pd.merge_ordered(vol_df, df_base_dates, on='Date').set_index('Date')
            ticker_df = ticker_df.cumsum().ffill()
            ticker_df = ticker_df[ticker_df['Volume'].isna() == False]

            # Update volumes based on stocksplits
            ticker_df = self.stocksplits(ticker_df,ticker)

            # Merge with close prices
            ticker_df = pd.merge_ordered(ticker_df, close[ticker], on='Date').set_index('Date')
            ticker_df.rename({ticker: 'Close'}, axis=1, inplace=True)

            # Merge ticker_df with portfolio_df
            ticker_df['Ticker'] = ticker # create ticker column for later indexing
            df_pfolio = pd.concat([df_pfolio, ticker_df])

        df_pfolio = df_pfolio.reset_index().set_index(['Date','Ticker']).sort_index()  # Finalise portfolio
        df_pfolio = df_pfolio.assign(Value = lambda x: x['Volume'] * x['Close'])  # Calculate portfolio value
        print('\nPortfolio successfully built')

        return df_pfolio

    def stocksplits(self, df, ticker):
        '''Adjusts volumes according to historical stocksplits

        Arguments:
            df {DataFrame} -- Dataframe with date (index) and volume (series) data
            ticker {String} -- Ticker to explore. Only ASX securities supported

        Returns:
            DataFrame -- Returns dataframe, but with volume scaled by the cumulative stocksplits
        '''
        # Terminal output
        print(f'\rChecking {ticker}', end='', flush=True)

        try:  # Dividends and stock splits
            stock = yf.Ticker(f'{ticker}.AX')
            df_actions = stock.actions
        except TypeError:
            print(f'\rSomething went wrong!',flush=True)
            return df

        # Add stocksplits column
        if len(df_actions[df_actions['Stock Splits'] > 0]) > 0:
            df['Stock Splits'] = df_actions['Stock Splits'].replace(0,np.nan)

            # Set all dates with 0 holdings to have np.nan holdings
            df['Volume'] = df['Volume'].replace(0,np.nan)

            stocksplit_cumulative = df.loc[df['Volume'].first_valid_index():]['Stock Splits'].cumprod()
            stocksplit_cumulative = stocksplit_cumulative.fillna(method='ffill').fillna(1)

            # Update holding volume
            df['Volume'] *= stocksplit_cumulative
            df['Volume'] = np.ceil(df['Volume'])

            # Update vwap
            # df['Vwap'] /= stocksplit_cumulative

            df = df.drop(columns='Stock Splits')  # Finalise dataframe to return by removing excess columns

        return df

    def plot(self, view='default'):
        # Portfolio value
        value_df = self.holdings.groupby('Date').sum()['Value']
        value_df = value_df[value_df > 0]

        # The portfolio cost for equity
        cost_df = self.trades.assign(Value = lambda x: x['Volume'] * x['EffectivePrice'])
        cost_df = cost_df.groupby('Date').sum()['Value'].cumsum()

        plot_df = pd.merge_ordered(value_df, cost_df, on='Date', fill_method='ffill').set_index('Date')
        plot_df.columns = ['Value', 'Cost']
        plot_df = plot_df.assign(pl_line = lambda x: x['Value'] - x['Cost'])
        plot_df = plot_df.assign(pl_pct = lambda x: x['pl_line']/x['Cost'])

        if view == 'default':
            cols = ['Value','Cost']
        elif view == 'pl':
            cols = ['pl_line']
        elif view == 'pl_pct':
            cols = ['pl_pct']

        plot_df[cols].plot()

        return plot_df


import os
import warnings
import yaml

import mlflow
import shutil
import uuid
import mlflow.utils.environment
import mlflow.utils.model_utils

from mlflow.exceptions import MlflowException, BAD_REQUEST
from mlflow import pyfunc
from mlflow.models import Model
from mlflow.models.model import MLMODEL_FILE_NAME
from mlflow.models.signature import ModelSignature
from mlflow.models.utils import ModelInputExample
from mlflow.utils.docstring_utils import format_docstring, LOG_MODEL_PARAM_DOCS
from mlflow.utils.environment import (
    _validate_env_arguments,
)
from mlflow.utils.requirements_utils import _get_pinned_requirement

from mlflow.utils.model_utils import (
    _validate_and_prepare_target_save_path,
)
from mlflow.utils.file_utils import (
    write_to,
)

from h2o_mlflow_flavors.utils import unzip_specific_file
from h2o_mlflow_flavors.utils import match_file_from_name_pattern
from h2o_mlflow_flavors.utils import unzip_specific_folder
from h2o_mlflow_flavors.utils import zip_folder

import h2o_mlflow_flavors

FLAVOR_NAME = "h2o_driverless_ai"
MOJO_FILE = "mojo-pipeline/pipeline.mojo"
MOJO_ZIP_FILE_NAME = "mojo"
PY_SCORING_WHL_FILE_PATTERN = "scoring-pipeline/scoring_h2oai_experiment.*\.whl"
PY_SCORING_SUMMARY_FILE_PATTERN = "scoring-pipeline/h2oai_experiment_summary.*\.zip"
PY_SCORING_FILE_NAME = "scorer"
PY_SCORING_CUSTOM_RECIPES_FOLDER = "scoring-pipeline/tmp"

def save_model(
        h2o_dai_artifact_location,
        h2o_dai_model_directory,
        path,
        model_type,
        conda_env=None,
        mlflow_model=None,
        settings=None,
        signature: ModelSignature = None,
        input_example: ModelInputExample = None,
        pip_requirements=None,
        extra_pip_requirements=None,
):
    _validate_env_arguments(conda_env, pip_requirements, extra_pip_requirements)

    path = os.path.abspath(path)
    _validate_and_prepare_target_save_path(path)
    model_data_subpath = "model.h2o.dai"
    model_data_path = os.path.join(path, model_data_subpath)

    shutil.copytree(h2o_dai_model_directory, model_data_path + "/")
    # os.makedirs(model_data_path)

    if mlflow_model is None:
        mlflow_model = Model()
    if signature is not None:
        mlflow_model.signature = signature
    if input_example is not None:
        _save_example(mlflow_model, input_example, path)

    if settings is None:
        settings = {}
    settings["dai_source_file"] = h2o_dai_artifact_location
    settings["dai_model"] = _get_file_name(h2o_dai_model_directory)
    with open(os.path.join(model_data_path, "h2o_dai.yaml"), "w") as settings_file:
        yaml.safe_dump(settings, stream=settings_file)

    mlflow_model.add_flavor(
        FLAVOR_NAME,  type=model_type
    )

    mlflow_model.save(os.path.join(path, MLMODEL_FILE_NAME))

def _get_file_name(path):
    return os.path.basename(path)

def _get_dir_path(path):
    return os.path.dirname(path)

def log_model(h2o_dai_artifact_location,
            artifact_path,
            model_type="dai/mojo_pipeline",
            h2o_dai_model_download_location="/tmp/"+str(uuid.uuid1()),
            conda_env=None,
            registered_model_name=None,
            signature: ModelSignature = None,
            input_example: ModelInputExample = None,
            pip_requirements=None,
            extra_pip_requirements=None,
            **kwargs,
    ):

        valid_dai_model_types = ['dai/mojo_pipeline', 'dai/scoring_pipeline']
        if model_type not in valid_dai_model_types:
            raise MlflowException.invalid_parameter_value("Invalid value for model_type. Valid values are pipeline/mojo or scoring-pipeline")

        h2o_dai_model_directory = determine_model_file(model_type, h2o_dai_artifact_location, h2o_dai_model_download_location)

        return Model.log(
            artifact_path=artifact_path,
            flavor=h2o_mlflow_flavors.driverless,
            registered_model_name=registered_model_name,
            h2o_dai_artifact_location=h2o_dai_artifact_location,
            conda_env=conda_env,
            signature=signature,
            input_example=input_example,
            pip_requirements=pip_requirements,
            extra_pip_requirements=extra_pip_requirements,
            model_type = model_type,
            h2o_dai_model_directory=h2o_dai_model_directory,
            **kwargs,
        )

def determine_model_file(model_type, h2o_dai_model, h2o_dai_model_download_location):
    if model_type == 'dai/mojo_pipeline':
        return _minimise_mojo_scoring_model(h2o_dai_model, h2o_dai_model_download_location)
    elif model_type == 'dai/scoring_pipeline':
        return _minimise_python_scoring_pipeline_model(h2o_dai_model, h2o_dai_model_download_location)

def _minimise_mojo_scoring_model(h2o_dai_model, h2o_dai_model_download_location):
    if match_file_from_name_pattern(h2o_dai_model, MOJO_FILE) is None:
        raise MlflowException.invalid_parameter_value("Not a valid DAI MOJO Pipeline - pipeline.mojo not present in the provided model.")
    location = h2o_dai_model_download_location + "/"
    minimal_model_file_location = location + "model"
    unzip_specific_file(h2o_dai_model, MOJO_FILE, directory=minimal_model_file_location)
    return minimal_model_file_location

def _minimise_python_scoring_pipeline_model(h2o_dai_model, h2o_dai_model_download_location):
    location = h2o_dai_model_download_location + "/"
    minimal_model_file_location = location + "model"
    wheel_file = match_file_from_name_pattern(h2o_dai_model, PY_SCORING_WHL_FILE_PATTERN)
    summary_file = match_file_from_name_pattern(h2o_dai_model, PY_SCORING_SUMMARY_FILE_PATTERN)

    if wheel_file is None or summary_file is None:
        raise MlflowException.invalid_parameter_value("Not a valid DAI Scoring Pipeline - Experiment wheel or Summary file not present in the provided model")

    unzip_specific_file(h2o_dai_model, wheel_file, summary_file, directory=minimal_model_file_location)
    unzip_specific_folder(h2o_dai_model, PY_SCORING_CUSTOM_RECIPES_FOLDER, directory=minimal_model_file_location)
    return minimal_model_file_location

def load_model(model_uri, dst_path=None):
    raise MlflowException(
        message="Executing DAI models within MLflow is currently not supported",
        error_code=BAD_REQUEST)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import uuid

from tensorflow.core.protobuf import saver_pb2
from tensorflow.python.eager import def_function
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor_spec
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gen_io_ops
from tensorflow.python.ops import io_ops
from tensorflow.python.ops import string_ops
from tensorflow.python.training.saving import saveable_object
from tensorflow.python.training.saving import saveable_object_util
from tensorflow.python.training import training_util
from tensorflow.python.util import nest

class _SingleDeviceSaver(object):
  """Saves and restores checkpoints from the current device."""

  def __init__(self, saveable_objects):
    """Specify a list of `SaveableObject`s to save and restore.

    Args:
      saveable_objects: A list of `SaveableObject`s.
    """
    saveable_objects = list(saveable_objects)
    for saveable in saveable_objects:
      if not isinstance(saveable, saveable_object.SaveableObject):
        raise ValueError(
            "Expected a list of SaveableObjects, got %s." % (saveable,))
    self._saveable_objects = saveable_objects

  def save(self, file_prefix):
    """Save the saveable objects to a checkpoint with `file_prefix`.

    Args:
      file_prefix: A string or scalar string Tensor containing the prefix to
        save under.
    Returns:
      An `Operation`, or None when executing eagerly.
    """
    tensor_names = []
    tensors = []
    tensor_slices = []
    ev_key_types = []
    has_ev = False
    for saveable in self._saveable_objects:
      if isinstance(saveable, saveable_object_util.EmbeddingVariableSaveable):
        tensors.append(training_util.get_or_create_global_step())
        tensor_names.append("global_step")
        tensor_slices.append("")
        has_ev = True
        break

    for saveable in self._saveable_objects:
      if isinstance(saveable, saveable_object_util.EmbeddingVariableSaveable):
        tensor_names.append(saveable.name)
        tensors.append(saveable.handle_op)
        tensor_slices.append("")
        ev_key_types.append(saveable.key_type)
        continue
      for spec in saveable.specs:
        tensor_names.append(spec.name)
        tensors.append(spec.tensor)
        tensor_slices.append(spec.slice_spec)
    with ops.device("cpu:0"):
      return io_ops.save_v2(file_prefix, tensor_names, tensor_slices, tensors,
              ev_key_types, has_ev)

  def restore(self, file_prefix):
    """Restore the saveable objects from a checkpoint with `file_prefix`.

    Args:
      file_prefix: A string or scalar string Tensor containing the prefix for
        files to read from.

    Returns:
      A dictionary mapping from SaveableObject names to restore operations.
    """
    restore_specs = []
    tensor_structure = []
    for saveable in self._saveable_objects:
      saveable_tensor_structure = []
      tensor_structure.append(saveable_tensor_structure)
      for spec in saveable.specs:
        saveable_tensor_structure.append(spec.name)
        restore_specs.append((spec.name, spec.slice_spec, spec.dtype))
    tensor_names, tensor_slices, tensor_dtypes = zip(*restore_specs)
    with ops.device("cpu:0"):
      restored_tensors = io_ops.restore_v2(
          file_prefix, tensor_names, tensor_slices, tensor_dtypes)
    structured_restored_tensors = nest.pack_sequence_as(
        tensor_structure, restored_tensors)
    restore_ops = {}
    for saveable, restored_tensors in zip(self._saveable_objects,
                                          structured_restored_tensors):
      restore_ops[saveable.name] = saveable.restore(
          restored_tensors, restored_shapes=None)
    return restore_ops

def sharded_filename(filename_tensor, shard, num_shards):
  """Append sharding information to a filename.

  Args:
    filename_tensor: A string tensor.
    shard: Integer.  The shard for the filename.
    num_shards: An int Tensor for the number of shards.

  Returns:
    A string tensor.
  """
  return gen_io_ops.sharded_filename(filename_tensor, shard, num_shards)

class MultiDeviceSaver(object):
  """Saves checkpoints directly from multiple devices.

  Note that this is a low-level utility which stores Tensors in the keys
  specified by `SaveableObject`s. Higher-level utilities for object-based
  checkpointing are built on top of it.
  """

  def __init__(self, saveable_objects):
    """Specify a list of `SaveableObject`s to save and restore.

    Args:
      saveable_objects: A list of `SaveableObject`s.
    """
    saveable_objects = list(saveable_objects)
    saveables_by_device = {}
    for saveable in saveable_objects:
      if not isinstance(saveable, saveable_object.SaveableObject):
        raise ValueError(
            "Expected a dictionary of SaveableObjects, got {}."
            .format(saveable))
      saveables_by_device.setdefault(saveable.device, []).append(saveable)
    self._single_device_savers = {
        device: _SingleDeviceSaver(saveables)
        for device, saveables in saveables_by_device.items()}

  def to_proto(self):
    """Serializes to a SaverDef referencing the current graph."""
    filename_tensor = array_ops.placeholder(
        shape=[], dtype=dtypes.string, name="saver_filename")
    save_tensor = self._traced_save(filename_tensor)
    restore_op = self._traced_restore(filename_tensor).op
    return saver_pb2.SaverDef(
        filename_tensor_name=filename_tensor.name,
        save_tensor_name=save_tensor.name,
        restore_op_name=restore_op.name,
        version=saver_pb2.SaverDef.V2)

  @def_function.function(
      input_signature=(tensor_spec.TensorSpec(shape=(), dtype=dtypes.string),),
      autograph=False)
  def _traced_save(self, file_prefix):
    save_op = self.save(file_prefix)
    with ops.device("cpu:0"):
      with ops.control_dependencies([save_op]):
        return array_ops.identity(file_prefix)

  @def_function.function(
      input_signature=(tensor_spec.TensorSpec(shape=(), dtype=dtypes.string),),
      autograph=False)
  def _traced_restore(self, file_prefix):
    restore_ops = self.restore(file_prefix)
    with ops.device("cpu:0"):
      with ops.control_dependencies(restore_ops.values()):
        return array_ops.identity(file_prefix)

  def save(self, file_prefix):
    """Save the saveable objects to a checkpoint with `file_prefix`.

    Args:
      file_prefix: A string or scalar string Tensor containing the prefix to
        save under.
    Returns:
      An `Operation`, or None when executing eagerly.
    """
    # IMPLEMENTATION DETAILS: most clients should skip.
    #
    # Suffix for any well-formed "checkpoint_prefix", when sharded.
    # Transformations:
    # * Users pass in "save_path" in save() and restore().  Say "myckpt".
    # * checkpoint_prefix gets fed <save_path><sharded_suffix>.
    #
    # Example:
    #   During runtime, a temporary directory is first created, which contains
    #   files
    #
    #     <train dir>/myckpt_temp/
    #        part-?????-of-?????{.index, .data-00000-of-00001}
    #
    #   Before .save() finishes, they will be (hopefully, atomically) renamed to
    #
    #     <train dir>/
    #        myckpt{.index, .data-?????-of-?????}
    #
    # Users only need to interact with the user-specified prefix, which is
    # "<train dir>/myckpt" in this case.  Save() and Restore() work with the
    # prefix directly, instead of any physical pathname.  (On failure and
    # subsequent restore, an outdated and orphaned temporary directory can be
    # safely removed.)
    sharded_suffix = "_temp_%s/part" % uuid.uuid4().hex

    with ops.device("cpu:0"):
      tmp_checkpoint_prefix = string_ops.string_join(
          [file_prefix, sharded_suffix])

    num_shards = len(self._single_device_savers)
    sharded_saves = []
    sharded_prefixes = []
    num_shards_tensor = constant_op.constant(num_shards, name="num_shards")
    last_device = None
    for shard, (device, saver) in enumerate(
        sorted(self._single_device_savers.items())):
      last_device = device
      with ops.device(saveable_object_util.set_cpu0(device)):
        shard_prefix = sharded_filename(tmp_checkpoint_prefix, shard,
                                        num_shards_tensor)
      sharded_prefixes.append(shard_prefix)
      with ops.device(device):
        # _SingleDeviceSaver will use the CPU device when necessary, but initial
        # read operations should be placed on the SaveableObject's device.
        sharded_saves.append(saver.save(shard_prefix))

    with ops.control_dependencies(sharded_saves):
      # Co-locates the merge step with the last device.
      with ops.device(saveable_object_util.set_cpu0(last_device)):
        # V2 format write path consists of a metadata merge step.  Once merged,
        # attempts to delete the temporary directory, "<user-fed prefix>_temp".
        return gen_io_ops.merge_v2_checkpoints(
            sharded_prefixes, file_prefix, delete_old_dirs=True)

  def restore(self, file_prefix):
    """Restore the saveable objects from a checkpoint with `file_prefix`.

    Args:
      file_prefix: A string or scalar string Tensor containing the prefix for
        files to read from.

    Returns:
      A dictionary mapping from SaveableObject names to restore operations.
    """
    restore_ops = {}
    # Sort by device name to avoid propagating non-deterministic dictionary
    # ordering in some Python versions.
    for device, saver in sorted(self._single_device_savers.items()):
      with ops.device(device):
        restore_ops.update(saver.restore(file_prefix))
    return restore_ops


from django.test import LiveServerTestCase
from selenium import webdriver

class HomepageTestCase(LiveServerTestCase):
    def setUp(self):
        self.selenium = webdriver.Chrome()
        self.selenium.maximize_window()
        super(HomepageTestCase, self).setUp()

    def tearDown(self):
        self.selenium.quit()
        super(HomepageTestCase, self).tearDown()

    def test_can_visit_homepage(self):
        self.selenium.get(
            '%s%s' % (self.live_server_url,  "/")
        )

        self.assertIn("Growth Studio - Enjoy Create & Share", self.selenium.title)


from __future__ import absolute_import, division, print_function

from marshmallow import fields

from polyaxon.schemas.base import BaseConfig, BaseSchema

class CelerySchema(BaseSchema):
    taskTrackStarted = fields.Bool(allow_none=True)
    brokerPoolLimit = fields.Int(allow_none=True)
    confirmPublish = fields.Bool(allow_none=True)
    workerPrefetchMultiplier = fields.Int(allow_none=True)
    workerMaxTasksPerChild = fields.Int(allow_none=True)
    workerMaxMemoryPerChild = fields.Int(allow_none=True)
    taskAlwaysEager = fields.Bool(allow_none=True)

    @staticmethod
    def schema_config():
        return CeleryConfig

class CeleryConfig(BaseConfig):
    SCHEMA = CelerySchema
    REDUCED_ATTRIBUTES = [
        "taskTrackStarted",
        "brokerPoolLimit",
        "confirmPublish",
        "workerPrefetchMultiplier",
        "workerMaxTasksPerChild",
        "workerMaxMemoryPerChild",
        "taskAlwaysEager",
    ]

    def __init__(
        self,  # noqa
        taskTrackStarted=None,
        brokerPoolLimit=None,
        confirmPublish=None,
        workerPrefetchMultiplier=None,
        workerMaxTasksPerChild=None,
        workerMaxMemoryPerChild=None,
        taskAlwaysEager=None,
    ):
        self.taskTrackStarted = taskTrackStarted
        self.brokerPoolLimit = brokerPoolLimit
        self.confirmPublish = confirmPublish
        self.workerPrefetchMultiplier = workerPrefetchMultiplier
        self.workerMaxTasksPerChild = workerMaxTasksPerChild
        self.workerMaxMemoryPerChild = workerMaxMemoryPerChild
        self.taskAlwaysEager = taskAlwaysEager


import gzip
import pickle
from pathlib import Path
import numpy as np
import torch
from torch import nn, optim
from torch.utils.data import DataLoader, TensorDataset
import torch.nn.functional as F

loss_func = F.cross_entropy

device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")

def loss_batch(model, loss_func, xb, yb, opt=None):
    loss = loss_func(model(xb), yb)

    if opt is not None:
        loss.backward()
        opt.step()
        opt.zero_grad()

    return loss.item(), len(xb)

class Mnist_CNN(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1, 16, kernel_size=3, stride=2, padding=1)
        self.conv2 = nn.Conv2d(16, 16, kernel_size=3, stride=2, padding=1)
        self.conv3 = nn.Conv2d(16, 10, kernel_size=3, stride=2, padding=1)

    def forward(self, xb):
        xb = xb.view(-1, 1, 28, 28)
        xb = F.relu(self.conv1(xb))
        xb = F.relu(self.conv2(xb))
        xb = F.relu(self.conv3(xb))
        xb = F.avg_pool2d(xb, 4)
        return xb.view(-1, xb.size(1))

class Lambda(nn.Module):
    def __init__(self, func):
        super().__init__()
        self.func = func

    def forward(self, x):
        return self.func(x)

def preprocess(x, y):
    return x.view(-1, 1, 28, 28).to(device), y.to(device)

class WrappedDataLoader:
    def __init__(self, dl, func):
        self.dl = dl
        self.func = func

    def __len__(self):
        return len(self.dl)

    def __iter__(self):
        batches = iter(self.dl)
        for b in batches:
            yield (self.func(*b))

def fit(epochs, model, loss_func, opt, train_dl, valid_dl):
    for epoch in range(epochs):
        model.train()
        for xb, yb in train_dl:
            loss_batch(model, loss_func, xb, yb, opt)

        model.eval()
        with torch.no_grad():
            losses, nums = zip(
                *[loss_batch(model, loss_func, xb, yb) for xb, yb in valid_dl]
            )
        val_loss = np.sum(np.multiply(losses, nums)) / np.sum(nums)

        print(epoch, val_loss)

def get_data(bs=100):
    DATA_PATH = Path("data")
    PATH = DATA_PATH / "mnist"
    FILENAME = "mnist.pkl.gz"
    with gzip.open((PATH / FILENAME).as_posix(), "rb") as f:
        ((x_train, y_train), (x_valid, y_valid), _) = pickle.load(f, encoding="latin-1")
    x_train, y_train, x_valid, y_valid = map(torch.tensor, (x_train, y_train, x_valid, y_valid))
    train_ds = TensorDataset(x_train, y_train)
    train_dl = DataLoader(train_ds, batch_size=100, shuffle=True)
    train_dl = WrappedDataLoader(train_dl, preprocess)

    valid_ds = TensorDataset(x_valid, y_valid)
    valid_dl = DataLoader(valid_ds, batch_size=bs * 2)
    valid_dl = WrappedDataLoader(valid_dl, preprocess)

    return (train_dl, valid_dl)

def get_model(lr=0.1):
    model = Mnist_CNN().to(device)
    opt = optim.SGD(model.parameters(), lr=lr, momentum=0.9)
    return model, opt

def seq():
    model = nn.Sequential(
        nn.Conv2d(1, 16, kernel_size=3, stride=2, padding=1),
        nn.ReLU(),
        nn.Conv2d(16, 16, kernel_size=3, stride=2, padding=1),
        nn.ReLU(),
        nn.Conv2d(16, 10, kernel_size=3, stride=2, padding=1),
        nn.ReLU(),
        nn.AdaptiveAvgPool2d(1),
        Lambda(lambda x: x.view(x.size(0), -1)),
    )
    return model

train_dl, valid_dl = get_data(bs=100)
model, opt = get_model()
epochs = 2
fit(epochs, model, loss_func, opt, train_dl, valid_dl)

""


'''
@Author: Pavani
@Date: 06-11-2021 13:56:00
@Last modified by: Pavani
@Last modified time : 06-11-2021 13:56:00
@Title : Print the Nth Harmonic number and N! = 0 ”
'''
# Nth harmonic number is the sum of the reciprocals of the first n natural numbers.
# harmonic numbers are the sums of the inverses of integers, forming the harmonic series.

def harmonic(num):
    """to generate harmonic value of the given number
       parameter : num
       return value : harmonic number"""

    # Base condition
    if (num < 2):
        return 1
    else:
        return 1 / num + (harmonic(num-1))
try:
  num = int(input("enter the number :"))
  if (num == 0):
    print("given value should not be zero ")
  else:
    print("the harmonic number is", (harmonic(num)))
except Exception:
    print("hey its an error because you entered invalid input")
finally:
  print("its an harmonic number program")


import requests
import sqlite3
import pandas as pd

engine = sqlite3.connect("pokedex.db")
cursor = engine.cursor()

names = []
types = []

for i in range(1,100):
    data = requests.get(f"https://pokeapi.co/api/v2/pokemon/{i}/").json()
    names.append(data["name"])
    types.append(data["types"][0]["type"]["name"])

df = pd.DataFrame({"name": names, "type": types})

df.to_sql("pokeinfo", engine, if_exists="replace", index=False)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""eq

Usage:
    eq [--density=<n>]
    eq (-h | --help)
    eq --version

Options:
    --density=<n>  PDF to PNG conversion image density [default: 300].
    -h --help      Show this screen.
    --version      Show version.
"""

import shutil
import os
import tempfile
import platform
import subprocess as sp
from docopt import docopt
import eq.template
from eq.version import __version__
from eq.log import log, warn, error

def main():
    arguments = docopt(__doc__, version=f'eq v{__version__}')
    system = platform.system()

    # Determine presence of and default executables

    latexmk = shutil.which('latexmk')
    if latexmk is None:
        error('The latexmk executable cannot be found.\n'
              'eq needs latexmk to compile your equation, '
              'please make sure you have a LaTeX distribution installed and '
              'available to run, and then install latexmk.')
        exit(1)

    if shutil.which('magick') is None:
        # convert could not be found; imagemagick may still exist, as newer
        # versions rely on the "magick" command.
        if system != 'Windows' and shutil.which('convert') is None:
            error('The imagemagick executable (magick or convert) could not '
                  'be found.\n'
                  'eq needs imagemagick to process your equation, '
                  'please make sure you have imagemagick installed and '
                  'available to run.')
            exit(1)
        # else:
        # The `convert` command was found.
        magick = ['convert']
    else:
        # `magick` was found.
        magick = ['magick', 'convert']

    first = lambda x: next((y for y in x if y is not None), None)
    editor = first([shutil.which('editor'),
                    os.environ.get('VISUAL'),
                    os.environ.get('EDITOR'),
                    shutil.which('vim'),
                    shutil.which('nano')])
    if editor is None:
        error('Could not find an editor to edit the equation with.\n'
              'Tried `editor`, $VISUAL, $EDITOR, `vim`, `nano`, in that order. '
              'Please ensure one of these exists, or set $VISUAL/$EDITOR to '
              'your preferred editor.')
        exit(1)

    if system == 'Windows':
        nircmd = shutil.which('nircmd')
        if nircmd is None:
            error('The nircmdc executable could not be found.\n'
                  'eq needs nircmd to copy the equation image into your '
                  'clipboard. Please ensure it is installed and available.')

        def copy(png_name):
            png_name = os.path.realpath(png_name)
            return sp.run([nircmd, 'clipboard', 'copyimage', png_name])
    else:
        if system != 'Linux':
            warn('Treating system as Linux')
        xclip = shutil.which('xclip')
        if xclip is None:
            error('The xclip executable could not be found.\n'
                  'eq needs xclip to copy the equation image into your '
                  'clipboard. Please ensure it is installed and available.')
            exit(1)

        def copy(png_name):
            return sp.run([xclip, '-in', '-selection', 'clipboard',
                            '-target', 'image/png', png_name])

    # Create a temporary LaTeX project.
    with tempfile.TemporaryDirectory() as proj_dir:
        # Fix needed for Windows:
        # The default temporary directory has a ~ in it, and LaTeX really hates
        # that.
        proj_dir = os.path.realpath(proj_dir)

        # Put default template into template directory
        file_name = eq.template.put(proj_dir)

        # Loop while the content changes on write
        with open(file_name, 'r') as old_file:
            old_content = old_file.read()

        while True:
            # Open template with editor
            sp.run([editor, file_name])

            # Compare with new content
            with open(file_name, 'r') as new_file:
                new_content = new_file.read()

            if new_content == old_content:
                break # out of edition loop
            # else:
            old_content = new_content

            # Do the rendering, and copy to the clipboard
            # Use latexmk to ensure full compilation
            result = sp.run(
                [latexmk, '-pdf', f'-output-directory="{proj_dir}"', file_name])
            if result.returncode != 0:
                # Something went wrong with the compilation (user error?)
                # Loop back to edition.
                error('Something went wrong with compiling the project.')
                continue # to another edition pass.

            # Finished producing a PDF.
            # Now convert it to a white background PNG with imagemagick.
            pdf_name = os.path.splitext(file_name)[0] + '.pdf'
            png_name = os.path.splitext(file_name)[0] + '.png'
            density = arguments.get('--density', 300)
            result = sp.run([*magick, '-flatten', '-colorspace', 'RGB',
                '-background', 'white', '-density', density, pdf_name,
                png_name])

            if result.returncode != 0:
                error('Something went wrong with converting the PDF to a PNG.')
                log('Press any key to go back to edition.')
                input()
                continue # to another edition pass.
            # else: success

            # Copy the PNG into the clipboard
            result = copy(png_name)
            if result.returncode != 0:
                error('Something went wrong with copying the PNG to your '
                      'clipboard.')
                log('Press any key to go back to edition.')
                input()
                continue # to another edition pass.

            # loop to another edition pass


import numpy as np
from random import uniform
import math
import pickle
from torch import nn
from torch.masked import masked_tensor
import torch
from bot import NeuralNetwork, inodes, onodes
from dataset import PositionDataset
from torch.utils.data import DataLoader

class Trainer():
    def __init__(self):
        return

    def loss_fn():
        return

    def train_one_epoch():
        return
"""
trainer = Trainer()

# Code to load NN parameters
model = NeuralNetwork().to("cpu")
model.load_state_dict(torch.load("./botModels/gen1.bot"))
model.eval()

# Code to load training data
data_file = open("./trainingData/pos1690.pickle", "rb")
posData = pickle.load(data_file)
data_file.close()

# Code to get the test run MCTS output, this should be done during the test runs for the future
indexes = posData[2].nonzero(as_tuple = False)
myOtherCoolTensor = []
nextToInsert = 0
for i in range(onodes - 1):
    if i in indexes:
        myOtherCoolTensor.append(posData[1][nextToInsert])
        nextToInsert += 1
    else:
        myOtherCoolTensor.append(0.)

actOutput = torch.tensor(myOtherCoolTensor)
print(actOutput)

# Code to get the normalized NN MCTS prediction
prediction = model(posData[0])
myCoolTensor = torch.where(posData[2], prediction[0], float('-inf'))
predictedOutput = nn.Softmax(dim=0)(myCoolTensor)

# Loss
print(nn.CrossEntropyLoss()(predictedOutput, actOutput))
"""

#mybatch = []
#for i in range(5):
#    data_file = open("./trainingData/pos" + str(i) + ".pickle", "rb")
#    posData = pickle.load(data_file)
#    data_file.close()
#    mybatch.append(posData[0])

#def AZLossFcn(predMTCS, actMTCS, predRes, actRes):
#    return nn.CrossEntropyLoss()(predMTCS, actMCTS)

#a,b,c,d = next(iter(loader))

#gamePos, train_MCTS, mask, train_result = list(next(iter(loader)))[0]
#print(gamePos)

def AZLossFcn(predMCTS, actMCTS, predRes, actRes):
    loss1 = nn.CrossEntropyLoss(reduction='mean')(predMCTS, actMCTS)
    loss2 = nn.MSELoss()(predRes, actRes.unsqueeze(dim=1))
    return (loss1 + loss2).sum()

def training(learnRate):
    training_data = PositionDataset("./AI/trainingData")
    loader = DataLoader(training_data, batch_size=32, shuffle=True)
    model = NeuralNetwork().to("cpu")
    model.load_state_dict(torch.load("./AI/botModels/currbot.bot"))
    model.eval()
    n_epochs = 500
    for epoch in range(n_epochs):
        gamePos, train_MCTS, mask, train_result = next(iter(loader))
        gamePos.requires_grad = True
        NNpred, NNvaluation = model(gamePos)
        predictedOutput = torch.where(mask, NNpred, float('-inf'))
        #predictedOutput = nn.Softmax(dim=1)(predictedOutput)
        loss = AZLossFcn(predictedOutput, train_MCTS, NNvaluation, train_result.float())

        optimizer = torch.optim.SGD(model.parameters(), lr=learnRate, momentum=0.9)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    torch.save(model.state_dict(), "./AI/botModels/nextbot.bot")

#training(0.1)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
@File    :   create_sketch.py
@Time    :   2023/03/28 10:10:16
@Author  :   Weihao Xia (xiawh3@outlook.com)
@Version :   1.0
@Desc    :   This script is used to generate sketches.
             please ensure torch==0.4.1 and torchvision==0.2.1
'''

import os
import cv2
import numpy as np
import argparse

import hashlib
import platform
import subprocess
# import urllib.request

from torchvision import transforms
from torchvision.utils import save_image
from torch.utils.serialization import load_lua

def sobel(img):
    opImgx = cv2.Sobel(img, cv2.CV_8U, 0, 1, ksize=3)
    opImgy = cv2.Sobel(img, cv2.CV_8U, 1, 0, ksize=3)
    return cv2.bitwise_or(opImgx, opImgy)

def sketch(frame):
    frame = cv2.GaussianBlur(frame, (3, 3), 0)
    invImg = 255 - frame
    edgImg0 = sobel(frame)
    edgImg1 = sobel(invImg)
    edgImg = cv2.addWeighted(edgImg0, 0.75, edgImg1, 0.75, 0)
    opImg = 255 - edgImg
    return opImg

def get_sketch_image(image_path):
    original = cv2.imread(image_path)
    original = cv2.cvtColor(original, cv2.COLOR_BGR2GRAY)
    sketch_image = sketch(original)
    return sketch_image[:, :, np.newaxis]

def download_model(modelname, filename, fileurl, filemd5):
    '''
    download the sketch simplification model (sketch_gan.t7) from official implementation:
    https://github.com/bobbens/sketch_simplification
    '''
    # check if the model already exists
    if os.path.isfile(filename):
        print (filename)
        print(f"Model '{modelname}' already exists. Skipping download.")
    else:
        #  call wget command to download the file
        print(f"Downloading the sketch simplification {modelname} model...")
        # urllib.request.urlretrieve(fileurl, filename)
        subprocess.call(["wget", "-q", "--show-progress", "--continue", "-O", filename, "--", fileurl])

        # verify the MD5 checksum
        print("checking integrity (md5sum)...")
        with open(filename, "rb") as f:
            md5 = hashlib.md5(f.read()).hexdigest()
        if md5 != filemd5:
            print("Integrity check failed. File is corrupt!")
            print(f"Try running this script again and if it fails remove '{filename}' before trying again.")
            os.remove(filename)
            raise ValueError("MD5 checksum does not match")

        print(f"Model '{modelname}' downloaded successfully")

if __name__ == "__main__":

    # define a dictionary to store the model information
    models = {
        "GAN": {
            "filename": "model_gan.t7",
            "fileurl": "https://esslab.jp/~ess/data/sketch_gan.t7",
            "filemd5": "3a5b4088f2490ca4b8140a374e80c878"
        },
        "MSE": {
            "filename": "model_mse.t7",
            "fileurl": "https://esslab.jp/~ess/data/sketch_mse.t7",
            "filemd5": "12317df9a0a2a7220629f5f361b45b82"
        },
        "PENCIL(1)": {
            "filename": "model_pencil1.t7",
            "fileurl": "https://esslab.jp/~ess/data/pencil_artist1.t7",
            "filemd5": "33d553ff3a50d6522e79a73002b0025c"
        },
        "PENCIL(2)": {
            "filename": "model_pencil2.t7",
            "fileurl": "https://esslab.jp/~ess/data/pencil_artist2.t7",
            "filemd5": "537b3ad9d46b2a82b65883be747a7ba9"
        }
    }

    # parse command-line arguments
    parser = argparse.ArgumentParser(description="sketch data generation")
    parser.add_argument("modelname", type=str, choices=models.keys(), help="name of the model to download")
    parser.add_argument("use_cuda", type=bool, default=True, help="use cuda or not")
    parser.add_argument("data_path", type=str, default="celeba_image", help="path to read images")
    parser.add_argument("save_path", type=str, default="celeba_sketch", help="path to save sketches")
    args = parser.parse_args()

    # download the specified model
    print("downloading pretrained models...")
    model = models[args.modelname]
    download_model(args.modelname, model["filename"], model["fileurl"], model["filemd5"])
    print("download finished!")

    # load the model
    cache = load_lua(args.modelname) # sketch_gan.t7
    model = cache.model
    immean = cache.mean
    imstd = cache.std
    model.evaluate()

    if not os.path.exists(args.save_path):
        os.makedirs(args.save_path)

    images = [os.path.join(args.data_path, f) for f in os.listdir(args.data_path)]

    for idx, image_path in enumerate(images):
        if idx % 50 == 0:
            print("{} out of {}".format(idx, len(images)))
        data = get_sketch_image(image_path)
        data = ((transforms.ToTensor()(data) - immean) / imstd).unsqueeze(0)
        if args.use_cuda:
            pred = model.cuda().forward(data.cuda()).float()
        else:
            pred = model.forward(data)
        save_image(pred[0], os.path.join(args.save_path, "{}_edges.jpg".format(image_path.split("/")[-1].split('.')[0])))


import cv2 as cv
import numpy as np

image = cv.imread('CVBVP-Daten/OpenCV-03/Img03a.jpg')
if not image.data:
    print("Error loading image")
    exit

## Aufgabe 1

#cv.imshow("Image:", image)
#cv.imshow("Image mit getauschten Farbkanaelen:", cv.cvtColor(image, cv.COLOR_BGR2RGB))

height = image.shape[0]
width = image.shape[1]

#https://pyimagesearch.com/2021/01/23/splitting-and-merging-channels-with-opencv/
(B, G, R) = cv.split(image)

grayscaleimage = np.empty((height, width))
fullgrayscaleimage = image.copy()

for y in range(height):
    for x in range (width):
        pixel = image[y][x]
        grayscalevalue = int((pixel[0]+pixel[1]+pixel[2])/3)
        fullgrayscaleimage[y][x][0] = grayscalevalue
        fullgrayscaleimage[y][x][1] = grayscalevalue
        fullgrayscaleimage[y][x][2] = grayscalevalue
        grayscaleimage[y][x] = grayscalevalue

cv.imshow("Image:", fullgrayscaleimage)
cv.imshow("Image2:", grayscaleimage)
cv.imshow("Image3:", cv.cvtColor(image, cv.COLOR_BGR2GRAY))
if cv.waitKey(0) == ord('q'):
        exit()
cv.destroyAllWindows()
cv.imwrite('CVBVP-Daten/OpenCV-03/out/blue.jpg', B)
cv.imwrite('CVBVP-Daten/OpenCV-03/out/green.tif', G)
cv.imwrite('CVBVP-Daten/OpenCV-03/out/red.png', R)
cv.imwrite('CVBVP-Daten/OpenCV-03/out/gray.png', fullgrayscaleimage)
cv.imwrite('CVBVP-Daten/OpenCV-03/out/convertedcolorspace.png', cv.cvtColor(image, cv.COLOR_BGR2RGB))

## Aufgabe 2


'''Train CIFAR10 with PyTorch.'''
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torch.backends.cudnn as cudnn

import torchvision
import torchvision.transforms as transforms
import torch.utils.data as data

from sklearn.metrics import matthews_corrcoef, f1_score, accuracy_score
from sklearn.metrics import precision_recall_fscore_support as score
from sklearn.metrics import roc_curve, roc_auc_score

import numba

import os
import argparse
from tqdm import tqdm, trange
from data import CelebA

from utils import get_scores, get_embeddings, get_scores_and_embeddings, get_neighbors,test_auc,get_dataset_img_info

import matplotlib
matplotlib.use('agg')
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.manifold import MDS, TSNE
from sklearn.decomposition import PCA
import umap
import time
from scipy import stats

from models.CustomModel import ResNet18, vgg11, AutoEncoder, SugiyamaNet, ComboModel, VAE, SugiyamaNet
from models.inception import inception_v3
import train

def write_data(args, data, clear=False):
    filename = f"{args.net}_{args.activation_name}_{args.lr}.csv"
    if clear: open(filename, 'w').close()
    f=open(filename, "a+");
    f.write(data);
    f.close()

attrs_default = [
    '5_o_Clock_Shadow', 'Arched_Eyebrows', 'Attractive', 'Bags_Under_Eyes',
    'Bald', 'Bangs', 'Big_Lips', 'Big_Nose', 'Black_Hair', 'Blond_Hair',
    'Blurry', 'Brown_Hair', 'Bushy_Eyebrows', 'Chubby', 'Double_Chin',
    'Eyeglasses', 'Goatee', 'Gray_Hair', 'Heavy_Makeup', 'High_Cheekbones',
    'Male', 'Mouth_Slightly_Open', 'Mustache', 'Narrow_Eyes', 'No_Beard',
    'Oval_Face', 'Pale_Skin', 'Pointy_Nose', 'Receding_Hairline',
    'Rosy_Cheeks', 'Sideburns', 'Smiling', 'Straight_Hair', 'Wavy_Hair',
    'Wearing_Earrings', 'Wearing_Hat', 'Wearing_Lipstick', 'Wearing_Necklace',
    'Wearing_Necktie', 'Young'
]

model_dict = {
        "classifier": (train.classifier,ResNet18),
        "ae":         (train.ae, AutoEncoder),
        "vae":        (train.vae, VAE),
        'gan':        (train.gan, None),
        "dre":        (train.dre_scratch,vgg11),
        "imagenet":   (train.imagenet, inception_v3)
    }

device = 'cuda' if torch.cuda.is_available() else 'cpu'

def get_args():
    parser = argparse.ArgumentParser(description='PyTorch celeba weird Training')
    parser.add_argument('--attrs', dest='attrs', default=attrs_default, nargs='+', help='attributes to learn')
    parser.add_argument('--lr', default=1e-3, type=float, help='learning rate')
    parser.add_argument('--img_size', default=64, type=int, help='img size')
    parser.add_argument('--epochs', default=20, type=int, help=' ')
    parser.add_argument('--batch_size', default=128, type=int, help='minibatch size')
    parser.add_argument('--data', default='CelebA', type=str,  choices=['CelebA', 'CelebA-HQ'])
    parser.add_argument('--net', default="classifier", type=str, choices=model_dict.keys())
    parser.add_argument('--resume', '-r', action='store_true', help='resume from checkpoint')
    parser.add_argument('--data_path', dest='data_path', type=str, default='./data/celebA')
    parser.add_argument('--attr_path', dest='attr_path', type=str, default='./data/celebA/list_attr_celeba.txt')
    parser.add_argument('--remove_att', type=str, default="")
    parser.add_argument('--remove_att_val', type=int, default=-1)
    parser.add_argument('--trainset_total', type=int, default=50000)
    parser.add_argument('--heldout_total', type=int, default=10000)
    parser.add_argument('--outlier_total', type=int, default=1000)
    parser.add_argument("--save_dir", type=str,  default="pics")
    parser.add_argument("--do_train", action='store_true', help="  ")
    parser.add_argument("--do_dre", action='store_true', help="")
    parser.add_argument("--cluster_eval_type", type=str, default="", help="??")
    return parser.parse_args()

@torch.no_grad()
def calc_inception_distr(model, dataset):
    model.eval()
    device = next(model.parameters()).device
    all_logits = torch.empty(0).to(device)

    print("pre-parsing the training data for average distribution")
    for images, labels, _ in tqdm(dataset):
        all_logits = torch.cat([all_logits, model(images.to(device))], dim = 0)

    model.average_distr = torch.log_softmax(all_logits, dim=1).mean(0)

def main():
    args = get_args()
    datasets = all_celeba(args)
    trainset_total = args.trainset_total
    heldout_total  = args.heldout_total
    outlier_total  = args.outlier_total
    inlier_total   = heldout_total - outlier_total

    # Model
    print('==> Building model..')

    train_func, modelfunc  = model_dict[args.net]

    model_path = args.net + ".pth"

    if args.net == 'imagenet':
        model = modelfunc(model_path).to(device)
        for name, param in model.named_parameters():
            param.requires_grad = False

        calc_inception_distr(model,  datasets.train_dataloader)
        bs = args.batch_size
    else:
        model = modelfunc().to(device)
        bs = 2048

        if args.do_train:
            optimizer  = optim.Adam(model.parameters(), lr=args.lr) if args.net !='dre' else optim.SGD(model.parameters(), lr=.01)
            for e in range(args.epochs):
                train_func(model, optimizer, datasets.train_dataloader, datasets.eval_dataloader)
            torch.save(model.state_dict(), model_path)
        else:
            model.load_state_dict(torch.load(model_path))
    model.eval()

    original_train_zs   = get_embeddings(model, datasets.train_dataloader)
    original_inlier_zs  = get_embeddings(model, datasets.inlier_dataloader)
    original_outlier_zs = get_embeddings(model, datasets.outlier_dataloader)

    if args.do_dre:
        model = get_2phase_dre(model,datasets, bs)
        model.eval()

    print("getting embeddings")

    train_scores,   train_zs   = get_scores_and_embeddings(model, datasets.train_dataloader)
    inlier_scores,  inlier_zs  = get_scores_and_embeddings(model, datasets.inlier_dataloader)
    outlier_scores, outlier_zs = get_scores_and_embeddings(model, datasets.outlier_dataloader)

    eval_scores    = np.concatenate([inlier_scores, outlier_scores])
    eval_zs        = np.concatenate([inlier_zs, outlier_zs])
    auc            = test_auc(inlier_scores, outlier_scores)

    top1k_scores = np.argpartition(eval_scores, outlier_total)[:outlier_total]
    inlier_total   =  heldout_total  - outlier_total
    outlier_selection_percent = (top1k_scores > inlier_total).sum() / outlier_total
    print(f"percentage of outlier selected: {outlier_selection_percent*100}")

    train_neighbors, eval_neighbors, train_subset = get_neighbors(model, datasets)

    if args.cluster_eval_type != "":
        print("kmeans or inception score or something")

    trans = umap.UMAP(n_neighbors=15,verbose=True).fit(eval_zs)
    embs_train      = trans.transform(train_zs[train_subset])
    embs_inlier     = trans.transform(inlier_zs)
    embs_outlier    = trans.transform(outlier_zs)

    total_points = embs_train.shape[0] + embs_inlier.shape[0] + embs_outlier.shape[0]

    train_scores,   tlabels, t_paths, _ = get_dataset_img_info(model, datasets.train_dataloader)  #this is probably what I want
    inlier_scores,  ilabels, i_paths, _ = get_dataset_img_info(model, datasets.inlier_dataloader)  #this is probably what I want
    outlier_scores, olabels, o_paths, _ = get_dataset_img_info(model, datasets.outlier_dataloader)  #this is probably what I want

    import pdb; pdb.set_trace()

    indices     = np.array(["indices"]+list(range(total_points)))
    data_type   = np.array(["datatype"]+["train"] * len(train_subset) + ["inlier"]*embs_inlier.shape[0] + ["outlier"]*embs_outlier.shape[0])
    labels      = np.concatenate([["data_label"], tlabels[train_subset], ilabels, olabels])
    paths       = np.concatenate([["data_path"],  t_paths[train_subset], i_paths, o_paths])
    scores      = np.concatenate([["shift_score"],  train_scores[train_subset], inlier_scores, outlier_scores])
    og_embs     = np.concatenate([[["emb_x","emb_y"]], embs_train, embs_inlier, embs_outlier])
    close_train = np.concatenate([[[str(i)+"_closest_train" for i in range(1, 21)]], train_neighbors])
    close_eval  = np.concatenate([[[str(i)+"_closest_eval"  for i in range(1, 21)]], eval_neighbors])

    savedata =  np.vstack([indices,data_type,labels,paths,scores,og_embs.T,close_train.T,close_eval.T]).T
    filename = f'dre_{args.net}.csv' if args.do_dre else f'{args.net}.csv'
    np.savetxt(filename,savedata, delimiter=',', fmt='%s')

    print("here")

    exit()
    do_plots(15, .1)

    '''for i, top10 in enumerate(np.argpartition(eval_scores, 10)[:10]):
        prep_img, _, path = eval_CelebA_data.__getitem__(top10)
        get_scorecam_example(net, prep_img.to(device), path, os.path.join("imgs",f"top{i}_outlier"), dr_model=dr_model, target_layer=2, invert_sigmoid=True)

    for i, top10 in enumerate(np.argpartition(eval_scores, -10)[-10:]):
        prep_img, _, path = eval_CelebA_data.__getitem__(top10)
        get_scorecam_example(net, prep_img.to(device), path, os.path.join("imgs",f"top{i}_inlier"), dr_model=dr_model, target_layer=2, invert_sigmoid=False)
    '''

def do_plots(neighbors,dist):
    #x_emb_dtest = TSNE(n_components=2, perplexity=neighbors, verbose=True).fit_transform(dr_eval_zs.cpu().numpy())
    x_emb_dtest = umap.UMAP(n_neighbors=neighbors,verbose=True).fit_transform(dr_eval_zs.cpu().numpy())
    #x_emb_dtest_small = x_emb_dtest[(x_emb_dtest[:,0] >= 12) & (x_emb_dtest[:,1] <= 0)]
    #make_plot2d(x_emb_dtest_small, ["test"] * x_emb_dtest_small.shape[0],        "true labels",                             "test.png")

    #x_emb_test = TSNE(n_components=2, perplexity=neighbors, verbose=True).fit_transform(torch.cat([inlier_zs, outlier_zs]).cpu().numpy())
    x_emb_test = umap.UMAP(n_neighbors=neighbors, verbose=True).fit_transform(torch.cat([inlier_zs, outlier_zs]).cpu().numpy())

    #do pretty graphs

    bins = 8
    graph_scores = np.array((stats.mstats.rankdata(eval_scores) / len(eval_scores)) * bins).astype(int)
    labels_true = ['inlier'] * len(inlier_zs) + ['outlier'] * len(outlier_zs)
    labels_selected = ["not selected"] * len(eval_scores)
    labled_intersected = ['excluded inlier'] * len(inlier_zs) + ['excluded outlier'] * len(outlier_zs)
    for s in top1k_scores:
        labels_selected[s] = "selected"
        if s > len(inlier_zs):
            labled_intersected[s] = 'intersection'

    make_plot2d(x_emb_dtest, labels_true,        "true labels",                             "true_labels_density.png")
    make_plot2d(x_emb_dtest[::-1], labels_true[::-1],        "true labels",                             "true_labels_density_flipped.png")
    make_plot2d(x_emb_dtest, labels_selected,    "labels selected by density_score",        "labled_selected_density.png")
    make_plot2d(x_emb_dtest, labled_intersected, "compare true labels to density selected", "labels_intersected_density.png")
    make_plot2d_scores(x_emb_dtest, labels_true, graph_scores, "density scores plot",       "score_density.png")

    make_plot2d(x_emb_test, labels_true,        "true labels",                             "true_labels_original.png")
    make_plot2d(x_emb_test[::-1], labels_true[::-1],        "true labels",                             "true_labels_original_flipped.png")
    make_plot2d(x_emb_test, labels_selected,    "labels selected by density_score",        "labled_selected_original.png")
    make_plot2d(x_emb_test, labled_intersected, "compare true labels to density selected", "labels_intersected_original.png")
    make_plot2d_scores(x_emb_test, labels_true, graph_scores, "density scores plot",       "score_plot_original.png")

def get_2phase_dre(net,datasets, batch_size = 2048):
    train_zs   = get_embeddings(net, datasets.train_dataloader)
    inlier_zs  = get_embeddings(net, datasets.inlier_dataloader)
    outlier_zs = get_embeddings(net, datasets.outlier_dataloader)

    epochs = 100
    size = net.get_latent_size()
    #compute dr data
    dr_train_loader        = torch.utils.data.DataLoader(train_zs, batch_size=batch_size, shuffle=True,drop_last=False)
    dr_test_loader         = torch.utils.data.DataLoader(torch.cat([inlier_zs, outlier_zs]), batch_size=batch_size, shuffle=True)
    dr_inlier_loader       = torch.utils.data.DataLoader(inlier_zs, batch_size=batch_size, shuffle=False, drop_last=False)
    dr_outlier_loader      = torch.utils.data.DataLoader(outlier_zs, batch_size=batch_size, shuffle=False, drop_last=False)

    torch.cuda.empty_cache()
    dr_model = train.dre(SugiyamaNet, size, dr_train_loader, dr_test_loader, dr_inlier_loader, dr_outlier_loader, epochs, device)

    return ComboModel(net, dr_model)

class all_celeba():
    def __init__(self, args):
        trainset_total = args.trainset_total
        heldout_total  = args.heldout_total
        outlier_total  = args.outlier_total
        inlier_total   = heldout_total - outlier_total
        batch_size = args.batch_size

        if args.attrs[0] == args.remove_att:
            exit("cant remove att to be classified")

        kwargs = {'num_workers': 6, 'pin_memory': True} if torch.cuda.is_available() else {}

        print(args.attrs, args.remove_att, args.remove_att_val, "loading dataset")

        train_dataset = CelebA(args.data_path, args.attr_path, args.img_size, 'train', args.attrs)

        outlier_train,       train_dataset = train_dataset.split_attribute(args.remove_att, args.remove_att_val)
        heldout_inlier_data, train_dataset = train_dataset.split(inlier_total)
        train_dataset, _ = train_dataset.split(trainset_total)

        heldout_outlier_data, _            = outlier_train.split(outlier_total)

        assert len(train_dataset)        == trainset_total
        assert len(heldout_inlier_data)  == inlier_total
        assert len(heldout_outlier_data) == outlier_total
        # Data
        self.train_dataloader = data.DataLoader(
            train_dataset, batch_size=args.batch_size,
            shuffle=False, drop_last=False, **kwargs
        )

        self.inlier_dataloader = data.DataLoader(
            heldout_inlier_data, batch_size=args.batch_size,
            shuffle=False, drop_last=False, **kwargs
        )
        self.outlier_dataloader = data.DataLoader(
            heldout_outlier_data, batch_size=args.batch_size,
            shuffle=False, drop_last=False, **kwargs
        )

        self.eval_CelebA_data = CelebA.combine(heldout_inlier_data, heldout_outlier_data)

        self.eval_dataloader = data.DataLoader(
            self.eval_CelebA_data, batch_size=args.batch_size,
            shuffle=True, drop_last=False, **kwargs
        )

        self.perfect_user_train_dataloader = data.DataLoader(
            CelebA.combine(train_dataset, heldout_outlier_data), batch_size=args.batch_size,
            shuffle=False, drop_last=False, **kwargs
        )

if __name__ == "__main__":
    main()


"""# Visualization
We use t-SNE plot to observe the distribution of extracted features.
"""

import argparse

import matplotlib.pyplot as plt
import numpy as np
import os
from model import Model
import torch
from sklearn import manifold
from model import source_transform, target_transform
from torch.utils.data import ImageFolder, DataLoader

"""## Step1: Load checkpoint and evaluate to get extracted features"""

# Hints:
# Set features_extractor to eval mode
# Start evaluation and collect features and labels

# Extract feature extractor from model
model = Model()

state_dict = torch.load('model.pth')['model']
model.load_state_dict(state_dict)
model = model.feature_extractor
model.eval()

root_dir = "/tmp2/edwardlee/dataset/real_or_drawing/"

src = ImageFolder(
    os.path.join(root_dir, 'train_data'), transform=source_transform
)[:]

tgt = ImageFolder(
    os.path.join(root_dir, 'test_data'), transform=target_transform
)[:5000]

img = torch.cat([src, tgt], 0)

"""## Step2: Apply t-SNE and normalize"""

# process extracted features with t-SNE
X = model(img)
X_tsne = manifold.TSNE(n_components=2, init='random', random_state=5, verbose=1).fit_transform(X)

# Normalization the processed features
x_min, x_max = X_tsne.min(0), X_tsne.max(0)
X_norm = (X_tsne - x_min) / (x_max - x_min)

x, y = X_norm[:, 0], X_norm[:, 1]

"""## Step3: Visualization with matplotlib"""

# Data Visualization
# Use matplotlib to plot the distribution
# The shape of X_norm is (N,2)

plt.figure(figsize=(10, 10))
plt.scatter(x, y, c='red', s=1, label='Source')
plt.savefig('tsne.png')


class Solution:
    def longestWord(self, words: List[str]) -> str:
        # one char at a time,
        # make word set
        # queue all letter starting, a to z,
        # then check if available, then append it into queue,
        # if curr is maximum, make it to max length, then move forward, update max length, then return
        queue = deque([])
        word_set = set(words)
        for i in range(26):
            c = chr(ord('a') + i)
            if c in word_set:
                queue.append(c)
        # now we have queue set up
        curr_max = ''
        while queue:
            popped = queue.popleft()
            # update curr max based on length
            if len(popped) > len(curr_max):
                curr_max = popped
            # put into queue if available in word set
            for i in range(26):
                c = chr(ord('a') + i)
                new_word = popped + c
                if new_word in word_set:
                    queue.append(new_word)
        return curr_max

720. Longest Word in Dictionary/solution.py


import sys

def main():
    print ('Hello there', sys.argv[1])

def repeat (s, exclaim):

    result = s + s + s
    if exclaim:
        result =  result + '!!!'
    return result

if __name__ == '__main__':
    main()
    s = repeat('Python',True)
    print(s)


from tkinter import*

def calcular():
    ubica=enEUbicacion.get()
    tipo=enETipoVenta.get()
    cantidad=int(enECantidad.get())

    if(ubica=="Palco" and tipo=="Pre-venta"):
        ValorVenta = 22000*cantidad
        aporte=ValorVenta*0.15

    enEValorVenta.insert(ValorVenta)
    enEAporte.insert(aporte)

teatro = Tk()
teatro.title("UV-Teatro")
teatro.geometry("200x160")

ventanaEntradas = Frame(teatro,pady = 10)
etEUbicacion = Label(ventanaEntradas, text="Ubicación")
etEUbicacion.grid(row=0, column=0)
enEUbicacion = Entry(ventanaEntradas, width = 10)
enEUbicacion.grid(row=0,column=1)
etETipoVenta = Label(ventanaEntradas, text="Tipo")
etETipoVenta.grid(row=1, column=0)
enETipoVenta = Entry(ventanaEntradas, width = 10)
enETipoVenta.grid(row=1,column=1)
etECantidad = Label(ventanaEntradas, text="Cantidad")
etECantidad.grid(row=2, column=0)
enECantidad = Entry(ventanaEntradas, width = 10)
enECantidad.grid(row=2,column=1)
etCalcular = Button(ventanaEntradas, text="Calcular Valores", command=calcular)
etCalcular.grid(row=3,columnspan=2)
etEValorVenta = Label(ventanaEntradas, text="Valor venta")
etEValorVenta.grid(row=4, column=0)
enEValorVenta = Entry(ventanaEntradas, width = 10)
enEValorVenta.grid(row=4,column=1)
etEAporte = Label(ventanaEntradas, text="Valor aporte")
etEAporte.grid(row=5, column=0)
enEAporte = Entry(ventanaEntradas, width = 10)
enEAporte.grid(row=5,column=1)

ventanaEntradas.pack()

teatro.mainloop()


"""
class Join:
----
This class represents the physical action of doing a JOIN in SQL databases.
"""

class Join:
    LEFT_JOIN = 0
    RIGHT_JOIN = 1

    def __init__(self, to_join_table, left_column, right_column, join_type=0):
        self.to_join_table = to_join_table
        self.left_column = left_column
        self.right_column = right_column
        self.join_type = join_type


N, T = map(int, input().split())

item = [tuple(map(int, input().split())) for _ in range(N)]
dp = [[0]*(T+1) for _ in range(N+1)]

for i in range(1, N+1):
    for j in range(1, T+1):
        curT,curS = item[i-1]
        if curT <= j:
            dp[i][j] = max(dp[i-1][j], curS+dp[i-1][j-curT])
        else:
            dp[i][j] = dp[i-1][j]

print (dp[N][T])


from setuptools import setup, find_packages

version = '0.1'

setup(name='fluegelform',
      version=version,
      description="Dexterity-based form generator for Plone.",
      long_description=open("README.rst").read() + "\n" +
                       open("CHANGES.rst").read(),
      # Get more strings from
      # http://pypi.python.org/pypi?:action=list_classifiers
      classifiers=[
        "Programming Language :: Python",
        "Framework :: Plone",
        ],
      keywords='plone form ploneformgen dexterity',
      author='David Glick',
      author_email='dglick@gmail.com',
      url='http://github.com/davisagli/fluegelform',
      license='GPL',
      packages=find_packages('src'),
      package_dir = {'': 'src'},
      include_package_data=True,
      zip_safe=False,
      install_requires=[
          'lxml',
          'repoze.xmliter',
          'setuptools',
          'Products.CMFPlone',
          'plone.app.blocks',
          'plone.app.deco',
          'plone.app.dexterity',
          'zope.interface',
          # -*- Extra requirements: -*-
      ],
      entry_points="""
      [z3c.autoinclude.plugin]
      target = plone
      """,
      )


import logging

from schema import Schema, And, Optional, Or

from appimagebuilder.recipe.roamer import Roamer

class RecipeSchema:
    def __init__(self):
        self.version = Schema(int, ignore_extra_keys=True)
        self.script = Schema(Or(str, [str]))

        self.v1_app_info = {
            "id": str,
            Optional("name"): str,
            Optional("icon"): str,
            "version": str,
            "exec": str,
            Optional("exec_args"): str,
        }

        self.v1_files = {
            Optional("include"): [str],
            Optional("exclude"): [str],
        }

        self.v1_runtime = {
            Optional("debug"): bool,
            Optional("version"): str,
            Optional("path_mappings"): [str],
            Optional("arch"): [Or("gnueabihf", "x86_64", "i386", "aarch64")],
            Optional("env"): {str: Or(str, int, bool)},
            Optional("preserve"): [str],
        }

        self.v1_tests = {
            str: {
                "image": str,
                "command": str,
                Optional("before_command"): self.script,
                Optional("use_host_x"): bool,
                Optional("env"): {str: Or(str, int, bool)},
            }
        }

        self.v1_apt = Schema(
            {
                "arch": Or(str, [str]),
                "sources": [{"sourceline": str, Optional("key_url"): str}],
                "include": [str],
                Optional("exclude"): [str],
                Optional("allow_unauthenticated"): bool,
            }
        )
        self.v1_pacman = Schema(
            {
                Optional("Architecture"): Or("auto", "x86_64", "i686", "aarch64"),
                Optional("repositories"): {str: [str]},
                Optional("keyrings"): [str],
                Optional("options"): {str: str},
                "include": [str],
                Optional("exclude"): [str],
            }
        )

        self.v1_appdir = Schema(
            {
                Optional("path"): str,
                "app_info": self.v1_app_info,
                Optional("files"): self.v1_files,
                Optional("apt"): self.v1_apt,
                Optional("pacman"): self.v1_pacman,
                Optional("runtime"): self.v1_runtime,
                Optional("test"): self.v1_tests,
                Optional("before_bundle"): self.script,
                Optional("after_bundle"): self.script,
                Optional("before_runtime"): self.script,
                Optional("after_runtime"): self.script,
            }
        )

        self.v1_appimage = Schema(
            {
                "arch": str,
                Optional("update-information"): str,
                Optional("sign-key"): str,
                Optional("file_name"): str,
            }
        )

        self.v1 = Schema(
            {
                "version": int,
                Optional("script"): self.script,
                "AppDir": self.v1_appdir,
                Optional("AppImage"): self.v1_appimage,
            }
        )

    def validate(self, recipe: Roamer):
        if recipe.version() == 1:
            return self.v1.validate(recipe(resolve_variables=False))
        else:
            logging.error("Unknown recipe version: %s" % recipe.version())
            logging.info(
                "Please make sure you're using the latest appimage-builder version"
            )
            exit(1)


# -*- coding: utf-8 -*-
import sys

import click

import click_spinner

from .. import api
from .. import cli
from .. import options
from ..helpers.datetime import parse_psql_date_str, reltime

@cli.cli.command()
@click.option('--limit', '-n', nargs=1, default=20,
              help='List N latest releases')
@options.app
def releases(app, limit):
    """
    List application releases

    More:

    $ asyncy releases:rollback
    """
    cli.user()
    cli.assert_project()

    # click.echo(click.style('Releases', fg='magenta'))
    # click.echo(click.style('========', fg='magenta'))

    with click_spinner.spinner():
        res = api.Releases.list(app, limit=limit)

    res = sorted(res, key=lambda elem: elem['id'])

    if res:
        from texttable import Texttable
        table = Texttable(max_width=800)
        table.set_deco(Texttable.HEADER)
        table.set_cols_align(['l', 'l', 'l', 'l'])
        all_releases = [['VERSION', 'STATUS', 'CREATED', 'MESSAGE']]
        for release in res:
            date = parse_psql_date_str(release['timestamp'])
            all_releases.append([
                f'v{release["id"]}',
                release['state'].capitalize(),
                reltime(date),
                release['message']
            ])
        table.add_rows(rows=all_releases)
        click.echo(table.draw())
    else:
        click.echo(f'No releases yet for app {app}.')

@cli.cli.command(aliases=['releases:rollback'])
@click.argument('version', nargs=1, required=False)
@options.app
def releases_rollback(version, app):
    """
    Rollback release to a previous release.
    """
    cli.user()
    cli.assert_project()

    if not version:
        click.echo(f'Getting latest release for app {cli.get_app_name()}... ',
                   nl=False)
        with click_spinner.spinner():
            res = api.Releases.get(app=app)
            version = int(res[0]['id']) - 1
        click.echo(click.style('√', fg='green'))

    if int(version) == 0:
        click.echo('Unable to rollback a release before v1.')
        sys.exit(1)

    click.echo(f'Rolling back to v{version}... ', nl=False)
    with click_spinner.spinner():
        res = api.Releases.rollback(version=version, app=app)
    click.echo(click.style('√', fg='green'))
    click.echo(f'Deployed new release... ' +
               click.style(f'v{res["id"]}', bold=True, fg='magenta'))


import os

import numpy as np

from livelossplot import PlotLosses
from livelossplot.outputs import TensorboardTFLogger

def test_tensorboard():
    groups = {'acccuracy': ['acc', 'val_acc'], 'log-loss': ['loss', 'val_loss']}
    logger = TensorboardTFLogger()

    liveplot = PlotLosses(groups=groups, outputs=(logger, ))

    for i in range(3):
        liveplot.update(
            {
                'acc': 1 - np.random.rand() / (i + 2.),
                'val_acc': 1 - np.random.rand() / (i + 0.5),
                'loss': 1. / (i + 2.),
                'val_loss': 1. / (i + 0.5)
            }
        )
        liveplot.send()

    assert all([f.startswith('events.out.tfevents.') for f in os.listdir(logger._path)])


import requests
import pandas as pd
import csv
import json
import datetime
from datetime import timezone
from time import gmtime, strftime

Headers = {
     'X-CSRFToken': 'token',
     'Authorization': 'Bearer Authorization'
}

url = 'https://sagebrush.pamdas.org/api/v1.0/observations/?subject_id=ee76e226-c05e-400b-9ae9-db8b11b5359b'

response= requests.get(url, headers=Headers, timeout=20)

data = response.json().get("results", []) or []

def extract_property_value(properties, prop_name):
    if not isinstance(properties, list):
        return "0"
    return next((prop["value"] for prop in properties if isinstance(prop, dict) and prop.get("name")== prop_name), '0')

df = pd.DataFrame({
    'humidity': [extract_property_value(obs['device_status_properties'], 'humidity') for obs in data],
    'recorded_at': [obs.get['recorded_at','0'] for obs in data],
    'battery': [extract_property_value(obs['device_status_properties'], 'battery')for obs in data],
    'external_temperature': [extract_property_value(obs['device_status_properties'],'external_temperature')for obs in data],
    'internal_temperature': [extract_property_value(obs['device_status_properties'],'internal_temperature')for obs in data]
})

# Print the dataframe
print(df)

# def get_date(df):
#     # Getting key
#     recorded_at_key = list(df['data']['results'][0].keys())[2]

#     print(recorded_at_key.capitalize())

#     # Getting all the recorded_at value
#     recorded_at = [df['data'][3][i][recorded_at_key] for i in range(0, len(df['data'][3]))]

#     for index, time in enumerate(recorded_at):

#         # Convert recorded_at to datetime format
#         new_timestamp = datetime.datetime.strptime(time, "%Y-%m-%dT%H:%M:%S%z")

#         #converts the new_timestamp object to GMT.
#         new_timestamp_gmt = new_timestamp.astimezone(timezone.utc)

#         # Format new_timestamp_gmt in GMT
#         new_timestamp_gmt_formatted = new_timestamp_gmt.strftime("%A, %Y-%m-%d %H:%M:%S")

#         dates_list = ('Timestamp: ', new_timestamp_gmt_formatted)

#         print(dates_list)

#     return dates_list

# print(get_date(df))


from socket import *
import _ip

port = 8890
address = (_ip.ip, port)
port1 = 8899
udp_socket = socket(AF_INET, SOCK_DGRAM)
udp_socket.bind(address)
while True:
    print('等待客户发送数据...')
    data, send_address = udp_socket.recvfrom(1024)
    print('来自:%s' % str(send_address))
    print('消息:%s' % data.decode('utf-8'))
    msg = input('请输入:')
    udp_socket.sendto(msg.encode('utf-8'), (_ip.ip, port1))
udp_socket.close()

网络通信 udp 服务器.py


"""A setuptools based setup module.

See:
https://packaging.python.org/en/latest/distributing.html
https://github.com/pypa/sampleproject
"""

# Always prefer setuptools over distutils
from setuptools import setup, find_packages
# To use a consistent encoding
from codecs import open
from os import path

here = path.abspath(path.dirname(__file__))

# Get the long description from the README file
with open(path.join(here, 'README.md'), encoding='utf-8') as f:
    long_description = f.read()

setup(
    name='oc_ica',
    description='OC_ICA models and analysis.',
    long_description=long_description,
    author='Jesse Livezey and Alex Bujan',
    author_email='jesse.livezey@gmail.com')


#collections module: counter, namedtuple, orderedDirect, defaultdict, deque

from collections import Counter

a= 'aaabbbccc'
counter_char= Counter(a)  #devuelve un diccionario con cada letra y su conteo
a.counter.keys() #devuelve una letra de cada
a.counter.values() #devuelve conteo de cada letra
a.most_common() #devuelve moda. si dentro del arg se pone num int, (1)-> más repetido, (2) -> segundo más repetido, etc
lista=list(a.elements()) #devuelve cada elemento dentro de una lista
#---
#namedtuple

from collections import namedtuple

Point= namedtuple ('Point', 'x,y')
pt= Point(1,-4)
print(pt.x, pt.y)

#--
# order Dict
from collections import OrderedDict

ordered_dict= OrderedDict()   #crea un dict ordenado
ordered_dict['a'] = 1
ordered_dict['b'] = 2
ordered_dict['c'] = 3
ordered_dict['d'] = 4
ordered_dict['e'] = 5

#--
# default dict

from collections import defaultdict

d= defaultdict(int)  #devuelve 0 si no existe
d['a'] = 1
d['b'] = 2

#---
d= defaultdict(float) #devuelve 0.0 si no existe
d['a'] = 1
d['b'] = 2

#----
#deque

from collections import deque
d=deque()  #devuelve lista
d.append(1)
d.append(2)
d.appendleft(0) #pone elementos en el punto0

d.pop() #remove last element
d.popleft() #remueve primer elemento
d.extend([4,5,6]) #igual que extend lista
d.extendleft({-1,0})

d.rotate()  #invierte


import configparser
import logging
import os

from telegram.common import options

logger = logging.getLogger(__name__)
config_stack = None

DEFAULT_CONF = 'ubuntu-system-tests.conf'
KEY_DEFAULT = 'default'

CONFIG_OPTIONS = [
    options.Option(
        'device_serial', help_string='Device serial'),
    options.Option(
        'device_password', help_string='Device password', mandatory=True),
    options.Option(
        'output_dir', mandatory=True,
        help_string='Directory to store the tests results and artifacts.\n'
                    'WARNING: This directory will have all existing content '
                    'deleted!'),
    options.Option(
        'sim_0_pin', mandatory=True,
        help_string='SIM pin for the card in the first slot'),
    options.Option(
        'sim_1_pin',
        help_string='SIM pin for the card in the second slot'),
    options.Option(
        'wifi_ssid', mandatory=True,
        help_string='Wi-Fi SSID used to connect during setup wizard'),
    options.Option(
        'wifi_password', mandatory=True,
        help_string='Wi-Fi password used to connect during setup wizard'),
    options.Option(
        'device_security', default='Passcode',
        help_string='The security method to select during setup wizard'),
    options.Option(
        'bluetooth_device_name',
        help_string='Name of a nearby bluetooth pairing device'),
    options.Option(
        'device_phone_number',
        help_string="The phone number of the SIM card inserted in the first "
                    "SIM slot. This must be formatted exactly as you would "
                    "dial it."),
    options.Option(
        'device_phone_number2',
        help_string="The phone number of the SIM card inserted in the second "
                    "SIM slot. If the device has only one SIM slot, you can "
                    "leave this blank. This must be formatted exactly as you "
                    "would dial it."),
    options.Option(
        'telephony_service_number1',
        help_string="The phone number to call and message from "
                    "in the telephony service. In Twilio go to "
                    "https://www.twilio.com/user/account/"
                    "phone-numbers/incoming and click on the number "
                    "you want to use, then copy the 'Phone Number' "
                    "field exactly."),
    options.Option(
        'telephony_service_number2',
        help_string="The second phone number to call and message from "
                    "in the telephony service. In Twilio go to "
                    "https://www.twilio.com/user/account/"
                    "phone-numbers/incoming and click on the number "
                    "you want to use, then copy the 'Phone Number' "
                    "field exactly."),
    options.Option(
        # this is a private key, that will not be exported to
        # autopilot directly
        '_twilio_account_sid', mandatory=True,
        help_string='Account SID for Twilio. Please log into '
                    'https://www.twilio.com/login '
                    'and select \'Dashboard -> Show API Credentials\'.'
                    'Then copy and paste the Account SID value here.'),
    options.Option(
        # this is a private key, that will not be exported to
        # autopilot directly
        '_twilio_auth_token', mandatory=True,
        help_string='Auth token for Twilio account. Please log into '
                    'https://www.twilio.com/login '
                    'and select \'Dashboard -> Show API Credentials\'.'
                    'Then copy and paste the Auth token value here.'),
    options.Option(
        # this is a private key, that will not be exported to
        # autopilot directly
        'max_unity8_retry_delay', default=30000,
        help_string='The boundary for how long we should be retrying until '
                    'unity8 is started'),
    options.Option(
        'app_startup_cold_runs', default=0,
        help_string='How many iterations to run for the cold start app '
                    'performance tests?'),
    options.Option(
        'app_startup_hot_runs', default=0,
        help_string='How many iterations to run for the hot start app '
                    'performance tests?'),
    options.Option(
        'email_outlook', default='platform_qa_test@outlook.com',
        help_string='@outlook.com email address to use for email tests'),
    options.Option(
        'password_outlook', mandatory=True,
        help_string='@outlook.com password to use for email tests'),
    options.Option(
        'email_yahoo', default='platform_qa_test_account@yahoo.com',
        help_string='@yahoo.com email address to use for email tests'),
    options.Option(
        'password_yahoo', mandatory=True,
        help_string='@yahoo.com password to use for email tests'),
    options.Option(
        'username_imap', default='platform-qa-test-account',
        help_string='IMAP username to use for email tests'),
    options.Option(
        'password_imap', mandatory=True,
        help_string='IMAP password to use for email tests'),
    options.Option(
        'pictures_scalability_runs',
        help_string='The different loads used for scalability pictures tests '
                    'separated by comma'),
    options.Option(
        '_practitest_project_id',
        help_string='Practitest project ID to use for retrieving test lists.'),
    options.Option(
        '_practitest_api_token',
        help_string='Practitest api token used for authentication.'),
    options.Option(
        'country_sim', default='United States',
        help_string='The name of the country of the SIM card to use for '
                    'telegram login'),
]

def get_user_config_dir():
    """Return the path to the user configuration directory."""
    conf_dir = os.environ.get(
        'XDG_CONFIG_HOME', os.path.expanduser('~/.config'))
    return conf_dir

def get_device_config_file_path():
    """Return the path of the config file copied to the device."""
    return '/tmp/ubuntu-system-tests.conf'

def get_device_config_section():
    """Return the name of config section to use."""
    return os.environ.get('CONFIG_SECTION')

def get_config_stack_from_file(file_path, config_section=None):
    """Return config stack from specified file path."""
    config = UbuntuSystemTestsConfig(file_path, section=config_section)
    config.get_config_from_file()
    return config

def get_device_config_stack():
    """Return the config stack variable"""
    global config_stack
    if not config_stack:
        config_stack = get_config_stack_from_file(
            get_device_config_file_path(), get_device_config_section())
    return config_stack

def get_test_config_values(config_stack):
    """ Return a string containing the comma separated key=value options.
    The list includes the config values and the temporal ones as well
    """
    return config_stack.get_autopilot_config_string()

class UbuntuSystemTestsConfig():

    """Class to get and save test configuration data."""

    def __init__(self, file_name=None, section=None, options=None):
        """
        Construct the config and parser object.
        :param file_name: Name of config file to load. Default file is used if
                          value is not specified.
        :param section: Name of section to use in config file. Default section
                        is used if not specified.
        :param options: List of options that should be specified in config.
                        Default list of options is used if not specified.
        """
        user_config_dir = get_user_config_dir()
        if not os.path.exists(user_config_dir):
            os.mkdir(user_config_dir)
        self.file_path = os.path.join(
            user_config_dir, file_name or DEFAULT_CONF)
        self.section = section or KEY_DEFAULT
        self.parser = configparser.ConfigParser(
            allow_no_value=True, empty_lines_in_values=False,
            default_section=KEY_DEFAULT)
        self.config = {}
        self.options = options or CONFIG_OPTIONS

    def set(self, option_name, value):
        """Set a config value.
        :param option_name: Name of config option.
        :param value: Value to set.
        """
        self.config[option_name] = value
        self.save()

    def get(self, option_name, default=''):
        """Get a config value.
        :param option_name: Name of config option to get.
        :param default: Value to return as default if item doesn't exist.
        :return: Config option value if it exists. If the config option itself
        doesn't exist, return None. Otherwise if the option exists but no value
        is defined return the default value."""
        return self.config.get(option_name, default)

    def save(self):
        """Save the config to file."""
        mode = 'r+' if os.path.exists(self.file_path) else 'w'
        with open(self.file_path, mode) as f:
            self.parser.write(f)

    def get_config_from_string(self, string):
        """Load config from a string. This is used for selftests only.
        :param string: Config string to load from.
        """
        self.parser.read_string(string)
        self._save_config_for_section()

    def get_config_from_file(self):
        """Load config from file."""
        self.parser.read(self.file_path)
        self._save_config_for_section()

    def _save_config_for_section(self):
        """Save the current config for the specified section only."""
        self.config = self.parser[self.section]

    def get_missing_options(self, silent=False):
        """Check through each required option and get a value for each one
        that is not present.
        :param silent: Whether to query stdin for an input value or take a
                       default value where it exists.
        """
        save_required = False
        for option in self.options:
            if option.name in self.config:
                # name exists, check if its mandatory and if a value exists
                if option.mandatory and not self.config.get(option.name):
                    # mandatory and no value exists, so get one
                    self._get_option_value(option, silent)
                    save_required = True
            else:
                # option name not in config, get a value
                self._get_option_value(option, silent)
                save_required = True
        self._save_if_required(save_required)

    def _save_if_required(self, save_required):
        """Save config if it is required.
        :param save_required: Whether to save or not.
        """
        if save_required:
            self.save()

    def _get_option_value(self, option, silent):
        """Request a value for specified option.
        :param option: Required option.
        :param silent: Whether to query stdin or take default value.
        """
        if silent:
            default = option.default
            if default:
                self.config[option.name] = option.default
                logger.warning(
                    'In silent mode no value specified for option: "{o}". '
                    'Using default value "{v}".'.format(
                        o=option.name, v=default))
            else:
                logger.warning(
                    'In silent mode no default value specified for option: '
                    '"{o}". Continuing with value unset.'.format(
                        o=option.name))
        else:
            self._get_option_value_from_stdin(option)

    def _get_option_value_from_stdin(self, option):
        """Request option value from stdin.
        :param option: Required option.
        """
        help_msg = self._get_option_help_text(option)
        print(help_msg, end='')
        value = self._get_value_from_user_or_default(option)
        self.config[option.name] = value

    def _get_option_help_text(self, option):
        """Return help message for specified option.
        :param option: Required option.
        """
        help_msg = option.help_string
        default = option.default
        if default:
            help_msg += ' (leave blank to use default value: {})'.format(
                default)
        elif not option.mandatory:
            help_msg += ' (leave blank if not required)'
        help_msg += ': '
        return help_msg

    def _get_value_from_user_or_default(self, option):
        """Return value from stdin or default value if no stdin value."""
        return input() or option.default

    def get_autopilot_config_string(self):
        """Return the configuration in a string usable by Autopilot.

        That is, a list of key=value pairs serparated by comma.

        Any keys starting with _ will be ignored.

        """
        config = ''
        for key, value in self.config.items():
            if not key.startswith('_'):
                config += '{k}={v},'.format(k=key, v=value)

        # remove trailing ',' if it exists
        if config.endswith(','):
            config = config[:-1]
        return config


import snakeoil3_gym
import numpy as np
import random as rd
from gym_torcs import TorcsEnv

PI= 3.14159265359

def get_n_clients(N, start_port):
	clients = []
	for i in range(N):
		c = snakeoil3_gym.Client(p=start_port+i)
		clients.append(c)
	return clients

def drive_example(c, target_speed, init_pos):
	u'''This is only an example. It will get around the track but the
	correct thing to do is write your own `drive()` function.'''
	S,R= c.S.d,c.R.d
	# target_speed=1000
	min_dist = 10
	# Steer To Corner
	R[u'steer']= S[u'angle']*10 / PI
	# Steer To the old pose
	R[u'steer'] += (init_pos - S[u'trackPos'])*.3

	if S[u'angle']*180/PI < 5:
		R[u'steer'] = np.clip(R[u'steer'], -.1, .1)

	# print(R[u'steer'])

	R[u'accel'] = 0.5

	if target_speed-10 < S[u'speedX']:
	    R[u'accel'] = 0.01

	if target_speed <= S[u'speedX']:
	    R[u'accel'] = 0.

	opponents = S['opponents']
	front = np.array([opponents[15], opponents[16], opponents[17], opponents[18], opponents[19], opponents[20]])
	back  = np.array([opponents[-3], opponents[-2], opponents[-1], opponents[0], opponents[1], opponents[2]])
	left  = np.array([opponents[6], opponents[7], opponents[8], opponents[9], opponents[10], opponents[11]])
	right = np.array([opponents[23], opponents[24], opponents[25], opponents[26], opponents[27], opponents[28], opponents[29]])

	# R[u'accel'] += np.random.normal(0, 0.1)

	# Throttle Control
	# if S[u'speedX'] < target_speed - (R[u'steer']*50):
	#     R[u'accel']+= .01
	# else:
	#     R[u'accel']-= .01
	# if S[u'speedX']<10:
	#    R[u'accel']+= 1/(S[u'speedX']+.1)

	if min(front) < 10:
		R[u'accel'] = 0.0

	if min(back) < 10:
		R[u'accel'] += 1.0
		R[u'brake'] = 0.

	# if min(front) > 100:
	# 	R[u'accel'] += 1.0

	# Traction Control System
	# if ((S[u'wheelSpinVel'][2]+S[u'wheelSpinVel'][3]) -
	#    (S[u'wheelSpinVel'][0]+S[u'wheelSpinVel'][1]) > 5):
	#    R[u'accel']-= .2

	# Automatic Transmission
	# R[u'gear']=1
	# if S[u'speedX']>50:
	#     R[u'gear']=2
	# if S[u'speedX']>80:
	#     R[u'gear']=3
	# if S[u'speedX']>110:
	#     R[u'gear']=4
	# if S[u'speedX']>140:
	#     R[u'gear']=5
	# if S[u'speedX']>170:
	#     R[u'gear']=6
	return

def block_driving(maxSteps):
	# env = TorcsEnv(vision=False, throttle=True, gear_change=False, main=1)

	velocities_range = [[100, 120], [70, 90], [40, 70]]

	clients = get_n_clients(10, 3101)

	for c in clients:
		c.get_servers_input(0)

	target_v1 = np.random.choice(range(velocities_range[0][0], velocities_range[0][1]))
	target_v2 = np.random.choice(range(velocities_range[1][0], velocities_range[1][1]))
	target_v3 = np.random.choice(range(velocities_range[2][0], velocities_range[2][1]))

	target_speed = [target_v1]*4 + [target_v2]*4 + [target_v3]*2

	init_track_pos = [c.S.d['trackPos'] for c in clients]
	print(init_track_pos)
	print(target_speed)
	# init_track_pos = [0.75, -0.75, 0.75, -0.75]*2 + [0.75, -0.75]

	for step in range(maxSteps,0,-1):
		for i, c in enumerate(clients):
			c.get_servers_input(step)
			drive_example(c, target_speed[i], init_track_pos[i])
			c.respond_to_server()

	for c in clients:
		c.shutdown()

def lane_driving(maxSteps):
	clients = get_n_clients(10, 3101)

	init_track_pos = 0.75

	for step in range(maxSteps,0,-1):
		for i, c in enumerate(clients):
			c.get_servers_input(step)
			drive_example(c, 100+i, init_track_pos)
			c.respond_to_server()

		if (maxSteps - step)%100 == 0:
			init_track_pos *= -1
	for c in clients:
		c.shutdown()

if __name__ == "__main__":
	block_driving(10000)


"""Main entry point
"""
from pyramid.config import Configurator

def main(global_config, **settings):
    config = Configurator(settings=settings)
    config.include("cornice")
    config.include('cornice_swagger')
    config.include("pyramid_chameleon")
    config.add_static_view("static", "static", cache_max_age=3600)
    config.add_route("swagger", "/swagger")
    config.scan("{{cookiecutter.repo_name}}.views")
    return config.make_wsgi_app()


import turtle as t
t.color("green")
t.pensize(3)
for j in range(0,361,10):
    t.seth(j)
    for i in range(4):
        t.forward(100)
        t.left(90)
t.exitonclick()


text = "http://www.pythonchallenge.com/pc/def/map.html"
output_text = ""
for letter in text:

	value = ord(letter)
	if value >= 97 and value <= 122:
		value += 2
		if value > 122:
			value -= 26
		output_text += chr(value)
	else:
		output_text += letter

translation_table = text.maketrans('abcdefghijklmnopqrstuvwxyz','cdefghijklmnopqrstuvwxyzab')
second_output_text = text.translate(translation_table)

print(second_output_text)

print(output_text)

print(ord('a'))

print(ord('z'))


# Create tuples
import csv
with open("wikiHow_categories_hierarchy.ttl") as f:
    reader = csv.reader(f)
    data = list(list(rec) for rec in csv.reader(f, delimiter='\t'))

# print(data[0])
import re
articles = ['a', 'an', 'of', 'the', 'is', 'and', 'for']

def title_except(s, exceptions):
    word_list = re.split('-', s)       # re.split behaves as expected
    final = [word_list[0].capitalize()]
    for word in word_list[1:]:
        final.append(word if word in exceptions else word.capitalize())
    return "-".join(final)

def cleanup(val):
    new_val = ""
    if val.startswith('ns1'):
        new_val = val[4:]
    elif val.startswith(''):
        new_val = val.replace("<http://www.wikihow.com/category:", "")
        new_val = new_val[:-1]

    return title_except(new_val, articles)

# print(cleanup('finance-and-business'))

new_data = [(cleanup(val[0]), cleanup(val[2][:-2])) for val in data]

print(new_data[1])

# a = [(1, 1), (4, 3), (5, 3), (6, 3), (7, 7), (8, 7), (9, 7), (2, 1), (3, 1)]

a = new_data
# print(a)
# pass 1: create nodes dictionary
nodes = {}
for i in a:
    id, parent_id = i
    nodes[id] = {'name': id}

# pass 2: create trees and parent-child relations
forest = []
for i in a:
    id, parent_id = i
    node = nodes[id]

    # either make the node a new tree or link it to its parent
    if id == parent_id:
        # start a new tree in the forest
        forest.append(node)
    else:
        # add new_node as child to parent
        parent = nodes[parent_id]
        if not 'children' in parent:
            # ensure parent has a 'children' field
            parent['children'] = []
        children = parent['children']
        children.append(node)

import json

with open('know-how.json', 'w') as f:
    json.dump(forest[0], f, indent=4)


import requests
from bs4 import BeautifulSoup

url = "https://www.clcoding.com/"
r = requests.get(url)

soup= BeautifulSoup(r.content, 'html.parser')
links = soup.find_all('a')
for link in links:
    print(link.get('href'))


i=0
a=[]
while i<=4:
    b=input("enter a method")
    if b=="append":
        c=int(input("enter a number"))
        a.append(int(c))
    elif b=="insert":
        index=int(input("enter a index number"))
        n=int(input("inter a number"))
        a.insert(index,n)
    elif b=="reverse":
        a.reverse()
    elif b=="pop":
        a.pop()
    elif b=="sort":
        a.sort()
    print(a)
    i=i+1