""" Clean the dataset """
import argparse
import hashlib
import logging
from urllib.parse import urlparse

import pandas as pd
import nltk
from nltk.corpus import stopwords
nltk.download('punkt')
nltk.download('stopwords')

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

def _read_data(filename):
    logger.info('Reading file %s', filename)

    return pd.read_csv(filename)

def _extract_newspaper_uid(filename):
    logger.info('Extracting newspaper uid')
    newspaper_uid = filename.split('_')[0]

    logger.info('Newspaper uid detected: %s', newspaper_uid)

    return newspaper_uid

def _add_newspaper_uid_column(df, newspaper_uid):
    logger.info('Filling newspaper_uid with %s', newspaper_uid)
    df['newspaper_uid'] = newspaper_uid

    return df

def _extract_host(df):
    logger.info('Extracting host from urls')
    df['host'] = df['url'].apply(lambda url: urlparse(url).netloc)

    return df

def _fill_missing_titles(df):
    logger.info('Filling missing titles.')
    missing_titles_mask = df['title'].isna()

    missing_titles = (df[missing_titles_mask]['url']
                      .str.extract(r'(?P<missing_titles>[^/]+)$')
                      .applymap(lambda title: title.split('-'))
                      .applymap(lambda title_word_list: ' '.join(title_word_list))
                      )

    df.loc[missing_titles_mask, 'title'] = missing_titles.loc[:, 'missing_titles']

    return df

def _generate_uids_for_rows(df):
    logger.info('Generating uids for each row')
    uids = (df
            .apply(lambda row: hashlib.md5(bytes(row['url'].encode())), axis=1)
            .apply(lambda hash_object: hash_object.hexdigest())
            )

    df['uid'] = uids

    return df.set_index('uid')

def _remove_new_lines_from_body(df):
    logger.info('Removing new lines from body.')

    stripped_body = (df
                     .apply(lambda row: row['body'], axis=1)
                     .apply(lambda body: list(body))
                     .apply(lambda letters: list(map(lambda letter: letter.replace('\n', ' '), letters)))
                     .apply(lambda letters: ''.join(letters)))

    df['body'] = stripped_body

    return df

def _tokenize_column(df, column_name):
    stop_words = set(stopwords.words('spanish'))

    column_names = []
    if type(column_name) is not list:
        column_names.append(column_name)
    else:
        column_names = column_name

    logger.info(
        f'Tokenizing column(s) {[column for column in column_names]}')

    for column in column_names:
        df['n_token_'+column] = (df
                                 .dropna()
                                 .apply(lambda row: nltk.word_tokenize(row[column]), axis=1)
                                 .apply(lambda tokens: list(filter(lambda token: token.isalpha(), tokens)))
                                 .apply(lambda tokens: list(map(lambda token: token.lower(), tokens)))
                                 .apply(lambda word_list: list(filter(lambda word: word not in stop_words, word_list)))
                                 .apply(lambda valid_word_list: len(valid_word_list)))

    return df

# noinspection PyShadowingNames
def main(filename):
    """ Funcion principal """
    logger.info('Starting cleaning process')

    df = _read_data(filename)
    print('Dataframe in firts step')
    print(df)

    newspaper_uid = _extract_newspaper_uid(filename)
    df = _add_newspaper_uid_column(df, newspaper_uid)
    df = _extract_host(df)
    df = _fill_missing_titles(df)
    df = _generate_uids_for_rows(df)
    df = _remove_new_lines_from_body(df)
    df = _tokenize_column(df, 'title')
    df = _tokenize_column(df, 'body')

    return df

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('filename', help='The path to the dirty data', type=str)

    arg = parser.parse_args()

    df = main(arg.filename)

    print(df)
    df.to_csv('cleaned_dataframe_el_universal.csv', sep='\t', encoding='utf-8')


"""
Utility to create tile pyramids out of an input raster.

Also provides various options to rescale data if necessary
"""

import click
import os
import rasterio

import mapchete
from mapchete.cli import utils
from mapchete.io import get_best_zoom_level

# ranges from rasterio
# https://github.com/mapbox/rasterio/blob/master/rasterio/dtypes.py#L61
DTYPE_RANGES = {
    'uint8': (0, 255),
    'uint16': (0, 65535),
    'int16': (-32768, 32767),
    'uint32': (0, 4294967295),
    'int32': (-2147483648, 2147483647),
    'float32': (-3.4028235e+38, 3.4028235e+38),
    'float64': (-1.7976931348623157e+308, 1.7976931348623157e+308)
}

@click.command(help="Create a tile pyramid from an input raster dataset.")
@utils.arg_input_raster
@utils.arg_out_dir
@utils.opt_pyramid_type_mercator
@utils.opt_output_format
@utils.opt_resampling_method
@utils.opt_scale_method
@utils.opt_zoom
@utils.opt_bounds
@utils.opt_overwrite
@utils.opt_debug
def pyramid(
    input_raster,
    output_dir,
    pyramid_type=None,
    output_format=None,
    resampling_method=None,
    scale_method=None,
    zoom=None,
    bounds=None,
    overwrite=False,
    debug=False
):
    """Create tile pyramid out of input raster."""
    bounds = bounds if bounds else None
    options = dict(
        pyramid_type=pyramid_type,
        scale_method=scale_method,
        output_format=output_format,
        resampling=resampling_method,
        zoom=zoom,
        bounds=bounds,
        overwrite=overwrite
    )
    raster2pyramid(input_raster, output_dir, options)

def raster2pyramid(input_file, output_dir, options):
    """Create a tile pyramid out of an input raster dataset."""
    pyramid_type = options["pyramid_type"]
    scale_method = options["scale_method"]
    output_format = options["output_format"]
    resampling = options["resampling"]
    zoom = options["zoom"]
    bounds = options["bounds"]
    mode = "overwrite" if options["overwrite"] else "continue"

    # Prepare process parameters
    minzoom, maxzoom = _get_zoom(zoom, input_file, pyramid_type)
    with rasterio.open(input_file, "r") as input_raster:
        output_bands = input_raster.count
        input_dtype = input_raster.dtypes[0]
        output_dtype = input_raster.dtypes[0]
        nodataval = input_raster.nodatavals[0]
        nodataval = nodataval if nodataval else 0
        if output_format == "PNG" and output_bands > 3:
            output_bands = 3
            output_dtype = 'uint8'
        scales_minmax = ()
        if scale_method == "dtype_scale":
            for index in range(1, output_bands+1):
                scales_minmax += (DTYPE_RANGES[input_dtype], )
        elif scale_method == "minmax_scale":
            for index in range(1, output_bands+1):
                band = input_raster.read(index)
                scales_minmax += ((band.min(), band.max()), )
        elif scale_method == "crop":
            for index in range(1, output_bands+1):
                scales_minmax += ((0, 255), )
        if input_dtype == "uint8":
            scale_method = None
            scales_minmax = ()
            for index in range(1, output_bands+1):
                scales_minmax += ((None, None), )

    # Create configuration
    config = dict(
        process="mapchete.processes.pyramid.tilify",
        output={
            "path": output_dir,
            "format": output_format,
            "bands": output_bands,
            "dtype": output_dtype
            },
        pyramid=dict(pixelbuffer=5, grid=pyramid_type),
        scale_method=scale_method,
        scales_minmax=scales_minmax,
        input={"raster": input_file},
        config_dir=os.getcwd(),
        zoom_levels=dict(min=minzoom, max=maxzoom),
        nodataval=nodataval,
        resampling=resampling,
        bounds=bounds,
        baselevel={"zoom": maxzoom, "resampling": resampling},
        mode=mode
    )

    # create process
    with mapchete.open(config, zoom=zoom, bounds=bounds) as mp:
        # prepare output directory
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)
        # run process
        mp.batch_process(zoom=[minzoom, maxzoom])

def _get_zoom(zoom, input_raster, pyramid_type):
    """Determine minimum and maximum zoomlevel."""
    if not zoom:
        minzoom = 1
        maxzoom = get_best_zoom_level(input_raster, pyramid_type)
    elif len(zoom) == 1:
        minzoom = zoom[0]
        maxzoom = zoom[0]
    elif len(zoom) == 2:
        if zoom[0] < zoom[1]:
            minzoom = zoom[0]
            maxzoom = zoom[1]
        else:
            minzoom = zoom[1]
            maxzoom = zoom[0]
    return minzoom, maxzoom


# 정보문화진흥원 정보 영재 동아리에서 동아리 활동을 하던 영수와 민혁이는 쉬는 시간을 틈타 숫자야구 게임을 하기로 했다.
#
#
#
#
#
# * 영수는 1에서 9까지의 서로 다른 숫자 세 개로 구성된 세 자리 수를 마음속으로 생각한다. (예: 324)
# * 민혁이는 1에서 9까지의 서로 다른 숫자 세 개로 구성된 세 자리 수를 영수에게 묻는다. (예: 123)
# * 민혁이가 말한 세 자리 수에 있는 숫자들 중 하나가 영수의 세 자리 수의 동일한 자리에 위치하면 스트라이크 한 번으로 센다.
# 숫자가 영수의 세 자리 수에 있긴 하나 다른 자리에 위치하면 볼 한 번으로 센다.
#
#
# 예) 영수가 324를 갖고 있으면
#
# 429는 1 스트라이크 1 볼이다.
# 241은 0 스트라이크 2 볼이다.
# 924는 2 스트라이크 0 볼이다.
#
#
#
#
# 영수는 민혁이가 말한 수가 몇 스트라이크 몇 볼인지를 답해준다.
#
#
#
# * 민혁이가 영수의 세 자리 수를 정확하게 맞추어 3 스트라이크가 되면 게임이 끝난다.
#
# 아니라면 민혁이는 새로운 수를 생각해 다시 영수에게 묻는다.
#
#
#
# 현재 민혁이와 영수는 게임을 하고 있는 도중에 있다.
#
# 민혁이가 영수에게 어떤 수들을 물어보았는지, 그리고 각각의 물음에 영수가 어떤 대답을 했는지가 입력으로 주어진다.
#
# 이 입력을 바탕으로 여러분은 영수가 생각하고 있을 가능성이 있는 수가 총 몇 개인지를 알아맞혀야 한다.
#
#
#
# 아래와 같은 경우를 생각해보자.
#
#
#
# 민혁: 123
#
# 영수: 1 스트라이크 1 볼.
#
# 민혁: 356
#
# 영수: 1 스트라이크 0 볼.
#
# 민혁: 327
#
# 영수: 2 스트라이크 0 볼.
#
# 민혁: 489
#
# 영수: 0 스트라이크 1 볼.
#
#
#
# 이 때 가능한 답은 324와 328, 이렇게 두 가지이다.
#
#
#
# 영수는 동아리의 규율을 잘 따르는 착한 아이라 민혁이의 물음에 곧이곧대로 정직하게 답한다.
#
# 그러므로 영수의 답들에는 모순이 없다.
#
#
#
# 민혁이의 물음들과 각각의 물음에 대한 영수의 답이 입력으로 주어질 때
#
# 영수가 생각하고 있을 가능성이 있는 답의 총 개수를 출력하는 프로그램을 작성하시오.
def check(n):
    global ans
    for num, s, b in temp:
        ns = 0
        nb = 0
        for j in range(3):
            if n[j] == num[j]:
                ns += 1
            elif n[j] in num:
                nb += 1
        if ns != s or nb != b:
            return 0
    ans += 1
    return 0

N = int(input())
temp = []
ans = 0
for i in range(N):
    num, s, b = input().split()
    s = int(s)
    b = int(b)
    num = list(map(int,num))
    temp.append([num, s, b])
temp.sort(key=lambda x:x[1],reverse=True)
for i in range(123, 988):
    n = [i // 100, (i // 10) % 10, i % 10]
    if n[0] == 0 or n[1] == 0 or n[2] == 0 or n[0] == n[1] or n[0] == n[2] or n[1] == n[2]:
        continue
    check(n)

print(ans)


import re
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from nltk.tokenize import word_tokenize, WhitespaceTokenizer, TweetTokenizer
np.random.seed(seed=234)
from io import StringIO

# The below reads in N lines of text from the 40-million-word news corpus I used (provided by SwiftKey for educational purposes).
N = 10000
with open("ptb.train.txt") as myfile:
    articles = [next(myfile) for x in range(N)]
joined_articles = [" ".join(articles)]

# The below takes out anything that's not a letter, replacing it with a space, as well as any single letter that is not the pronoun "I" or the article "a."
def clean_article(article):
    art1 = re.sub("[^A-Za-z]", ' ', article)
    art2 = re.sub("\s[B-HJ-Zb-hj-z]\s", ' ', art1)
    art3 = re.sub("^[B-HJ-Zb-hj-z]\s", ' ', art2)
    art4 = re.sub("\s[B-HJ-Zb-hj-z]$", ' ', art3)
    return art4.lower()

# The below breaks up the words into n-grams of length 1 to 5 and puts their counts into a Pandas dataframe with the n-grams as column names.  The maximum number of n-grams can be specified if a large corpus is being used.
ngram_bow = CountVectorizer(stop_words = None, preprocessor = clean_article, tokenizer = WhitespaceTokenizer().tokenize, ngram_range=(1,5), max_features = None, max_df = 1.0, min_df = 1, binary = False)
ngram_count_sparse = ngram_bow.fit_transform(joined_articles)
ngram_count = pd.DataFrame(ngram_count_sparse.toarray())
ngram_count.columns = ngram_bow.get_feature_names()

# The below turns the n-gram-count dataframe into a Pandas series with the n-grams as indices for ease of working with the counts.  The second line can be used to limit the n-grams used to those with a count over a cutoff value.
sums = ngram_count.sum(axis = 0)
sums = sums[sums > 0]
ngrams = list(sums.index.values)

# The function below gives the total number of occurrences of 1-grams in order to calculate 1-gram frequencies
def number_of_onegrams(sums):
    onegrams = 0
    for ng in ngrams:
        ng_split = ng.split(" ")
        if len(ng_split) == 1:
            onegrams += sums[ng]
    return onegrams

# The function below makes a series of 1-gram frequencies.  This is the last resort of the back-off algorithm if the n-gram completion does not occur in the corpus with any of the prefix words.
def base_freq(og):
    freqs = pd.Series()
    for ng in ngrams:
        ng_split = ng.split(" ")
        if len(ng_split) == 1:
            freqs[ng] = sums[ng] / og
    return freqs

# For use in later functions so as not to re-calculate multiple times:
bf = base_freq(number_of_onegrams(sums))

# The function below finds any n-grams that are completions of a given prefix phrase with a specified number (could be zero) of words 'chopped' off the beginning.  For each, it calculates the count ratio of the completion to the (chopped) prefix, tabulating them in a series to be returned by the function.  If the number of chops equals the number of words in the prefix (i.e. all prefix words are chopped), the 1-gram base frequencies are returned.
def find_completion_scores(prefix, chops, factor = 0.4):
    cs = pd.Series()
    prefix_split = prefix.split(" ")
    l = len(prefix_split)
    prefix_split_chopped = prefix_split[chops:l]
    new_l = l - chops
    if new_l == 0:
        return factor**chops * bf
    prefix_chopped = ' '.join(prefix_split_chopped)
    for ng in ngrams:
        ng_split = ng.split(" ")
        if (len(ng_split) == new_l + 1) and (ng_split[0:new_l] == prefix_split_chopped):
            cs[ng_split[-1]] = factor**chops * sums[ng] / sums[prefix_chopped]
    return cs

# The below tries different numbers of 'chops' up to the length of the prefix to come up with a (still unordered) combined list of scores for potential completions of the prefix.
def score_given(given, fact = 0.4):
    sg = pd.Series()
    given_split = given.split(" ")
    given_length = len(given_split)
    for i in range(given_length+1):
        fcs = find_completion_scores(given, i, fact)
        for i in fcs.index:
            if i not in sg.index:
                sg[i] = fcs[i]
    return sg

#The below takes the potential completion scores, puts them in descending order and re-normalizes them as a percentage (pseudo-probability).
def score_output(given, fact = 0.4):
    sg = score_given(given, fact)
    ss = sg.sum()
    sg = 100 * sg / ss
    sg.sort_values(axis=0, ascending=False, inplace=True)
    sg = round(sg,1)[0:15]
    return sg.to_json()


#!/usr/bin/env python
# coding: utf-8

from django.utils.safestring import mark_safe
from table.columns import Column

class CheckboxColumn(Column):
    def __init__(self, header=None, **kwargs):
        kwargs["field"] = None
        kwargs["safe"] = False
        kwargs["sortable"] = False
        kwargs["searchable"] = False
        super(CheckboxColumn, self).__init__(header=header, **kwargs)

    def render(self, obj):
        return mark_safe('<input type="checkbox">')


#calss header
class _SCRAPE():
	def __init__(self,):
		self.name = "SCRAPE"
		self.definitions = [u'a difficult or slightly dangerous situation that you cause by your own silly behaviour: ', u'a slight injury or an unpleasant noise produced by rubbing against a surface: ']

		self.parents = []
		self.childen = []
		self.properties = []
		self.jsondata = {}

		self.specie = 'nouns'

	def run(self, obj1 = [], obj2 = []):
		return self.jsondata


"""
Utility functions for pulse2percept
"""
import numpy as np
import multiprocessing
import random
import copy
import logging

from scipy import misc as spm
from scipy import interpolate as spi
from scipy import signal as sps

# Rather than trying to import these all over, try once and then remember
# by setting a flag.
try:
    import joblib
    joblib.Parallel
    has_joblib = True
except (ImportError, AttributeError):
    has_joblib = False

try:
    import dask
    import dask.multiprocessing
    dask.delayed
    has_dask = True
except (ImportError, AttributeError):
    has_dask = False

try:
    from numba import jit
    has_jit = True
except ImportError:
    has_jit = False

def deprecated(arg):
    """Decorator used to mark functions as deprecated

    This is a decorator which can be used to mark functions as deprecated.
    It will result in a warning being emitted when the function is used.

    Adapted from: http://www.artima.com/weblogs/viewpost.jsp?thread=240845

    Parameters
    ----------
    alt_func : str, optional
        The function to use instead of the deprecated one.
        Be sure to list a string, not a function object.
        Default: None.
    """
    def wrap(func):
        """The function performing the decoration

        This function is perfoming the actual decoration process. It can only
        have a single argument, which is the function object.

        Parameters
        ----------
        func : function object
            The function that is deprecated.
        """
        e_s = "Call to deprecated function %s." % func.__name__
        if alt_func:
            e_s += " Use %s instead." % alt_func
        logging.getLogger(__name__).warn(e_s)

        def wrapped_func(*args, **kwargs):
            return func(*args, **kwargs)
        return wrapped_func

    alt_func = None
    if callable(arg):
        # Direct decoration
        return wrap(arg)
    else:
        # Alternative function given
        alt_func = arg
        return wrap

class Parameters(object):
    """Container to wrap a MATLAB array in a Python dict

    This function is deprecated as of v0.2 and will be completely removed
    in v0.3.

    .. deprecated:: 0.2
    """

    @deprecated
    def __init__(self, **params):
        for k, v in params.items():
            self.__dict__[k] = v

    def __repr__(self):
        my_list = []
        for k, v in self.__dict__.items():
            my_list.append("%s : %s" % (k, v))
        my_list.sort()
        my_str = "\n".join(my_list)
        return my_str

    def __setattr(self, name, values):
        self.__dict__[name] = values

class TimeSeries(object):

    def __init__(self, tsample, data):
        """Container for time series data

        Provides a container for time series data. Every time series has a
        sampling step `tsample`, and some `data` sampled at that rate.

        Parameters
        ----------
        tsample : float
            Sampling time step (seconds).
        data : array_like
            Time series data sampled at every `tsample` seconds.
        """
        self.data = data
        self.tsample = tsample
        self.duration = self.data.shape[-1] * tsample
        self.shape = data.shape

    def __getitem__(self, y):
        return TimeSeries(self.tsample, self.data[y])

    def append(self, other):
        """Appends the data of another TimeSeries object (in time)

        This function concatenates the data of another TimeSeries object to
        the current object, along the last dimension (time). To make this work,
        all but the last dimension of the two objects must be the same.

        If the two objects have different time sampling steps, the other object
        is resampled to fit the current `tsample`.

        Parameters
        ----------
        other : p2p.utils.TimeSeries
            A TimeSeries object whose content should be appended.

        Examples
        --------
        >>> from pulse2percept import utils
        >>> pt = utils.TimeSeries(1.0, np.zeros((2, 2, 10)))
        >>> num_frames = pt.shape[-1]
        >>> pt.append(pt)
        >>> pt.shape[-1] == 2 * num_frames
        True
        """
        # Make sure type is correct
        if not isinstance(other, TimeSeries):
            raise TypeError("Other object must be of type "
                            "p2p.utils.TimeSeries.")

        # Make sure size is correct for all but the last dimension (number
        # of frames)
        if self.shape[:-1] != other.shape[:-1]:
            raise ValueError("Shape mismatch: ", self.shape[:-1], " vs. ",
                             other.shape[:-1])

        # Then resample the other to current `tsample`
        resampled = other.resample(self.tsample)

        # Then concatenate the two
        self.data = np.concatenate((self.data, resampled.data), axis=-1)
        self.duration = self.data.shape[-1] * self.tsample
        self.shape = self.data.shape

    def max(self):
        """Returns the time and value of the largest data point

        This function returns the the largest value in the TimeSeries data,
        as well as the time at which it occurred.

        Returns
        -------
        t : float
            time (s) at which max occurred
        val : float
            max value
        """
        # Find index and value of largest element
        idx = self.data.argmax()
        val = self.data.max()

        # Find frame that contains the brightest data point using `unravel`,
        # which maps the flat index `idx_px` onto the high-dimensional
        # indices (x,y,z).
        # What we want is index `z` (i.e., the frame index), given by the last
        # dimension in the return argument.
        idx_frame = np.unravel_index(idx, self.data.shape)[-1]

        # Convert index to time
        t = idx_frame * self.tsample

        return t, val

    def max_frame(self):
        """Returns the time frame that contains the largest data point

        This function returns the time frame in the TimeSeries data that
        contains the largest data point, as well as the time at which
        it occurred.

        Returns
        -------
        t : float
            time (s) at which max occurred
        frame : TimeSeries
            A TimeSeries object.
        """
        # Find index and value of largest element
        idx = self.data.argmax()

        # Find frame that contains the brightest data point using `unravel`,
        # which maps the flat index `idx_px` onto the high-dimensional
        # indices (x,y,z).
        # What we want is index `z` (i.e., the frame index), given by the last
        # dimension in the return argument.
        idx_frame = np.unravel_index(idx, self.data.shape)[-1]
        t = idx_frame * self.tsample
        frame = self.data[..., idx_frame]

        return t, TimeSeries(self.tsample, copy.deepcopy(frame))

    def resample(self, tsample_new):
        """Returns data sampled according to new time step

        This function returns a TimeSeries object whose data points were
        resampled according to a new time step `tsample_new`. New values
        are found using linear interpolation.

        Parameters
        ----------
        tsample_new : float
            New sampling time step (s)

        Returns
        -------
        TimeSeries object
        """
        if tsample_new is None or tsample_new == self.tsample:
            return TimeSeries(self.tsample, self.data)

        # Try to avoid rounding errors in arr size by making sure `t_old` is
        # too long at first, then cutting it to the right size
        y_old = self.data
        t_old = np.arange(0, self.duration + self.tsample, self.tsample)
        t_old = t_old[:y_old.shape[-1]]
        f = spi.interp1d(t_old, y_old, axis=-1, fill_value='extrapolate')

        t_new = np.arange(0, self.duration, tsample_new)
        y_new = f(t_new)

        return TimeSeries(tsample_new, y_new)

def _sparseconv(data, kernel, mode):
    """
    Returns the discrete, linear convolution of two one-dimensional sequences.
    output is of length len(kernel) + len(data) -1 (same as the default for
    numpy.convolve).

    Can run faster than numpy.convolve if:
    (1) a is much longer than v
    (2) a is sparse (has lots of zeros)
    """
    kernel_len = kernel.shape[-1]
    data_len = data.shape[-1]
    out = np.zeros(data_len + kernel_len - 1)

    pos = np.where(data != 0)[0]
    # add shifted and scaled copies of kernel only where data is nonzero
    for p in pos:
        out[p:p + kernel_len] = out[p:p + kernel_len] + kernel * data[p]

    if mode.lower() == 'full':
        return out
    elif mode.lower() == 'valid':
        return center_vector(out, data_len - kernel_len + 1)
    elif mode.lower() == 'same':
        return center_vector(out, data_len)
    else:
        raise ValueError("Acceptable mode flags are 'valid',"
                         " 'same', or 'full'.")

if has_jit:
    _sparseconvj = jit(_sparseconv)

def sparseconv(data, kernel, mode='full', dojit=True):
    """
    Returns the discrete, linear convolution of two one-dimensional sequences.

    Can run faster than numpy.convolve if:
    (1) `data` is much longer than `kernel`
    (2) `data` is sparse (has lots of zeros)

    Parameters
    ----------
    data : array_like
        First input, typically the data array.
    kernel : array_like
        Second input, typically the kernel.
    mode : str {'full', 'valid', 'same'}, optional
        A string indicating the size of the output:
        ``full``
        The output is the full discrete linear convolution of the inputs.
        (Default)
        ``valid``
        The output consists only of those elements that do not rely on
        zero-padding.
        ``same``
        The output is the same size as `data`, centered with respect to the
        'full' output.
    dojit : boolean
        A flag indicating whether to use numba's just-in-time compilation
        option.

    """
    if dojit and not has_jit:
        e_s = ("You do not have numba ",
               "please run sparseconv with dojit=False")
        raise ImportError(e_s)
    else:
        return _sparseconv(data, kernel, mode)

def conv(data, kernel, tsample, mode='full', method='fft', dojit=True):
    """Convoles data with a kernel using either FFT or sparseconv

    This function convolves data with a kernel, relying either on the
    fast Fourier transform (FFT) or a sparse convolution function.

    Parameters
    ----------
    data : array_like
        First input, typically the data array
    kernel : array_like
        Second input, typically the kernel
    mode : str {'full', 'valid', 'same'}, optional
        A string indicating the size of the output:
        ``full``
        The output is the full discrete linear convolution of the inputs.
        (Default)
        ``valid``
        The output consists only of those elements that do not rely on
        zero-padding.
        ``same``
        The output is the same size as `data`, centered with respect to the
        'full' output.
    method : str {'fft', 'sparse'}, optional
        A string indicating the convolution method:
        ``fft``
        Use the fast Fourier transform (FFT). (Default)
        ``sparse``
        Use the sparse convolution.
    dojit : bool, optional
        A flag indicating whether to use numba's just-in-time compilation
        option (only relevant for `method`=='sparse').
    """
    if method.lower() == 'fft':
        # Use FFT: faster on non-sparse data
        conved = tsample * sps.fftconvolve(data, kernel, mode)
    elif method.lower() == 'sparse':
        # Use sparseconv: faster on sparse data
        conved = tsample * sparseconv(data, kernel, mode, dojit)
    else:
        raise ValueError("Acceptable methods are: 'fft', 'sparse'.")
    return conved

def center_vector(vec, newlen):
    """
    Returns the center `newlen` portion of a vector.

    Adapted from scipy.signal.signaltools._centered:
    github.com/scipy/scipy/blob/v0.18.0/scipy/signal/signaltools.py#L236-L243

    """
    currlen = vec.shape[-1]
    startind = (currlen - newlen) // 2
    endind = startind + newlen
    return vec[startind:endind]

def parfor(func, in_list, out_shape=None, n_jobs=-1, engine='joblib',
           backend='threading', func_args=[], func_kwargs={}):
    """
    Parallel for loop for numpy arrays

    Parameters
    ----------
    func : callable
        The function to apply to each item in the array. Must have the form:
        func(arr, idx, *args, *kwargs) where arr is an ndarray and idx is an
        index into that array (a tuple). The Return of `func` needs to be one
        item (e.g. float, int) per input item.

    in_list : list
       All legitimate inputs to the function to operate over.

    n_jobs : integer, optional
        The number of jobs to perform in parallel. -1 to use all cpus
        Default: 1

    engine : str
        {"dask", "joblib", "serial"}
        The last one is useful for debugging -- runs the code without any
        parallelization.

    backend : str
        What dask backend to use. Irrelevant for other engines.

    func_args : list, optional
        Positional arguments to `func`

    func_kwargs : list, optional
        Keyword arguments to `func`

    Returns
    -------
    ndarray of identical shape to `arr`

    Notes
    -----
    Imported from pyAFQ (blob e20eaa0 from June 3, 2016):
    https://github.com/arokem/pyAFQ/blob/master/AFQ/utils/parallel.py

    Examples
    --------
    """
    if n_jobs == -1:
        n_jobs = multiprocessing.cpu_count()
        n_jobs = n_jobs - 1

    if engine.lower() == 'joblib':
        if not has_joblib:
            err = "You do not have `joblib` installed. Consider setting"
            err += "`engine` to 'serial' or 'dask'."
            raise ImportError(err)

        p = joblib.Parallel(n_jobs=n_jobs, backend=backend)
        d = joblib.delayed(func)
        d_l = []
        for in_element in in_list:
            d_l.append(d(in_element, *func_args, **func_kwargs))
        results = p(d_l)
    elif engine.lower() == 'dask':
        if not has_dask:
            err = "You do not have `dask` installed. Consider setting"
            err += "`engine` to 'serial' or 'joblib'."
            raise ImportError(err)

        def partial(func, *args, **keywords):
            def newfunc(in_arg):
                return func(in_arg, *args, **keywords)
            return newfunc
        p = partial(func, *func_args, **func_kwargs)
        d = [dask.delayed(p)(i) for i in in_list]
        if backend == 'multiprocessing':
            results = dask.compute(*d, get=dask.multiprocessing_get,
                                   workers=n_jobs)
        elif backend == 'threading':
            results = dask.compute(*d, get=dask.threaded.get, workers=n_jobs)
    elif engine.lower() == 'serial':
        results = []
        for in_element in in_list:
            results.append(func(in_element, *func_args, **func_kwargs))

    if out_shape is not None:
        return np.array(results).reshape(out_shape)
    else:
        return results

@deprecated
def mov2npy(movie_file, out_file):
    """Converts a movie file to a NumPy array

    This function is deprecated as of v0.2 and will be removed completely
    in v0.3.

    Parameters
    ----------
    movie_file : array
        The movie file to be read
    out_file: str
        Name of .npy file to be created.

    .. deprecated:: 0.2
    """
    # Don't import cv at module level. Instead we'll use this on python 2
    # sometimes...
    try:
        import cv
    except ImportError:
        e_s = "You do not have opencv installed. "
        e_s += "You probably want to run this in Python 2"
        raise ImportError(e_s)

    capture = cv.CaptureFromFile(movie_file)
    frames = []
    img = cv.QueryFrame(capture)
    while img is not None:
        tmp = cv.CreateImage(cv.GetSize(img), 8, 3)
        cv.CvtColor(img, tmp, cv.CV_BGR2RGB)
        frames.append(np.asarray(cv.GetMat(tmp)))
        img = cv.QueryFrame(capture)
    frames = np.fliplr(np.rot90(np.mean(frames, -1).T, -1))
    np.save(out_file, frames)

def gamma(n, tau, tsample, tol=0.01):
    """Returns the impulse response of `n` cascaded leaky integrators

    This function calculates the impulse response of `n` cascaded
    leaky integrators with constant of proportionality 1/`tau`:
    y = (t/theta).^(n-1).*exp(-t/theta)/(theta*factorial(n-1))

    Parameters
    ----------
    n : int
        Number of cascaded leaky integrators
    tau : float
        Decay constant of leaky integration (seconds).
        Equivalent to the inverse of the constant of proportionality.
    tsample : float
        Sampling time step (seconds).
    tol : float
        Cut the kernel to size by ignoring function values smaller
        than a fraction `tol` of the peak value.
    """
    n = int(n)

    # Allocate a time vector that is long enough for sure.
    # Trim vector later on.
    t = np.arange(0, 5 * n * tau, tsample)

    # Calculate gamma
    y = (t / tau) ** (n - 1) * np.exp(-t / tau)
    y /= (tau * spm.factorial(n - 1))

    # Normalize to unit area
    y /= np.trapz(np.abs(y), dx=tsample)

    # Cut off tail where values are smaller than `tol`.
    # Make sure to start search on the right-hand side of the peak.
    peak = y.argmax()
    small_vals = np.where(y[peak:] < tol * y.max())[0]
    t = t[:small_vals[0] + peak]
    y = y[:small_vals[0] + peak]

    return t, y

def traverse_randomly(seq):
    """Traverses a list in random order

    Parameters
    ----------
    seq : list
        An iterable list of elements.
    Returns
    -------
    A list iterator.

    Examples
    --------
    Shuffle a list:

    >>> list_ordered = [0, 1, 2, 3]
    >>> list_shuffled = [l for l in traverse_randomly(list_ordered)]

    Traverse a list in random order:

    >>> list_ordered = [0, 1, 2, 3]
    >>> for idx, val in traverse_randomly(enumerate(list_ordered)):
    ...     pass

    Notes
    -----
    From: http://stackoverflow.com/a/9253366
    Note that even for rather small `len(x)`, the total number of permutations
    of `seq` can quickly grow larger than the period of most random number
    generators. For example, a sequence of length 2080 is the largest that can
    fit within the period of the Mersenne Twister random number generator.
    """
    # Make sure we are dealing with a list
    shuffled = list(seq)

    # Import `random` at module level (for performance)
    random.shuffle(shuffled)

    return iter(shuffled)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

def checksum(orbits):
    satellites = []
    maps = {'COM':None}                 # 'satellite': 'orbit'
    count = 0
    for orbit in orbits:
        a,b = orbit.split(')')
        maps[b] = a
        satellites.append(b)

    # count indirect orbits
    for satellite in satellites:
        orbit = maps[satellite]          # including 'COM'
        while orbit is not None:
            count += 1
            orbit = maps[orbit]
    return count, satellites, maps

def minimum_transfer(satellites, maps):
    # strategy: find the first cross point
    you, san = maps['YOU'], maps['SAN']
    t_you, t_san = {}, {}                # trajectory, orbit:step

    # make trajectories
    orbit = you
    count = 0
    while orbit is not None:
        count += 1
        orbit = maps[orbit]
        t_you[orbit] = count

    orbit = san
    count = 0
    while orbit is not None:
        count += 1
        orbit = maps[orbit]
        t_san[orbit] = count

    movements = 99999
    for i in set(t_you.keys()) & set(t_san.keys()):
        movements = min(movements, t_you[i]+t_san[i])

    return movements

if __name__ == "__main__":
    with open("input.txt", 'r') as file:
        orbits = []
        for line in file:
            orbits.append(line.strip())

    count, satellites, maps = checksum(orbits)
    print("Part 1:", count)
    print("Part 2:", minimum_transfer(satellites, maps))


def _real_extract(self, url):
    mobj = re.match(self._VALID_URL, url)
    (video_id, site_id) = mobj.group('id', 'site_id')
    webpage = self._download_webpage(url, video_id)
    config = self._parse_json(self._search_regex("videoJSConfig\\s*=\\s*JSON\\.parse\\(\\'({.+?})\\'\\);", webpage, 'config', default='{}'), video_id, transform_source=(lambda s: s.replace('\\\\', '\\').replace('\\"', '"').replace("\\'", "'")))
    vod_id = (config.get('vodId') or self._search_regex(('\\\\"vodId\\\\"\\s*:\\s*\\\\"(.+?)\\\\"', '"vodId"\\s*:\\s*"(.+?)"', '<[^>]+id=["\\\']vod-(\\d+)'), webpage, 'video_id', default=None))
    if (not vod_id):
        player = self._parse_json(self._search_regex('vmmaplayer\\(({.+?})\\);', webpage, 'vmma player', default=''), video_id, transform_source=(lambda s: ('[%s]' % s)), fatal=False)
        if player:
            video = player[(- 1)]
            if (video['videoUrl'] in ('http', 'https')):
                return self.url_result(video['url'], MedialaanIE.ie_key())
            info = {
                'id': video_id,
                'url': video['videoUrl'],
                'title': video['title'],
                'thumbnail': video.get('imageUrl'),
                'timestamp': int_or_none(video.get('createdDate')),
                'duration': int_or_none(video.get('duration')),
            }
        else:
            info = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls')[0]
            info.update({
                'id': video_id,
                'title': self._html_search_meta('description', webpage),
                'duration': parse_duration(self._html_search_meta('duration', webpage)),
            })
    else:
        if (not self._logged_in):
            self._login()
        settings = self._parse_json(self._search_regex('jQuery\\.extend\\(Drupal\\.settings\\s*,\\s*({.+?})\\);', webpage, 'drupal settings', default='{}'), video_id)

        def get(container, item):
            return (try_get(settings, (lambda x: x[container][item]), compat_str) or self._search_regex(('"%s"\\s*:\\s*"([^"]+)' % item), webpage, item, default=None))
        app_id = (get('vod', 'app_id') or self._SITE_TO_APP_ID.get(site_id, 'vtm_watch'))
        sso = (get('vod', 'gigyaDatabase') or 'vtm-sso')
        data = self._download_json(('http://vod.medialaan.io/api/1.0/item/%s/video' % vod_id), video_id, query={
            'app_id': app_id,
            'user_network': sso,
            'UID': self._uid,
            'UIDSignature': self._uid_signature,
            'signatureTimestamp': self._signature_timestamp,
        })
        formats = self._extract_m3u8_formats(data['response']['uri'], video_id, entry_protocol='m3u8_native', ext='mp4', m3u8_id='hls')
        self._sort_formats(formats)
        info = {
            'id': vod_id,
            'formats': formats,
        }
        api_key = get('vod', 'apiKey')
        channel = get('medialaanGigya', 'channel')
        if api_key:
            videos = self._download_json('http://vod.medialaan.io/vod/v2/videos', video_id, fatal=False, query={
                'channels': channel,
                'ids': vod_id,
                'limit': 1,
                'apikey': api_key,
            })
            if videos:
                video = try_get(videos, (lambda x: x['response']['videos'][0]), dict)
                if video:

                    def get(container, item, expected_type=None):
                        return try_get(video, (lambda x: x[container][item]), expected_type)

                    def get_string(container, item):
                        return get(container, item, compat_str)
                    info.update({
                        'series': get_string('program', 'title'),
                        'season': get_string('season', 'title'),
                        'season_number': int_or_none(get('season', 'number')),
                        'season_id': get_string('season', 'id'),
                        'episode': get_string('episode', 'title'),
                        'episode_number': int_or_none(get('episode', 'number')),
                        'episode_id': get_string('episode', 'id'),
                        'duration': (int_or_none(video.get('duration')) or int_or_none(video.get('durationMillis'), scale=1000)),
                        'title': get_string('episode', 'title'),
                        'description': get_string('episode', 'text'),
                        'timestamp': unified_timestamp(get_string('publication', 'begin')),
                    })
        if (not info.get('title')):
            info['title'] = (try_get(config, (lambda x: x['videoConfig']['title']), compat_str) or self._html_search_regex('\\\\"title\\\\"\\s*:\\s*\\\\"(.+?)\\\\"', webpage, 'title', default=None) or self._og_search_title(webpage))
    if (not info.get('description')):
        info['description'] = self._html_search_regex('<div[^>]+class="field-item\\s+even">\\s*<p>(.+?)</p>', webpage, 'description', default=None)
    return info


import numpy as np
import pylab as py
import matplotlib

rho = np.zeros((500,500))
gs = (6000.0*2.0/500.0)
v_e_1 = 950.0
v_e_2 = 300.0
v_out_2 = 600.0
vz_off_1 = 250.0
vz_off_2 = -1500.0

n1 = 'twoclump.dat'
n2 = 'OI_features/OI_268d.txt'

scale = 2.
fig = py.figure( figsize=(5.5*scale, 4.5*scale), facecolor='w', edgecolor='k' )
font = {'weight' : 'bold',
        'size'   : 10*scale}
matplotlib.rc('font', **font)  #set the font properties for entire plot
matplotlib.rc('xtick', labelsize=8*scale) #but have the tickmarks a little smaller

[x1,y1] = np.genfromtxt(n2,usecols=(0,1),unpack=True)
[x2,y2] = np.genfromtxt(n1,usecols=(0,1),unpack=True)
max1 = np.max(y1)
max2 = np.max(y2)
py.axis([6100, 6800,-.5,10.5])
py.plot(x1,y1*10/max1, 'k', lw=1, label='2011dh, 268 days')
py.plot(x2,y2*10/max2, 'k--', lw=2, label='Model')
leg = py.legend(loc=(.57,.86),fancybox=True,fontsize='small')
leg.get_frame().set_alpha(0.0)
py.xlabel(r'Wavelength ($\AA$)')
py.ylabel(r'Scaled F$_\lambda$')

for i in range(500):
    for j in range(500):
        z = gs*(1.0*i - 250)
        x = gs*(1.0*j - 250)+vz_off_1
        r = (x**2 + z**2.0)**0.5
        #FIRST SPHERE / TORUS
        sinsq = 0
        if (r != 0): sinsq = (1 - (z/r)**2.0)
        rho[i][j] = np.exp(-r/v_e_1) #*sinsq

        #SECOND SPHERE
        onoff = 1
        z = gs*(1.0*i - 250)
        x = gs*(1.0*j - 250)+vz_off_2
        r = (x**2 + z**2.0)**0.5
        if (r < v_out_2): rho[i][j] +=  4.0*np.exp(-r/v_e_2)*onoff

a = py.axes([.561, .5, .27, .27])
a.axis([138, 363, 150, 350])
py.setp(a, xticks=[], yticks=[])
a.imshow(np.log10(rho))#, cmap=py.cm.afmhot_r, py.cm.gray_r
a.scatter( 250,250, c='k', alpha=.5, marker='+', s=1000)

b = py.axes([.57, .23, .25, .25])
b.tick_params(labelsize='x-small')
b.set_xticks([6300+(40*l) for l in range(4)])
b.set_yticks([2*l for l in range(1,6)])
b.axis([6290, 6420, 1,10.5])
#b.set_xlabel(r'($\AA$)', size='x-small')
b.plot(x1,y1*10/max1,'k-', lw=1)
b.plot(x2,y2*10/max2,'k--', lw=2)

# add the eye
import matplotlib.image as image
im = image.imread('Eye_Symbol.png')[125:475,:,3]
c = py.axes([.38, .595, .3, .08], frameon=False)
c.axis([0,750,375,0])
py.setp(c, xticks=[], yticks=[])
c.imshow(im, cmap=py.cm.binary)
c.scatter( 450,176, marker=r'$\rightarrow$', s=500 )

py.savefig( 'twoclump.png', dpi=300 )


# -*- coding: utf-8 -*-
"""
Created on Tue May 28 15:53:48 2019

@author: Josh
"""

import spacy

#Define the custom component
def length_component(doc):
    #Get the doc's length
    doc_length = len(doc)
    print("This document is {} tokens long.".format(doc_length))
    #Return the doc
    return doc

#Load the small English model
nlp = spacy.load("en_core_web_sm")

#Add the component first in the pipeline and print the pipe names
nlp.add_pipe(length_component, first=True)
print(nlp.pipe_names)

#Process a text
doc = nlp("This is a sentence.")

Chapter 3 - Processing Pipelines/6 - Simple components.py


n = int(input())
a = [int(x) for x in input().split()]
for i in range(n):
    if i!=n-1 and a[i] > 0 < a[i+1]:
        print("YES")
        exit()
    elif i!=n-1 and a[i] < 0 > a[i+1]:
        print("YES")
        exit()
print("NO")


from urllib.request import Request, urlopen
import requests, zipfile, io
import pandas as pd
from pvlib import solarposition
from datetime import time, datetime, timedelta, timezone

default_url = "https://energyplus.net/weather-download/north_and_central_america_wmo_region_4/USA/CA/USA_CA_Oakland.Intl.AP.724930_TMY/USA_CA_Oakland.Intl.AP.724930_TMY.epw"

def get_data(url):
    """ Return a list of the data from api call.
    """
    if url[-3:] == "zip" or url[-3:] == "all":
        request = requests.get(url)
        zf = zipfile.ZipFile(io.BytesIO(request.content))
        for i in zf.namelist():
            if i[-3:] == 'epw':
                epw_name = i
        data = zf.read(epw_name)
        data = repr(data).split("\\n")
        return data
    headers = {'User-Agent': 'Mozilla/5.0'}
    req = Request(url, headers = headers)
    epw = urlopen(req).read().decode()
    lines = epw.split("\n")
    return lines

def create_df(default_url):
    """ Extract and clean the data. Return a pandas data from a url.
    """
    lst = get_data(default_url)
    meta = lst[0].strip().split(',')
    city = meta[1]
    country = meta[3]
    latitude = float(meta[-4])
    longitude = float(meta[-3])
    time_zone = float(meta[-2])
    site_elevation = meta[-1]
    location_name = (city + ", " + country)

    lst = lst[8:len(lst) - 1]
    lst = [line.strip().split(',') for line in lst]

    # Each data row exlude index 4 and 5, and everything afterdayssnow
    for line in lst:
        del line[4:6]
        del line[9]
        del line[-1]
        del line[-1]
        del line[-1]

    col_names = ['year', 'month', 'day', 'hour', 'DBT', 'DPT', 'RH',
                'Apressure', 'EHrad', 'HIRrad', 'GHrad',
                'DNrad', 'DifHrad', 'GHillum', 'DNillum', 'DifHillum',
                'Zlumi', 'Wdir', 'Wspeed', 'Tskycover', 'Oskycover',
                'Vis', 'Cheight', 'PWobs', 'PWcodes', 'Pwater',
                'AsolOptD', 'SnowD', 'DaySSnow']

    epw_df = pd.DataFrame(columns = col_names, data = lst)

    # Add fake_year
    epw_df['fake_year'] = 'year'

    # Add in month names
    month_look_up = {'1': 'Jan', '2': 'Feb', '3': 'Mar', '4': 'Apr', '5': 'May',
            '6': 'Jun', '7': 'Jul', '8': 'Aug', '9': 'Sep', '10': 'Oct', '11': 'Nov', '12': 'Dec'}
    epw_df['month_names'] = epw_df['month'].apply(lambda x: month_look_up[x])

    # Add in DOY
    def doy_helper(row):
        """ Helper function for the DOY column.
        """
        month = int(row['month'])
        year = int(row['year'])
        day = int(row['day'])
        period = pd.Period(day = day, month = month, year = year, freq = 'D')
        return period.dayofyear

    epw_df['DOY'] = epw_df.apply(lambda row: doy_helper(row), axis = 1)

    # Change to int type
    change_to_int = ['year', 'day', 'month', 'hour']
    for col in change_to_int:
        epw_df[col] = epw_df[col].astype(int)

    # Change to float type
    change_to_float = ['DBT', 'DPT', 'RH', 'Apressure', 'EHrad', 'HIRrad', 'GHrad',
    'DNrad', 'DifHrad', 'GHillum', 'DNillum', 'DifHillum', 'Zlumi', 'Wdir', 'Wspeed', 'Tskycover',
    'Oskycover', 'Vis', 'Cheight', 'PWobs', 'PWcodes', 'Pwater', 'AsolOptD', 'SnowD', 'DaySSnow']
    for col in change_to_float:
        epw_df[col] = epw_df[col].astype(float)

    # Add in times df
    date = datetime(2000, 6, 21, 12 - 1, 0, 0, 0, tzinfo = timezone.utc)
    tz = timedelta(days = 0, hours = time_zone - 1, minutes = 0)
    date = date - tz
    tz = 'UTC'
    times = pd.date_range(
                '2019-01-01 00:00:00',
                '2020-01-01',
                closed = 'left',
                freq = 'H',
                tz = tz
            )
    delta = timedelta(days = 0, hours = time_zone - 1, minutes = 0)
    times = times - delta
    times_df = pd.DataFrame(times, columns = ['times'])
    epw_df = pd.concat([epw_df, times_df], axis = 1)
    epw_df.set_index("times", drop = False, append = False, inplace = True, verify_integrity = False)

    # Add in solpos df
    solpos = solarposition.get_solarposition(times, latitude, longitude)
    epw_df = pd.concat([epw_df, solpos], axis = 1)

    return epw_df, meta


from flask import Flask, render_template, request
from book_parser import BookParser
from book_request_handler import BookAPIRequestHandler
application = Flask(__name__, template_folder='templates', static_folder='static')

@application.route('/', methods=['GET'])
def open():
    return render_template('index.html', book_data=None)

@application.route('/', defaults={'term':None, 'page': 1}, methods=['POST'])
@application.route('/<term>/<page>', methods=['GET'])
def search(term, page):
    if not term:
        term = request.form['term']

    request_handler = BookAPIRequestHandler()
    response = request_handler.request_book_search(term, page)

    if ('items' in response):
        page = int(page)+1
        parser = BookParser()
        books = parser.get_book_data_from_json(response)
        return render_template('index.html', search_word=term, book_data=books, page_number=page)

    elif ('error' in response):
        return response['error']

    else:
        return render_template('index.html', book_data=None)


from CONFIG import *

def sum_dicts(dict1, dict2):
    """
    :param dict1: dictionary with integers as values
    :param dict2: dictionary with integers as values
    :return: dictionary, with the sum of the keys.
    """
    new_keys = np.unique(np.append(dict1.keys(), dict2.keys()))
    new_dict = {}
    for k in new_keys:
        new_dict[k] = dict1.get(k, 0) + dict2.get(k, 1)
    return new_dict

def constract_finished_items_dict():
    ks = ITEMS_NAMES
    return dict(zip(ks, np.zeros(len(ks))))

def get_round(n, r=0.2):
    if n == 0:
        return n
    n_mod = np.round(n % int(n), 2)
    gaps_arr = np.arange(0 - r, 1 + r, r)
    for i in gaps_arr:
        if n_mod <= i:
            return int(n) + np.round(i, 2)
    return


## -*- coding: utf-8 -*-
import argparse
import os
import torch
import datetime
import logging
from pathlib import Path
import sys
import importlib
import shutil
from tqdm import tqdm
from utils import *
from ShapeNetDataLoader import PartNormalDataset
from concurrent.futures import ThreadPoolExecutor

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
ROOT_DIR = BASE_DIR
sys.path.append(os.path.join(ROOT_DIR, 'models'))

def parse_args():
    parser = argparse.ArgumentParser('Model')
    parser.add_argument('--model', type=str, default='transformer', help='model name [default: transformer]')
    parser.add_argument('--batch_size', type=int, default=4, help='Batch Size during training [default: 8]')
    parser.add_argument('--epoch',  default=128, type=int, help='Epoch to run [default: 128]')
    parser.add_argument('--learning_rate', default=0.002, type=float, help='Initial learning rate [default: 0.002]')
    parser.add_argument('--gpu', type=str, default='1', help='GPU to use [default: GPU 0]')
    parser.add_argument('--optimizer', type=str, default='Adam', help='Adam or SGD [default: Adam]')
    parser.add_argument('--log_dir', type=str, default=None, help='Log path [default: None]')
    parser.add_argument('--decay_rate', type=float, default=1e-4, help='weight decay [default: 1e-4]')
    parser.add_argument('--npoint', type=int,  default=1024, help='Point Number [default: 2048]')
    parser.add_argument('--normal', action='store_true', default=False, help='Whether to use normal information [default: False]')
    parser.add_argument('--step_size', type=int,  default=20, help='Decay step for lr decay [default: every 20 epochs]')
    parser.add_argument('--lr_decay', type=float,  default=0.5, help='Decay rate for lr decay [default: 0.5]')
    return parser.parse_args()

def main(args):
    logger=logging.getLogger("Transformer")
    def log_string(str):
        logger.info(str)
        print(str)

    '''HYPER PARAMETER'''
    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
    timestr = str(datetime.datetime.now().strftime('%Y-%m-%d_%H-%M'))
    experiment_dir = Path('./log/')
    experiment_dir.mkdir(exist_ok=True)
    experiment_dir = experiment_dir.joinpath('transformer')
    experiment_dir.mkdir(exist_ok=True)
    if args.log_dir is None:
        experiment_dir = experiment_dir.joinpath(timestr)
    else:
        experiment_dir = experiment_dir.joinpath(args.log_dir)
    experiment_dir.mkdir(exist_ok=True)
    visual_dir=experiment_dir.joinpath("visual")
    visual_dir.mkdir(exist_ok=True)
    checkpoints_dir = experiment_dir.joinpath('checkpoints/')
    checkpoints_dir.mkdir(exist_ok=True)
    log_dir = experiment_dir.joinpath('logs/')
    log_dir.mkdir(exist_ok=True)

    '''LOG'''
    args = parse_args()
    logger.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    file_handler = logging.FileHandler('%s/%s.txt' % (log_dir, args.model))
    file_handler.setLevel(logging.INFO)
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)
    log_string('INIT PARAMETERS ...')
    log_string(args)
    root = 'data/shapenetcore_partanno_segmentation_benchmark_v0_normal/'

    TRAIN_DATASET = PartNormalDataset(root = root, npoints=args.npoint, split='trainval', normal_channel=args.normal,class_choice=["Bag","Cap","Earphone","Rocket"])
    trainDataLoader = torch.utils.data.DataLoader(TRAIN_DATASET, batch_size=args.batch_size,shuffle=True, num_workers=4,drop_last=True)
    TEST_DATASET = PartNormalDataset(root = root, npoints=args.npoint, split='test', normal_channel=args.normal,class_choice=["Bag","Cap","Earphone","Rocket"])
    testDataLoader = torch.utils.data.DataLoader(TEST_DATASET, batch_size=args.batch_size,shuffle=False, num_workers=4,drop_last=True)
    log_string("The number of training data is: %d" % len(TRAIN_DATASET))
    log_string("The number of test data is: %d" %  len(TEST_DATASET))

    MODEL = importlib.import_module(args.model)
    shutil.copy('models/%s.py' % args.model, str(experiment_dir))
    shutil.copy('utils.py', str(experiment_dir))
    channel=6 if args.normal else 3
    classifier = MODEL.get_model(d_model=128,channel=channel).cuda()
    def weights_init(m):
        classname = m.__class__.__name__
        if classname.find('Conv2d') != -1:
            torch.nn.init.xavier_normal_(m.weight.data)
            torch.nn.init.constant_(m.bias.data, 0.0)
        elif classname.find('Linear') != -1:
            torch.nn.init.xavier_normal_(m.weight.data)
            torch.nn.init.constant_(m.bias.data, 0.0)

    try:
        checkpoint = torch.load(str(experiment_dir) + '/checkpoints/best_model.pth')
        epoch = checkpoint['epoch']
        model_loss=checkpoint['loss']
        classifier.load_state_dict(checkpoint['model_state_dict'])
        log_string('Use pretrain model')
    except:
        log_string('No existing model, starting training from scratch...')
        epoch = 0
        model_loss=999.0
        classifier = classifier.apply(weights_init)

    if args.optimizer == 'Adam':
        optimizer = torch.optim.Adam(
            classifier.parameters(),
            lr=args.learning_rate,
            betas=(0.9, 0.999),
            eps=1e-08,
            weight_decay=args.decay_rate
        )
    else:
        optimizer = torch.optim.SGD(classifier.parameters(), lr=args.learning_rate, momentum=0.9)

    def bn_momentum_adjust(m, momentum):
        if isinstance(m, torch.nn.BatchNorm2d) or isinstance(m, torch.nn.BatchNorm1d):
            m.momentum = momentum

    LEARNING_RATE_CLIP = 1e-5
    MOMENTUM_ORIGINAL = 0.1
    MOMENTUM_DECCAY = 0.5
    MOMENTUM_DECCAY_STEP = args.step_size

    t = ThreadPoolExecutor(args.batch_size * 5)

###   仔细检查训练流程
    for epoch in range(epoch, args.epoch):
        log_string('Epoch %d (%d/%s):' % (epoch + 1, epoch + 1, args.epoch))
        '''Adjust learning rate and BN momentum'''
        lr = max(args.learning_rate * (args.lr_decay ** (epoch // args.step_size)), LEARNING_RATE_CLIP)
        log_string('Learning rate:%f' % lr)
        for param_group in optimizer.param_groups:
            param_group['lr'] = lr
        momentum = MOMENTUM_ORIGINAL * (MOMENTUM_DECCAY ** (epoch // MOMENTUM_DECCAY_STEP))
        if momentum < 0.01:
            momentum = 0.01
        print('BN momentum updated to: %f' % momentum)
        classifier = classifier.apply(lambda x: bn_momentum_adjust(x,momentum))

#  Train one epoch
        total_loss1 ,total_loss2,i=0,0,0
        for  data in tqdm(trainDataLoader, total=len(trainDataLoader), smoothing=0.9):
            i+=1
            points1, points2, _ = data
            points1,points2=points1.cuda(),points2.cuda()
            optimizer.zero_grad()
            classifier = classifier.train()
            warped,loss1,loss2 = classifier(points1, points2)
            if epoch <10:
                # loss = (loss1 + loss2).mean()
                loss=loss1.mean()
            else:
                # loss = (loss1 + 0.1*loss2).mean()
                loss=loss1.mean()
            total_loss1+=float(loss1.mean())
            total_loss2+=float(loss2.mean())
            loss.backward()
            optimizer.step()

        log_string('EPOCH %d train: loss1 is: %.5f   loss2 is: %.5f ' % (epoch+1,total_loss1/i,total_loss2/i))
        with torch.no_grad():
            total_loss1, total_loss2, i = 0, 0, 0
            for batch_id, (points1, points2, cls) in tqdm(enumerate(testDataLoader), total=len(testDataLoader), smoothing=0.9):
                i+=1
                points1, points2 = points1.cuda(), points2.cuda()
                classifier = classifier.eval()
                warped, loss1, loss2 = classifier(points1, points2)
                total_loss1 += float(loss1.mean())
                total_loss2 += float(loss2.mean())

            log_string('EPOCH %d test: loss1 is: %.5f   loss2 is: %.5f ' % (epoch+1, total_loss1 / i, total_loss2 / i))

            if (total_loss1/batch_id) < model_loss :
                model_loss=total_loss1/batch_id
                logger.info('Save model...')
                savepath = str(checkpoints_dir) + '/best_model.pth'
                log_string('Saving at %s'% savepath)
                state = {
                    'epoch': epoch,
                    'loss' :model_loss,
                    'model_state_dict': classifier.state_dict(),
                    'optimizer_state_dict': optimizer.state_dict(),
                }
                torch.save(state, savepath)

    #   save the last epoch
    model_loss = total_loss1 / batch_id
    logger.info('Save last model...')
    savepath = str(checkpoints_dir) + '/last_model.pth'
    log_string('Saving at %s' % savepath)
    state = {
        'epoch': epoch,
        'loss': model_loss,
        'model_state_dict': classifier.state_dict(),
        'optimizer_state_dict': optimizer.state_dict(),
    }
    torch.save(state, savepath)

if __name__ == '__main__':
    args = parse_args()
    main(args)


#!/usr/bin/python
__author__ = 'madsens'
# Runs the RFID Tester in the lobby. When triggered spins a wheel with sounds.

import time, sys, os
sys.path.append("/home/pi/PiClasses")
import Logging
import GPIOLib
from termios import tcflush, TCIOFLUSH
from random import randint

# Instantiate Logging and GPIO Classes (11 - power drill, 13 - Change Polarity)
dbConn = Logging.Logging()
gpio = GPIOLib.GPIOLib("BOARD", "LOW", [11])

while True:    # Runs until break is encountered. We want to set it to break on a particular ID.
    n = raw_input("Scanned ID: ")
    currentScan = time.time()
    if n == "STOP":
        break  # stops the loop
    else :
        dbConn.logAccess(n)

        # Play Wraith on even n, Wilhelm on odd n
        if (int(n) % 2 == 0):
            audio = "/home/pi/Assets/wraith.mp3"
        else:
            audio = "/home/pi/Assets/wilhelm.mp3"

        # Trigger GPIO Pins. Need to make a random number of "pulses" with a random number of "micro-pulses" each with a random duration pulse, flipping the "polarity pin" with each micro pulse
        # Play an mp3 during each "pulse"
        pulses = randint(1,4)
        for x in range (0, pulses):
            os.system('mpg321 ' + audio + ' -q &')
            #Micropulses also random between 4 and 10
            micropulse = randint(6,10)
            for y in range (0, micropulse):
                gpio.on([11])
                time.sleep(.15)
                gpio.off([11])
                time.sleep(.1)

            time.sleep(2)

        #flush keyboard buffer
        sys.stdout.flush();
        tcflush(sys.stdin, TCIOFLUSH)


from pathlib import Path
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from sklearn.decomposition import PCA

if __name__ == '__main__':

    # set paths
    dir_path = Path('./data')
    csv_path = dir_path / 'パラメーターテスト4.txt'
    out_dir_path = Path('./out')
    out_pc_path = out_dir_path / 'principal_components_100dim_to_6val.csv'
    out_dir_path.mkdir(parents=True, exist_ok=True)

    # set params
    j_time_frames = ['5M','15M','1H','4H','Daily']
    i_periods = ['10','20','40','80','200']
    indicators = ['wkRSI','wkBB','wkATR','wkSMA']
    y_label = 'wkFixPips'
    indicator_start_label = 'wkSMA[0][0]'

    # load csv
    df = pd.read_csv(str(csv_path))
    keys = df.keys().values
    st_indicator = list(keys).index(indicator_start_label)

    # set flags
    is_buy = df['strbuysell'].values == 'close_buy'

    pc = []
    for bs_i, buy_or_sell in enumerate([is_buy, ~is_buy]):

        if bs_i == 0:
            bs = 'buy'
        else:
            bs = 'sell'

        # pca to compress indicator value
        X = np.array([df.iloc[i,st_indicator:].values for i in range(len(df)) if buy_or_sell[i]])
        X = np.array([(X[:,i]-np.mean(X[:,i]))/np.std(X[:,i]) for i in range(len(X[0]))]).T
        pca = PCA(n_components=6)
        pca.fit(X)
        Xd = pca.transform(X)
        print('Cumulative contribution rate from 1st to 6th PC:')
        print(np.cumsum(pca.explained_variance_ratio_))

        pc.append(pca.components_)

        for xi in range(6):

            # sliding window
            w_gp_mean, w_gp_std, w_cent, w_num = [], [], [], []
            w_min, w_max = Xd[:,xi].min(), Xd[:,xi].max()
            w_stride = (w_max-w_min)/50
            w_width = w_stride*2
            for w_st in np.arange(w_min,w_max,w_stride):
                w_cent.append(w_st + (w_width/2))
                is_w = (Xd[:,xi] >= w_st) & (Xd[:,xi] < w_st+10)
                w_num.append(is_w.sum())
                w_gp_mean.append(np.mean(df[y_label].values[buy_or_sell][is_w]))
                w_gp_std.append(np.std(df[y_label].values[buy_or_sell][is_w]))
            w_cent = np.array(w_cent)
            w_gp_mean = np.array(w_gp_mean)
            w_gp_std = np.array(w_gp_std)
            w_num = np.array(w_num)
            err = np.array([w_gp_mean + w_gp_std, w_gp_mean-w_gp_std]).T

            # plot
            plt.close('all')
            plt.figure()
            plt.scatter(
                Xd[:,xi],
                df[y_label].values[buy_or_sell],
                label='samples',
                marker='+')
            plt.plot(w_cent,w_gp_mean*10,color='r',label=y_label+'_mean x 10')
            # plt.plot(w_cent,err*10,color='r',label='$\pm$std x 10',linestyle='dashed', alpha=0.5)
            plt.plot(w_cent, w_num, label='sample_num',linestyle='dashed',color='g',alpha=0.6)
            plt.grid()
            plt.xlabel('{}th-pcv-projected value'.format(xi+1))
            # plt.ylim([-100,100])
            plt.ylabel(y_label)
            plt.legend()
            plt.title('{}-th-pcv-projected value VS gain/loss ({})'.format(xi+1,bs))
            fname = str(out_dir_path/ 'pc_vs_gl_w_{}_pc{:02}.png'.format(bs,xi))
            print('saving: {}'.format(fname))
            plt.savefig(fname)

    # save principal components
    pc = np.array(pc).reshape(len(pc[0])*2,-1)
    df_pc= pd.DataFrame(pc.T,
        columns = ['buy_pc{:02}'.format(num+1) for num in range(6)]+['sell_pc{:02}'.format(num+1) for num in range(6)],
        index = keys[st_indicator:])
    print('saving: {}'.format(str(out_pc_path)))
    df_pc.to_csv(str(out_pc_path))


import argparse
import os
import time

import gym
import numpy as np
import rc_gym
import torch

from agents.ddpg_selfplay import DDPGActor
from agents.sac import GaussianPolicy

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--cuda", default=False,
                        action="store_true", help="Enable cuda")
    parser.add_argument("-c_atk", "--checkpoint_atk", required=True,
                        help="checkpoint to load attacker")
    parser.add_argument("-c_gk", "--checkpoint_gk", required=True,
                        help="checkpoint to load goalkeeper")
    args = parser.parse_args()
    device = "cuda" if args.cuda else "cpu"

    checkpoint_atk = torch.load(args.checkpoint_atk)
    checkpoint_gk = torch.load(args.checkpoint_gk)

    env = gym.make(checkpoint_atk['ENV_NAME'])

    if checkpoint_atk['AGENT'] == 'ddpg_async':
        pi_atk = DDPGActor(checkpoint_atk['N_OBS'], checkpoint_atk['N_ACTS']).to(device)
        pi_gk = DDPGActor(checkpoint_gk['N_OBS'], checkpoint_gk['N_ACTS']).to(device)
    elif checkpoint['AGENT'] == 'sac_async':
        pi_atk = GaussianPolicy(checkpoint_atk['N_OBS'], checkpoint_atk['N_ACTS'],
                            checkpoint_atk['LOG_SIG_MIN'],
                            checkpoint_atk['LOG_SIG_MAX'], checkpoint['EPSILON']).to(device)
        pi_gk = GaussianPolicy(checkpoint_gk['N_OBS'], checkpoint_gk['N_ACTS'],
                            checkpoint_gk['LOG_SIG_MIN'],
                            checkpoint_gk['LOG_SIG_MAX'], checkpoint_gk['EPSILON']).to(device)
    else:
        raise AssertionError

    pi_atk.load_state_dict(checkpoint_atk['pi_state_dict'])
    pi_gk.load_state_dict(checkpoint_gk['pi_state_dict'])
    pi_atk.eval()
    pi_gk.eval()

    while True:
        done = False
        s = env.reset()
        info = {}
        ep_steps = 0
        ep_rw_atk = 0
        ep_rw_gk = 0
        st_time = time.perf_counter()
        for i in range(checkpoint_atk['MAX_EPISODE_STEPS']):
            # Step the environment
            s_v_atk = torch.Tensor(s['observation_atk']).to(device)
            s_v_gk = torch.Tensor(s['observation_gk']).to(device)
            a_atk = pi_atk.get_action(s_v_atk)
            a_gk = pi_gk.get_action(s_v_gk)
            s_next, r, done, info = env.step({'action_atk': a_atk,'action_gk': a_gk})
            ep_steps += 1
            ep_rw_atk += r['reward_atk']
            ep_rw_gk += r['reward_gk']
            env.render()
            if done:
                break

            # Set state for next step
            s = s_next

        info['fps'] = ep_steps / (time.perf_counter() - st_time)
        info['ep_steps'] = ep_steps
        info['ep_rw_atk'] = ep_rw_atk
        info['ep_rw_gk'] = ep_rw_gk
        print(info)


import hlt
import logging
import micro
from objective import Objective

# GAME START
# Here we define the bot's name as Settler and initialize the game, including communication with the Halite engine.
game = hlt.Game("Version2")
# Then we print our start message to the logs
logging.info("Starting my Version2 bot!")
'''
def attack_docked_ships(ship, planet, own_ships, game_map):
    docked_ships = planet.all_docked_ships()
    for opponent_ship in docked_ships:
        navigate_command = ship.navigate(ship.closest_point_to(opponent_ship, min_distance=0), game_map, speed=int(hlt.constants.MAX_SPEED), entities=own_ships)

        #if you can't attack nearest ship just return None
        return navigate_command

    return None

def attack_ships(ship, entities_by_distance, own_ships, game_map):
    for ship_distance in sorted(entities_by_distance):
        nearest_ship = next((nearest_entity for nearest_entity in entities_by_distance[ship_distance] if isinstance(nearest_entity, hlt.entity.Ship)), None)

        if nearest_ship == None:
            continue
        if nearest_ship.owner==game_map.get_me():
            continue
        navigate_command = ship.navigate(ship.closest_point_to(nearest_ship, min_distance=0), game_map, speed=int(hlt.constants.MAX_SPEED), entities=own_ships)
        return navigate_command
    return None
'''

# objtype = ['attack','defend','dock_owned', 'dock_unowned']
def updateObjectives(game_map, opponent_ships):
    objs = []
    for p in planets:
        en_list = sorted(opponent_ships, key=lambda x: p.calculate_distance_between(x))
        for i in range(len(en_list)):
            if p.calculate_distance_between(en_list[i]) > 40 - p.radius:
                break

        if not p.is_owned() and p.remaining_resources > 0:
            objs.append(Objective(p, 'dock_unowned'))
            objs[-1].addEnShip(en_list[:i])
        elif p.owner == me:
            objs.append(Objective(p, 'defend'))
            objs[-1].addEnShip(en_list[:i])
            if not p.is_full() and p.remaining_resources > 0:
                objs.append(Objective(p,'dock_owned'))
                objs[-1].addEnShip(en_list[:i])
        elif p.owner != me and p.is_owned():
            objs.append(Objective(p, 'attack'))
            objs[-1].addEnShip(en_list[:i])

    return objs

def assignObjectives(objectives, my_ships):
    if len(objectives) == 0:
        return objectives
    for ship in my_ships:
        if (ship.docking_status == 1 or ship.docking_status ==2) and ship.planet.remaining_resources > 0:
            continue
        bestObj = objectives[0]
        bestScore = -100000
        for obj in objectives:
            score = obj.priority - ship.calculate_distance_between(obj.entity) * 2
            if score > bestScore:
                bestScore = score
                bestObj = obj
        bestObj.addMyShip(ship)
        logging.info('i am at ' + str(ship.x) + ' ' + str(ship.y) + ' my obj is to ' + str(bestObj.objtype) + ' at ' + str(bestObj.entity.x) + ' ' + str(bestObj.entity.y) + ' against ' + str(len(bestObj.en_ships)) + ' ships')
    return objectives

#flee_pos = [hlt.entity.Position(0,0), hlt.entity.Position(game_map.width-1,0),
#                    hlt.entity.Position(0,game_map.height-1),
#                    hlt.entity.Position(game_map.width-1,game_map.height-1)]
def flee(game_map):
    for ship in my_ships:
        navigate_command = None
        if (ship.docking_status != ship.DockingStatus.UNDOCKED):
            navigate_command = ship.undock()
        else:
            if (ship.y <= 5) and (ship.x > 5):
                flee_pos = hlt.entity.Position(0,3)
            elif (ship.x <= 5) and (ship.y < game_map.height - 5):
                flee_pos = hlt.entity.Position(3,game_map.height)
            elif (ship.y >= game_map.height - 5) and (ship.x < game_map.width - 5):
                flee_pos = hlt.entity.Position(game_map.width, game_map.height - 3)
            elif (ship.x >= game_map.width - 5) and (ship.y > 5):
                flee_pos = hlt.entity.Position(game_map.width - 3, 0)
            else:
                flee_pos = min( hlt.entity.Position(ship.x, 0),
                                hlt.entity.Position(0, ship.y),
                                hlt.entity.Position(ship.x, game_map.height),
                                hlt.entity.Position(game_map.width, ship.y),
                                key = lambda p: ship.calculate_distance_between(p))

            navigate_command = ship.navigate(flee_pos,
                                            game_map,
                                            speed=int(hlt.constants.MAX_SPEED),
                                            entities=own_ships_nav)

        if navigate_command is not None:
            command_queue.append(navigate_command)

    return command_queue

me = None
turn = 0

while True:
    # TURN START
    # Update the map for the new turn and get the latest version
    turn+=1
    logging.info(str(turn))
    game_map = game.update_map()
    me = game_map.get_me()
    # Here we define the set of commands to be sent to the Halite engine at the end of the turn

    own_ships_nav = [] #list of our ships by position just for navigation purposes
    command_queue = []
    planets = game_map.all_planets()
    opponent_ships = []
    my_ships = game_map.get_me().all_ships()

    for player in game_map.all_players():
        if player.id != game_map.get_me().id:
            opponent_ships += player.all_ships()

    if (0.95*len(my_ships) < 0.1*len(opponent_ships)):
        command_queue = flee(game_map)
        game.send_command_queue(command_queue)
        continue

    objectives = updateObjectives(game_map, opponent_ships)

    objectives = assignObjectives(objectives, my_ships)

    for objective in objectives:
        command_queue += micro.getMovesForObjective(objective, game_map, own_ships_nav, opponent_ships)

    '''
    # For every ship that I control
    for ship in game_map.get_me().all_ships():
        # If the ship is docked
        if ship.docking_status != ship.DockingStatus.UNDOCKED:
            # Skip this ship
            continue

        entities_by_distance = game_map.nearby_entities_by_distance(ship)
        nearest_planet = None

        if len(game_map.all_players()) == 2:
            navigate_command = attack_ships(ship, entities_by_distance, own_ships, game_map)

            if (navigate_command != None):
                command_queue.append(navigate_command)
                continue

        for distance in sorted(entities_by_distance):
            nearest_planet = next((nearest_entity for nearest_entity in entities_by_distance[distance] if isinstance(nearest_entity, hlt.entity.Planet)), None)

            if nearest_planet == None:
                continue

            navigate_command = None
            #if nearest planet is opponents, attack the docked ships there
            if nearest_planet.is_owned() and nearest_planet.owner!=game_map.get_me():
                navigate_command = attack_docked_ships(ship, nearest_planet, own_ships, game_map)

            # If we can dock at a planet, let's (try to) dock. If two ships try to dock at once, neither will be able to.
            elif ship.can_dock(nearest_planet) and not (nearest_planet.is_full()):
                # We add the command by appending it to the command_queue
                command_queue.append(ship.dock(nearest_planet))

            elif nearest_planet.owner==game_map.get_me():
                #if we can't doc and it is our planet, just continue
                continue

            else:
                navigate_command = ship.navigate(
                    ship.closest_point_to(nearest_planet),
                    game_map,
                    speed=int(hlt.constants.MAX_SPEED), entities=own_ships_nav)

            if (navigate_command != None):
                command_queue.append(navigate_command)
            break

    '''
    # Send our set of commands to the Halite engine for this turn
    game.send_command_queue(command_queue)
    # TURN END

# GAME END


from typing import Union
import warnings
from pathlib import Path

import click

from audiomidi import audio_utils, params

import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

warnings.filterwarnings(action='ignore', category=UserWarning)

@click.command()
@click.option('-m', '--max-files', default=None, type=int)
def main(max_files: Union[int, None]):

    output_dir = params.features_dir
    output_dir.mkdir(exist_ok=True, parents=True)

    # process training files

    train_dir = params.nsynth_train_audio
    test_dir = params.nsynth_test_audio
    valid_dir = params.nsynth_valid_audio

    dirs = [train_dir, test_dir, valid_dir]
    dir_names = ['train', 'test', 'valid']

    for d, d_name in zip(dirs, dir_names):

        names, chroma_stfts, mfcc_stfts, _ = audio_utils.process_files(
            d,
            max_files=max_files,
            calc_chroma_stft=True,
            calc_mfcc_stft=True,
            calc_mfcc=False,
            label=d_name)

        names_file = audio_utils.dump_to_file(names, d_name + '_name',
                                              output_dir)

        if not Path(names_file[0]).exists():
            print('Error writing names')

        chroma_stft_file = audio_utils.dump_to_file(
            chroma_stfts, d_name + '_chroma_stft', output_dir)

        if not Path(chroma_stft_file[0]).exists():
            print(f'Error writing chroma_stft_{d_name}')

        mfcc_stft_file = audio_utils.dump_to_file(
            mfcc_stfts, d_name + '_mfcc_stft', output_dir)

        if not Path(mfcc_stft_file[0]).exists():
            print(f'Error writing mfcc_stft_{d_name}')

if __name__ == "__main__":
    main()


# coding=utf-8
"""LU decomposition module.

Module for lower-upper factorization of matrix using Doolittle algorithm.
"""

import numpy as np
from fractions import Fraction
from random import randrange

class DecompositionError(Exception):
    """Signals problems with matrix factorization.
    """
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

def lu_decomposition(matrix, silent=True):
    """Factors matrix into a lower triangular matrix and an upper triangular matrix.

    Args:
        matrix: numpy double subscriptable square object.
        silent: whether to display factorization steps (bool).

    Returns:
        Tuple (L, U):
            L - lower triangular matrix (1 on diagonal)
            U - upper triangular matrix
    """

    factored_matrix = np.array([[Fraction(y, 1) for y in x] for x in matrix])
    rank = factored_matrix[0].__len__()
    lower_matrix = np.array([[Fraction(1, 1) if x == y else Fraction() for x in range(rank)] for y in range(rank)])
    upper_matrix = np.array([[Fraction(1, 1) if x == y else Fraction() for x in range(rank)] for y in range(rank)])
    step_count = 0

    for i in range(rank):

        # calculating upper matrix
        for j in range(i, rank):
            sub_sum = 0
            formula = "u" + str(i + 1) + str(j + 1) + " = a" + str(i + 1) + str(j + 1)
            step_count += 1
            for k in range(i):
                sub_sum += lower_matrix[i][k] * upper_matrix[k][j]
                formula += " - " + "l" + str(i + 1) + str(k + 1) + "u" + str(k + 1) + str(j + 1)
            upper_matrix[i][j] = Fraction(factored_matrix[i][j] - sub_sum, 1)

            if not silent:
                print(str(step_count) + " / " + str(rank**2))
                print(step_count, formula, upper_matrix[i][j])

        # calculating lower matrix
        for j in range(i + 1, rank):
            if i != rank:
                sub_sum = 0
                formula = "l" + str(j + 1) + str(i + 1) + " = (a" + str(j + 1) + str(i + 1)
                step_count += 1
                for k in range(i):
                    sub_sum += lower_matrix[j][k] * upper_matrix[k][i]
                    formula += " - " + "l" + str(j + 1) + str(k + 1) + "u" + str(k + 1) + str(i + 1)
                try:
                    lower_matrix[j][i] = Fraction((factored_matrix[j][i] - sub_sum), upper_matrix[i][i])
                except ZeroDivisionError:
                    raise DecompositionError("Zero found in matrix.")
                formula += ") / u" + str(i + 1) + str(i + 1)

                if not silent:
                    print(str(step_count) + " / " + str(rank**2))
                    print(step_count, formula, lower_matrix[j][i])
    lower_counted = [[float(fraction) for fraction in row] for row in lower_matrix]
    upper_counted = [[float(fraction) for fraction in row] for row in upper_matrix]

    return lower_counted, upper_counted


import copy
import itertools
from typing import List, Dict, Tuple

import torch

from modules.block import Block
from modules.cell import Cell
from modules.model import Model
from search_space import Operation

MAX_NUMBER_OF_BLOCKS_PER_CELL = 0

def _prediction_accuracy(inputs, targets) -> float:
    n = len(inputs)
    assert n == len(targets)

    mse = torch.nn.MSELoss()

    la_tensor = torch.FloatTensor(inputs)
    lb_tensor = torch.FloatTensor(targets)

    return mse(la_tensor, lb_tensor).mean().item()

def _get_input_word(input_num: int) -> str:
    """
    convert this input number into the input word in the dictionary
    :param input_num: the input num
    :return: the input word
    """
    return 'input-'.format(input_num)

def _get_op_word(op: Operation) -> str:
    """
    get the operation identifier in the dictionary
    :param op: the operation
    :return: the operation identifier
    """
    return 'op-{}'.format(op.__str__())

def _get_vocabulary_size() -> int:
    """
    get the vocabulary size
    :return: the size of the vocabulary for the embeddings
    """
    return len(_get_vocabulary())

def _get_vocabulary() -> Dict[str, int]:
    """
    get the full vocabulary
    :return: the vocabulary
    """
    vocabulary = [_get_input_word(i) for i in range(MAX_NUMBER_OF_BLOCKS_PER_CELL + 2)]
    vocabulary += [_get_op_word(op) for op in list(Operation)]

    return {w: i for i, w in enumerate(vocabulary)}

def _get_all_possible_blocks(current_cell_size: int) -> List[Block]:
    """
    get's all possible blocks
    :param current_cell_size: the current cell sizes from (1 to ...), it is necessary, to calculate all possible inputs
    :return: all possible blocks
    """
    possible_first_input_operations = list(Operation)
    possible_second_input_operations = possible_first_input_operations.copy()
    possible_first_input = list(range(current_cell_size))
    possible_second_input = possible_first_input.copy()

    combinations = itertools.product(*[possible_first_input, possible_first_input_operations, possible_second_input,
                                       possible_second_input_operations])

    blocks = []

    for c in combinations:
        b = Block(current_cell_size)
        b.first_input_block = c[0]
        b.first_input_op = c[1]

        b.second_input_block = c[2]
        b.second_input_op = c[3]

        blocks.append(b)

    return blocks

def _expand_cell(cell: Cell) -> List[Cell]:
    """
    Expand the cell by a single block
    :param cell: the cell with block-size b
    :return: all descendant cells with block-size b + 1
    """
    expanded_cells = []
    possible_blocks = _get_all_possible_blocks(len(cell.blocks) + 2)

    for block in possible_blocks:
        tmp_cell = copy.deepcopy(cell)
        tmp_cell.blocks.append(block)
        expanded_cells.append(tmp_cell)

    return expanded_cells

def _create_initial_cells() -> List[Cell]:
    """
    create all initial cells with size b = 1
    :return: the initially created cells
    """
    possible_blocks = _get_all_possible_blocks(2)
    cells = []

    for block in possible_blocks:
        c = Cell()
        c.blocks.append(block)

        cells.append(c)

    return cells

def _expand_cells(cells: List[Cell]) -> List[Cell]:
    """
    expand the cells by one more block
    :param cells: the cells that should be expanded
    :return: the expanded cells
    """
    if len(cells) == 0:
        return _create_initial_cells()

    return_cells = []

    for cell in cells:
        expanded_cells = _expand_cell(cell)
        return_cells += expanded_cells

    return return_cells

def _get_normal_and_reduction_cells(cells: List[Cell]) -> List[Tuple[Cell, Cell]]:
    """
    get a pairwise combination of normal and reduction cells
    :param cells: the cells
    :return: normal and reduction cell combination
    """
    return [(cell, cell) for cell in cells]

def _to_architecture_tensor(model: Model) -> List[int]:
    """

    :return:
    """
    normal_cell_tokens = model.normal_cell.__str__().split(' ')
    reduction_cell_tokens = model.normal_cell.__str__().split(' ')

    architecture_tokens = normal_cell_tokens + reduction_cell_tokens

    tokens = [int(t) for t in architecture_tokens]

    expected_block_count = 2 * MAX_NUMBER_OF_BLOCKS_PER_CELL * 4

    # apply padding
    if len(tokens) < expected_block_count:
        tokens += [-1] * (expected_block_count - len(tokens))

    return tokens

class PNASDataset(torch.utils.data.Dataset):
    def __init__(self, inputs, targets=None, train=True):
        super(PNASDataset, self).__init__()
        if targets is not None:
            assert len(inputs) == len(targets)
        self.inputs = copy.deepcopy(inputs)
        self.targets = copy.deepcopy(targets)
        self.train = train

    def __getitem__(self, index):
        surrogate_input = self.inputs[index]
        surrogate_target = None
        if self.targets is not None:
            surrogate_target = [self.targets[index]]
        if self.train:
            sample = {
                'surrogate_input': torch.LongTensor(surrogate_input),
                'surrogate_target': torch.FloatTensor(surrogate_target)
            }
        else:
            sample = {
                'surrogate_input': torch.LongTensor(surrogate_input)
            }
            if surrogate_target is not None:
                sample['surrogate_target'] = torch.FloatTensor(surrogate_target)
        return sample

    def __len__(self):
        return len(self.inputs)


from bin.common.AppConfigurations import mongo_host, mongo_port, db_name, cart_collection
from bin.core.utilities.mongo_utility import MongoUtility
from datetime import date, datetime

class CartFormation(object):
    def __init__(self):
        self.mongo_client = MongoUtility(_mongo_port=mongo_port, _mongo_host=mongo_host)

    def list_cart_item_from_collection(self):
        try:
            print("Fetching endpoint details from mongo")
            mongo_content = self.mongo_client.find_all(database_name=db_name,
                                                       collection_name=cart_collection)
            response = list()
            for content in mongo_content:
                del content["_id"]
                response.append(content)
            return response
        except Exception as e:
            import traceback
            print("EROR :", traceback.print_exc())
            print("Failure to fetch cart collection --> {}".format(str(e)))
            raise Exception("Error when fetching cart collection!")

    def add_item_into_collection(self, input_json):
        try:
            if input_json.get("user_name") == "admin":
                mongo_content = self.mongo_client.insert_one(json_data=input_json, database_name=db_name,
                                                             collection_name=cart_collection)
                print("mongo_content : ", mongo_content)
                return True
            else:
                return False
        except Exception as e:
            import traceback
            print("EROR :", traceback.print_exc())
            print("Failure to fetch cart collection --> {}".format(str(e)))
            raise Exception("Error when fetching cart collection!")

    def add_multiple_item_into_collection(self, input_json):
        try:
            print("Type InputJson : ", type(input_json))
            flag = False
            for data in input_json:
                print("data : ", data)
                if data.get("user_name", "") == "admin":
                    flag = True
                    if not data.get("id", ""):
                        data["timestamp"] = str(datetime.utcnow()).split('.')[0]
                        data["id"] = self.mongo_client.generating_ramdon_id(cart_collection)
                else:
                    flag = False
                    break
            print("FLAG : ", flag)
            print("INPUT JSON MULTI : ", input_json)
            if flag:
                mongo_content = self.mongo_client.insert_many(json_data=input_json, database_name=db_name,
                                                          collection_name=cart_collection)
                return True
            else:
                return False

        except Exception as e:
            import traceback
            print("ERROR :", traceback.print_exc())
            print("Failure to fetch cart collection --> {}".format(str(e)))
            raise Exception("Error when fetching cart collection!")

    def update_cart_data(self, input_json):
        try:
            print("Type InputJson : ", type(input_json))
            input_json["timestamp"] = str(datetime.utcnow()).split('.')[0]
            condition = dict(id=input_json.get("id", ""))
            mongo_content = self.mongo_client.update_one(condition=condition, json_data=input_json,
                                                         _database_name=db_name,
                                                         collection_name=cart_collection)
            return True
        except Exception as e:
            import traceback
            print("EROR :", traceback.print_exc())
            print("Failure to fetch cart collection --> {}".format(str(e)))
            raise Exception("Error when fetching cart collection!")

    def remove_cart(self, input_json):
        try:
            print("Type InputJson : ", type(input_json))
            # condition = dict(id=input_json.get("id", ""))
            mongo_content = self.mongo_client.remove(json_data=input_json,
                                                     _database_name=db_name,
                                                     collection_name=cart_collection)
            print("mongo_content : ", mongo_content)
            # flask.jsonify(message="success", insertedIds=todo_many.inserted_ids)
            response = list()
            print("REMOVE CART mongoCOntent : ", mongo_content)
            return mongo_content
        except Exception as e:
            import traceback
            print("EROR :", traceback.print_exc())
            print("Failure to fetch cart collection --> {}".format(str(e)))
            raise Exception("Error when fetching cart collection!")


# -*- coding: future_annotations -*-
import ast

from nbsafety.tracing.ast_eavesdrop import AstEavesdropper
from nbsafety.utils import KeyDict

PROGRAM = """
for i in [foo(x) for x in [1, 2, 3]]:
    try:
        logging.info(i)
    except:
        logging.warning("warning!")
    finally:
        sys.exit(0)

if False:
    asdf(qwer(1, 2, a().b[c,d,e](f.g()).h))[a, 7] = foo.bar
"""

def test_ast_rewrite():
    """
    No asserts; just make sure we don't throw an error.
    """
    rewriter = AstEavesdropper(KeyDict())
    assert rewriter.visit(ast.parse(PROGRAM)) is not None


"""
find all a, b, c, d in q such that
f(a) + f(b) = f(c) - f(d)
"""
from pprint import pprint

#q = set(range(1, 10))
q = set(range(1, 80))
# print(q)
# q = (1, 3, 4, 7, 12)
# print(q)
q = list(q)

# breakpoint()

# TODO: cache this?
print(q)
# f = {}
def f(x):
    return x * 4 + 6

def pad(x):
    return (3-len(str(x)))*'0'+str(x)

dict_lhs = {}
# Your code here
i = 0
j = 1
gap = 1
while i < len(q)-1:
    while j < len(q):
        # print(i,j, gap)
        x = (3-len(str(i)))*'0'+str(i)
        # key1 = pad(i)+'-'+pad(j)
        # key2 = pad(j)+'-'+pad(i)

        key1 = str(i)+'-'+str(j)
        key2 = str(j)+'-'+str(i)

        # if neither key is in the dictionary, then compute f(i) + f(j). then create 2 entries in the mappign table
        if key1 not in dict_lhs and key2 not in dict_lhs:
            dict_lhs[key1] = f(q[i]) + f(q[j])
            dict_lhs[key2] = f(q[i]) + f(q[j])

        # if either key is in the dictionary, then lookup the value from dictionary and populate the sum ans assign to the empty
        elif key1 not in dict_lhs or key2 in dict_lhs:
            dict_lhs[key1] = dict_lhs[key2]

        elif key1 in dict_lhs or key2 not in dict_lhs:
            dict_lhs[key2] = dict_lhs[key1]

        else:
            continue

        i += 1
        j += 1
    gap += 1
    i = 0 if gap < len(q) else len(q)
    j = gap + i

# pprint(dict_lhs)

dict_rhs = {}
# Your code here
i = 0
j = 1
gap = 1
while i < len(q)-1:
    while j < len(q):
        # print(i,j, gap)
        # key3 = pad(i)+'-'+pad(j)
        # key4 = pad(j)+'-'+pad(i)
        key3 = str(i)+'-'+str(j)
        key4 = str(j)+'-'+str(i)
        # diff_3 = f(q[i]) - f(q[j])
        # diff_4 = f(q[j]) - f(q[i])
        # if neither key is in the dictionary, then compute f(i) + f(j). then create 2 entries in the mappign table
        if key3 not in dict_rhs:# and key4 not in dict_rhs:
            dict_rhs[key3] = f(q[i]) - f(q[j])
        if key4 not in dict_rhs:
            dict_rhs[key4] = abs(f(q[i]) - f(q[j]))

        i += 1
        j += 1
    gap += 1
    i = 0 if gap < len(q) else len(q)
    j = gap + i

# pprint(dict_rhs)

# for key_l,value_l in dict_lhs.items():
#     # breakpoint()

#     a = int(key_l[:1])
#     b = int(key_l[2:3])
#     a_or_b = [a,b]
#     # print(a,b, a_or_b)

#     for key_r, value_r in dict_rhs.items():
#         c = int(key_r[:1])
#         d = int(key_r[2:3])
#         # print(c,d)
#         # if c not in a_or_b and d not in a_or_b:
#         #     print(value_l,value_r)
#         #     if value_l == value_r:
#         #         print("f({q[a]}) + f({q[b]} = f){q[c]} - f({q[d]})")
#         if value_l == value_r:

combined = {}
# Your code here
i = 0
j = 1
gap = 1
keys = [i for i in dict_lhs.keys()]

while i < len(dict_lhs)-1:
    while j < len(dict_lhs):
        # print(i,j, gap)
        key = keys[i]+'-'+keys[j]
        # key2 = keys[j]+'-'+keys[i]
        # if neither key is in the dictionary, then compute f(i) + f(j). then create 2 entries in the mappign table
        if key not in combined:
            # breakpoint()
            combined[key] = dict_lhs[keys[i]] - dict_rhs[keys[j]]
            # combined[key2] = dict_lhs[key2] + dict_rhs[key2]

        # if either key is in the dictionary, then lookup the value from dictionary and populate the sum ans assign to the empty
        # elif key1 not in dict_lhs or key2 in dict_lhs:
        #     combined[key1] = combined[key2]

        # elif key1 in dict_lhs or key2 not in dict_lhs:
        #     combined[key2] = combined[key1]

        else:
            continue

        i += 1
        j += 1

    gap += 1
    i = 0 if gap < len(dict_lhs) else len(dict_lhs)
    j = gap + i

# print('*** combined')
# pprint(combined)

# print('zippp')
# combined = {}
# for (k_l, v_l), (k_r, v_r) in zip(dict_lhs.items(), dict_rhs.items()):
#     # combining both lhs and rhs, but keeping the same name
#     print(k_l, k_r)
#     print(v_l, v_r)
#     combined[k_l+'-'+k_r] = v_l - v_r

# make this a list comprehension
combined_f = {}
for key,val in combined.items():
    if val == 0:
        # breakpoint()
        # a = int(key[0:1]) #LOGIC WILL FAIL FOR SIZE > 10!!!! PAD INDICES
        # b = int(key[2:3])
        # c = int(key[4:5])
        # d = int(key[6:7])
        pos = key.split('-')
        # breakpoint()
        # print('**', key,val)
        print(f"f({q[int(pos[0])]}) + f({q[int(pos[1])]}) = f({q[int(pos[2])]}) - f({q[int(pos[3])]})    {f(q[int(pos[0])])} + {f(q[int(pos[1])])} = {f(q[int(pos[2])])} - {f(q[int(pos[3])])}")
        # combined_f[key] = val

# print('combined')
# pprint(combined)

# breakpoint()

# if neither key is in the dictionary, then compute f(i) + f(j)

# elseit
    # pair-up i and i+1; slide right O(n)

    # pair-up i and i+2; slide right O(n)

    # pair-up i and i+3; slide right O(n)