from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import math
def calc(x):
      return str(math.log(abs(12*math.sin(int(x)))))

link = "http://suninjuly.github.io/explicit_wait2.html"
browser = webdriver.Chrome(executable_path="C:\\ChromeDriver\\ChromeDriver.exe")
browser.get(link)

bookbut = WebDriverWait(browser,12).until(EC.text_to_be_present_in_element((By.ID,'price'),'$100'))
browser.find_element_by_xpath('//*[@id="book"]').click()

x = browser.find_element_by_xpath('//*[@id="input_value"]').text
int(x)
result = calc(x)
browser.find_element_by_xpath('//*[@id="answer"]').send_keys(result)
browser.find_element_by_xpath('/html/body/form/div/div/button').click()


from bs4 import BeautifulSoup
import zipfile
import glob
import os
import re
import csv
import pickle
import datetime

# Chiki Chiki Checker

#曜日変換
WEEKDAY_JA_TO_EN = {
    '月': 'Mon',
    '火': 'Tue',
    '水': 'Wed',
    '木': 'Thu',
    '金': 'Fri',
    '土': 'Sat',
    '日': 'Sun'
}

WEEKDAY_EN_TO_JA = {
    'Mon': '月',
    'Tue': '火',
    'Wed': '水',
    'Thu': '木',
    'Fri': '金',
    'Sat': '土',
    'Sun': '日'
}

#ふたばスレッド futaba_thread……list
#ふたばレス futaba_res……dict

#ふたばレス futaba_res

FUTABA_RESPONSE_COLOMN = {
    r'url': r'URL',
    r'thread_number': r'スレッド番号',
    r'thread_title': r'スレッドタイトル',
    r'number': r'レス番号',
    r'munen': r'無念',
    r'name': r'名前',
    r'date': r'年月日',
    r'weekday': r'曜日',
    r'time': r'時刻',
    r'ip': r'IP',
    r'domain': r'ドメイン',
    r'res_id': r'レスID',
    r'soudane': r'そうだね',
    r'res': r'レス'
}

FUTABA_RESPONSE_TEMPLATE = {
    'url': '',
    'thread_number': 0,
    'thread_title': '',
    'number': 0,
    'munen': '無念',
    'name': 'としあき',
    'date': '00/01/01',
    'weekday': '月',
    'time': '00:00:00',
    'ip': '000.000.*',
    'domain': 'xxx.com',
    'res_id': 0,
    'soudane': 0,
    'res': ''
}

# コピー
# dest = list(map(lambda d: d.copy(), target))

CONDITION_METHOD = {
    'none': 0,
    'freeword': 1,
    'range': 2
}

re_datetime = re.compile(r'\d\d/\d\d/\d\d\([日月火水木金土]\)\d\d:\d\d:\d\d')
re_datetime_search = re.compile(r'(\d\d)/(\d\d)/(\d\d)%s*(\d\d):(\d\d):(\d\d)')
re_date = re.compile(r'(\d\d)/(\d\d)/(\d\d)')
re_time = re.compile(r'(\d\d):(\d\d):(\d\d)')
re_weekday = re.compile(r'[日月火水木金土]')

re_ip = re.compile(r'IP:([0-9a-fA-F]+[\.:])+\*?')
re_domain = re.compile(r'\([0-9a-zA-Z\-\.]+\)')

re_brace = re.compile(r'[\(\)]')
re_ip_letters = re.compile(r'IP:')
re_numbers = re.compile(r'\d+')
re_link = re.compile(r'\[\nlink\n\]')
re_res_ip = re.compile(r'([0-9a-fA-F]+[\.:]?)+')
re_res_ip_reform = re.compile(r'\[\n(([0-9a-fA-F]+[\.:]?)+)\n]')

def split_time_ip_domain(data):
    #21/01/24(日)20:47:21 IP:101.141.*(eonet.ne.jp)
    #21/01/24(日)20:47:07 IP:240f:45.*(ipv6)
    time = re_datetime.search(data)
    ip = re_ip.search(data)
    domain = re_domain.search(data)

    if domain:
        ret_domain = re_brace.sub('', domain.group(0))
    else:
        ret_domain = None

    if time:
        ret_time = time.group(0)
    else:
        ret_time = None

    if ip:
        ret_ip = re_ip_letters.sub('', ip.group(0))
    else:
        ret_ip = None

    return ret_time, ret_ip, ret_domain

def split_date_weekday_time(data):
    #21/01/24(日)20:47:07
    date = re_date.search(data)
    weekday = re_weekday.search(data)
    time = re_time.search(data)

    if date:
        ret_date = date.group(0)
    else:
        ret_date = None

    if weekday:
        ret_weekday = weekday.group(0)
    else:
        ret_weekday = None

    if time:
        ret_time = time.group(0)
    else:
        ret_time = None

    return ret_date, ret_weekday, ret_time

def convert_soudane(data):
    ret = 0
    chk = re_numbers.search(data)
    if chk:
        ret = int(chk.group(0))
    return ret

def convert_res_id(data):
    ret = 0
    chk = re_numbers.search(data)
    if chk:
        ret = int(chk.group(0))
    return ret

def check_res_ip(soup):
    data = soup.select('blockquote font[color="#ff0000"]')
    if not data :
        return None

    data = data[0].get_text()
    chk = re_res_ip.search(data)

    if chk:
        return chk.group(0)
    else:
        return None
    return

def convert_res(data):
    ret = data
    ret = re_link.sub('', ret)
    ret = re_res_ip_reform.sub(r'[\1]', ret)
    return ret

def convert_to_futaba_res_dict(
    url='',
    thread_number='0',
    thread_title='',
    number='0',
    munen='無念',
    name='としあき',
    timestamp='00/01/01(月) IP:000.000.*(xxx.com)',
    res_id='0',
    soudane='+',
    res=''):

    ret = FUTABA_RESPONSE_TEMPLATE.copy()
    ret['url'] = url
    ret['thread_number'] = int(thread_number)
    ret['thread_title'] = thread_title
    ret['number'] = int(number)
    ret['munen'] = munen
    ret['name'] = name

    dtm, ip, domain = split_time_ip_domain(timestamp)
    date, weekday, time = split_date_weekday_time(dtm)

    ret['date'] = date
    ret['weekday'] = weekday
    ret['time'] = time
    ret['ip'] = ip
    ret['domain'] = domain

    ret['res_id'] = convert_res_id(res_id)
    ret['soudane'] = convert_soudane(soudane)
    ret['res'] = convert_res(res)
    return ret

def split_thread_meta(soup):
    thread_title = soup.select('title')[0].get_text()
    thread_number = soup.select('span.thre')[0]['data-res']
    return thread_title, thread_number

def split_thread_dominus(soup):
    dom_munen = soup.select('span.csb')[0].get_text()
    dom_name = soup.select('span.cnm')[0].get_text()
    dom_time = soup.select('span.cnw')[0].get_text()
    dom_number = soup.select('span.cno')[0].get_text()
    dom_sod = soup.select('a.sod')[0].get_text()
    dom_content = soup.select('blockquote')[0].get_text('\n')
    return dom_munen, dom_name, dom_time, dom_number, dom_sod, dom_content

def split_thread_res(res):
    res_count = res.select('span.rsc')[0].get_text()
    res_munen = res.select('span.csb')[0].get_text()
    res_name = res.select('span.cnm')[0].get_text()
    res_time = res.select('span.cnw')[0].get_text()
    res_number = res.select('span.cno')[0].get_text()
    res_sod = res.select('a.sod')[0].get_text()
    res_content = res.select('blockquote')[0].get_text('\n')
    return res_count, res_munen, res_name, res_time, res_number, res_sod, res_content

def is_contains_word(res, word):
    return word.lower() in res.lower()

def is_res_in_range(res, fore=None, apre=None):
    if fore == apre:
        if res == fore:
            return True
        else:
            return False
    elif fore and apre:
        if res >= fore and res <= apre:
            return True
        else:
            return False
    elif not fore and apre:
        if res <= apre:
            return True
        else:
            return False
    elif fore and not apre:
        if res >= fore:
            return True
        else:
            return False
    else:
        return True

def is_res_matched_re(res, word):
    ret = re.search(word, res)
    if ret:
        return True
    else:
        return False

#filt=(mode, fore, apre)
def filter_range(res, filt):
    return is_res_in_range(res, filt[1], filt[2])

#連言
def filter_res(res, filt):
    for key, value in filt.items():
        if condition_handler[CONDITION_METHOD[value[0]]](res[key], value) == False:
            return False

    return True

#filt=(mode, words, isReg)
def filter_free_word(res, filt):
    if filt[2] == True:
        return is_res_matched_re(res, filt[1])
    else:
        #print(str(filt[0]) + str(filt[1]) + str(filt[2]))
        for x in filt[1].split():
            if is_contains_word(res, x) == False:
                return False
        return True

condition_handler = [
    None,
    filter_free_word,
    filter_range
]

def load_thread_dict(filename):
    with open(filename, 'rb') as f:
        ret = pickle.load(f)
    return ret

def set_list_to_write(thread, url, to_write):
    soup = BeautifulSoup(thread, 'html.parser')
    thread_title, thread_number = split_thread_meta(soup)

    dom_munen, dom_name, dom_time, dom_number, dom_sod, dom_content = split_thread_dominus(soup)

    dom = convert_to_futaba_res_dict(
        url,
        thread_number,
        thread_title,
        '0',
        dom_munen,
        dom_name,
        dom_time,
        dom_number,
        dom_sod,
        dom_content
        )
    chk = check_res_ip(soup)
    if chk:
        dom['ip'] = chk

    to_write.append(dom)

    reses = [s for s in soup.select('td.rtd')]
    for res in reses:
        res_count, res_munen, res_name, res_time, res_number, res_sod, res_content = split_thread_res(res)

        res_dict = convert_to_futaba_res_dict(
            url,
            thread_number,
            thread_title,
            res_count,
            res_munen,
            res_name,
            res_time,
            res_number,
            res_sod,
            res_content
            )
        chk = check_res_ip(res)
        if chk:
            res_dict['ip'] = chk

        to_write.append(res_dict)

    return

def write_thread_dict(filename, to_write):
    with open(filename, 'wb') as f:
        pickle.dump(to_write, f)
    return

def write_csv(result_filename, to_write):
    with open(result_filename, 'w', errors='xmlcharrefreplace') as f:
        writer = csv.writer(f, lineterminator='\n')
        writer.writerows([x.values() for x in to_write])

def extract_ftbucket_zip(filename):
    with zipfile.ZipFile(filename) as log:
        log_list = log.namelist()
        log_index = [s for s in log_list if(r'index.htm' in s and r'gz' not in s)][0]
        with log.open(log_index) as log_content:
            ret = log_content.read()
    return ret

def fetch_html_file(filename):
    with open(filename, 'r') as f:
        ret = f.read()
    return ret


import pandas as pd
import os
import matplotlib
from tqdm import tqdm
matplotlib.use('Agg')
import matplotlib.pyplot as plt

def main():
    root = './result/salinas/'
    paths = os.listdir(root)
    save_root = './result/salinas_plot/'

    for path in tqdm(paths):
        df = pd.read_csv(root + path)
        false_alarm_rate = df['False alarm rate'][0]
        plt.bar(range(len(df)), df['Recognition rate'])
        plt.title(path.split('.')[0] + ',false_alarm_rate:{}%'.format(round(false_alarm_rate*100, 2)))
        plt.xticks(range(len(df)), df['class'])
        plt.ylabel('Recognition rate')
        plt.xlabel('other classes')
        save_path = save_root + path.split('.')[0] + '.png'
        plt.savefig(save_path)
        plt.clf()

if __name__ == '__main__':
    main()


import matplotlib.pyplot as plt
from matplotlib.colors import Normalize
import numpy as np
from numpy import rot90
from os.path import split
from collections import defaultdict

class GridPlotBase:

    def extract(self, *args):
        self.extracts += args

    def _parse_legend_param(self, legend):
        legend_defaults = {'loc': 'best', 'fontsize': 8, 'frameon': True}
        if legend:
            try:
                legend_defaults.update(legend)
            except TypeError:
                pass
            finally:
                legend = legend_defaults
        return legend

    @staticmethod
    def get_grid_xy(paths):
        """Get size of grid required to plot data by choosing the
        smallest square grid that will fit all of the data.
        """
        x = np.ceil(np.sqrt(len(paths)))
        return int(x), int(x)

    @staticmethod
    def hideaxes(axarr):
        for ax in axarr:
            for curr in ax:
                curr.spines["left"].set_visible(False)
                curr.spines["right"].set_visible(False)
                curr.spines["top"].set_visible(False)
                curr.spines["bottom"].set_visible(False)
                curr.xaxis.set_visible(False)
                curr.yaxis.set_visible(False)

    def _title_from(self, fpath, level, maxchars=None, ellipsis=False):
        dir, base = split(fpath)
        if level == 0:
            title = base
        else:
            title = self._title_from(dir, level - 1)
        if maxchars is not None and len(title) > maxchars:
            title = title[:maxchars]
            if ellipsis:
                title += r'$\ldots$'
        return title

    def _empty_2darr(self, m, n=None):
        if n is None:
            n = m
        return [[None for x in range(n)] for y in range(m)]

    def rotated_indices(self, row, col, nrows, ncols, ostring):
        '''Function that takes (iy, ix) coordinate in original matrix and returns
        the (iyy, ixx) coordinate where that element should go in a new matrix
        that has been rotated by 'angle'°
        '''
        smat = self.rotated_serial_matrix(ostring, nrows, ncols)
        scoord = self.row_col_to_serial(row, col, nrows, ncols)
        return np.argwhere(smat == scoord)[0]

    def rotated_serial_matrix(self, ostring, nrows, ncols):
        ostring = ostring.lower()
        mat = np.arange(nrows * ncols).reshape(nrows, ncols)
        trans_dict ={
         'nwes': (0, False),
         'nwse': (0, True),
         'nesw': (270, False),
         'news': (270, True),
         'sewn': (180, False),
         'senw': (180, True),
         'swne': (90, False),
         'swen': (90, True)}
        angle, transp = trans_dict[ostring]
        if transp:
            mat = mat.T
        if angle == 90:
            mat = rot90(mat)
        elif angle == 180:
            mat = rot90(rot90(mat))
        elif angle == 270:
            mat = rot90(rot90(rot90(mat)))
        return mat

    def rotated_shape(self, ostring, nrows, ncols):
        return self.rotated_serial_matrix(ostring, nrows, ncols).shape

    def row_col_to_serial(self, row_y, col_x, nrows, ncols):
        return col_x + ncols * row_y

class GridPlot(GridPlotBase):
    """Plot a 2d grid of HLoops that lack ordering. Plotting on a
    grid is just a convenient way of visualizing many HLoops.

    Args:
        hloops (sequence): hloopy.HLoops to include in the grid plot.
        legend: If `None` or `False` then no legend. If `True` then
                a legend is put in one axes. If a dict is passed, the
                legend is shown and the dict is passed to `ax.legend()`
                as its kwargs.
        lablevel (int): Determines how each axis is labeled. Default
                        is no label. Value of 0 means label with the
                        datafile basename, 1 means datafile dirname,
                        2 means grandparent and so on.
        titleparams (dict): Optional kwargs for :code:`ax.set_title()`
    """
    def __init__(self, hloops, hideaxes=True, legend=True, lablevel=None,
                 titleparams={}, title_chars=10):
        if hloops is None or len(hloops) == 0:
            raise ValueError("Must have at least 1 hloop to make a GridPlot.")
        self.hloops = hloops
        self.mx, self.my = self.get_grid_xy(self.hloops)
        self.fig, self.axarr = plt.subplots(self.mx, self.my, squeeze=False)
        self.legend = self._parse_legend_param(legend)
        if hideaxes:
            self.hideaxes(self.axarr)
        self.extracts = []
        self.lablevel = lablevel
        self.titleparams = titleparams
        self.title_chars = title_chars

    def plot(self, title_chars=20, **kwargs):
        """Plot all of `self.hloops` onto a 2d array of `matploblib.axes`.

        Args:
            The `kwargs` are passed to ax.plot().

        Returns:

        """
        self.lines_plotted = self._empty_2darr(self.mx)
        self.extract_instances = defaultdict(list)
        for q, hl in enumerate(self.hloops):
            x, y = q // self.mx, q % self.mx
            ax = self.axarr[x][y]
            if self.lablevel is not None:
                title_style = {'fontsize': 12}
                title_style.update(self.titleparams)
                title = self._title_from(hl.fpath, level=self.lablevel,
                                         maxchars=self.title_chars,
                                         ellipsis=True)
                ax.set_title(title, **title_style)
            ln = hl.plot(ax)
            self.lines_plotted[x][y] = ln
            for e in self.extracts:
                e_instance = e(hl)
                e_instance.plot(ax, **kwargs)
                self.extract_instances[hl.fpath].append(e_instance)
            if q == 0 and self.legend:
                try:
                    ax.legend(**self.legend)
                except TypeError:
                    ax.legend()
        return self.lines_plotted

    def extracts_as_df(self):
        """After plotting, get all extracts in a dataframe with rows like:

        "/Path/to/hloop/data, label, avg_val, xcoords, ycoords, indices"

        If you want the actual extracts just grab them from
        GridPlot.extract_instances, which is a dict keyed by fpaths and with
        lists of extacts as values.
        """
        from pandas import DataFrame
        d = defaultdict(list)
        for fpath, extracts in self.extract_instances.items():
            for e in extracts:
                d['fpath'].append(fpath)
                d['label'].append(e.label)
                d['avg_val'].append(e.avg_val)
                d['xcoords'].append(e.xcoords)
                d['ycoords'].append(e.ycoords)
                d['indices'].append(e.indices)
        return DataFrame(d)

class HLoopGridPlot(GridPlotBase):
    def __init__(self, hloop_grid, legend=None, extracts=None, lablevel=None,
                 titleparams={}, hideaxes=True):
        """Plot an HLoopGrid object
        """
        self.hloop_grid = hloop_grid
        self.hloops = hloop_grid.hloops
        self.nrows = self.hloop_grid.nrows
        self.ncols = self.hloop_grid.ncols
        self.nloops = self.hloop_grid.nloops
        self.hg = hloop_grid
        self.extracts = extracts
        self.legend = self._parse_legend_param(legend)
        self.lablevel = lablevel
        self.titleparams = titleparams
        self.extracts = []
        self.hideaxes_switch = hideaxes

    def plot(self, simple_label=False, extract_plot_kwargs={},
             ostring='nwes', **kwargs):
        """Plot the HLoops on a grid.

        Args:
            - ostring (string): A four character string that determines the
                orientation transformation applied. Letters correspond to
                cardinal directions n, e, s, w. The first two characters
                are the location of (0, 0), the third character is the +x
                direction and the fourth character is the +y direction. The
                x and y reffered to are that of typical computer image
                coordinates. Top left is (0, 0), +x is to the right and +y is
                down. The default orientation is 'nwes'.
            - extract_plot_kwargs (dict): kwargs for pyplot.plot when used to
                plot any extracts that may have been added.
            - kwargs: passed to pyplot.plot call that plots the HLoop data.
        """
        ostring = ostring.lower()
        final_nrows, final_ncols = self.rotated_shape(ostring, self.nrows,
                                                      self.ncols)
        self.fig, self.axarr = plt.subplots(nrows=final_nrows,
                                            ncols=final_ncols)
        if self.hideaxes_switch:
            self.hideaxes(self.axarr)
        self.extract_instances = defaultdict(list)
        self.plotted_lines = []
        for i, hl in enumerate(self.hloops):
            # Plot hloop
            row_init, col_init = self.hg.mapping[i]
            row, col = self.rotated_indices(row_init, col_init, self.nrows,
                                            self.ncols, ostring)
            ax = self.axarr[row][col]
            self.plotted_lines.append(hl.plot(ax, **kwargs))
            # Maybe add a title
            title_style = {'fontsize': 12}
            title_style.update(self.titleparams)
            if self.lablevel is not None:
                title = self._title_from(hl.fpath, level=self.lablevel,
                                         maxchars=20, ellipsis=True)
                ax.set_title(title, **title_style)
            if simple_label:
                ax.set_title('{}, {}'.format(col_init, row_init),
                             **title_style)
            # Plot extracts
            for e in self.extracts:
                e_instance = e(hl)
                e_instance.plot(ax, **extract_plot_kwargs)
                self.extract_instances[hl.fpath].append(e_instance)
            # Maybe add a legend
            if i == 0 and self.legend:
                try:
                    ax.legend(**self.legend)
                except TypeError:
                    ax.legend()
        return self.plotted_lines

class ExtractGridPlot(GridPlotBase):
    def __init__(self, hloop_grid, extract):
        """Plot an HLoopGrid object
        """
        self.hloop_grid = hloop_grid
        self.hloops = hloop_grid.hloops
        self.nrows = self.hloop_grid.nrows
        self.ncols = self.hloop_grid.ncols
        self.nloops = self.hloop_grid.nloops
        self.hg = hloop_grid
        self.extract = extract

    def plot(self, ostring='nwes', colorbar={}, clim=None, hideaxes=False,
             missing_val=0.0, **kwargs):
        """Plot the HLoops on a grid.

        Also adds extract_instances 2d array to this ExtractGridPlot instance
        if the instances are needed for something morecomplicated than just an
        imshow of the avg_vals.

        Args:
            - ostring (string): A four character string that determines the
                orientation transformation applied. Letters correspond to
                cardinal directions n, e, s, w. The first two characters
                are the location of (0, 0), the third character is the +x
                direction and the fourth character is the +y direction. The
                x and y reffered to are that of typical computer image
                coordinates. Top left is (0, 0), +x is to the right and +y is
                down. The default orientation is 'nwes'.
            - colorbar (dict): To add a colorbar pass True or a dict of params
                to be passed to fig.colorbar().
            - clim (tuple): vmin and vmax to be used for colorbar. Overrides
                the norm parameter if it is also passed in kwargs below.
            - hideaxes (bool): Hides the axes.
            - kwargs: passed to pyplot.imshow call. FOr example:
                    im = plt.imshow(Hcs, norm=norm, aspect='equal')
        """
        ostring = ostring.lower()
        final_nrows, final_ncols = self.rotated_shape(ostring, self.nrows,
                                                      self.ncols)
        self.extract_avg_vals = self._empty_2darr(final_nrows, final_ncols)
        self.extract_instances = self._empty_2darr(final_nrows, final_ncols)
        for i, hl in enumerate(self.hloops):
            # Plot hloop
            row_init, col_init = self.hg.mapping[i]
            row, col = self.rotated_indices(row_init, col_init, self.nrows,
                                            self.ncols, ostring)
            ext = self.extract(hl)
            self.extract_instances[row][col] = ext
            self.extract_avg_vals[row][col] = ext.avg_val
        self.fig, self.ax = plt.subplots()
        if clim is not None:
            norm = Normalize(*clim)
            kwargs.update(norm=norm)
        avg_vals_cleaned = [[x if x is not None else missing_val for x in y]
                             for y in self.extract_avg_vals]
        res = self.ax.imshow(avg_vals_cleaned, **kwargs)
        self.extract_avg_vals = avg_vals_cleaned
        if colorbar:
            colorbar = {} if colorbar is True else colorbar
            self.fig.colorbar(res, **colorbar)
        return res

    def hideaxes(self, ax):
        ax.spines["left"].set_visible(False)
        ax.spines["right"].set_visible(False)
        ax.spines["top"].set_visible(False)
        ax.spines["bottom"].set_visible(False)
        ax.xaxis.set_visible(False)
        ax.yaxis.set_visible(False)

    def histogram(self, xslice=(None, None), yslice=(None, None), ax=None, hist_kwargs={}):
        """Plot histogram of extract values. Must be run
        after self.plot()!!!

        Args:
            xslice / yslice (tuple): A tuple of ints or None. Used to select
                which subregion of the 2d grid is to be turned into a
                histogram. (None, None) will select the whole range.
                (1, -1) will be like list[1:-1].
            ax (mpl.Axes): Provide if you want this func to plot onto an
                existing axes. If None, new axes will be created.
        """
        if ax is None:
            fig, ax = plt.subplots()
            ret_fig = True
        else:
            ret_fig = False
        vals = np.array(self.extract_avg_vals)
        vals = vals[xslice[0]:xslice[1], yslice[0]:yslice[1]]
        vals_flat = vals.reshape(vals.shape[0] * vals.shape[1])
        ax.hist(vals_flat, **hist_kwargs)
        return ax if not ret_fig else fig, ax

class SaturationGridPlot(ExtractGridPlot):
    def plot(self, ostring='nwes', colorbar={}, clim=None, hideaxes=False,
             **kwargs):
        """Plot the HLoops on a grid.

        Also adds extract_instances 2d array to this ExtractGridPlot instance
        if the instances are needed for something morecomplicated than just an
        imshow of the avg_vals.

        Args:
            - ostring (string): A four character string that determines the
                orientation transformation applied. Letters correspond to
                cardinal directions n, e, s, w. The first two characters
                are the location of (0, 0), the third character is the +x
                direction and the fourth character is the +y direction. The
                x and y reffered to are that of typical computer image
                coordinates. Top left is (0, 0), +x is to the right and +y is
                down. The default orientation is 'nwes'.
            - colorbar (dict): To add a colorbar pass True or a dict of params
                to be passed to fig.colorbar().
            - clim (tuple): vmin and vmax to be used for colorbar. Overrides
                the norm parameter if it is also passed in kwargs below.
            - hideaxes (bool): Hides the axes.
            - kwargs: passed to pyplot.imshow call. FOr example:
                    im = plt.imshow(Hcs, norm=norm, aspect='equal')
        """
        ostring = ostring.lower()
        final_nrows, final_ncols = self.rotated_shape(ostring, self.nrows,
                                                      self.ncols)
        self.extract_avg_vals = self._empty_2darr(final_nrows, final_ncols)
        self.extract_instances = self._empty_2darr(final_nrows, final_ncols)
        for i, hl in enumerate(self.hloops):
            # Plot hloop
            row_init, col_init = self.hg.mapping[i]
            row, col = self.rotated_indices(row_init, col_init, self.nrows,
                                            self.ncols, ostring)
            ext = self.extract(hl)
            self.extract_instances[row][col] = ext
            avg_val = (ext.xcoords[1] - ext.xcoords[0]) / 2.0
            self.extract_avg_vals[row][col] = avg_val
        self.fig, self.ax = plt.subplots()
        if clim is not None:
            norm = Normalize(*clim)
            kwargs.update(norm=norm)
        res = self.ax.imshow(self.extract_avg_vals, **kwargs)
        if colorbar:
            colorbar = {} if colorbar is True else colorbar
            self.fig.colorbar(res, **colorbar)
        return res

class ExtractsPlotBase:
    def __init__(self, extract_instances, quantity_to_plot, labels=None,
                 xfunc=None, xsortfunc=None):
        """Plot extracts on their own axis

        Args:
            extract_instances (dict):  A mapping from
                fpath --> hloopy.ExtractBase. This type of object
                is produced by a GridPlot with extracts added
                (GridPlot.extract_instances attribute). The fpath keys are
                the paths to the datafiles where HLoop objects got their data.
                There need not be only one type of extract in this dict.
            quantity_to_plot (callable): Function that gets the attr from the
                extract that you want to plot. i.e. `lambda x: x.avg_val`,
                `lambda x: x.xcoords[0]`, `lambda x: x.ycoords[1]`. This
                callable takes one argument, the Extract object, and returns
                one number.
            labels (collection): Extract objects have a label attr. If that
                attr is not in this collection, the extract will not be
                included in this plot. If `None` then do not filter by label.
            xfunc (callable): A function that returns the x-axis value for
                each extract_instance. (fpath, extract) --> number. If `None`
                then the x axis order will just be a counter index.
        """
        self.extract_instances = extract_instances
        self.es = self._flatten_and_filter(extract_instances, labels)
        self.quantity_to_plot = quantity_to_plot
        self.labels = labels
        self.xfunc = xfunc
        self.xsortfunc = xsortfunc

    def x(self):
       return np.array([self.xfunc(fp, e) for fp, e in self.es])

    def y(self):
        return np.array([getattr(e, self.quantity_to_plot)
                         for (fp, e) in self.es])

    def xy_sorted(self):
        x = self.x()
        y = self.y()
        tsort = sorted(enumerate(self.es), key=lambda x: self.xsortfunc(*x[1]))
        isort = [x[0] for x in tsort]
        return x[isort], y[isort]

    def plot(self, *args, **kwargs):
        x, y = self.xy_sorted()
        fig, ax = plt.subplots()
        ax.plot(x, y, *args, **kwargs)
        return fig, ax

    @staticmethod
    def _flatten_and_filter(instances, labels=None):
        if labels is not None:
           return [(fp, e) for fp, es in instances.items()
                           for e in es if e.label in labels]
        else:
           return [(fp, e) for fp, es in instances.items() for e in es]


from distutils.core import setup, Extension
import numpy as np
import sys
import os
from glob import glob

PYHSMM_VERSION = "0.1.3"

###########################
#  compilation arguments  #
###########################

extra_link_args = []
extra_compile_args = []

if '--with-old-clang' in sys.argv:
    sys.argv.remove('--with-old-clang')
    extra_compile_args.append('-stdlib=libc++')
    extra_link_args.append('-stdlib=libc++')

if '--with-openmp' in sys.argv:
    sys.argv.remove('--with-openmp')
    extra_compile_args.append('-fopenmp')
    extra_link_args.append('-fopenmp')

if '--with-native' in sys.argv:
    sys.argv.remove('--with-native')
    extra_compile_args.append('-march=native')

if '--with-mkl' in sys.argv:
    sys.argv.remove('--with-mkl')
    # NOTE: there's no way this will work on Windows
    extra_compile_args.extend(['-m64','-I' + os.environ['MKLROOT'] + '/include','-DEIGEN_USE_MKL_ALL'])
    extra_link_args.extend(('-Wl,--start-group %(MKLROOT)s/lib/intel64/libmkl_intel_lp64.a %(MKLROOT)s/lib/intel64/libmkl_core.a %(MKLROOT)s/lib/intel64/libmkl_sequential.a -Wl,--end-group -lm' % {'MKLROOT':os.environ['MKLROOT']}).split(' '))

if '--with-assembly' in sys.argv:
    sys.argv.remove('--with-assembly')
    extra_compile_args.extend(['--save-temps','-masm=intel','-fverbose-asm'])

if '--with-cython' in sys.argv:
    sys.argv.remove('--with-cython')
    use_cython = True
else:
    use_cython = False

#######################
#  extension modules  #
#######################

cython_pathspec = os.path.join('pyhsmm','**','*.pyx')

if use_cython:
    from Cython.Build import cythonize
    ext_modules = cythonize(cython_pathspec)
else:
    paths = [os.path.splitext(fp)[0] for fp in glob(cython_pathspec)]
    names = ['.'.join(os.path.split(p)) for p in paths]
    ext_modules = [
        Extension(name,
                  sources=[path + '.cpp'],
                  include_dirs=[os.path.join('pyhsmm','deps','Eigen3')],
                  extra_compile_args=['-O3','-std=c++11','-DNDEBUG','-w',
                                      '-DHMM_TEMPS_ON_HEAP'])
        for name, path in zip(names,paths)]

for e in ext_modules:
    e.extra_compile_args.extend(extra_compile_args)
    e.extra_link_args.extend(extra_link_args)

############
#  basics  #
############

setup(name='pyhsmm',
      version=PYHSMM_VERSION,
      description="Bayesian inference in HSMMs and HMMs",
      author='Matthew James Johnson',
      author_email='mattjj@csail.mit.edu',
      maintainer='Matthew James Johnson',
      maintainer_email='mattjj@csail.mit.edu',
      url="https://github.com/mattjj/pyhsmm",
      packages=['pyhsmm',
                'pyhsmm.basic',
                'pyhsmm.internals',
                'pyhsmm.plugins',
                'pyhsmm.util'],
      platforms='ALL',
      keywords=['bayesian', 'inference', 'mcmc', 'time-series',
                'monte-carlo'],
      install_requires=[
          "Cython >= 0.20.1",
          "numpy",
          "scipy",
          "matplotlib",
          "nose",
          "pybasicbayes",
      ],
      package_data={"pyhsmm": [os.path.join("examples", "*.txt")]},
      ext_modules=ext_modules,
      include_dirs=[np.get_include(),],
      classifiers=[
          'Intended Audience :: Science/Research',
          'Programming Language :: Python',
          'Programming Language :: C++',
      ])


from tkinter.ttk import Combobox

from .FuncLib import *

class PasteWin:
    def __init__(self, data: DataTree) -> None:
        self.dataset: DataTree = data

        # create widgets
        self.top_window: Toplevel = Toplevel(master=self.dataset.rootwin)
        self.vm_picker_label: Label = Label(master=self.top_window, text='Select a VM to paste into:')
        self.vm_picker: Combobox = Combobox(master=self.top_window, values=sorted(list(self.dataset.vmdict.keys())))
        self.pasted_text: Text = Text(master=self.top_window, width=80, height=24)
        self.paste_button: Button = Button(master=self.top_window, text='Paste to VM', command=lambda: self.click_handle())

        # place widgets
        self.vm_picker_label.grid(row=0, column=0)
        self.vm_picker.grid(row=1, column=0, padx=10)
        self.pasted_text.grid(row=2, column=0, padx=10, pady=10)
        self.paste_button.grid(row=3, column=0, columnspan=1, pady=10)

    def click_handle(self):
        print("button clicked")
        # get VM name
        vm_name: str = self.vm_picker.get()
        # lookup VM Object
        vm_object: vim.VirtualMachine = self.dataset.vmdict.get(vm_name)
        # get text from box
        raw_text: str= self.pasted_text.get(1.0, END)
        # translate text to usb_code
        trans_text: vim.UsbScanCodeSpec = str_to_usb(raw_text)
        # send codes to VM
        vm_object.PutUsbScanCodes(trans_text)
        return


"""
Hacer un programa que diga cuantos alumnos reprobaron y cuantos aprobaron

"""
alumno = 1
aprobados = 0
reprobados = 0

while alumno <= 5:
    calificacion = int(input("Ingrese la calificacion del alumno: "))
    alumno += 1
    if calificacion >=7:
        print(f"El alumno sacó {calificacion} está aprobado")
        aprobados +=1
    else:
        print(f"El alumno sacó {calificacion} está reprobado")
        reprobados += 1
print(f"La cantidad de aprobados es de {aprobados} y de reprobados es de {reprobados}")


from PyQt5.QtWidgets import QMessageBox

class BaseApplicationForm:
    @staticmethod
    def show_message(error, window_title):
        msg = QMessageBox()
        msg.setStyleSheet('background-color:#4caf50')
        msg.setIcon(QMessageBox.Critical)
        msg.setText(error)
        msg.setWindowTitle(window_title)
        msg.setStandardButtons(QMessageBox.Ok)
        msg.exec_()


'''
 @Author: Ritika Patidar
 @Date: 2021-02-23 21:44:10
 @Last Modified by: Ritika Patidar
 @Last Modified time: 2021-02-23 21:44:38
 @Title : check numbe into sequence
'''
def check(sequence,value):
    """
    Description:
        cheack value in sequence or not
    Parameter:
        sequence (list) : group of element
    Return:
        True (boolean): if value in sequence
        False (boolean): if value not in sequence
    """
    if(value in sequence):
        return True
    else:
        return False

test_case=[[3,[1, 5, 8, 3]],[-1,[1, 5, 8, 3]]]
for test in test_case:
    print("{0} -> {1} : {2}".format(test[0],test[1],check(test[1],test[0])))


#!/usr/bin/python3

with open('../input', 'r') as fp:
    data = [int(x) for x in fp.readlines()]
print(sum(data))     # part 1

twice = False
allFreq = set([0])
freq = 0
# part 2
while not twice:
    for i in data:
        freq += i
        if freq in allFreq:
            twice = True
            print(freq)
            break

        allFreq.add(freq)


class Solution:
    """
    @param senate: a string
    @return: return a string
    """
    def predictPartyVictory(self, senate):
        senators = list(senate)
        parties = {'R':'Radiant', 'D':'Dire'}

        i = 0
        while senators:
            if len(set(senators)) == 1:
                return parties[senators[0]]

            now = senators[i]

            if now == 'R':
                senators.remove('D')
            else:
                senators.remove('R')
            if i >= len(senators) - 1:
                i = 0
            else:
                i += 1


# Definition for a binary tree node.
class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

class Solution:
    def __init__(self):
            self.preIndex = 0

    def buildTree(self, preorder, inorder) -> TreeNode:

        if not inorder:
            return None

        node = TreeNode()
        node.val = preorder[self.preIndex]; self.preIndex+=1
        inIndex = inorder.index(node.val)
        node.left, node.right = self.buildTree(preorder, inorder[:inIndex]), self.buildTree(preorder, inorder[inIndex+1:])
        return node


#신수민
def solution(input):
    answer=0
    x=0
    y=0
    for i in input:
        if i=='w': y-=1
        elif i=='s':y+=1
        elif i=='a':x-=1
        elif i=='d':x+=1
        if x==y:answer+=1
    return answer

input="wsadsdwasd"
ret=solution(input)
print(ret)


import random

import numpy as np
import pytest
import tensorflow as tf

from tests.conftest import sample_state, sample_action

def test_deceleration(navigation, batch_size):
    center = navigation.deceleration["center"]
    decay = navigation.deceleration["decay"]

    state1 = sample_state(navigation)
    state2 = sample_state(navigation, batch_size)

    for state in [state1, state2]:
        batch_size = tf.shape(state)[0]

        lambda_ = navigation._deceleration(state)
        lambda_ = navigation._deceleration(state)

        assert lambda_.shape == [batch_size,]

        for i in range(batch_size):
            state_ = state[i]
            lambda_i = lambda_[i]

            expected = 1.0
            for c, d in zip(center, decay):
                delta = np.squeeze(state_ - c)
                distance = np.sqrt(np.sum(delta ** 2, axis=-1))
                expected *= 2 / (1.0 + np.exp(-d * distance)) - 1.0

            assert isinstance(lambda_i, tf.Tensor)
            assert lambda_i.shape == []
            assert lambda_i.dtype == tf.float32
            assert tf.reduce_all(tf.abs(lambda_i - expected) < 1e-4)

    fn = navigation._deceleration
    cfn1 = fn.get_concrete_function(state1)
    cfn2 = fn.get_concrete_function(state2)
    assert cfn1 == cfn2

def test_transition(navigation, batch_size):
    state1 = sample_state(navigation)
    action1 = sample_action(navigation)

    state2 = sample_state(navigation, batch_size)
    action2 = sample_action(navigation, batch_size)

    for state, action in [(state1, action1), (state2, action2)]:
        batch_size = tf.shape(state)[0]

        lambda_ = navigation._deceleration(state)

        next_state = navigation.transition(state, action)
        assert next_state.shape == state.shape

        for i in range(batch_size):
            assert tf.reduce_all(
                next_state[i] == state[i] + lambda_[i] * action[i])

    fn = navigation.transition

    cfn1 = fn.get_concrete_function(state1, action1)
    cfn2 = fn.get_concrete_function(state2, action2)

    cfn3 = fn.get_concrete_function(state1, action1)
    cfn4 = fn.get_concrete_function(state2, action2)

    assert cfn1 == cfn2 == cfn3 == cfn4

def test_linear_transition(navigation, batch_size):
    goal = navigation.goal
    decay = navigation.deceleration["decay"]
    center = navigation.deceleration["center"]

    state1 = sample_state(navigation)
    action1 = sample_action(navigation)
    state2 = sample_state(navigation, batch_size)
    action2 = sample_action(navigation, batch_size)

    for state, action in [(state1, action1), (state2, action2)]:
        batch_size = tf.shape(state)[0]

        lambda_ = navigation._deceleration(state)

        model = navigation.get_linear_transition(state, action)
        f = model.f
        f_x = model.f_x
        f_u = model.f_u

        assert f.shape == state.shape

        assert tf.reduce_all(f == navigation.transition(state, action))

        assert f_u.shape == [batch_size] + [navigation.action_size] * 2
        assert tf.reduce_all(f_u == tf.reshape(lambda_, [-1, 1, 1]) * tf.eye(2))
        assert f_x.shape == [batch_size] + [navigation.state_size] * 2

        for i in range(batch_size):
            state_ = state[i]
            action_ = action[i]

            lambda_x = tf.zeros_like(state_)
            for c, d in zip(center, decay):
                dist = tf.norm(state_ - c)
                l = 2 / (1.0 + tf.exp(-d * dist)) - 1.0

                h = 2 * d * tf.exp(-d * dist) / ((1 + tf.exp(-d * dist)) ** 2)
                lambda_x += h * (state_ - c) / dist * lambda_[i] / l

            expected_f_x = (tf.eye(navigation.state_size)
                            + tf.tensordot(tf.squeeze(action_),
                                           tf.squeeze(lambda_x), axes=0))

            assert tf.reduce_all(tf.abs(f_x[i] - expected_f_x) < 1e-4)

    fn = navigation.get_linear_transition
    cfn1 = fn.get_concrete_function(state1, action1)
    cfn2 = fn.get_concrete_function(state2, action2)
    assert cfn1 == cfn2

def test_cost(navigation, batch_size):
    goal = navigation.goal

    state1 = sample_state(navigation)
    action1 = sample_action(navigation)
    state2 = sample_state(navigation, batch_size=batch_size)
    action2 = sample_action(navigation, batch_size=batch_size)

    for state, action in [(state1, action1), (state2, action2)]:
        batch_size = tf.shape(state)[0]

        cost = navigation.cost(state, action)

        assert isinstance(cost, tf.Tensor)
        assert cost.shape == [batch_size,]
        assert tf.reduce_all(cost >= 0.0)

        cost2 = navigation.cost(tf.expand_dims(goal, axis=0), action)
        assert tf.reduce_all(cost2 == 0.0)

    fn = navigation.cost
    cfn1 = fn.get_concrete_function(state1, action1)
    cfn2 = fn.get_concrete_function(state2, action2)
    assert cfn1 == cfn2

def test_quadratic_cost(navigation, batch_size):
    goal = navigation.goal

    state1 = sample_state(navigation)
    action1 = sample_action(navigation)
    state2 = sample_state(navigation, batch_size=batch_size)
    action2 = sample_action(navigation, batch_size=batch_size)

    for state, action in [(state1, action1), (state2, action2)]:
        batch_size = tf.shape(state)[0]
        state_size = navigation.state_size
        action_size = navigation.action_size

        model = navigation.get_quadratic_cost(state, action)
        l = model.l
        l_x = model.l_x
        l_u = model.l_u
        l_xx = model.l_xx
        l_uu = model.l_uu
        l_xu = model.l_xu
        l_ux = model.l_ux

        assert l.shape == [batch_size,]
        assert tf.reduce_all(l == navigation.cost(state, action))

        assert tf.reduce_all(l_x == 2 * (state - goal))

        assert tf.reduce_all(l_u == tf.zeros_like(action))

        assert tf.reduce_all(l_xx == tf.reshape(
            tf.tile(2 * tf.eye(state_size), [batch_size, 1]),
            [batch_size, state_size, state_size]))

        assert tf.reduce_all(l_uu == tf.zeros([batch_size, action_size, action_size]))

        assert tf.reduce_all(l_ux == tf.zeros([batch_size, state_size, action_size]))

        assert tf.reduce_all(l_xu == tf.zeros([batch_size, action_size, state_size]))

    fn = navigation.get_quadratic_cost
    cfn1 = fn.get_concrete_function(state1, action1)
    cfn2 = fn.get_concrete_function(state2, action2)
    assert cfn1 == cfn2

def test_quadratic_final_cost(navigation):
    goal = navigation.goal

    state = navigation.obs_space.sample()

    model = navigation.get_quadratic_final_cost(state)
    l = model.l
    l_x = model.l_x
    l_xx = model.l_xx

    assert l.shape == []
    assert tf.reduce_all(l == navigation.final_cost(state))

    assert tf.reduce_all(l_x == 2 * (state - goal))
    assert tf.reduce_all(l_xx == 2 * tf.eye(navigation.state_size))


from typing import Any
from unittest.mock import PropertyMock, create_autospec, patch

from api.admin.controller.collection_settings import CollectionSettingsController
from api.controller import CirculationManager
from core.integration.goals import Goals
from core.integration.registry import IntegrationRegistry
from tests.fixtures.database import DatabaseTransactionFixture

class TestCollectionSettingsController:
    def test_duplicate_protocol_settings(self, db: DatabaseTransactionFixture):
        """Dedupe protocol settings using the last settings of the same value"""
        manager = create_autospec(spec=CirculationManager)
        manager._db = PropertyMock(return_value=db.session)

        class MockProviderAPI:
            NAME = "NAME"
            SETTINGS = [
                dict(key="k1", value="v1"),
                dict(key="k2", value="v2"),  # This should get overwritten
                dict(key="k2", value="v3"),  # Only this should remain
            ]

        controller = CollectionSettingsController(manager)
        with patch.object(
            controller, "registry", IntegrationRegistry[Any](Goals.LICENSE_GOAL)
        ) as registry:
            registry.register(MockProviderAPI, canonical=MockProviderAPI.NAME)
            protocols = controller._get_collection_protocols()

        k2_list = list(filter(lambda x: x["key"] == "k2", protocols[0]["settings"]))
        assert len(k2_list) == 1
        assert k2_list[0]["value"] == "v3"

        class MockProviderAPIMulti:
            NAME = "NAME"
            SETTINGS = [
                dict(key="k1", value="v0"),  # This should get overwritten
                dict(key="k1", value="v1"),  # Only this should remain
                dict(key="k2", value="v1"),  # This should get overwritten
                dict(key="k2", value="v2"),  # This should get overwritten
                dict(key="k2", value="v4"),  # Only this should remain
            ]

        with patch.object(
            controller, "registry", IntegrationRegistry[Any](Goals.LICENSE_GOAL)
        ) as registry:
            registry.register(MockProviderAPIMulti, canonical=MockProviderAPIMulti.NAME)
            protocols = controller._get_collection_protocols()

        k2_list = list(filter(lambda x: x["key"] == "k2", protocols[0]["settings"]))
        assert len(k2_list) == 1
        assert k2_list[0]["value"] == "v4"

        k1_list = list(filter(lambda x: x["key"] == "k1", protocols[0]["settings"]))
        assert len(k1_list) == 1
        assert k1_list[0]["value"] == "v1"


# 引入selenium UI自动化测试模块 pip3 install selenium
# 下载对应浏览器驱动：此处使用chromedriver  地址：http://npm.taobao.org/mirrors/chromedriver/
# 需求：爬取 百度首页热搜信息

from selenium import webdriver
from lxml import etree

# 实例化之前可以先做一些配置
options = webdriver.ChromeOptions()
options.add_argument('--headless')  # 后台运行模式

# 实例化驱动对象
driver = webdriver.Chrome(executable_path='./chromedriver', options=options)

# 打开网页
driver.get('https://www.baidu.com/')

# 获取网页页面源代码
page_text = driver.page_source

# 关闭当前窗口
driver.close()

# 关闭整个浏览器
driver.quit()

# 解析文本
tree = etree.HTML(page_text)
li_list = tree.xpath('//div[@id="s-hotsearch-wrapper"]/ul/li')
hot_text = list()
for li in li_list:
    text = li.xpath('./a/span[2]/text()')[0]
    hot_text.append(text)

print(hot_text)


import pandas as pd
import requests
from utils import convert_attribute_column, handle_hyper_file

class MercadoLibreData:
    def __init__(self, product_name, num_items):
        self.product_name = product_name
        self.num_items = num_items
        self.df = pd.DataFrame()

    def retrieve_data(self):
        # Set the desired number of items to retrieve.
        num_items = self.num_items

        # Variables to function and data
        api_url = ''
        offset = 0
        limit = 50

        # Calculate the number of requests needed to retrieve the desired number of items.
        num_requests = num_items // limit

        # Make the requests to the API.
        for i in range(num_requests):
            # Calculate the current offset.
            offset = i * limit

            # Make a request to the API with the current offset and limit.
            editable_product_name = self.product_name.upper().replace(' ', '%20')
            api_url = f'https://api.mercadolibre.com/sites/MLA/search?q={editable_product_name}&limit={limit}&offset={offset}#json'
            data = requests.get(api_url)
            transformedData = data.json()

            # Convert the results to a DataFrame and append it to the existing DataFrame.
            new_df = pd.DataFrame(pd.json_normalize(transformedData['results']))
            # Find all columns that have the desired prefix.
            cols_to_drop = [col for col in new_df.columns if col.startswith('variations_data.')]

            # Drop the columns from the DataFrame.
            new_df = new_df.drop(cols_to_drop, axis=1)
            self.df = pd.concat([self.df, new_df], ignore_index=True)

            # Convert the attribute column.
            self.df = convert_attribute_column(self.df)

    def generate_hyper_file(self):
        # Generate the Hyper file.
        hyper_file = handle_hyper_file(self.df)

        # Read the contents of the Hyper file.
        with open(hyper_file, 'rb') as f:
            data = f.read()

        return data


"""
Contains throttling middleware implementation.

.. class:: ThrottlingMiddleware(BaseMiddleware)

.. decorator:: rate_limit
    Set particular rate limit for handler (anti-flood)

.. const:: THROTTLING_RATE_LIMIT_KEY
.. const:: THROTTLING_KEY
"""

import asyncio
from typing import (
    Callable,
    Optional
)

from aiogram import (
    Dispatcher,
    types
)
from aiogram.dispatcher import DEFAULT_RATE_LIMIT
from aiogram.dispatcher.handler import (
    CancelHandler,
    current_handler
)
from aiogram.dispatcher.middlewares import BaseMiddleware
from aiogram.utils.exceptions import Throttled

THROTTLING_RATE_LIMIT_KEY = 'throttling_rate_limit'
THROTTLING_KEY = 'throttling_key'

def rate_limit(limit: float, key: Optional[str] = None) -> Callable:
    """
    Decorator for configuring rate limit and key in different functions.

    :param limit: time limit in seconds
    :type limit: float
    :param key: throttling key
    :type key: Optional[str]

    :return: decorator
    :rtype: Callable
    """

    def decorator(func: Callable) -> Callable:
        """
        Set throttle attrs in handler.

        :param func: handler
        :type func: Callable
        :return: handler with set attrs
        :rtype: Callable
        """

        setattr(func, THROTTLING_RATE_LIMIT_KEY, limit)
        if key:
            setattr(func, THROTTLING_KEY, key)

        return func

    return decorator

class ThrottlingMiddleware(BaseMiddleware):
    """
    Implements anti-flood middleware
    """

    def __init__(self, limit: float = DEFAULT_RATE_LIMIT, key_prefix: str = 'antiflood_'):
        self.rate_limit = limit
        self.prefix = key_prefix

        super(ThrottlingMiddleware, self).__init__()

    async def on_process_message(self, message: types.Message, data: dict):
        """
        This handler is called when dispatcher receives a message.
        Manage messages to avoid flood.
        """

        handler = current_handler.get()
        dispatcher = Dispatcher.get_current()

        if handler:
            limit = getattr(handler, THROTTLING_RATE_LIMIT_KEY, self.rate_limit)
            key = getattr(handler, THROTTLING_KEY, f"{self.prefix}_{handler.__name__}")
        else:
            limit = self.rate_limit
            key = f"{self.prefix}_message"

        try:
            await dispatcher.throttle(key, rate=limit)
        except Throttled as t:
            await self.message_throttled(message, t)

            raise CancelHandler()

    async def message_throttled(self, message: types.Message, throttled: Throttled):
        """
        Notify user only on first exceed and notify about unlocking only on last exceed
        """

        handler = current_handler.get()
        dispatcher = Dispatcher.get_current()

        if handler:
            key = getattr(handler, THROTTLING_KEY, f"{self.prefix}_{handler.__name__}")
        else:
            key = f"{self.prefix}_message"

        delta = throttled.rate - throttled.delta

        if throttled.exceeded_count <= 2:
            await message.reply('Too many requests! Don`t flood, please!')

        await asyncio.sleep(delta)

        thr = await dispatcher.check_key(key)

        if thr.exceeded_count == throttled.exceeded_count:
            await message.reply('Unlocked. You can continue!')


#!/bin/usr python3

import util
import numpy as np

# CNN(Convolutional Neural Network)の部品となる畳み込み層とプーリング層の実装
class Convolution:
    # 重み、バイアス、ストライド、パディングを引数にもつ
    def __init__(self, W, b, stride=1, pad=0):
        self.W = W
        self.b = b
        self.stride = stride
        self.pad = pad
    def forward(self, x):
        # 重み（ここではフィルター）の数、チャンネル、高さ、幅を設定する
        FN, C, FH, FW = self.W.shape
        # 入力データの値も設定する
        N, C, H, W = x.shape
        # 出力するときの高さと幅を設定する
        out_h = int(1 + (H + 2 * self.pad - FH) / self.stride)
        out_w = int(1 + (W + 2 * self.pad -FW) / self.stride)

        # im2colメソッドを使用して行列に展開する
        col = im2col(x, FH, FW, self.stride, self.pad)
        # -1を指定することで環境に合うように自動で値を設定してくれる。Tは行と列を入れ替える
        # 処理内容は計算しやすいように上記の処理で行列に変換しているため、重みも整形している
        col_W = self.W.reshape(FN, -1).T
        out = np.dot(col, col_W) + self.b
        # 出力データも整形して出力する(transposeは設定したインデックスの順に整形する)
        out = out.rehape(N, out_h, out_w, -1).transpose(0, 3, 1, 2)
        return out

class Pooling:
    def __init__(self, pool_h, pool_w, stride=1, pad=0):
        self.pool_h = pool_h
        self.pool_w = pool_w
        self.stride = stride
        self.pad = pad
    def forward(self, x):
        N, C, H, H = x.shape
        out_h = int(1 + (H - self.pool_h) / self.stride)
        out_w = int(1 + (W - self.pool_w) / self.stride)
        # 最大値を取得しやすいように展開する
        col = im2col(x, self.pool_h, self.pool_w, self.stride, self.pad)
        col = col.reshape(-1, self.pool_h * self.pool_w)
        # 最大値を取得する(1次元目から取得する)
        # 行列の中からの最大値を取得するため多少の変化にはロバストである（頑健性）
        out = np.max(col, axis=1)
        # 出力データに合わせて整形する
        out = out.reshape(N, out_h, out_w, C).transpose(0, 3, 1, 2)
        return out


from models import *
from flask_script import Manager
from flask_migrate import Migrate, MigrateCommand

def run_migration_manager():
    migrate = Migrate(app, db)
    manager = Manager(app)

    manager.add_command('db', MigrateCommand)
    manager.run()

if __name__ == '__main__':
    run_migration_manager()


"""
This module is the core tracking system
"""
import json
import re
from os import listdir
from os.path import isfile, join

import imageio
import numpy as np
from PIL import Image
from pygifsicle import optimize

from tracker.objects import DetectedObject
from tracker.objects import Vehicle

def numerical_sort(value):
    """
    :param value:
    :return:
    """
    numbers = re.compile(r'(\d+)')
    parts = numbers.split(value)
    parts[1::2] = map(int, parts[1::2])
    return parts

def track(images_folder, bb_folder, detection_threshold, memory_frames_number=10):
    """
    :param images_folder: str, path to the images folder
    :param bb_folder: str, path to bounding_boxes folder
    :param detection_threshold:
    :param memory_frames_number: int,
    :return:
    """
    images = sorted([f for f in listdir(images_folder) if isfile(join(images_folder, f))],
                    key=numerical_sort)

    img_array = []
    img_pil = []
    bb_array = []
    vehicle_count = 0
    output_data = {}

    for image in images:
        img = Image.open(images_folder + image)
        img_pil.append(img)
        img_array.append(np.asarray(img))

    # Instantiate bounding_boxes
    if str(bb_folder)[-5:] == ".json":
        # Open as it is a json
        try:
            data_file = json.load(open(bb_folder, 'r'))
            for content in data_file.values():
                bb_array.append(content)
        except IsADirectoryError:
            print("Provide a json file or similar")
            return "Provide a json file or similar"
    else:
        # Open as it is a dictionnary
        try:
            data_file = json.loads(bb_folder)
        except TypeError:
            data_file = bb_folder
        for content in data_file.values():
            bb_array.append(content)

    detected_vehicles = []
    for i, (img, pil, bbs) in enumerate(zip(img_array, img_pil, bb_array)):
        detected_objects = []

        # Reset visibility
        for d_v in detected_vehicles:
            d_v.update_counter(False)
            if d_v.get_counter() > memory_frames_number:
                detected_vehicles.remove(d_v)

        # Retrieve the different objects
        for obj in bbs:
            detected_objects.append(DetectedObject(obj, img))

        potential_vehicles_indexes = [i for i in range(len(detected_vehicles))]

        for d_o in detected_objects:
            found = False
            distances = []

            # Distances calculation
            for j in potential_vehicles_indexes:
                distances.append(d_o.get_distance_from(detected_vehicles[j]))

            if distances:
                shortest_distance = min(distances)
                shortest_distance_index = distances.index(shortest_distance)

                if shortest_distance < detection_threshold:
                    found = True
                    vehicle_index = potential_vehicles_indexes[shortest_distance_index]
                    detected_vehicles[vehicle_index].update_vehicle(d_o)
                    potential_vehicles_indexes.remove(
                        potential_vehicles_indexes[shortest_distance_index])

            if not found:
                detected_vehicles.append(Vehicle(d_o, img, vehicle_count))
                vehicle_count += 1

        for d_v in detected_vehicles:
            d_v.update_visibility()
        for d_v in detected_vehicles:
            if d_v.get_visible():
                d_v.draw(pil)
            else:
                d_v.draw_prob(pil)

        output_data["frame " + str(i)] = [d_v.get_id()
                                          for d_v in detected_vehicles if d_v.get_visible()]

    return output_data

    imageio.mimsave('output.gif', img_pil)
    optimize("output.gif")


import rospy
from sensor_msgs.msg import Image

class image_time_sync:

    def __init__(self, topic1, topic2):

        rospy.init_node('img_time_sync_node')

        self.image1 = None
        self.image2 = None
        self.pending1 = False
        self.pending2 = False

        self.finished = False
        self.img_sub1 = rospy.Subscriber(topic1, Image, callback=self.image1_cb)
        self.img_sub2 = rospy.Subscriber(topic2, Image, callback=self.image2_cb)
        self.img_pub1 = rospy.Publisher('/camera1_image', Image)
        self.img_pub2 = rospy.Publisher('/camera2_image', Image)

        self.run()

    def image1_cb (self, img):
        self.image1 = img
        self.pending1 = True

    def image2_cb (self, img):
        self.image2 = img
        self.pending2 = True

    def run (self):
        sleeper = rospy.Rate(30)
        while not self.finished:
            if self.pending1 and self.pending2:
                ts = rospy.Time.now()
                self.image1.header.stamp = ts
                self.image2.header.stamp = ts
                self.img_pub1.publish(self.image1)
                self.img_pub2.publish(self.image2)
                self.pending1 = False
                self.pending2 = False
            sleeper.sleep()

    def set_finished (self, finished=True):
        self.finished = finished

if __name__=='__main__':
    image_time_sync('/camera1/rgb/image_raw', '/camera2/rgb/image_raw')


import tarski.model
from tarski import fstrips as fs
from tarski.syntax import land

def create_sample_language():
    # The standard, untyped version of gripper
    lang = fs.language("gripper")
    object_t = lang.get_sort('object')

    [lang.predicate(p, object_t) for p in ["room", "ball", "gripper", "at-robby", "free"]]
    [lang.predicate(p, object_t, object_t) for p in ["at", "carry"]]

    [lang.constant(name, object_t) for name in ['rooma', 'roomb', 'ball4', 'ball3', 'ball2', 'ball1', 'left', 'right']]
    return lang

def create_sample_problem():
    lang = create_sample_language()
    init = tarski.model.create(lang)

    room, ball, at_robby, free, at, gripper, carry = lang.get(
        "room", "ball", "at-robby", "free", "at", "gripper", "carry")
    rooma, roomb, ball4, ball3, ball2, ball1, left, right = lang.get(
        'rooma', 'roomb', 'ball4', 'ball3', 'ball2', 'ball1', 'left', 'right')

    init.add(room, rooma)
    init.add(room, roomb)
    init.add(ball, ball1)
    init.add(ball, ball2)
    init.add(ball, ball3)
    init.add(ball, ball4)
    init.add(gripper, left)
    init.add(gripper, right)

    init.add(at_robby, rooma)
    init.add(free, left)
    init.add(free, right)
    init.add(at, ball1, rooma)
    init.add(at, ball2, rooma)
    init.add(at, ball3, rooma)
    init.add(at, ball4, rooma)

    problem = fs.create_fstrips_problem(lang, problem_name="sample", domain_name="gripper-strips")
    problem.init = init
    problem.goal = land(at(ball1, roomb), at(ball2, roomb), at(ball3, roomb), at(ball4, roomb))

    from_, to, o, r, g = [lang.variable(x, lang.Object) for x in ["from", "to", "o", "r", "g"]]

    problem.action("move", [from_, to],
                   precondition=land(from_ != to, room(from_), room(to), at_robby(from_), flat=True),
                   effects=[fs.AddEffect(at_robby(to)), fs.DelEffect(at_robby(from_))])

    problem.action("pick", [o, r, g],
                   precondition=land(ball(o), room(r), gripper(g), at(o, r), at_robby(r), free(g), flat=True),
                   effects=[fs.AddEffect(carry(o, g)), fs.DelEffect(at(o, r)), fs.DelEffect(free(g))])

    problem.action("drop", [o, r, g],
                   precondition=land(ball(o), room(r), gripper(g), carry(o, g), at_robby(r), flat=True),
                   effects=[fs.DelEffect(carry(o, g)), fs.AddEffect(at(o, r)), fs.AddEffect(free(g))])

    return problem


import csv, json

with open("students.json") as f:
    data = json.load(f)
ids = [(record["short_name"], record["id"])
       for record in data if record["short_name"] != "Test Student"]
with open("students.csv", "w") as f:
    writer = csv.writer(f)
    for record in sorted(ids):
        writer.writerow(record)


import requests

class currency_convert:
    rates = {}
    def __init__(self,url):
        data = requests.get(url).json()
        self.rates = data['rates']

    def convert(self,from_currency,to_currency,amount):
        initial_amount = amount
        if from_currency != 'EUR':
            amount = amount / self.rates[from_currency]
            amount= round(amount*self.rates[to_currency],2)
            print(initial_amount,from_currency,amount,to_currency)

if __name__=='__main__':
    YOUR_ACCESS_KEY = fixer.io
    url = str.__add__('http://data.fixer.io/api/latest?access_key=')


import os
import logging
import timeit
import pickle
from multiprocessing import Process, Queue
import queue  # For the exception queue.Empty which is not in the multiprocessing package
from scipy import sparse

import utility
import features_space

features2int_dict = None

class Analysis:
    """ To store the analysis results. """

    def __init__(self, pdg_path, label=None):
        self.pdg_path = pdg_path
        self.file_path = None
        self.features = None
        self.label = label
        self.prediction = None

    def set_file_path(self, file_path):
        self.file_path = file_path

    def set_features(self, features):
        self.features = features

    def set_prediction(self, prediction):
        self.prediction = prediction

def main_analysis(js_dirs, js_files, labels_files, labels_dirs, n,
                  features2int_dict_path):
    """
        Main function, performs a static analysis (syntactic using the AST)

    """

    start = timeit.default_timer()

    global features2int_dict
    features2int_dict = pickle.load(open(features2int_dict_path, 'rb'))

    if js_dirs is None and js_files is None:
        logging.error('Please, indicate a directory or a JS file to be analyzed')
        return None

    if js_files is not None:
        files2do = js_files
        if labels_files is None:
            labels_files = ['?' for _, _ in enumerate(js_files)]
        labels = labels_files
    else:
        files2do, labels = [], []
    if js_dirs is not None:
        i = 0
        if labels_dirs is None:
            labels_dirs = ['?' for _, _ in enumerate(js_dirs)]
        for cdir in js_dirs:
            for cfile in os.listdir(cdir):
                files2do.append(os.path.join(cdir, cfile))
                if labels_dirs is not None:
                    labels.append(labels_dirs[i])
            i += 1

    analyses = get_features(files2do, labels, n)
    logging.debug('Got all features')

    features_repr = get_features_representation(analyses)

    return features_repr

def worker_get_features_vector(my_queue, out_queue, except_queue):
    """ Worker to get the features."""

    while True:
        try:
            [analysis, n] = my_queue.get(timeout=2)
            try:
                features = features_space.features_vector(analysis.pdg_path,
                                                          n, features2int_dict)
                analysis.set_features(features)
                out_queue.put(analysis)  # To share modified analysis object between processes
            except Exception as e:  # Handle exception occurring in the processes spawned
                logging.error('Something went wrong with %s', analysis.pdg_path)
                print(e)
                except_queue.put([analysis.pdg_path, e])
        except queue.Empty:  # Empty queue exception
            break

def get_features(files2do, labels, n):
    """
        Returns an analysis object with its features attribute filled
    """

    my_queue = Queue()
    out_queue = Queue()
    except_queue = Queue()
    workers = list()

    logging.debug('Preparing processes to get all features')

    for i, _ in enumerate(files2do):
        analysis = Analysis(pdg_path=files2do[i], label=labels[i])
        my_queue.put([analysis, n])

    for i in range(utility.NUM_WORKERS):
        p = Process(target=worker_get_features_vector, args=(my_queue, out_queue, except_queue))
        p.start()
        workers.append(p)

    analyses = list()

    while True:
        try:
            analysis = out_queue.get(timeout=0.01)
            analyses.append(analysis)
        except queue.Empty:
            pass
        all_exited = True
        for w in workers:  # Instead of join, as the worker cannot be joined when elements
            if w.exitcode is None:
                all_exited = False
                break
        if all_exited & out_queue.empty():
            break

    return analyses

def worker_features_representation(my_queue, out_queue):
    """ Worker to represent the features in the corresponding form (list or CSR). """

    analyses = list()
    tab_res0 = list()
    tab_res2 = list()
    concat_features = None

    while True:
        try:
            analysis = my_queue.get(timeout=2)
            analyses.append(analysis)
        except queue.Empty:  # Empty queue exception
            break

    for analysis in analyses:
        features = analysis.features
        if features is not None:
            tab_res0.append(analysis.pdg_path)
            concat_features = sparse.vstack((concat_features, features), format='csr')
            if concat_features is None or concat_features.nnz == 0:
                logging.error('Something strange occurred for %s with the features ',
                              analysis.pdg_path)
                logging.error(concat_features)
            tab_res2.append(analysis.label)

    logging.debug('Merged features in subprocess')

    out_queue.put([tab_res0, tab_res2, concat_features])

def get_features_representation(analyses):
    """
        Returns the features representation used in the ML modules.
    """

    my_queue = Queue()
    out_queue = Queue()
    workers = list()

    tab_res = [[], [], []]
    concat_features = None

    logging.debug('Preparing processes to merge all features efficiently')

    for i, _ in enumerate(analyses):
        analysis = analyses[i]
        my_queue.put(analysis)

    for i in range(utility.NUM_WORKERS):
        p = Process(target=worker_features_representation, args=(my_queue, out_queue))
        p.start()
        workers.append(p)

    while True:
        try:
            # Get modified analysis objects
            [tab_res0, tab_res2, features] = out_queue.get(timeout=0.01)
            if features is not None and features.nnz > 0:
                tab_res[0].extend(tab_res0)
                tab_res[2].extend(tab_res2)
                try:
                    concat_features = sparse.vstack((concat_features, features), format='csr')
                except ValueError:
                    logging.error('Problem to merge %s with %s', concat_features, features)
            logging.debug('Merged features in main process')
        except queue.Empty:
            pass
        all_exited = True
        for w in workers:  # Instead of join, as the worker cannot be joined when elements
            if w.exitcode is None:
                all_exited = False
                break
        if all_exited & out_queue.empty():
            break

    tab_res[1].append(concat_features)
    tab_res[1] = tab_res[1][0]

    if len(tab_res[0]) != tab_res[1].shape[0] or len(tab_res[0]) != len(tab_res[2])\
            or tab_res[1].shape[0] != len(tab_res[2]):
        logging.error('Got %s files to analyze, %s features and %s labels; do not match',
                      str(len(tab_res[0])), str(tab_res[1].shape[0]), str(len(tab_res[2])))
    logging.debug('Finished to merge features, will move to ML stuff :)')

    return tab_res


# -*- coding: utf-8 -*-
import scrapy
from jobparser.items import superjobItem
import re

class SjruSpider(scrapy.Spider):
    name = 'sjru'
    allowed_domains = ['superjob.ru']
    search_str = 'Python'
    start_urls = ['https://www.superjob.ru/vacancy/search/?keywords=' + search_str +
                  '&geo%5Bc%5D%5B0%5D=1']

    def parse(self, response):
        next_page = response.css('a.f-test-link-dalshe::attr(href)').extract_first()
        if not next_page:
            pass
        else:
            yield response.follow(next_page, callback=self.parse)

        vacansy = response.css(
            'div.f-test-vacancy-item a[target="_blank"]::attr(href)').extract()

        for link in vacansy:
            yield response.follow(link, self.vacansy_parse)  # Переходим на страницы вакансий

    def vacansy_parse(self, response):
        job_name = response.css('div._3MVeX h1::text').extract_first().strip()
        salary = response.css('span[class="_3mfro _2Wp8I ZON4b PlM3e _2JVkc"] *::text').extract()
        salary = str(''.join(salary))
        salary = salary.replace(u'\xa0', u'')
        if '—' in salary:
            salary_min = salary.split('—')[0]
            salary_min = re.sub(r'[^0-9]', '', salary_min)
            salary_max = salary.split('—')[1]
            salary_max = re.sub(r'[^0-9]', '', salary_max)
            salary_min = int(salary_min)
            salary_max = int(salary_max)
        elif 'от' in salary:
            salary_min = salary[2:]
            salary_min = re.sub(r'[^0-9]', '', salary_min)
            salary_min = int(salary_min)
            salary_max = None
        elif 'договорённости' in salary:
            salary_min = None
            salary_max = None
        elif 'до' in salary:
            salary_min = None
            salary_max = salary[2:]
            salary_max = re.sub(r'[^0-9]', '', salary_max)
            salary_max = int(salary_max)
        else:
            salary_min = int(re.sub(r'[^0-9]', '', salary))
            salary_max = int(re.sub(r'[^0-9]', '', salary))
        site = 'superjob.ru'
        job_link = response.css('link[rel="canonical"]::attr(href)').extract_first()
        yield superjobItem(name=job_name, min_salary=salary_min, max_salary=salary_max, link=job_link, site=site)

Методы сбора и обработки данных из сети Интернет/5/scrapy_jobs/jobparser/spiders/sjru.py


import os
import re
import json
import pandas as pd

import tensorflow as tf
from tensor2tensor.data_generators import problem
from tensor2tensor.data_generators import text_problems, text_encoder
from tensor2tensor.layers import modalities
from tensor2tensor.utils import registry
from . import config
from .utils import extract_flatworld_tags, get_line_context, normalize_text, split_lines

@registry.register_problem
class HansardLineType4(text_problems.Text2ClassProblem):
    f"""Predicts the type of each line in a Hansard transcript.

    Line types: {config.LINE_LABEL_CODES.keys()}.
    """

    CONTEXT_SEPARATOR = "<EOC>"
    CONTEXT_SEPARATOR_ID = 2

    @property
    def additional_reserved_tokens(self):
        return [self.CONTEXT_SEPARATOR]

    @property
    def vocab_type(self):
        return text_problems.VocabType.SUBWORD

    @property
    def approx_vocab_size(self):
        """Approximate vocab size to generate. Only for VocabType.SUBWORD."""
        return 2**16  # ~64k

    @property
    def is_generate_per_split(self):
        return True

    @property
    def num_classes(self):
        """The number of classes."""
        return len(config.LINE_LABEL_CODES)

    def class_labels(self, data_dir):
        """String representation of the classes."""
        del data_dir
        return sorted(config.LINE_LABEL_CODES.keys(), key=lambda x: config.LINE_LABEL_CODES[x])

    @property
    def dataset_splits(self):
        """Splits of data to produce and number of output shards for each."""
        # 10% evaluation data
        return [{
            "split": problem.DatasetSplit.TRAIN,
            "shards": 9,
        }, {
            "split": problem.DatasetSplit.EVAL,
            "shards": 1,
        }]

    def feature_encoders(self, data_dir):
        encoder = self.get_or_create_vocab(data_dir, None, force_get=True)
        encoders = {'inputs': encoder, 'context': encoder}
        return encoders

    def generate_text_for_vocab(self, data_dir, tmp_dir):
        for i, sample in enumerate(self.generate_samples(data_dir, tmp_dir, problem.DatasetSplit.TRAIN)):
            yield sample["inputs"]
            yield sample["context"]
            # yield sample["targets"]
            if self.max_samples_for_vocab and (i + 1) >= self.max_samples_for_vocab:
                break

    def generate_encoded_samples(self, data_dir, tmp_dir, dataset_split):
        generator = self.generate_samples(data_dir, tmp_dir, dataset_split)
        encoder = self.get_or_create_vocab(data_dir, tmp_dir)
        encoders = {'inputs': encoder, 'context': encoder}
        # vocab = self.feature_encoders(data_dir)["context"]
        for sample in generator:
            encoded_sample = self.encode_example(sample, encoders)
            yield encoded_sample

    def encode_example(self, example, encoders):
        """encodes an example.

        example is a dict containing the inputs (str), context (str), and label (int).
        """
        inputs = encoders['inputs'].encode(example["inputs"])
        inputs.append(text_encoder.EOS_ID)
        context = encoders['context'].encode(example["context"])
        context.append(text_encoder.EOS_ID)
        label = example["label"]
        return {"inputs": inputs, "context": context, "targets": [label]}

    def hparams(self, defaults, unused_model_hparams):
        (super(HansardLineType4, self)
         .hparams(defaults, unused_model_hparams))
        p = defaults
        p.modality["context"] = modalities.ModalityType.SYMBOL
        p.vocab_size["context"] = self._encoders["context"].vocab_size
        if self.packed_length:
            raise NotImplementedError("HansardLineType4 does not "
                                      "support packed_length")

    def example_reading_spec(self):
        data_fields, data_items_to_decoders = (super().example_reading_spec())
        data_fields["context"] = tf.VarLenFeature(tf.int64)
        return (data_fields, data_items_to_decoders)

    def generate_samples(self, data_dir, tmp_dir, dataset_split):
        is_train = dataset_split == problem.DatasetSplit.TRAIN
        assert 'train.csv' in os.listdir(tmp_dir)
        if is_train:
            dataset_path = os.path.join(tmp_dir, 'train.csv')
        else:
            dataset_path = os.path.join(tmp_dir, 'dev.csv')
        samples = self._generate_csv_samples(dataset_path, is_train)
        for sample in samples:
            yield sample

    def preprocess_example(self, example, mode, hparams):
        sep = tf.convert_to_tensor([self.CONTEXT_SEPARATOR_ID],
                                    dtype=example["inputs"].dtype)
        example["inputs"] = tf.concat([example["inputs"][:-1], sep, example["context"]], 0)
        example = super().preprocess_example(example, mode, hparams)
        return example

    def _generate_csv_samples(self, dataset_path, is_train):
        lines = pd.read_csv(dataset_path)
        # keeps only lines where label exists and is not equal to 'blank'.
        lines = lines[(lines.label.notnull()) & (~lines.label.isin(['blank']))]
        # keeps lines without a note.
        lines = lines[lines.notes.isnull()]
        lines['text'] = lines.text.apply(normalize_text)
        assert lines.text.isnull().sum() == 0, "Every line must have a non-null text value."
        assert (lines.text.str.len() > 0).all(), "Every line must have a text value with len > 0."
        assert lines.label.isnull().sum() == 0, "Every line must have a non-null label."
        allowed_labels = set(config.LINE_LABEL_CODES.keys())
        assert len(set(lines.label.unique()).difference(allowed_labels)) == 0, \
            (f"Training data contains a non-allowable label. Labels in training "
             f"data: {lines.label.unique().tolist()}")
        assert len(allowed_labels.difference(set(lines.label.unique()))) == 0, \
            ("One or more labels does not appear in training data.")
        if is_train:
            # duplicates data, without `garbage` lines.
            lines_wo_garbage = lines[~lines.label.isin(['garbage'])]
            lines_wo_garbage['file'] += f'_wo_garbage'
            lines = pd.concat([lines, lines_wo_garbage], axis=0, ignore_index=True, sort=True)
            if config.N_SPLIT_LINES_PASSES > 0:
                # randomly splits lines to create more training data.
                lines_split = []
                for i in range(config.N_SPLIT_LINES_PASSES):
                    lines_split_temp = split_lines(lines)
                    lines_split_temp['file'] += f'_{i}'
                    lines_split.append(lines_split_temp)
                lines_split = pd.concat(lines_split, axis=0, ignore_index=True, sort=True)
                # idx = np.random.randint(low=0, high=lines_split.shape[0])
                # lines_split.iloc[idx:idx+10,:]
                lines = pd.concat([lines, lines_split], axis=0, ignore_index=True, sort=True)
            lines.sort_values(by=['year', 'file', 'line'], inplace=True)
        # retrieves context surrounding each line.
        contexts = get_line_context(lines, n=config.CONTEXT_N_LINES)
        assert contexts.prev_context.isnull().sum() == 0
        assert contexts.next_context.isnull().sum() == 0
        lines = pd.merge(lines, contexts, on=['year', 'file', 'line'], how='left', validate='1:1')
        # drops lines that have a null prev_context and are not the first N lines.
        lines = lines[~((lines.prev_context.str.len() == 0) & (lines.line > config.CONTEXT_N_LINES))]
        assert (lines[lines.line == 1].prev_context.str.len() == 0).all()
        # drops lines that have a null next_context and are not an end-of-document line.
        lines = lines[~((lines.next_context.str.len() == 0) & ~lines.text.str.contains(r'<EOD>$', flags=re.IGNORECASE))]
        lines[lines.text.str.contains(r'<EOD>$', flags=re.IGNORECASE)]['next_context'] = ''
        # lines['line_length'] = lines.text.str.len()
        # lines['line_context_length'] = lines.text.str.len() + lines.prev_context.str.len() + lines.next_context.str.len()
        # lines.line_length.quantile(q=pd.np.arange(0, 1.1, 0.1))
        # lines.line_context_length.quantile(q=pd.np.arange(0, 1.1, 0.1))
        if is_train and config.UPSAMPLE:
            lines['label_prop'] = lines.groupby('label').label.transform(lambda x: x.shape[0] / float(lines.shape[0]))
            lines['weight'] = lines.shape[0] / lines['label_prop']
            lines['weight'] /= lines['weight'].sum()
            assert lines['weight'].isnull().sum() == 0
            # sanity checks:
            # pd.crosstab(lines['label'], lines['label_prop'])
            # lines['label'].value_counts() / lines.shape[0]
            # pd.crosstab(lines['label'], lines['weight'])
            # samples 10x the number of examples in `lines`.
            lines = lines.sample(n=lines.shape[0] * config.UPSAMPLE_FACTOR, replace=True, weights=lines.weight.values, random_state=config.SEED)
            # sanity check: number of examples per label class should be roughly equal.
            assert all(abs(lines['label'].value_counts() / lines.shape[0] - 1.0 / lines['label'].nunique()) < 0.01), \
                ('There should be roughly equal numbers of examples in each label class after upsampling.')
        # lines = [{'text': l['text'], '_id': f"{l['_id']}-{l['line']}"} for ix, l in lines.iterrows()]
        assert lines.label.isnull().sum() == 0
        for _, line in lines.iterrows():
            line_text = line['text']
            prev_context = line['prev_context'] if pd.notnull(line['prev_context']) else ''
            next_context = line['next_context'] if pd.notnull(line['next_context']) else ''
            if config.RM_FLATWORLD_TAGS:
                # removes Flatworld tags from lines.
                line_text, _ = extract_flatworld_tags(line_text)
                prev_context, _ = extract_flatworld_tags(prev_context)
                next_context, _ = extract_flatworld_tags(next_context)
            context = '\n'.join([prev_context, next_context])
            label = config.LINE_LABEL_CODES[line['label']]
            yield {'inputs': line_text, 'context': context, 'label': label}

@registry.register_problem
class HansardLineType4Char(HansardLineType4):
    @property
    def vocab_type(self):
        return text_problems.VocabType.CHARACTER


from math import log
import time

from django.test import TestCase

from desecapi import crypto

class CryptoTestCase(TestCase):
    context = "desecapi.tests.test_crypto"

    def test_retrieved_key_is_reproducible(self):
        keys = (
            crypto.retrieve_key(label="test", context=self.context) for _ in range(2)
        )
        self.assertEqual(*keys)

    def test_retrieved_key_depends_on_secret(self):
        keys = []
        for secret in ["abcdefgh", "hgfedcba"]:
            with self.settings(SECRET_KEY=secret):
                keys.append(crypto.retrieve_key(label="test", context=self.context))
        self.assertNotEqual(*keys)

    def test_retrieved_key_depends_on_label(self):
        keys = (
            crypto.retrieve_key(label=f"test_{i}", context=self.context)
            for i in range(2)
        )
        self.assertNotEqual(*keys)

    def test_retrieved_key_depends_on_context(self):
        keys = (
            crypto.retrieve_key(label="test", context=f"{self.context}_{i}")
            for i in range(2)
        )
        self.assertNotEqual(*keys)

    def test_encrypt_has_high_entropy(self):
        def entropy(value: str):
            result = 0
            counts = [value.count(char) for char in set(value)]
            for count in counts:
                count /= len(value)
                result -= count * log(count, 2)
            return result * len(value)

        ciphertext = crypto.encrypt(b"test", context=self.context)
        self.assertGreater(entropy(ciphertext), 100)  # arbitrary

    def test_encrypt_decrypt(self):
        plain = b"test"
        ciphertext = crypto.encrypt(plain, context=self.context)
        timestamp, decrypted = crypto.decrypt(ciphertext, context=self.context)
        self.assertEqual(plain, decrypted)
        self.assertTrue(0 <= time.time() - timestamp <= 1)

    def test_encrypt_decrypt_raises_on_tampering(self):
        ciphertext = crypto.encrypt(b"test", context=self.context)

        with self.assertRaises(ValueError):
            ciphertext_decoded = ciphertext.decode()
            ciphertext_tampered = (
                ciphertext_decoded[:30] + "TAMPERBEEF" + ciphertext_decoded[40:]
            ).encode()
            crypto.decrypt(ciphertext_tampered, context=self.context)

        with self.assertRaises(ValueError):
            crypto.decrypt(ciphertext, context=f"{self.context}2")