"""
Script for labeling purposes

Usage :
    python label_processing.py -d [PATH TO IMAGES FOLDER] [PARAMS]

Parameters:
    -d : Path to the folder where the images is stored
    -l : Lower all labels in xml files.
    -c : Count each label of all the xml files
    -s : Find images with specific label
    -lm : Create label_map.pbtxt
    -ar : Remove xml piles without images

RangRang - Machine Learning - 2021
"""

import os, argparse, glob
import xml.etree.ElementTree as ET

def auto_remove(path):
    """ Menghapus file xml tanpa gambar """
    images = [os.path.splitext(x) for x in os.listdir(path)]
    images = [x for x, y in images]
    for x in set(images):
        if images.count(x) < 2:
            os.remove(os.path.join(path, x + '.xml'))

def make_labelmap(path, export_dir):
    """ Membuat label_map.pbtxt """
    labels = []
    for xml_file in glob.glob(path + '/*.xml'):
        root = ET.parse(xml_file).getroot()
        for member in root.findall('object'):
            if member[0].text not in labels:
                labels.append(member[0].text)

    with open(os.path.join(export_dir, 'label_map.pbtxt'), 'w') as w:
        for i, label in enumerate(labels):
            w.writelines('item {\n    id: ' + str(i + 1) +  "\n    name: '" + label + "'\n}\n\n")
    print(f'[INFO] label_map.pbtxt exported to {export_dir}')

def counter(path):
    """ Menghitung jumlah masing - masing label dari setiap xml file """
    count = {}
    for xml_file in glob.glob(path + '/*.xml'):
        root = ET.parse(xml_file).getroot()
        for member in root.findall('object'):
            if member[0].text in count:
                count[member[0].text] += 1
            else:
                count[member[0].text] = 1

    print('[INFO] Label Counter')
    for i, (key, value) in enumerate(count.items()):
        print(f'  {i + 1}. {key} : {value}')

def search(path, indexs):
    """ Mencari image yang memiliki label tertentu """
    images = {}
    for xml_file in glob.glob(path + '/*.xml'):
        images[os.path.basename(xml_file)] = []
        root = ET.parse(xml_file).getroot()
        for member in root.findall('object'):
            images[os.path.basename(xml_file)].append(member[0].text)

    print('[INFO] Label Finder')
    for label in indexs.split(','):
        print(f' {label} '.center(20, '#'))
        for img in [x for x, y in images.items() if label in y]:
            print(f' - {img}')
        print()

def lower(path):
    """ lowering all label in the xml files """
    for xml_file in glob.glob(path + '/*.xml'):
        root = ET.parse(xml_file).getroot()
        for member in root.findall('object'):
            member[0].text = member[0].text.lower()
        element = ET.tostring(root)

        with open(xml_file, "wb") as w:
            w.write(element)

def UA(path):
    """ lowering all label in the xml files """
    for xml_file in glob.glob(path + '/*.xml'):
        root = ET.parse(xml_file).getroot()
        img_file = list(os.path.splitext(os.path.basename(root.find('filename').text)))
        xml_file_ = list(os.path.splitext(os.path.basename(xml_file)))
        if img_file[0] != xml_file_[0]:
            img_file[0] = xml_file_[0]
            if os.path.isfile(os.path.join(path, img_file[0] + '.jpg')):
                img_file[1] = '.jpg'
            else:
                img_file[1] = '.jpeg'
            img_file = img_file[0] + img_file[1]
            root.find('filename').text = img_file

            print(f'[INFO] Writing {xml_file}')
            element = ET.tostring(root)
            with open(xml_file, "wb") as w:
                w.write(element)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Labeling helper script")
    parser.add_argument("-d",
                        "--img_dir",
                        help="Path to the folder where the images is stored.",
                        type=str)
    parser.add_argument("-l",
                        "--to_lower",
                        help="Lower all labels in xml files.",
                        action='store_true')
    parser.add_argument("-c",
                        "--counter",
                        help="Count each label of all the xml files",
                        action='store_true')
    parser.add_argument("-s",
                        "--search",
                        help="Find images with specific label",
                        type=str)
    parser.add_argument("-lm",
                        "--label_map",
                        help="Create label_map.pbtxt",
                        type=str)
    parser.add_argument("-ar",
                        "--auto_remove",
                        help="Delete xlm files without img",
                        action='store_true')
    parser.add_argument("-ua",
                        "--update_annotation",
                        help="Update annotation",
                        action='store_true')

    args = parser.parse_args()

    if args.img_dir is None:
        raise KeyError('Harus menyertakan -d argument atau folder dimana images disimpan')

    if args.to_lower:
        lower(args.img_dir)

    if args.counter:
        counter(args.img_dir)

    if args.search:
        search(args.img_dir, args.search)

    if args.label_map:
        make_labelmap(args.img_dir, args.label_map)

    if args.auto_remove:
        auto_remove(args.img_dir)

    if args.update_annotation:
        UA(args.img_dir)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# PYTHON_ARGCOMPLETE_OK
# Pass --help flag for help on command-line interface

import sympy as sp
import numpy as np

from pyneqsys.symbolic import SymbolicSys

def solve(guess_a, guess_b, power, solver='scipy'):
    """ Constructs a pyneqsys.symbolic.SymbolicSys instance and returns from its ``solve`` method. """
    # The problem is 2 dimensional so we need 2 symbols
    x = sp.symbols('x:2', real=True)
    # There is a user specified parameter ``p`` in this problem:
    p = sp.Symbol('p', real=True, negative=False, integer=True)
    # Our system consists of 2-non-linear equations:
    f = [x[0] + (x[0] - x[1])**p/2 - 1,
         (x[1] - x[0])**p/2 + x[1]]
    # We construct our ``SymbolicSys`` instance by passing variables, equations and parameters:
    neqsys = SymbolicSys(x, f, [p])  # (this will derive the Jacobian symbolically)

    # Finally we solve the system using user-specified ``solver`` choice:
    return neqsys.solve([guess_a, guess_b], [power], solver=solver)

def main(guess_a=1., guess_b=0., power=3, savetxt='None', verbose=False):
    """
    Example demonstrating how to solve a system of non-linear equations defined as SymPy expressions.

    The example shows how a non-linear problem can be given a command-line interface which may be
    preferred by end-users who are not familiar with Python.
    """
    x, sol = solve(guess_a, guess_b, power)  # see function definition above
    assert sol.success
    if savetxt != 'None':
        np.savetxt(x, savetxt)
    else:
        if verbose:
            print(sol)
        else:
            print(x)

if __name__ == '__main__':  # are we running from the command line (or are we being imported from)?
    try:
        import argh
        argh.dispatch_command(main, output_file=None)
    except ImportError:
        import sys
        if len(sys.argv) > 1:
            import warnings
            warnings.warn("Ignoring parameters run "
                          "'pip install --user argh' to fix.")
        main()


#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @Time  : 2020/10/15 10:19 PM
# @Author: Zechen Li
# @File  : aug.py.py
from glue.tasks import get_task
import pandas as pd
import numpy as np
import os
from augment.eda import *
import argparse

parser = argparse.ArgumentParser()

parser.add_argument('--alpha', default=0.1, type=float)
parser.add_argument('--num_aug', default=1, type=int)
parser.add_argument('--num_type', default=4, type=int)
parser.add_argument('--task_name', default='CoLA')
parser.add_argument('--dataroot', default='./glue_data/')
parser.add_argument('--aug_dataroot', default='./aug_data/')

args = parser.parse_args()

alpha = args.alpha
num_aug = args.num_aug
num_type = args.num_type
task_name = args.task_name
task_dir = os.path.join(args.dataroot, task_name)
task = get_task(task_name.lower(), task_dir)
output_dir = os.path.join(args.aug_dataroot, task_name)

try:
    os.makedirs(output_dir)
except OSError:
    pass

ori_train_df = task.get_train_df()
ori_dev_df = task.get_dev_df()

aug_train_df = pd.DataFrame(columns=["sentence", "label"])

print("Trainning dataset preview:")
print("train sentences num:", len(ori_train_df))
print("Original:", ori_train_df.head())

for i in ori_train_df.sentence:
    ori_train_sentence = i

    method_label = np.random.randint(0, num_type, 1)[0]
    method = augment_single_with_label(method_label)

    aug_train_sentences = eda(ori_train_sentence, alpha=alpha, num_aug=num_aug, method=method)
    for aug_sentence in aug_train_sentences:
        aug_train_df = aug_train_df.append({'sentence': aug_sentence, 'label': method}, ignore_index=True)

print("Augment:", aug_train_df.head())
print(aug_train_df['label'].value_counts(normalize=True) * 100)
aug_train_df.to_csv(os.path.join(output_dir, "train.tsv"), sep='\t', index=False)

print('---------------------------------------------------------')

aug_dev_df = pd.DataFrame(columns=["sentence", "label"])

print("Dev dataset preview:")
print("dev sentences num:", len(ori_dev_df))
print("Original:", ori_dev_df.head())

for i in ori_dev_df.sentence:
    ori_dev_sentence = i

    method_label = np.random.randint(0, num_type, 1)[0]
    method = augment_single_with_label(method_label)

    aug_dev_sentences = eda(ori_dev_sentence, alpha=alpha, num_aug=num_aug, method=method)
    for aug_sentence in aug_dev_sentences:
        aug_dev_df = aug_dev_df.append({'sentence': aug_sentence, 'label': method}, ignore_index=True)

print("Augment:", aug_dev_df.head())
print(aug_dev_df['label'].value_counts(normalize=True) * 100)
aug_dev_df.to_csv(os.path.join(output_dir, "dev.tsv"), sep='\t', index=False)

print("generated augmented sentences finished.")


def get_input_lines():
    return [line.strip() for line in open(0).readlines()]

def get_req():
    # 12 red cubes, 13 green cubes, and 14 blue cubes
    return {
        'red':12,
        'green':13,
        'blue':14,
    }

def main():
    input_lines = get_input_lines()
    req = get_req()
    total = 0
    for line in input_lines:
        label, info = line.split(':')
        id = int(label.split()[-1])
        line_valid = True
        for rev_cycle in info.split(';'):
            for ball_info in rev_cycle.split(','):
                count, color = ball_info.split()
                if int(count) > req[color]:
                    line_valid = False
                    break
        if not line_valid:
            continue
        total += id
    print(total)

if __name__=='__main__':
    main()


from tkinter import *
from tkinter import messagebox
from tkColorPicker import *
from tkinter import filedialog
from tkinter.filedialog import askopenfilename
from  tkinter import  filedialog
from tkinter import scrolledtext
import fileinput
import os

class menuBar(object):
    def __init__(self):
        self.bgcolor = None
        # 打开的文件路径；打开的文件路径
        self.file = None
        self.f = None
        # 创建主窗口
        self.root = Tk()
        self.root.title('菜单栏')
        self.root.geometry('500x500')
        # self.fram = Tk.frame(self.root)
        # self.fram.pack()

        self.create_menu()
        self.root.mainloop()
    #创建菜单栏
    def create_menu(self):
        # 创建菜单栏
        self.menuBar = Menu(self.root)
        self.root.config(menu=self.menuBar)
        # 创建第一级菜单项fill
        self.fillBar = Menu(self.menuBar, tearoff=0)
        self.menuBar.add_cascade(label='第一个菜单栏', menu=self.fillBar)
        self.twoBar = Menu(self.menuBar, tearoff=0)
        self.menuBar.add_cascade(label='Open', menu=self.twoBar)
        # 创建fill下面的二级菜单项
        self.fillBar.add_command(label='new', command=self.winOne)
        self.fillBar.add_separator()
        self.fillBar.add_command(label='Exit', command=self.ExitClick)

        self.twoBar.add_command(label='OpenColor', command=self.Open_Color)
        self.twoBar.add_separator()
        # accelerator 在子菜单后面追加文字
        self.twoBar.add_command(label='打开', command=self.Open_File, accelerator='Ctr + O')
        self.twoBar.bind_all("<Control-o>", self.Open_File)
        self.twoBar.add_separator()
        self.twoBar.add_command(label='保存', command=self.Save_File)
        self.twoBar.add_command(label='另存为',command=self.Saves_File)

        # 创建文本域
        self.textnew = scrolledtext.ScrolledText(self.root, fg=self.bgcolor, width=100, height=100, wrap=tk.WORD)
        # textnew.grid()
        self.textnew.pack()

    # 关闭事件
    def ExitClick(self):
        # 关闭窗口
        self.root.quit()
        # 将所有控件销毁，内存回收
        self.root.destroy()
        exit()
    # 创建子窗口
    def winOne(self):
        top1 = Toplevel()
        top1.title('子窗口')
        top1.geometry('400x300')
        but1 = Button(top1, text='子窗体按钮', command=self.help_test)
        but1.pack()
        # 取消窗口的最大化最小化按钮
        # top1.attributes('-toolwindow', 1)
        top1.focus_get()
        # 窗口顶置
        top1.wm_attributes('-topmost', 1)

    def help_test(self):
        '''
        askokcancel 确定、取消对话框,按钮返回值 True/False
        askquestion 是、否对话框，按钮返回值 yes/no
        askyesno    是、否对话框，按钮返回值 True/False
        showerror   错误对话框，按钮返回值 ok
        showinfo    按钮返回值 ok
        showwarning 警告对话框 按钮返回值 ok
        :return:
        '''
        ask = messagebox.askokcancel('点击','按钮被点击了',)
        if ask:
            print('确定',ask)
        else:
            print('取消')
    # 打开颜色控制面板
    def Open_Color(self):
        # color = askcolor('red',0,'颜色面板')
        # if color[1] != None:
        #     print('选中的颜色是：',color)
        #     self.bgcolor = color[1]
        # else:
        #     print('没有选择颜色！',color)
        color = askcolor()
        self.textnew["foreground"] = color[1]
    def Open_File(self,event=None):
        self.file = askopenfilename(filetypes=[('txt','*.txt')])
            # 给文本框赋值
        for line in fileinput.input(self.file):
            self.textnew.insert("1.0",line)

    def Save_File(self):
        # 获取文本域中的数据
        test = self.textnew.get(1.0,END)
        # print(test)
        # 判断文件是否是文件
        # if os.path.isfile(self.file):
        if self.file!=None:
            self.f = open(self.file, 'w')
            # 将文本域中修改后的值保存
            self.f.write(str(test))
            self.f.close()
        else:
            # 另存为打开文件夹
            name = filedialog.asksaveasfilename(title='保存文件', filetypes=[('保存文件','*.txt')],defaultextension='.txt')
            print('aa', name)
            if name:
                fop = open(name, 'w')
                fop.write(str(test))
                # fop.fileno()
                fop.close()
    def Saves_File(self):
        # 获取文本域中的数据
        test = self.textnew.get(1.0, END)
        # 另存为打开文件夹
        name = filedialog.asksaveasfilename(title='保存文件', filetypes=[('保存文件', '*.txt')], defaultextension='.txt')
        print('aa', name)
        if name:
            fop = open(name, 'w')
            fop.write(str(test))
            # fop.fileno()
            fop.close()
    def delete(self):
        self.textnew.insert(tk.INSERT, '')
        self.textnew.update()

menu_bar = menuBar()


import os
#os.environ['GLOG_minloglevel'] = '2'
from tqdm import trange
import caffe
import argparse
import pandas as pd

def main(solver_proto, out_dir):
    caffe.set_mode_gpu()
    solver = caffe.SGDSolver(solver_proto)
    train_loss, test_loss = [], []
    test_loss.append(solver.test_nets[0].blobs['loss'].data.copy())
    for ix in trange(solver.param.max_iter, desc='overall progress'): #):
        for jx in trange(solver.param.test_interval, desc='until next test'):
            solver.step(1)
            train_loss.append(solver.net.blobs['loss'].data.ravel()[0])
        test_loss.append(solver.test_nets[0].blobs['loss'].data.ravel()[0])
        if ix % 1 == 0:
            solver.snapshot()
            pd.DataFrame(train_loss, columns=['train_loss']).to_csv(os.path.join(out_dir, 'train_loss.csv'))
            pd.DataFrame(test_loss, columns=['test_loss']).to_csv(os.path.join(out_dir, 'test_loss.csv'))

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--solver', required=True, type=str)
    parser.add_argument('--out_dir', default='.')
    args = parser.parse_args()

    main(args.solver, args.out_dir)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Feb 12 22:29:57 2020

@author: b
https://www.kaggle.com/kanncaa1/data-sciencetutorial-for-beginners#1.-INTRODUCTION-TO-PYTHON
3.CLEANING DATA

DIAGNOSE DATA for CLEANING
We need to diagnose and clean data before exploring.
Unclean data:

    Column name inconsistency like upper-lower case letter or space between words
    missing data
    different language

We will use head, tail, columns, shape and info methods to diagnose data
"""

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt
import seaborn as sns  # visualization tool

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

from subprocess import check_output
print(check_output(["ls"]).decode("utf8"))

# Any results you write to the current directory are saved as output.
import os
os.chdir ('/home/b/Documents/Python/Data/Pokemon Challenge')
data = pd.read_csv('pokemon.csv')

# tail shows last 5 rows
data.tail()

# columns gives column names of features
data.columns

# shape gives number of rows and columns in a tuble
data.shape

# info gives data type like dataframe, number of sample or row, number of feature or column, feature types and memory usage
data.info()

"""
EXPLORATORY DATA ANALYSIS

value_counts(): Frequency counts
outliers: the value that is considerably higher or lower from rest of the data

    Lets say value at 75% is Q3 and value at 25% is Q1.
    Outlier are smaller than Q1 - 1.5(Q3-Q1) and bigger than Q3 + 1.5(Q3-Q1). (Q3-Q1) = IQR
    We will use describe() method. Describe method includes:
    count: number of entries
    mean: average of entries
    std: standart deviation
    min: minimum entry
    25%: first quantile
    50%: median or second quantile
    75%: third quantile
    max: maximum entry
"""

# For example lets look frequency of pokemom types
print(data['Type 1'].value_counts(dropna =False))  # if there are nan values that also be counted
# As it can be seen below there are 112 water pokemon or 70 grass pokemon

# For example max HP is 255 or min defense is 5
data.describe() #ignore null entries

"""
VISUAL EXPLORATORY DATA ANALYSIS

    Box plots: visualize basic statistics like outliers, min/max or quantiles
"""
# For example: compare attack of pokemons that are legendary  or not
# Black line at top is max
# Blue line at top is 75%
# Red line is median (50%)
# Blue line at bottom is 25%
# Black line at bottom is min
# There are no outliers
data.boxplot(column='Attack',by = 'Legendary')

"""
TIDY DATA
We tidy data with melt(). Describing melt is confusing. Therefore lets make example to understand it.
"""
# Firstly I create new data from pokemons data to explain melt nore easily.
data_new = data.head()    # I only take 5 rows into new data
data_new

# lets melt
# id_vars = what we do not wish to melt
# value_vars = what we want to melt
melted = pd.melt(frame=data_new,id_vars = 'Name', value_vars= ['Attack','Defense'])
melted

"""
PIVOTING DATA
Reverse of melting.
"""
# Index is name
# I want to make that columns are variable
# Finally values in columns are value
melted.pivot(index = 'Name', columns = 'variable',values='value')

"""
DATA TYPES

There are 5 basic data types: object(string),booleab, integer, float and categorical.
We can make conversion data types like from str to categorical or from int to float
Why is category important:

    make dataframe smaller in memory
    can be utilized for anlaysis especially for sklear(we will learn later)
"""
data.dtypes

# lets convert object(str) to categorical and int to float.
data['Type 1'] = data['Type 1'].astype('category')
data['Speed'] = data['Speed'].astype('float')

# As you can see Type 1 is converted from object to categorical
# And Speed ,s converted from int to float
data.dtypes

"""
MISSING DATA and TESTING WITH ASSERT
If we encounter with missing data, what we can do:

    leave as is
    drop them with dropna()
    fill missing value with fillna()
    fill missing values with test statistics like mean
    Assert statement: check that you can turn on or turn off when you are done with your testing of the program
"""
# Lets look at does pokemon data have nan value
# As you can see there are 800 entries. However Type 2 has 414 non-null object so it has 386 null object.
data.info()

# Lets chech Type 2
data["Type 2"].value_counts(dropna =False)
# As you can see, there are 386 NAN value

# Lets drop nan values
data1=data   # also we will use data to fill missing value so I assign it to data1 variable
data1["Type 2"].dropna(inplace = True)  # inplace = True means we do not assign it to new variable. Changes automatically assigned to data
# So does it work ?

#  Lets check with assert statement
# Assert statement:
assert 1==1 # return nothing because it is true

# In order to run all code, we need to make this line comment
# assert 1==2 # return error because it is false

assert  data['Type 2'].notnull().all() # returns nothing because we drop nan values

data["Type 2"].fillna('empty',inplace = True)

assert  data['Type 2'].notnull().all() # returns nothing because we do not have nan values

# # With assert statement we can check a lot of thing. For example
# assert data.columns[1] == 'Name'
# assert data.Speed.dtypes == np.int


#!/local/cluster/bin/python
#biopython take three regions, invert middle, then put together
import sys
from Bio import SeqIO

strain=sys.argv[1]
#include 50 bp margin so as not to interrupt att site in rotated genome
largestart=int(sys.argv[2]) + 50
largeend=int(sys.argv[3]) - 50
infile="../" + strain + ".gbk"
outfile=strain + ".noSI.gbk"

record = SeqIO.read(infile, "genbank")
firstpart = record[1:largestart]
endpart = record[largeend:len(record.seq)]
newrecord = firstpart + endpart
fw=open(outfile,'w')
SeqIO.write(newrecord,fw,"genbank")


from simbolo import Simbolo, TOKENS, ZONA_DE_CODIGO, TIPO_DATO
from error import Error
import json

palabras_reservadas = [
  'bool',
  'call',
  'char',
  'do',
  'else',
  'float',
  'for',
  'function',
  'if',
  'int',
  'main',
  'read',
  'return',
  'string',
  'then',
  'to',
  'void',
  'while',
  'write',
  'false',
  'true']

#? Lista de Palabras reservadas

class Lexico:
  def __init__(self, codigo):             #Constructor del Analizador Lexico.
    self.codigo = " " + codigo + " "    #codigo fuente a compilar.
    self.tablaSimb = []                 #tabla de Simbolos
    self.index = 0                      #indice del caracter actual
    self.inicioLex = 1                  #inicio del lexema actual
    self.Lexema = ""                    #Ultimo lexema encontrado
    self.num_linea = 1                  #numero de linea del codigo fuente
    self.estado = 0
    self.caracteres_permitidos = "(){}[],;+-*/\\%&|!"     #estado actual en los automatas.
    self.tipo_de_dato_actual = 0  # Registra el tipo de dato de los identificadores.

    self.zona_de_codigo = ZONA_DE_CODIGO['DEF_VARIABLES_GLOBALES']  # Indica la zona del codigo
    # fuente que se esa procesando.

    self.fin_definicion_palabras_reservadas = None  # Indica donde termina la definicion de
    # Palabra Reservadas
    self.fin_definicion_variables_globales = None  # Inidica donde termina la definicion de
    # Variables Globales
    self.inicio_definicion_variables_locales = None  # Indica donde inicia la definicion de
    # Variables Locales en la funcion actual
    self.fin_definicion_variables_locales = None  # Indica donde finaliza la definicion de
    # Variables locales en la funcion actual

    self.error = Error()
    self.cargar_palabras_reservadas()   #Cargar las palabras reservadas en
                      #la tabla de simbolos.

  def insertar_simbolo(self, simbolo):    #inserta un nuevo simbolor en la TS.
    if simbolo:
      self.tablaSimb.append(simbolo)
      return self.tablaSimb[len(self.tablaSimb)-1]
    else:
      raise ValueError("Se esperaba un simbolo")

  def cargar_palabras_reservadas(self):   #Carga las palabras reservadas en TS
    for p in palabras_reservadas:
      self.insertar_simbolo(Simbolo(p, TOKENS[p.upper()]))
    self.fin_definicion_palabras_reservadas = len(self.tablaSimb)

  def mostrar_tabla_simbolos(self):       #muestra el contenido de la TS.
    for s in self.tablaSimb:
      print(s)

  def buscar_lexema(self, lexema):        #busca un lexema en la TS.
    if self.zona_de_codigo == ZONA_DE_CODIGO['DEF_VARIABLES_GLOBALES']:
      for simb in self.tablaSimb:
        if lexema == simb.Lexema:
          return simb
      return None

    elif self.zona_de_codigo == ZONA_DE_CODIGO['DEF_VARIABLES_LOCALES']:
      for simb in self.tablaSimb[self.inicio_definicion_variables_locales:]:
        if lexema == simb.Lexema:
          return simb
      for simb in self.tablaSimb[:self.fin_definicion_palabras_reservadas]:
        if lexema == simb.Lexema:
          return simb
      return None

    elif self.zona_de_codigo == ZONA_DE_CODIGO['CUERPO_FUNCION_LOCAL']:
      for simb in self.tablaSimb[self.inicio_definicion_variables_locales:]:
        if lexema == simb.Lexema:
          return simb
      for simb in self.tablaSimb[:self.fin_definicion_variables_globales]:
        if lexema == simb.Lexema:
          return simb
      return None

    elif self.zona_de_codigo == ZONA_DE_CODIGO['CUERPO_PRINCIPAL']:
      for simb in self.tablaSimb[:self.fin_definicion_variables_globales]:
        if lexema == simb.Lexema:
          return simb
      return None

  def tablaSimb2JSON(self):               #regresa el contenido de TS en JSON
    return json.dumps([obj.__dict__ for obj in self.tablaSimb])

  def siguiente_caracter(self):           #regresa el siguiente caracter del
    self.index += 1                     #codigo fuente.
    try:
      return self.codigo[self.index]
    except IndexError:
      return '\0'

  def saltar_caracter(self):              #ignora el caracter actual, por eje-
    self.index += 1                     #mplo: tabs, espacios, enters, etc.
    self.inicioLex = self.index

  def leer_lexema(self):                  #regresa la cadena que se encuentra
    self.Lexema = self.codigo[self.inicioLex:self.index + 1]
    self.estado = 0                     #entre inicioLex y el index.
    self.avanza_inicio_lexema()
    return self.Lexema

  def regresa_caracter(self):             #Representa el (*) en un estado de
    self.index -= 1                     #aceptacion.

  def avanza_inicio_lexema(self):         #mueve el incioLex un caracter hacia
    self.inicioLex = self.index + 1     #adelante

  def deshacer_automata(self):
    self.index = self.inicioLex
    return self.codigo[self.index]

  def siguiente_componente_lexico(self):  #regresa el siguiente simbolo encon-
    while(True):                        #trado en el codigo fuente.
      if self.estado == 0:
        c = self.siguiente_caracter()

        if c ==' ' or c =='\t' or c == '\n':
          self.avanza_inicio_lexema() #Ignorar todo tipo de espacios
          if c == '\n':               #en blanco
            self.num_linea += 1     #incrementar num_line en enter.
        elif c == '\0':
          return None
        elif c == '<':
          self.estado = 1
        elif c == '=':
          self.estado = 5
        elif c == '>':
          self.estado = 6
        else:
          self.estado = self.fallo()  #Probar el siguiente automata.
      elif self.estado == 1:
        c = self.siguiente_caracter()   #Todos los estados intermedios
        if c == '=':                    #deben llamar a siguiente_caracter
          self.estado = 2
        elif c == '>':
          self.estado = 3
        else:
          self.estado = 4
      elif self.estado == 2:
        self.leer_lexema()
        return(Simbolo(self.Lexema,TOKENS['MEI']))
      elif self.estado == 3:
        self.leer_lexema()
        return(Simbolo(self.Lexema,TOKENS['DIF']))
      elif self.estado == 4:
        self.regresa_caracter()
        self.leer_lexema()
        return(Simbolo(self.Lexema,TOKENS['MEN']))
      elif self.estado == 5:
        self.leer_lexema()
        return(Simbolo(self.Lexema,TOKENS['IGU']))
      elif self.estado == 6:
        c = self.siguiente_caracter()
        if c == '=':
          self.estado = 7
        else:
          self.estado = 8
      elif self.estado == 7:
        self.leer_lexema()
        return Simbolo(self.Lexema, TOKENS['MAI'])
      elif self.estado == 8:
        self.regresa_caracter()
        self.leer_lexema()
        return Simbolo(self.Lexema, TOKENS['MAY'])
      elif self.estado == 9:
        if c.isalpha():
          self.estado = 10
        else:
          self.estado = self.fallo()
      elif self.estado == 10:
        c = self.siguiente_caracter()
        if not c.isalnum():
          self.estado = 11
      elif self.estado == 11:
        self.regresa_caracter()
        self.leer_lexema()
        simb = self.buscar_lexema(self.Lexema)

        if self.zona_de_codigo == 0 or self.zona_de_codigo == 1:
          if simb and simb.Token != TOKENS['ID']:
            return simb
          elif simb is None:
            return self.insertar_simbolo(Simbolo(self.Lexema, TOKENS['ID'], self.tipo_de_dato_actual))
          elif simb.Token == TOKENS['ID']:
            self.error.reportar_error(self.num_linea, "Semantico", "La variable '{}' ya fue definida en el ambito actual.".format(self.Lexema))
            return simb
        elif self.zona_de_codigo == 2 or self.zona_de_codigo == 3:
          if simb:
            return simb
          else:
            self.error.reportar_error(self.num_linea, "Semantico", "La variable '{}' no fue declarada.".format(self.Lexema))
            return self.insertar_simbolo(Simbolo(self.Lexema, TOKENS['ID'], TIPO_DATO['na']))
      elif self.estado == 12:
        if c.isdigit():
          self.estado = 13
        else:
          self.estado = self.fallo()
      elif self.estado == 13:
        c = self.siguiente_caracter()
        if c == 'E' or c == 'e':
          self.estado = 16
        elif c == '.':
          self.estado = 14
        elif not c.isdigit():
          self.estado = 20
      elif self.estado == 14:
        c = self.siguiente_caracter()
        if c.isdigit():
          self.estado = 15
        else:
          self.estado = self.fallo()
      elif self.estado == 15:
        c = self.siguiente_caracter()
        if c == 'E' or c == 'e':
          self.estado = 16
        elif not c.isdigit():
          self.estado = 21
      elif self.estado == 16:
        c = self.siguiente_caracter()
        if c == '+' or c == '-':
          self.estado = 17
        elif c.isdigit():
          self.estado = 18
        else:
          self.es = self.fallo()
      elif self.estado == 17:
        c = self.siguiente_caracter()
        if c.isdigit():
          self.estado = 18
        else: self.estado = self.fallo()
      elif self.estado == 18:
        c =self.siguiente_caracter()
        if not c.isdigit():
          self.estado = 19
      elif self.estado == 19:
        self.regresa_caracter()
        self.leer_lexema()
        return Simbolo(self.Lexema, TOKENS['NUMF'])
      elif self.estado == 20:
        self.regresa_caracter()
        self.leer_lexema()
        return Simbolo(self.Lexema, TOKENS['NUM'])
      elif self.estado == 21:
        self.regresa_caracter()
        self.leer_lexema()
        return Simbolo(self.Lexema, TOKENS['NUMF'])
      elif self.estado == 22:
        if c == '"':
          self.estado = 23
        else:
          self.estado = self.fallo()
      elif self.estado == 23:
        c = self.siguiente_caracter()
        if c == "\\":
          self.estado = 24
        elif c == '"':
          self.estado = 25
      elif self.estado == 24:
        c = self.siguiente_caracter()
        if c in 'nta"\\r':
          self.estado = 23
        else:
          self.estado = self.fallo()
      elif self.estado == 25:
        self.leer_lexema()
        return Simbolo(self.Lexema, TOKENS['CONST_STRING'])
      elif self.estado == 26:
        if c == "'":
          self.estado = 27
        else:
          self.estado = self.fallo()
      elif self.estado == 27:
        c = self.siguiente_caracter()
        if c == '\\':
          self.estado = 28
        else:
          self.estado = 29
      elif self.estado == 28:
        c = self.siguiente_caracter()
        if c in "nta'\\r":
          self.estado = 29
        else:
          self.estado = self.fallo()
      elif self.estado == 29:
        c = self.siguiente_caracter()
        if c == "'":
          self.estado = 30
        else:
          self.estado = self.fallo()
      elif self.estado == 30:
        self.leer_lexema()
        return Simbolo(self.Lexema, TOKENS['CONST_CHAR'])
      elif self.estado == 31:
        if c == "/":
          self.estado = 32
        else:
          self.estado = self.fallo()
      elif self.estado == 32:
        c = self.siguiente_caracter()
        if c == "/":
          self.estado = 34
        elif c == "*":
          self.estado = 33
        else:
          c = self.deshacer_automata()
          self.estado = self.fallo()
      elif self.estado == 33:
        c = self.siguiente_caracter()
        if c == "*":
          self.estado = 35
      elif self.estado == 34:
        c = self.siguiente_caracter()
        if c == "\n" or c == "\0":
          self.estado = 36
      elif self.estado == 35:
        c = self.siguiente_caracter()
        if c == "/":
          self.estado = 37
        else:
          self.estado = 33
      elif self.estado == 36:
        self.regresa_caracter()
        self.leer_lexema()
      elif self.estado == 37:
        self.leer_lexema()
      elif self.estado == 38:
        if c in self.caracteres_permitidos:
          self.leer_lexema()
          return Simbolo(c,ord(c))
        else:
          self.estado = self.fallo()
      else:
        self.leer_lexema()
        self.error.reportar_error(self.num_linea, "Lexico", "Simbolo no permitido '{}'.".format(self.Lexema))

  def fallo(self):
    if self.estado <= 8:
      return 9
    elif self.estado <= 11:
      return 12
    elif self.estado <= 21:
      return 22
    elif self.estado <= 25:
      return 26
    elif self.estado <= 30:
      return 31
    elif self.estado <= 37:
      return 38
    else:
      return 99


from PySide2.QtCore import QUrl, QObject, Slot

from PySide2.QtGui import QGuiApplication

from PySide2.QtQuick import QQuickView

class MyClass(QObject):

    @Slot(int, result=str)  # 声明为槽，输入参数为int类型，返回值为str类型
    def returnValue(self, value):
        return str(value + 10)

if __name__ == '__main__':
    path = 'src/demo1.qml'

    app = QGuiApplication([])

    view = QQuickView()

    con = MyClass()

    context = view.rootContext()

    context.setContextProperty("con", con)

    view.engine().quit.connect(app.quit)

    view.setSource(QUrl(path))

    view.show()

    app.exec_()


from django.shortcuts import render

product_price = {"라면":980,"홈런볼":1500,"칙촉":2300, "식빵":1800}

# Create your views here.
def price(request, thing, cnt):
    if thing in product_price:
        y_n = 'y'
        price = product_price[thing]
    else:
        y_n = 'n'
        price = 0
    context = {
        'y_n' : y_n,
        'price' : price,
        'thing' : thing,
        'cnt' : cnt,
        'total' : price * cnt
    }
    return render(request, 'prices/price.html', context)


"""project URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import (
    include,
    url,
)
from django.contrib import admin

from classifier.views import CreateClassification
from healthcheck.views import HealthCheck
from report.views import CreateReport
from secret.views import Secret

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^auth/', include('djoser.urls')),
    url(r'^auth/', include('djoser.urls.authtoken')),
    url(
        r'^classify-image',
        CreateClassification.as_view(),
        name='classify-image',
    ),
    url(r'^health', HealthCheck.as_view(), name='healthcheck'),
    url(r'^report', CreateReport.as_view(), name='report'),
    url(r'^secret', Secret.as_view(), name='secret'),
]


#!/usr/bin/env python3

import numpy as np
from tensorflow.keras.preprocessing.image import load_img, img_to_array
import rospy
from std_msgs.msg import UInt16
import os
from tensorflow.keras.models import load_model

def process_image(img_path):
    img = load_img(img_path, target_size=(30, 30))
    img = img_to_array(img)
    img = np.expand_dims(img, axis=0)
    return img

def predict_model(img_path, model):
    labels = ['20 km/h', '30 km/h', '50 km/h', '60 km/h', '70 km/h', '80 km/h', '80 km/h end', '100 km/h',
              '120 km/h', 'No overtaking', 'No overtaking for tracks', 'Crossroad with secondary way',
              'Main road', 'Give way', 'Stop', 'Road up', 'Road up for track', 'Brock', 'Other dangerous',
              'Turn left', 'Turn right', 'Winding road', 'Hollow road', 'Slippery road', 'Narrowing road',
              'Roadwork', 'Traffic light', 'Pedestrian', 'Children', 'Bike', 'Snow', 'Deer', 'End of the limits',
              'Only right', 'Only left', 'Only straight', 'Only straight and right', 'Only straight and left',
              'Take right', 'Take left', 'Circle crossroad', 'End of overtaking limit', 'End of overtaking limit for track']

    image = process_image(img_path)
    pred = np.argmax(model.predict(image), axis=1)

    prediction = labels[pred[0]]
    print(prediction)
    return prediction

def choose_way(sign):
    msg = 0
    if sign == 'Stop':
        msg = 2
        return msg
    if sign == 'Only right':
        msg = 3
        return msg
    if sign == 'Only left':
        msg = 4
        return msg
    else:
        msg = 1
        return msg

os.environ["CUDA_VISIBLE_DEVICES"] = "-1"

MODEL_PATH = '/home/adam/catkin_ws/src/robot_tsr/script/model-1.h5'

# Loading the model back:
model = load_model(MODEL_PATH)

def main():
    file_path = f'/home/adam/catkin_ws/src/robot_tsr/script/uploads/stop.jpeg'
    sign = predict_model(file_path, model)
    return sign

def talker():
    signs_publisher = rospy.Publisher('dc_motors', UInt16, queue_size=10)
    rospy.init_node('talker', anonymous=True)
    rate = rospy.Rate(10)  # 1hz
    while not rospy.is_shutdown():
        new_sign = main()
        # Choose to right value
        msg = choose_way(new_sign)
        rospy.loginfo(msg)
        signs_publisher.publish(msg)
        rate.sleep()

if __name__ == '__main__':
    try:
        talker()
    except rospy.ROSInterruptException:
        pass


import wiredtiger, wttest, string, random, time
from wtbound import *
from enum import Enum
from wtscenario import make_scenarios

class operations(Enum):
    UPSERT = 1
    REMOVE = 2
    TRUNCATE = 3

class key_states(Enum):
    UPSERTED = 1
    DELETED = 2
    NONE = 3

class bound_scenarios(Enum):
    NEXT = 1
    PREV = 2
    SEARCH = 3
    SEARCH_NEAR = 4

class bound_type(Enum):
    LOWER = 1
    UPPER = 2

class key():
    key_state = key_states.NONE
    data = -1
    value = "none"
    prepared = False
    timestamp = 0

    def __init__(self, data, value, key_state, timestamp):
        self.key_state = key_state
        self.data = data
        self.value = value
        self.timestamp = timestamp

    def clear_prepared(self):
        self.prepared = False

    def is_prepared(self):
        return self.prepared

    def is_deleted(self):
        return self.key_state == key_states.DELETED

    def is_out_of_bounds(self, bound_set):
        return not bound_set.in_bounds_key(self.data)

    def is_deleted_or_oob(self, bound_set):
        return self.is_deleted() or self.is_out_of_bounds(bound_set)

    def update(self, value, key_state, timestamp, prepare):
        self.value = value
        self.key_state = key_state
        self.timestamp = timestamp
        self.prepared = prepare

    def to_string(self):
        return "Key: " + str(self.data) + ", state: " + str(self.key_state) + ", prepared: " + str(self.prepared)

    def equals(self, key, value):
        if (self.key_state == key_states.UPSERTED and self.data == key and self.value == value):
            return True
        else:
            return False

# test_cursor_bound_fuzz.py
#    A python test fuzzer that generates a random key range and applies bounds to it, then runs
#    randomized operations and validates them for correctness.
class test_cursor_bound_fuzz(wttest.WiredTigerTestCase):
    file_name = 'test_fuzz.wt'

    iteration_count = 200 if wttest.islongtest() else 50
    # For each iteration we do search_count searches that way we test more cases without having to
    # generate as many key ranges.
    search_count = 20
    key_count = 10000 if wttest.islongtest() else 1000
    # Large transactions throw rollback errors so we don't use them in the long test.
    transactions_enabled = False if wttest.islongtest() else True
    value_size = 100000 if wttest.islongtest() else 100
    prepare_frequency = 5/100
    update_frequency = 2/10

    min_key = 1
    # Max_key is not inclusive so the actual max_key is max_key - 1.
    max_key = min_key + key_count
    # A lot of time was spent generating values, to achieve some amount of randomness we pre
    # generate N values and keep them in memory.
    value_array = []
    value_array_size = 20
    current_ts = 1
    applied_ops = False
    key_range = {}

    types = [
        ('file', dict(uri='file:')),
        ('table', dict(uri='table:'))
    ]

    data_format = [
        ('row', dict(key_format='i')),
        ('column', dict(key_format='r'))
    ]
    scenarios = make_scenarios(types, data_format)

    # Iterates valid keys from min_key to max_key, the maximum key is defined as max_key - 1.
    # Python doesn't consider the end of the range as inclusive.
    def key_range_iter(self):
        for i in range(self.min_key, self.max_key):
            yield i

    def dump_key_range(self):
        for i in self.key_range_iter():
            self.pr(self.key_range[i].to_string())

    # Generate a random ascii value.
    def generate_value(self):
        return ''.join(random.choice(string.ascii_lowercase) for _ in range(self.value_size))

    # Get a value from the value array.
    def get_value(self):
        return self.value_array[random.randrange(self.value_array_size)]

    # Get a key within the range of min_key and max_key.
    def get_random_key(self):
        return random.randrange(self.min_key, self.max_key)

    # Update a key using the cursor and update its in memory representation.
    def apply_update(self, cursor, key_id, prepare):
        value = self.get_value()
        cursor[key_id] = value
        self.key_range[key_id].update(value, key_states.UPSERTED, self.current_ts, prepare)
        self.verbose(3, "Updating " + self.key_range[key_id].to_string())

    # Remove a key using the cursor and mark it as deleted in memory.
    # If the key is already deleted we skip the remove.
    def apply_remove(self, cursor, key_id, prepare):
        if (self.key_range[key_id].is_deleted()):
            return
        cursor.set_key(key_id)
        self.assertEqual(cursor.remove(), 0)
        self.key_range[key_id].update(None, key_states.DELETED, self.current_ts, prepare)
        self.verbose(3, "Removing " + self.key_range[key_id].to_string())

    # Apply a truncate operation to the key range.
    def apply_truncate(self, session, cursor, cursor2, prepare):
        lower_key = self.get_random_key()
        if (lower_key + 1 < self.max_key):
            upper_key = random.randrange(lower_key + 1, self.max_key)
            cursor.set_key(lower_key)
            cursor2.set_key(upper_key)
            self.assertEqual(session.truncate(None, cursor, cursor2, None), 0)

            # Mark all keys from lower_key to upper_key as deleted.
            for key_id in range(lower_key, upper_key + 1):
                self.key_range[key_id].update(None, key_states.DELETED, self.current_ts, prepare)

            self.verbose(3, "Truncated keys between: " + str(lower_key) + " and: " + str(upper_key))

    # Each iteration calls this function once to update the state of the keys in the database and
    # in memory.
    def apply_ops(self, session, cursor, prepare):
        op = random.choice(list(operations))
        if (op is operations.TRUNCATE and self.applied_ops):
            cursor2 = session.open_cursor(self.uri + self.file_name)
            self.apply_truncate(session, cursor, cursor2, prepare)
        else:
            for i in self.key_range_iter():
                if (random.uniform(0, 1) < self.update_frequency):
                    continue
                op = random.choice(list(operations))
                if (op is operations.TRUNCATE):
                    pass
                elif (op is operations.UPSERT):
                    self.apply_update(cursor, i, prepare)
                elif (op is operations.REMOVE):
                    self.apply_remove(cursor, i, prepare)
                else:
                    raise Exception("Unhandled operation generated")
        self.applied_ops = True

    # As prepare throws a prepare conflict exception we wrap the call to anything that could
    # encounter a prepare conflict in a try except, we then return the error code to the caller.
    def prepare_call(self, func):
        try:
            ret = func()
        except wiredtiger.WiredTigerError as e:
            if wiredtiger.wiredtiger_strerror(wiredtiger.WT_PREPARE_CONFLICT) in str(e):
                ret = wiredtiger.WT_PREPARE_CONFLICT
            else:
                raise e
        return ret

    # Once we commit the prepared transaction, update and clear the prepared flags.
    def clear_prepare_key_ranges(self):
        for i in self.key_range_iter():
            self.key_range[i].clear_prepared()

    # Given a bound, this functions returns the start or end expected key of the bounded range.
    # Note the type argument determines if we return the start or end limit. e.g. if we have a lower
    # bound then the key would be the lower bound, however if the lower bound isn't enabled then the
    # lowest possible key would be min_key. max_key isn't inclusive so we subtract 1 off it.
    def get_expected_limit_key(self, bound_set, type):
        if (type == bound_type.LOWER):
            if (bound_set.lower.enabled):
                if (bound_set.lower.inclusive):
                    return bound_set.lower.key
                return bound_set.lower.key + 1
            return self.min_key
        if (bound_set.upper.enabled):
            if (bound_set.upper.inclusive):
                return bound_set.upper.key
            return bound_set.upper.key - 1
        return self.max_key - 1

    # When a prepared cursor walks next or prev it can skip deleted records internally before
    # returning a prepare conflict, we don't know which key it got to so we need to validate that
    # we see a series of deleted keys followed by a prepared key.
    def validate_deleted_prepared_range(self, start_key, end_key, next):
        if (next):
            step = 1
        else:
            step = -1
        self.verbose(3, "Walking deleted range from: " + str(start_key) + " to: " + str(end_key))
        for i in range(start_key, end_key, step):
            self.verbose(3, "Validating state of key: " + self.key_range[i].to_string())
            if (self.key_range[i].is_prepared()):
                return
            elif (self.key_range[i].is_deleted()):
                continue
            else:
                self.assertTrue(False)

    # Validate a prepare conflict in the cursor->next scenario.
    def validate_prepare_conflict_next(self, current_key, bound_set):
        self.verbose(3, "Current key is: " + str(current_key) + " min_key is: " + str(self.min_key))
        start_range = None
        if current_key == self.min_key:
            # We hit a prepare conflict while walking forwards before we stepped to a valid key.
            # Therefore validate all the keys from start of the range are deleted followed by a prepare.
            start_range = self.get_expected_limit_key(bound_set, bound_type.LOWER)
        else:
            # We walked part of the way through a valid key range before we hit the prepared
            # update. Therefore validate the range between our current key and the
            # end range.
            start_range = current_key
        end_range = self.get_expected_limit_key(bound_set, bound_type.UPPER)

        # Perform validation from the start range to end range.
        self.validate_deleted_prepared_range(start_range, end_range, True)

    # Validate a prepare conflict in the cursor->prev scenario.
    def validate_prepare_conflict_prev(self, current_key, bound_set):
        self.verbose(3, "Current key is: " + str(current_key) + " max_key is: " + str(self.max_key))
        start_range = None
        if current_key == self.max_key - 1:
            # We hit a prepare conflict while walking backwards before we stepped to a valid key.
            # Therefore validate all the keys from start of the range are deleted followed by a
            # prepare.
            start_range = self.get_expected_limit_key(bound_set, bound_type.UPPER)
        else:
            # We walked part of the way through a valid key range before we hit the prepared
            # update. Therefore validate the range between our current key and the
            # end range.
            start_range = current_key
        end_range = self.get_expected_limit_key(bound_set, bound_type.LOWER)

        # Perform validation from the start range to end range.
        self.validate_deleted_prepared_range(start_range, end_range, False)

    # Walk the cursor using cursor->next and validate the returned keys.
    def run_next(self, bound_set, cursor):
        # This array gives us confidence that we have validated the full key range.
        checked_keys = []
        self.verbose(3, "Running scenario: NEXT")
        key_range_it = self.min_key - 1
        ret = self.prepare_call(lambda: cursor.next())
        while (ret != wiredtiger.WT_NOTFOUND and ret != wiredtiger.WT_PREPARE_CONFLICT):
            current_key = cursor.get_key()
            current_value = cursor.get_value()
            self.verbose(3, "Cursor next walked to key: " + str(current_key) + " value: " + current_value)
            self.assertTrue(bound_set.in_bounds_key(current_key))
            self.assertTrue(self.key_range[current_key].equals(current_key, current_value))
            checked_keys.append(current_key)
            # If the cursor has walked to a record that isn't +1 our current record then it
            # skipped something internally.
            # Check that the key range between key_range_it and current_key isn't visible
            if (current_key != key_range_it + 1):
                for i in range(key_range_it + 1, current_key):
                    self.verbose(3, "Checking key is deleted or oob: " + str(i))
                    checked_keys.append(i)
                    self.assertTrue(self.key_range[i].is_deleted_or_oob(bound_set))
            key_range_it = current_key
            ret = self.prepare_call(lambda: cursor.next())
        key_range_it = key_range_it + 1
        # If we were returned a prepare conflict it means the cursor has found a prepared key/value.
        # We need to validate that it arrived there correctly using the in memory state of the
        # database. We cannot continue from a prepare conflict so we return.
        if (ret == wiredtiger.WT_PREPARE_CONFLICT):
            self.validate_prepare_conflict_next(key_range_it, bound_set)
            return
        # If key_range_it is < key_count then the rest of the range was deleted
        # Remember to increment it by one to get it to the first not in bounds key.
        for i in range(key_range_it, self.max_key):
            checked_keys.append(i)
            self.verbose(3, "Checking key is deleted or oob: " + str(i))
            self.assertTrue(self.key_range[i].is_deleted_or_oob(bound_set))
        self.assertTrue(len(checked_keys) == self.key_count)

    # Walk the cursor using cursor->prev and validate the returned keys.
    def run_prev(self, bound_set, cursor):
        # This array gives us confidence that we have validated the full key range.
        checked_keys = []
        self.verbose(3, "Running scenario: PREV")
        ret = self.prepare_call(lambda: cursor.prev())
        key_range_it = self.max_key
        while (ret != wiredtiger.WT_NOTFOUND and ret != wiredtiger.WT_PREPARE_CONFLICT):
            current_key = cursor.get_key()
            current_value = cursor.get_value()
            self.verbose(3, "Cursor prev walked to key: " + str(current_key) + " value: " + current_value)
            self.assertTrue(bound_set.in_bounds_key(current_key))
            self.assertTrue(self.key_range[current_key].equals(current_key, current_value))
            checked_keys.append(current_key)

            # If the cursor has walked to a record that isn't -1 our current record then it
            # skipped something internally.
            # Check that the key range between key_range_it and current_key isn't visible
            if (current_key != key_range_it - 1):
                # Check that the key range between key_range_it and current_key isn't visible
                for i in range(current_key + 1, key_range_it):
                    self.verbose(3, "Checking key is deleted or oob: " + str(i))
                    checked_keys.append(i)
                    self.assertTrue(self.key_range[i].is_deleted_or_oob(bound_set))
            key_range_it = current_key
            ret = self.prepare_call(lambda: cursor.prev())
        # If key_range_it is > key_count then the rest of the range was deleted
        key_range_it -= 1
        if (ret == wiredtiger.WT_PREPARE_CONFLICT):
            self.validate_prepare_conflict_prev(key_range_it, bound_set)
            return
        for i in range(self.min_key, key_range_it + 1):
            checked_keys.append(i)
            self.verbose(3, "Checking key is deleted or oob: " + str(i))
            self.assertTrue(self.key_range[i].is_deleted_or_oob(bound_set))
        self.assertTrue(len(checked_keys) == self.key_count)

    # Run basic cursor->search() scenarios and validate the outcome.
    def run_search(self, bound_set, cursor):
        # Choose a N random keys and perform a search on each
        for i in range(0, self.search_count):
            search_key = self.get_random_key()
            cursor.set_key(search_key)
            ret = self.prepare_call(lambda: cursor.search())
            if (ret == wiredtiger.WT_PREPARE_CONFLICT):
                self.assertTrue(self.key_range[search_key].is_prepared())
            elif (ret == wiredtiger.WT_NOTFOUND):
                self.assertTrue(self.key_range[search_key].is_deleted_or_oob(bound_set))
            elif (ret == 0):
                # Assert that the key exists, and is within the range.
                self.assertTrue(self.key_range[search_key].equals(cursor.get_key(), cursor.get_value()))
                self.assertTrue(bound_set.in_bounds_key(cursor.get_key()))
            else:
                raise Exception('Unhandled error returned by search')

    # Check that all the keys within the given bound_set are deleted.
    def check_all_within_bounds_not_visible(self, bound_set):
        for i in range(bound_set.start_range(self.min_key), bound_set.end_range(self.max_key)):
            self.verbose(3, "checking key: " +self.key_range[i].to_string())
            if (not self.key_range[i].is_deleted()):
                return False
        return True

    # Run a cursor->search_near scenario and validate that the outcome was correct.
    def run_search_near(self, bound_set, cursor):
        # Choose N random keys and perform a search near.
        for i in range(0, self.search_count):
            search_key = self.get_random_key()
            cursor.set_key(search_key)
            self.verbose(3, "Searching for key: " + str(search_key))
            ret = self.prepare_call(lambda: cursor.search_near())
            if (ret == wiredtiger.WT_NOTFOUND):
                self.verbose(3, "Nothing visible checking.")
                # Nothing visible within the bound range.
                # Validate.
            elif (ret == wiredtiger.WT_PREPARE_CONFLICT):
                # Due to the complexity of the search near logic we will simply check if there is
                # a prepared key within the range.
                found_prepare = False
                for i in range(bound_set.start_range(self.min_key), bound_set.end_range(self.max_key)):
                    if (self.key_range[i].is_prepared()):
                        found_prepare = True
                        break
                self.assertTrue(found_prepare)
                self.verbose(3, "Received prepare conflict in search near.")
            else:
                key_found = cursor.get_key()
                self.verbose(3, "Found a key: " + str(key_found))
                current_key = key_found
                # Assert the value we found matches.
                # Equals also validates that the key is visible.
                self.assertTrue(self.key_range[current_key].equals(current_key, cursor.get_value()))
                if (bound_set.in_bounds_key(search_key)):
                    # We returned a key within the range, validate that key is the one that
                    # should've been returned.
                    if (key_found == search_key):
                        # We've already determined the key matches. We can return.
                        pass
                    if (key_found > search_key):
                        # Walk left and validate that all isn't visible to the search key.
                        while (current_key != search_key):
                            current_key = current_key - 1
                            self.assertTrue(self.key_range[current_key].is_deleted())
                    if (key_found < search_key):
                        # Walk right and validate that all isn't visible to the search key.
                        while (current_key != search_key):
                            current_key = current_key + 1
                            self.assertTrue(self.key_range[current_key].is_deleted())
                else:
                    # We searched for a value outside our range, we should return whichever value
                    # is closest within the range.
                    if (bound_set.lower.enabled and search_key <= bound_set.lower.key):
                        # We searched to the left of our bounds. In the equals case the lower bound
                        # must not be inclusive.
                        # Validate that the we returned the nearest value to the lower bound.
                        if (bound_set.lower.inclusive):
                            self.assertTrue(key_found >= bound_set.lower.key)
                            current_key = bound_set.lower.key
                        else:
                            self.assertTrue(key_found > bound_set.lower.key)
                            current_key = bound_set.lower.key + 1
                        while (current_key != key_found):
                            self.assertTrue(self.key_range[current_key].is_deleted())
                            current_key = current_key + 1
                    elif (bound_set.upper.enabled and search_key >= bound_set.upper.key):
                        # We searched to the right of our bounds. In the equals case the upper bound
                        # must not be inclusive.
                        # Validate that the we returned the nearest value to the upper bound.
                        if (bound_set.upper.inclusive):
                            self.assertTrue(key_found <= bound_set.upper.key)
                            current_key = bound_set.upper.key
                        else:
                            self.assertTrue(key_found < bound_set.upper.key)
                            current_key = bound_set.upper.key - 1
                        while (current_key != key_found):
                            self.assertTrue(self.key_range[current_key].is_deleted())
                            current_key = current_key - 1
                    else:
                        raise Exception('Illegal state found in search_near')

    # Choose a scenario and run it.
    def run_bound_scenarios(self, bound_set, cursor):
        scenario = random.choice(list(bound_scenarios))
        if (scenario is bound_scenarios.NEXT):
            self.run_next(bound_set, cursor)
        elif (scenario is bound_scenarios.PREV):
            self.run_prev(bound_set, cursor)
        elif (scenario is bound_scenarios.SEARCH):
            self.run_search(bound_set, cursor)
        elif (scenario is bound_scenarios.SEARCH_NEAR):
            self.run_search_near(bound_set, cursor)
        else:
            raise Exception('Unhandled bound scenario chosen')

    # Generate a set of bounds and apply them to the cursor.
    def apply_bounds(self, cursor):
        cursor.reset()
        lower = bound(self.get_random_key(), bool(random.getrandbits(1)), bool(random.getrandbits(1)))
        upper = bound(random.randrange(lower.key, self.max_key), bool(random.getrandbits(1)), bool(random.getrandbits(1)))
        # Prevent invalid bounds being generated.
        if (lower.key == upper.key and lower.enabled and upper.enabled):
            lower.inclusive = upper.inclusive = True
        bound_set = bounds(lower, upper)
        if (lower.enabled):
            cursor.set_key(lower.key)
            cursor.bound("bound=lower,inclusive=" + lower.inclusive_str())
        if (upper.enabled):
            cursor.set_key(upper.key)
            cursor.bound("bound=upper,inclusive=" + upper.inclusive_str())
        return bound_set

    # The primary test loop is contained here.
    def test_bound_fuzz(self):
        uri = self.uri + self.file_name
        create_params = 'value_format=S,key_format={}'.format(self.key_format)
        # Reset the key range for every scenario.
        self.key_range = {}
        # Setup a reproducible random seed.
        # If this test fails inspect the file WT_TEST/results.txt and replace the time.time()
        # with a given seed. e.g.:
        # seed = 1660215872.5926154
        # Additionally this test is configured for verbose logging which can make debugging a bit
        # easier.
        seed = time.time()
        self.pr("Using seed: " + str(seed))
        random.seed(seed)
        self.session.create(uri, create_params)
        read_cursor = self.session.open_cursor(uri)

        write_session = self.setUpSessionOpen(self.conn)
        write_cursor = write_session.open_cursor(uri)

        # Initialize the value array.
        self.verbose(3, "Generating value array")
        for i in range(0, self.value_array_size):
            self.value_array.append(self.generate_value())

        # Initialize the key range.
        for i in self.key_range_iter():
            key_value = self.get_value()
            self.key_range[i] = key(i, key_value, key_states.UPSERTED, self.current_ts)
            self.current_ts += 1
            if (self.transactions_enabled):
                write_session.begin_transaction()
            write_cursor[i] = key_value
            if (self.transactions_enabled):
                write_session.commit_transaction('commit_timestamp=' + self.timestamp_str(self.key_range[i].timestamp))
        self.session.checkpoint()

        # Begin main loop
        for  i in range(0, self.iteration_count):
            self.verbose(3, "Iteration: " + str(i))
            bound_set = self.apply_bounds(read_cursor)
            self.verbose(3, "Generated bound set: " + bound_set.to_string())

            # Check if we are doing a prepared transaction on this iteration.
            prepare = random.uniform(0, 1) <= self.prepare_frequency and self.transactions_enabled
            if (self.transactions_enabled):
                write_session.begin_transaction()
            self.apply_ops(write_session, write_cursor, prepare)
            if (self.transactions_enabled):
                if (prepare):
                    self.verbose(3, "Preparing applied operations.")
                    write_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(self.current_ts))
                else:
                    write_session.commit_transaction('commit_timestamp=' + self.timestamp_str(self.current_ts))

            # Use the current timestamp so we don't need to track previous versions.
            if (self.transactions_enabled):
                self.session.begin_transaction('read_timestamp=' + self.timestamp_str(self.current_ts))
            self.run_bound_scenarios(bound_set, read_cursor)
            if (self.transactions_enabled):
                self.session.rollback_transaction()
                if (prepare):
                    write_session.commit_transaction(
                        'commit_timestamp=' + self.timestamp_str(self.current_ts) +
                        ',durable_timestamp='+ self.timestamp_str(self.current_ts))
                    self.clear_prepare_key_ranges()
            self.current_ts += 1
            if (i % 10 == 0):
                # Technically this is a write but easier to do it with this session.
                self.session.checkpoint()

if __name__ == '__main__':
    wttest.run()


from django.contrib import auth
from django.core.exceptions import ObjectDoesNotExist

from .models import Member

class EcosystemActivityMiddleware:
    EXTENSIONS_EXCLUDED = ['js', 'map', 'css']
    PATHS_EXCLUDED = ['/api/graphql-jwt']

    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        user = self._get_user(request)
        if user.is_authenticated and \
                request.method in ['POST', 'PUT', 'DELETE']:

            try:
                member = Member.objects.get(user=user)
                member.update_activity()
            except ObjectDoesNotExist:
                pass

        return self.get_response(request)

    def _get_user(self, request):
        if not hasattr(request, '_cached_user'):
            request._cached_user = auth.get_user(request)
        return request._cached_user


# ---
# jupyter:
#   jupytext:
#     cell_metadata_filter: -all
#     comment_magics: true
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.3'
#       jupytext_version: 1.13.7
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# %% [markdown]
# # Exploratory cluster analysis

# %%
# Exploratory cluster analysis

# %load_ext autoreload
# %autoreload 2

# %% [markdown]
# ## Preamble

# %%
import altair as alt
import numpy as np
from scipy.stats import ttest_ind
import pandas as pd
import requests
from toolz import pipe

from afs_neighbourhood_analysis.getters.clustering import clustering_diagnostics
from afs_neighbourhood_analysis.pipeline.lad_clustering.cluster_utils import (
    extract_clusters,
    clustering_params,
)
from afs_neighbourhood_analysis.getters.clustering import (
    early_years_for_clustering,
    public_health_for_clustering,
)

# %%
phf_long

# %%
def fetch_geojson(url):

    return pipe(
        requests.get(url),
        lambda req: [e["properties"] for e in req.json()["features"]],
        pd.DataFrame,
    )

def make_code_name_lookup():

    county_json = pipe(
        fetch_geojson(
            "https://services1.arcgis.com/ESMARspQHYMw9BZ9/arcgis/rest/services/CTY_APR_2019_EN_NC/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson"
        ),
        lambda df: df.set_index("CTY19CD")["CTY19NM"].to_dict(),
    )

    lad_json = (
        pd.read_excel(
            "https://www.arcgis.com/sharing/rest/content/items/c4f647d8a4a648d7b4a1ebf057f8aaa3/data"
        )
        .set_index(["LAD21CD"])["LAD21NM"]
        .to_dict()
    )

    return {**county_json, **lad_json}

def get_code_nuts_lookup():

    lad_nuts = (
        pd.read_excel(
            "https://www.arcgis.com/sharing/rest/content/items/c110087ae04a4cacb4ab0aef960936ce/data"
        )
        .set_index("LAD20CD")["ITL121NM"]
        .to_dict()
    )

    lad_county = pipe(
        fetch_geojson(
            "https://services1.arcgis.com/ESMARspQHYMw9BZ9/arcgis/rest/services/LAD21_CTY21_EN_LU/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson"
        ),
        lambda df: df.set_index("LAD21CD")["CTY21CD"],
    )

    return {
        **{code: name for code, name in lad_nuts.items()},
        **{lad_county[lad]: lad_nuts[lad] for lad in set(lad_county.keys())},
    }

def plot_ey_perf(ey, year, gender):
    """LALALA"""

    return (
        alt.Chart(ey.query(f"year=={year}").query(f"gender=='{gender}'"))
        .mark_boxplot()
        .encode(
            y="cluster:O",
            color="cluster:N",
            x="zscore",
            tooltip=["la_name"],
            column="indicator",
        )
    ).properties(width=200)

def plot_ey_trend(ey, gender="Total"):
    """ """

    return (
        alt.Chart(ey_bench.query(f"gender=='{gender}'"))
        .mark_boxplot()
        .encode(
            x="year:O",
            y="zscore",
            column="cluster",
            color="cluster:N",
            row="indicator",
            tooltip=["la_name", "cluster"],
        )
    ).properties(width=100, height=100)

def plot_ey_year_comp(ey, gender="Total"):
    """ """

    return (
        alt.Chart(ey_bench.query(f"gender=='{gender}'"))
        .mark_boxplot()
        .encode(
            column="year:O",
            y="zscore",
            x="cluster:N",
            color="cluster:N",
            row="indicator",
            tooltip=["la_name", "cluster"],
        )
    ).properties(width=100, height=100)

def plot_ey_evol(ey_table, gender):
    """ """

    # return (alt.Chart(ey_table.query(f"gender=='{gender}'"))
    #         .mark_line(point=True)
    #         .encode(x="year",y="zscore",color="cluster:N",
    #                 row="indicator",column="cluster",
    #                 detail="la_name",tooltip=["la_name","year","zscore"])
    #         .properties(width=100,height=100))

    all_lads = (
        alt.Chart()
        .mark_line(point=True)
        .encode(
            x="year:N",
            y=alt.Y("zscore", scale=alt.Scale(zero=False)),
            color="cluster:N",
            detail="la_name",
            tooltip=["la_name", "zscore"],
        )
    ).properties(width=100, height=100)

    mean = (
        alt.Chart()
        .mark_line(point=False, color="black")
        .encode(
            x="year:N",
            y=alt.Y("median(zscore)", scale=alt.Scale(zero=False), title="Score"),
        )
    ).properties(width=100, height=100)

    return (
        alt.layer(all_lads, mean, data=ey_table.query(f"gender=='{gender}'"))
        .facet(row="indicator", column="cluster:N")
        .resolve_scale(y="independent")
    )

def phf_for_analysis(ph_table, cluster_lookup, code_name_lookup):
    return (
        phf.stack()
        .reset_index(name="score")
        .assign(cluster=lambda df: df["area_code"].map(cluster_lookup))
        .assign(area_name=lambda df: df["area_code"].map(code_name_lookup))
    )

def calc_mean_ph(ph_long):

    ph_agg = pd.concat(
        [
            ph_long.rename(columns={"score": name})
            .groupby(["cluster", "indicator_name_expanded"])[name]
            .apply(lambda x: function(x))
            for function, name in zip([np.mean, np.std], ["mean", "std"])
        ],
        axis=1,
    ).reset_index()

    return ph_agg.assign(
        rank=lambda df: df["indicator_name_expanded"].map(
            ph_agg.groupby("indicator_name_expanded")["mean"].std().rank(ascending=True)
        )
    )

    # return (pd.concat([ph_long
    #                  .rename(columns={"score":name})
    #                  .groupby(["cluster","indicator_name_expanded"])[name].apply(lambda x: function(x)) for
    #                  function,name in zip([np.mean, np.std],["mean","std"])],axis=1)
    #        .reset_index()
    #        .assign(rank = lambda df: df["indicator_name_expanded"]
    #                .map(ph_mean
    #                     .groupby("indicator_name_expanded")["mean"]
    #                     .std().rank(ascending=True))))

def phf_ttest(phf_long, sig_level=0.05, equal_var=True):
    """ """

    test_results = []

    for ind in phf_long["indicator_name_expanded"].unique():

        ind_df = phf_long.query(f"indicator_name_expanded == '{ind}'").reset_index(
            drop=True
        )

        for cl in ind_df["cluster"].unique():

            ttest = ttest_ind(
                ind_df.query(f"cluster=={cl}")["score"],
                ind_df.query(f"cluster!={cl}")["score"],
                equal_var=equal_var,
            )

            test_results.append([ind, cl, ttest.pvalue])

    return pd.DataFrame(
        test_results, columns=["indicator", "cluster", "ttest_sign"]
    ).assign(is_sig=lambda df: df["ttest_sign"] < sig_level)

def plot_phf_differences(phf_long, sig_level=0.05, equal_var=True):

    return (
        alt.Chart(
            pipe(phf_long, calc_mean_ph)
            .merge(
                phf_ttest(phf_long, sig_level, equal_var),
                left_on=["indicator_name_expanded", "cluster"],
                right_on=["indicator", "cluster"],
            )
            .query("is_sig == True")
        )
        .mark_rect(filled=True)
        .encode(
            x=alt.X(
                "indicator_name_expanded",
                sort=alt.EncodingSortField("rank", order="descending"),
                axis=alt.Axis(labels=False, ticks=False),
            ),
            y="cluster:N",
            color=alt.Color("mean", scale=alt.Scale(scheme="Redblue", reverse=True)),
            tooltip=["cluster", "indicator_name_expanded", "mean"],
        )
        .properties(width=800, height=300)
    )

def plot_gender_gap_trend(gender_gap):
    """ """

    all_lads = (
        alt.Chart()
        .mark_line(point=True)
        .encode(
            x="year:N",
            y=alt.Y("ratio", scale=alt.Scale(zero=False)),
            color="cluster:N",
            detail="new_la_code",
            tooltip=["la_name", "ratio"],
        )
    ).properties(width=100, height=100)

    mean = (
        alt.Chart()
        .mark_line(point=False, color="black")
        .encode(
            x="year:N",
            y=alt.Y("median(ratio)", scale=alt.Scale(zero=False), title="Gender ratio"),
        )
    ).properties(width=100, height=100)

    return (
        alt.layer(all_lads, mean, data=gender_gap)
        .facet(row="indicator", column="cluster:N")
        .resolve_scale(y="independent")
    )

def get_gender_gap(ey):
    """Calculates gender gap between boys and girls"""

    return (
        ey.groupby(["year", "indicator"])
        .apply(
            lambda df: df.pivot_table(
                index="new_la_code", columns="gender", values="score"
            ).assign(ratio=lambda df_2: (df_2["Girls"] / df_2["Boys"]))["ratio"]
        )
        .reset_index(drop=False)
        .assign(cluster=lambda df: df["new_la_code"].map(clust_lu))
        #  .assign(la_name = lambda df: df["new_la_code"].map(names_codes_lookup_2))
        .assign(la_name=lambda df: df["new_la_code"].map(code_name_lookup))
        #  .dropna(axis=0,subset=["cluster"])
    )

def plot_gender_gap_comp(gender_gap, year=2019):
    """Boxplot comparing gender gap across clusters"""

    return (
        alt.Chart(gender_gap.query(f"year=={year}"))
        .mark_boxplot()
        .encode(
            y="cluster:N",
            x=alt.X("ratio", scale=alt.Scale(zero=False)),
            column="indicator",
            tooltip=["la_name", "ratio"],
            color="cluster:N",
        )
        .resolve_axis(x="independent")
        .properties(width=200)
    )

def plot_gender_gap_trend(gender_gap):
    """Visualise trends in gender gap"""

    all_lads = (
        alt.Chart()
        .mark_line(point=True)
        .encode(
            x="year:N",
            y=alt.Y("ratio", scale=alt.Scale(zero=False)),
            color="cluster:N",
            detail="new_la_code",
            tooltip=["la_name", "ratio"],
        )
    ).properties(width=100, height=100)

    mean = (
        alt.Chart()
        .mark_line(point=False, color="black")
        .encode(
            x="year:N",
            y=alt.Y("median(ratio)", scale=alt.Scale(zero=False), title="Gender ratio"),
        )
    ).properties(width=100, height=100)

    return (
        alt.layer(all_lads, mean, data=gender_gap)
        .facet(row="indicator", column="cluster:N")
        .resolve_scale(y="independent")
    )

# %% [markdown]
# ## Clustering diagnostics

# %%
diag = clustering_diagnostics()

# %%
alt.Chart(diag).mark_point(filled=True).encode(
    x="pca", y="value", row="diagnostic_var", color="comm_resolution:O"
).resolve_scale(y="independent")

# %%
pca_mean = (
    diag.groupby(["pca", "diagnostic_var"])["value"].median().reset_index(drop=False)
)

alt.Chart(pca_mean).mark_line(point=True).encode(
    x="pca", y="value", color="diagnostic_var"
)

# %%
com_res_mean = (
    diag.groupby(["comm_resolution", "diagnostic_var"])["value"]
    .median()
    .reset_index(drop=False)
)

alt.Chart(com_res_mean).mark_line(point=True).encode(
    x="comm_resolution", y="value", color="diagnostic_var"
)

# %% [markdown]
# ## Extract clusters

# %%
ey = early_years_for_clustering()
phf = public_health_for_clustering()

# %%
clust = extract_clusters(phf, 5, 0.9, clustering_params)

# %%
code_name_lookup = make_code_name_lookup()
code_nut_lookup = get_code_nuts_lookup()

# %%
cluster_df = (
    pd.Series(clust[1])
    .reset_index(name="cluster")
    .assign(geo_name=lambda df: df["index"].map(code_name_lookup))
    .assign(nuts_name=lambda df: df["index"].map(code_nut_lookup))
    .rename(columns={"index": "geo_code"})
)

# %% [markdown]
# ## Explore cluster results

# %% [markdown]
# ### Regional differences

# %%
clust_region_shares = (
    cluster_df.groupby("cluster")["nuts_name"]
    .apply(lambda x: x.value_counts(normalize=True))
    .unstack()
    .fillna(0)
    .stack()
    .reset_index(name="share")
    .rename(columns={"level_1": "region"})
)
reg_bar = (
    alt.Chart(clust_region_shares)
    .mark_bar()
    .encode(y="cluster:O", x="share", color="region")
)
reg_bar

# %% [markdown]
# ### EFSYP performance differences

# %%
clust_lu = cluster_df.set_index("geo_code")["cluster"].to_dict()

# %%
ey_bench = ey.assign(cluster=lambda df: df["new_la_code"].map(clust_lu)).dropna(
    axis=0, subset=["cluster"]
)

# %%
ey_comp = (
    ey.query("year==2019")
    .query("gender=='Total'")
    .assign(cluster=lambda df: df["new_la_code"].map(clust_lu))
    .dropna(axis=0, subset=["cluster"])
)

# %%
plot_ey_perf(ey_bench, 2019, "Total")

# %%
plot_ey_perf(ey_bench, 2019, "Boys")

# %% [markdown]
# ### Evolution of differences

# %%
plot_ey_trend(ey_bench)

# %% [markdown]
# ### Year on year comparisons

# %%
plot_ey_year_comp(ey_bench)

# %%
(
    ey_bench.query("gender=='Total'")
    .groupby("indicator")
    .apply(
        lambda x: x.pivot_table(
            index="new_la_code", columns="year", values="zscore"
        ).corr()
    )[2019]
    .unstack()
)

# %% [markdown]
# ### Differences between clusters

# %%
phf_long = phf_for_analysis(phf, clust_lu, code_name_lookup)

# %%
plot_phf_differences(phf_long, sig_level=0.01)

# %% [markdown]
# ### Improvements in performance inside clusters

# %%
# Where have we seen the greatest improvements inside clusters?

# %%
plot_ey_evol(ey_bench, "Girls")

# %%
# Other things to do:
# 1. Calculate SHAPLEY values for variables
# 2. measure gender gap in performance inside clusters
# 3. Create choropleth

# %%
# Gender gap

# %%

# %%
gender_gap = get_gender_gap(ey)

# %%
gender_gap.columns

# %%
gender_gap.loc[gender_gap["la_name"].isna()]

# %%
plot_gender_gap_comp(gender_gap, year=2019)

# %%
plot_gender_gap_trend(gender_gap)

# %%
la = fetch_geojson(
    "https://services1.arcgis.com/ESMARspQHYMw9BZ9/arcgis/rest/services/Counties_and_Unitary_Authorities_December_2021_UK_BGC/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson"
)

# %%
names_codes_lookup_2 = la.set_index("CTYUA21CD")["CTYUA21NM"].to_dict()

# %%
names_codes_lookup_2

# %%


"""
WRITE ME

Tests for the R operator / L operator

For the list of op with r op defined, with or without missing test
see this file: doc/library/tensor/basic.txt

For function to automatically test your Rop implementation, look at
the docstring of the functions: check_mat_rop_lop, check_rop_lop,
check_nondiff_rop,
"""
from __future__ import absolute_import, print_function, division
import unittest
from theano.tests import unittest_tools as utt
from theano import function
import theano
from theano import tensor
import itertools
import numpy as np
from theano.gof import Op, Apply
from theano.gradient import grad_undefined
from theano.tests.unittest_tools import SkipTest
from theano.tensor.signal.pool import Pool
from theano.tensor.nnet import conv, conv2d

'''
Special Op created to test what happens when you have one op that is not
differentiable in the computational graph
'''

class BreakRop(Op):
    """
    @note: Non-differentiable.
    """
    __props__ = ()

    def make_node(self, x):
        return Apply(self, [x], [x.type()])

    def perform(self, node, inp, out_):
        x, = inp
        out, = out_
        out[0] = x

    def grad(self, inp, grads):
        return [grad_undefined(self, 0, inp[0])]

    def R_op(self, inputs, eval_points):
        return [None]

break_op = BreakRop()

class RopLop_checker(unittest.TestCase):
    """
    Don't peform any test, but provide the function to test the
    Rop to class that inherit from it.
    """
    def setUp(self):
        utt.seed_rng()
        # Using vectors make things a lot simpler for generating the same
        # computations using scan
        self.x = tensor.vector('x')
        self.v = tensor.vector('v')
        self.rng = np.random.RandomState(utt.fetch_seed())
        self.in_shape = (5 + self.rng.randint(3),)
        self.mx = tensor.matrix('mx')
        self.mv = tensor.matrix('mv')
        self.mat_in_shape = (5 + self.rng.randint(3),
                             5 + self.rng.randint(3))

    def check_nondiff_rop(self, y):
        """
        If your op is not differentiable(so you can't define Rop)
        test that an error is raised.
        """
        raised = False
        try:
            tensor.Rop(y, self.x, self.v)
        except ValueError:
            raised = True
        if not raised:
            self.fail((
                'Op did not raise an error even though the function'
                ' is not differentiable'))

    def check_mat_rop_lop(self, y, out_shape):
        """
        Test the Rop/Lop when input is a matrix and the output is a vector

        :param y: the output variable of the op applied to self.mx
        :param out_shape: Used to generate a random tensor
                          corresponding to the evaluation point of the Rop
                          (i.e. the tensor with which you multiply the
                          Jacobian). It should be a tuple of ints.

        If the Op has more than 1 input, one of them must be mx, while
        others must be shared variables / constants. We will test only
        against the input self.mx, so you must call
        check_mat_rop_lop/check_rop_lop for the other inputs.

        We expect all inputs/outputs have dtype floatX.

        If you want to test an Op with an output matrix, add a sum
        after the Op you want to test.
        """
        vx = np.asarray(self.rng.uniform(size=self.mat_in_shape),
                        theano.config.floatX)
        vv = np.asarray(self.rng.uniform(size=self.mat_in_shape),
                        theano.config.floatX)
        yv = tensor.Rop(y, self.mx, self.mv)
        rop_f = function([self.mx, self.mv], yv, on_unused_input='ignore')
        sy, _ = theano.scan(lambda i, y, x, v:
                            (tensor.grad(y[i], x) * v).sum(),
                            sequences=tensor.arange(y.shape[0]),
                            non_sequences=[y, self.mx, self.mv])
        scan_f = function([self.mx, self.mv], sy, on_unused_input='ignore')

        v1 = rop_f(vx, vv)
        v2 = scan_f(vx, vv)

        assert np.allclose(v1, v2), ('ROP mismatch: %s %s' % (v1, v2))

        self.check_nondiff_rop(theano.clone(y, replace={self.mx: break_op(self.mx)}))

        vv = np.asarray(self.rng.uniform(size=out_shape), theano.config.floatX)
        yv = tensor.Lop(y, self.mx, self.v)
        lop_f = function([self.mx, self.v], yv)

        sy = tensor.grad((self.v * y).sum(), self.mx)
        scan_f = function([self.mx, self.v], sy)

        v1 = lop_f(vx, vv)
        v2 = scan_f(vx, vv)
        assert np.allclose(v1, v2), ('LOP mismatch: %s %s' % (v1, v2))

    def check_rop_lop(self, y, out_shape):
        """
        As check_mat_rop_lop, except the input is self.x which is a
        vector. The output is still a vector.
        """
        # TEST ROP
        vx = np.asarray(self.rng.uniform(size=self.in_shape),
                        theano.config.floatX)
        vv = np.asarray(self.rng.uniform(size=self.in_shape),
                        theano.config.floatX)

        yv = tensor.Rop(y, self.x, self.v)
        rop_f = function([self.x, self.v], yv, on_unused_input='ignore')
        J, _ = theano.scan(lambda i, y, x: tensor.grad(y[i], x),
                           sequences=tensor.arange(y.shape[0]),
                           non_sequences=[y, self.x])
        sy = tensor.dot(J, self.v)

        scan_f = function([self.x, self.v], sy, on_unused_input='ignore')

        v1 = rop_f(vx, vv)
        v2 = scan_f(vx, vv)
        assert np.allclose(v1, v2), ('ROP mismatch: %s %s' % (v1, v2))
        known_fail = False
        try:
            self.check_nondiff_rop(theano.clone(y, replace={self.x: break_op(self.x)}))
        except AssertionError:
            known_fail = True

        # TEST LOP

        vx = np.asarray(self.rng.uniform(size=self.in_shape),
                        theano.config.floatX)
        vv = np.asarray(self.rng.uniform(size=out_shape),
                        theano.config.floatX)

        yv = tensor.Lop(y, self.x, self.v)
        lop_f = function([self.x, self.v], yv, on_unused_input='ignore')
        J, _ = theano.scan(lambda i, y, x: tensor.grad(y[i], x),
                           sequences=tensor.arange(y.shape[0]),
                           non_sequences=[y, self.x])
        sy = tensor.dot(self.v, J)

        scan_f = function([self.x, self.v], sy)

        v1 = lop_f(vx, vv)
        v2 = scan_f(vx, vv)
        assert np.allclose(v1, v2), ('LOP mismatch: %s %s' % (v1, v2))

        if known_fail:
            raise SkipTest('Rop does not handle non-differentiable inputs '
                           'correctly. Bug exposed by fixing Add.grad method.')

class test_RopLop(RopLop_checker):
    def test_shape(self):
        self.check_nondiff_rop(self.x.shape[0])

    def test_specifyshape(self):
        self.check_rop_lop(tensor.specify_shape(self.x, self.in_shape),
                           self.in_shape)

    def test_max(self):
        # If we call max directly, we will return an CAReduce object
        # which doesn't have R_op implemented!
        # self.check_mat_rop_lop(tensor.max(self.mx, axis=[0,1])[0], ())
        self.check_mat_rop_lop(tensor.max(self.mx, axis=0),
                               (self.mat_in_shape[1],))
        self.check_mat_rop_lop(tensor.max(self.mx, axis=1),
                               (self.mat_in_shape[0],))

    def test_argmax(self):
        self.check_nondiff_rop(tensor.argmax(self.mx, axis=1))

    def test_subtensor(self):
        self.check_rop_lop(self.x[:4], (4,))

    def test_incsubtensor1(self):
        tv = np.asarray(self.rng.uniform(size=(3,)),
                        theano.config.floatX)
        t = theano.shared(tv)
        out = tensor.inc_subtensor(self.x[:3], t)
        self.check_rop_lop(out, self.in_shape)

    def test_incsubtensor2(self):
        tv = np.asarray(self.rng.uniform(size=(10,)),
                        theano.config.floatX)
        t = theano.shared(tv)
        out = tensor.inc_subtensor(t[:4], self.x[:4])
        self.check_rop_lop(out, (10,))

    def test_setsubtensor1(self):
        tv = np.asarray(self.rng.uniform(size=(3,)),
                        theano.config.floatX)
        t = theano.shared(tv)
        out = tensor.set_subtensor(self.x[:3], t)
        self.check_rop_lop(out, self.in_shape)

    def test_print(self):
        out = theano.printing.Print('x', attrs=('shape',))(self.x)
        self.check_rop_lop(out, self.in_shape)

    def test_setsubtensor2(self):
        tv = np.asarray(self.rng.uniform(size=(10,)),
                        theano.config.floatX)
        t = theano.shared(tv)
        out = tensor.set_subtensor(t[:4], self.x[:4])
        self.check_rop_lop(out, (10,))

    def test_dimshuffle(self):
        # I need the sum, because the setup expects the output to be a
        # vector
        self.check_rop_lop(self.x[:4].dimshuffle('x', 0).sum(axis=0),
                           (4,))

    def test_rebroadcast(self):
        # I need the sum, because the setup expects the output to be a
        # vector
        self.check_rop_lop(tensor.unbroadcast(
            self.x[:4].dimshuffle('x', 0), 0).sum(axis=1),
            (1,))

    def test_downsample(self):
        rng = np.random.RandomState(utt.fetch_seed())
        # ws, shp
        examples = (
            ((2,), (16,)),
            ((2,), (4, 16,)),
            ((2,), (4, 2, 16,)),
            ((1, 1), (4, 2, 16, 16)),
            ((2, 2), (4, 2, 16, 16)),
            ((3, 3), (4, 2, 16, 16)),
            ((3, 2), (4, 2, 16, 16)),
            ((3, 2, 2), (3, 2, 16, 16, 16)),
            ((2, 3, 2), (3, 2, 16, 16, 16)),
            ((2, 2, 3), (3, 2, 16, 16, 16)),
            ((2, 2, 3, 2), (3, 2, 6, 6, 6, 5)),
        )

        for example, ignore_border in itertools.product(examples, [True, False]):
            (ws, shp) = example
            vx = rng.rand(*shp)
            vex = rng.rand(*shp)

            x = theano.shared(vx)
            ex = theano.shared(vex)

            maxpool_op = Pool(ignore_border, ndim=len(ws))
            a_pooled = maxpool_op(x, ws).flatten()
            yv = tensor.Rop(a_pooled, x, ex)
            mode = None
            if theano.config.mode == "FAST_COMPILE":
                mode = "FAST_RUN"
            rop_f = function([], yv, on_unused_input='ignore', mode=mode)
            sy, _ = theano.scan(lambda i, y, x, v:
                                (tensor.grad(y[i], x) * v).sum(),
                                sequences=tensor.arange(a_pooled.shape[0]),
                                non_sequences=[a_pooled, x, ex],
                                mode=mode)
            scan_f = function([], sy, on_unused_input='ignore', mode=mode)
            v1 = rop_f()
            v2 = scan_f()
            assert np.allclose(v1, v2), ("Rop mismatch: %s %s" % (v1, v2))

    def test_conv(self):
        for conv_op in [conv.conv2d, conv2d]:
            for border_mode in ['valid', 'full']:
                image_shape = (2, 2, 4, 5)
                filter_shape = (2, 2, 2, 3)
                image_dim = len(image_shape)
                filter_dim = len(filter_shape)
                input = tensor.TensorType(
                    theano.config.floatX,
                    [False] * image_dim)(name='input')
                filters = tensor.TensorType(
                    theano.config.floatX,
                    [False] * filter_dim)(name='filter')
                ev_input = tensor.TensorType(
                    theano.config.floatX,
                    [False] * image_dim)(name='ev_input')
                ev_filters = tensor.TensorType(
                    theano.config.floatX,
                    [False] * filter_dim)(name='ev_filters')

                def sym_conv2d(input, filters):
                    return conv_op(input, filters, border_mode=border_mode)
                output = sym_conv2d(input, filters).flatten()
                yv = tensor.Rop(output, [input, filters], [ev_input, ev_filters])
                mode = None
                if theano.config.mode == "FAST_COMPILE":
                    mode = "FAST_RUN"
                rop_f = function([input, filters, ev_input, ev_filters],
                                 yv, on_unused_input='ignore', mode=mode)
                sy, _ = theano.scan(lambda i, y, x1, x2, v1, v2:
                                    (tensor.grad(y[i], x1) * v1).sum() +
                                    (tensor.grad(y[i], x2) * v2).sum(),
                                    sequences=tensor.arange(output.shape[0]),
                                    non_sequences=[output, input, filters,
                                                   ev_input, ev_filters],
                                    mode=mode)
                scan_f = function([input, filters, ev_input, ev_filters], sy,
                                  on_unused_input='ignore', mode=mode)
                dtype = theano.config.floatX
                image_data = np.random.random(image_shape).astype(dtype)
                filter_data = np.random.random(filter_shape).astype(dtype)
                ev_image_data = np.random.random(image_shape).astype(dtype)
                ev_filter_data = np.random.random(filter_shape).astype(dtype)
                v1 = rop_f(image_data, filter_data, ev_image_data, ev_filter_data)
                v2 = scan_f(image_data, filter_data, ev_image_data, ev_filter_data)
                assert np.allclose(v1, v2), ("Rop mismatch: %s %s" % (v1, v2))

    def test_join(self):
        tv = np.asarray(self.rng.uniform(size=(10,)),
                        theano.config.floatX)
        t = theano.shared(tv)
        out = tensor.join(0, self.x, t)
        self.check_rop_lop(out, (self.in_shape[0] + 10,))

    def test_dot(self):
        insh = self.in_shape[0]
        vW = np.asarray(self.rng.uniform(size=(insh, insh)),
                        theano.config.floatX)
        W = theano.shared(vW)
        self.check_rop_lop(tensor.dot(self.x, W), self.in_shape)

    def test_elemwise0(self):
        self.check_rop_lop((self.x + 1) ** 2, self.in_shape)

    def test_elemwise1(self):
        self.check_rop_lop(self.x + tensor.cast(self.x, 'int32'),
                           self.in_shape)

    def test_reshape(self):
        new_shape = tensor.constant(np.asarray([
            self.mat_in_shape[0] * self.mat_in_shape[1]],
            dtype='int64'))

        self.check_mat_rop_lop(self.mx.reshape(new_shape),
                               (self.mat_in_shape[0] * self.mat_in_shape[1],))

    def test_flatten(self):
        self.check_mat_rop_lop(self.mx.flatten(),
                               (self.mat_in_shape[0] * self.mat_in_shape[1],))

    def test_sum(self):
        self.check_mat_rop_lop(self.mx.sum(axis=1), (self.mat_in_shape[0],))

    def test_softmax(self):
        # Softmax adds an extra dimnesion !
        self.check_rop_lop(tensor.nnet.softmax(self.x)[0], self.in_shape[0])

    def test_alloc(self):
        # Alloc of the sum of x into a vector
        out1d = tensor.alloc(self.x.sum(), self.in_shape[0])
        self.check_rop_lop(out1d, self.in_shape[0])

        # Alloc of x into a 3-D tensor, flattened
        out3d = tensor.alloc(self.x, self.mat_in_shape[0], self.mat_in_shape[1], self.in_shape[0])
        self.check_rop_lop(out3d.flatten(), self.mat_in_shape[0] * self.mat_in_shape[1] * self.in_shape[0])

    def test_invalid_input(self):
        success = False

        try:
            tensor.Rop(0., [tensor.matrix()], [tensor.vector()])
            success = True
        except ValueError:
            pass

        assert not success

    def test_multiple_outputs(self):
        m = tensor.matrix('m')
        v = tensor.vector('v')
        m_ = tensor.matrix('m_')
        v_ = tensor.vector('v_')

        mval = self.rng.uniform(size=(3, 7)).astype(theano.config.floatX)
        vval = self.rng.uniform(size=(7,)).astype(theano.config.floatX)
        m_val = self.rng.uniform(size=(3, 7)).astype(theano.config.floatX)
        v_val = self.rng.uniform(size=(7,)).astype(theano.config.floatX)

        rop_out1 = tensor.Rop([m, v, m + v], [m, v], [m_, v_])
        assert isinstance(rop_out1, list)
        assert len(rop_out1) == 3
        rop_out2 = tensor.Rop((m, v, m + v), [m, v], [m_, v_])
        assert isinstance(rop_out2, tuple)
        assert len(rop_out2) == 3

        all_outs = []
        for o in rop_out1, rop_out2:
            all_outs.extend(o)
        f = theano.function([m, v, m_, v_], all_outs)
        f(mval, vval, m_val, v_val)

    def test_Rop_dot_bug_18Oct2013_Jeremiah(self):
        # This test refers to a bug reported by Jeremiah Lowin on 18th Oct
        # 2013. The bug consists when through a dot operation there is only
        # one differentiable path (i.e. there is no gradient wrt to one of
        # the inputs).
        x = tensor.arange(20.0).reshape([1, 20])
        v = theano.shared(np.ones([20]))
        d = tensor.dot(x, v).sum()
        tensor.Rop(tensor.grad(d, v), v, v)


#
# @lc app=leetcode id=24 lang=python3
#
# [24] Swap Nodes in Pairs
#

# @lc code=start
# Definition for singly-linked list.
from typing import Optional

class ListNode:
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

class Solution:
    def swapPairs(self, head: Optional[ListNode]) -> Optional[ListNode]:
        dummy = ListNode(0)
        dummy.next = head
        pre = dummy
        while head and head.next:
            left = head
            right = head.next
            pre.next = right
            left.next = right.next
            right.next = left
            pre = left
            head = pre.next
        if head:
            pre.next = head

        return dummy.next

# @lc code=end