from django.db import models
from django.core.validators import MaxValueValidator, MinValueValidator
from decimal import Decimal
from .tasks import get_imdb_rating

class Movie(models.Model):
    name = models.CharField(max_length=214)
    director = models.CharField(max_length=214)
    subject = models.TextField(max_length=10000)
    release_date = models.DateField()
    imdb = models.DecimalField(max_digits=3,
                               decimal_places=1,
                               validators=[MinValueValidator(Decimal('1.0')), MaxValueValidator(Decimal('10.0'))],
                               null=True,
                               blank=True)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def __str__(self) -> str:
        return f"{self.name} - {self.release_date}"

    def save(self, *args, **kwargs) -> None:
        super().save(*args, **kwargs)
        if self.imdb is None: # This is obstacle of recursive.
            get_imdb_rating.delay(self.name, instance=self.id)


from PySide6 import QtCore, QtWidgets
from PySide6.QtWidgets import QGraphicsOpacityEffect, QHBoxLayout, QMainWindow, QVBoxLayout, QWidget
from PySide6.QtCore import Property, QPropertyAnimation, QSize

class Ventana(QMainWindow):
    def __init__(self):
        super().__init__()
        layoutCompleto = QVBoxLayout()
        layoutBotones = QHBoxLayout()

        self.botonTransparente = Boton("Transparente")
        self.botonOpaco = Boton("Opaco")
        self.cuadro = Cuadro()

        layoutBotones.addWidget(self.botonTransparente)
        layoutBotones.addWidget(self.botonOpaco)
        layoutCompleto.addWidget(self.cuadro)

        layoutCompleto.addLayout(layoutBotones)

        contenedor = QWidget()
        contenedor.setLayout(layoutCompleto)
        contenedor.resize(100, 100) #Tamaño inicial del contenedor.
        self.setCentralWidget(contenedor)

        self.cuadro.setStyleSheet("background-color:red;border-radius:15px;")
        self.cuadro.resize(100, 100)

        self.botonTransparente.clicked.connect(self.cuadro.hacer_transparente)
        self.botonOpaco.clicked.connect(self.cuadro.hacer_opaco)

class Boton(QtWidgets.QWidget):
    def __init__(self, texto = "Pulsa aquí", parent=None):
        super().__init__(parent)
        layout = QtWidgets.QHBoxLayout() # Establecemos un layout horizontal

        self.boton = QtWidgets.QPushButton() # Creamos el widget Boton
        self.boton.setText(texto)
        layout.addWidget(self.boton)
        self.setLayout(layout)

    def __getattr__(self, name):
        return getattr(self.boton, name)

    def sizeHint(self):
        return QSize(40, 40)

    def getTexto(self):
        return self.boton.text()

    def setTexto(self, texto):
        self.boton.setText(texto)

    texto = Property(str, getTexto, setTexto)

class Cuadro(QWidget):
    cambio_opacidad = QtCore.Signal(float)

    def __init__(self):
        super().__init__()
        self.resize(100, 100)
        self.cuadro = QWidget(self)
        self.cuadro.setStyleSheet("background-color:blue;border-radius:15px;")
        self.cuadro.resize(100, 100)
        self.opacidad = 1.0

        self.cambio_opacidad.connect(self.setOpacidad)

    def getOpacidad(self):
        return self.opacidad

    def setOpacidad(self, opacidad):
        if opacidad != self.opacidad:
            self.opacidad = opacidad

    value = Property(float, getOpacidad, setOpacidad)

    def cambiar_opacidad(self, valor):
        effect = QGraphicsOpacityEffect(self)
        self.setGraphicsEffect(effect)

        self.anim = QPropertyAnimation(effect, b"opacity")
        self.anim.setStartValue(self.getOpacidad())
        self.anim.setEndValue(valor)
        self.anim.setDuration(800)
        self.anim.start()

        #self.setOpacidad(valor)
        self.cambio_opacidad.emit(valor)  #Emitimos la señal

    def hacer_transparente(self):
        self.cambiar_opacidad(0.25)

    def hacer_opaco(self):
        self.cambiar_opacidad(1.0)

    def sizeHint(self):
        return QSize(100, 100)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import sys

if '__file__' in globals():
    path = os.path.dirname(os.path.abspath(__file__))
elif hasattr(sys, 'frozen'):
    path = os.path.dirname(os.path.abspath(sys.executable)) # for py2exe
else: #should never happen
    path = os.getcwd()
os.chdir(path)

sys.path = [path]+[p for p in sys.path if not p == path]

# import gluon.import_all ##### This should be uncommented for py2exe.py
import gluon.widget

# Start Web2py and Web2py cron service!
if __name__ == '__main__':
    try:
        from multiprocessing import freeze_support
        freeze_support()
    except:
        sys.stderr.write('Sorry, -K only supported for python 2.6-2.7\n')
    gluon.widget.start(cron=True)


import sys

from flexmock import flexmock

# Mocking neutron.common.constants
mock_neutron_pkg = flexmock(name='mock_neutron')
mock_common_pkg = flexmock(name='mock_common')
mock_neutron_pkg.common = mock_common_pkg
mock_constants_mod = flexmock(name='mock_constant')
mod_attrs = {
    'PROTO_NUM_TCP': 6,
    'PROTO_NAME_TCP': 'tcp',
    'PROTO_NUM_UDP': 17,
    'PROTO_NAME_UDP': 'udp',
    'PROTO_NUM_ICMP': 1,
    'PROTO_NAME_ICMP': 'icmp',
    'NET_STATUS_ACTIVE': 'ACTIVE',
    'NET_STATUS_DOWN': 'DOWN',
    'PORT_STATUS_ACTIVE': 'ACTIVE',
    'PORT_STATUS_DOWN': 'DOWN',
    'DEVICE_OWNER_ROUTER_GW': 'network:router_gateway',
    'DEVICE_OWNER_ROUTER_INTF': 'network:router_interface',
}
for attr, value in mod_attrs.items():
    mock_constants_mod.__setattr__(attr, value)
mock_common_pkg.constants = mock_constants_mod
sys.modules['neutron'] = mock_neutron_pkg
sys.modules['neutron.common'] = mock_common_pkg


swap1 = 0
srav = 0
arrpuz = []
arrvstavka = []
arrvibor = []
arrqs = []
arrshell = []

def puz(x):
    sr = 0
    sw = 0
    for i in range(len(x)):
        for j in range(len(x) - 1):
            sr += 1
            if x[j] > x[j + 1]:
                tmp = x[j]
                x[j] = x[j + 1]
                x[j + 1] = tmp
                sw += 1
    print("Пузырёк")
    print("Количество сравнений", sr, "Количество перестановок", sw)

def vstavka1(x):
    sw = 0
    sr = 0
    for i in range(1, len(x)):
        temp = x[i]
        tmp = i
        sr += 1
        while tmp > 0 and x[tmp - 1] > temp:
            x[tmp] = x[tmp - 1]
            tmp -= 1
            sw += 1
        x[tmp] = temp
    print("Вставками")
    print("Количество сравнений", sr, "Количество перестановок", sw)

def vibor(x):
    sw = 0
    sr = 0
    for i in range(len(x) - 1):
        mini = x[i]
        minindex = i
        for j in range(i + 1, len(x)):
            sr += 1
            if x[j] < mini:
                mini = x[j]
                minindex = j
                sw += 1
        x[i], x[minindex] = x[minindex], x[i]
    print("Выбором")
    print("Количество сравнений", sr, "Количество перестановок", sw)

def shell(x):
    sr = 0
    sw = 0
    step = len(x) // 2
    while step > 0:
        for i in range(len(x)):
            while i >= step:
                sr += 1
                if x[i] < x[i - step]:
                    x[i], x[i - step] = x[i - step], x[i]
                    sw += 1
                else:
                    break
                i -= step
        step //= 2
    print("Шелл")
    print("Количество сравнений", sr, "Количество перестановок", sw)

def qsort(x, nstart, nend):
    global srav, swap1
    if nstart >= nend:
        return
    L = nstart
    R = nend
    X = x[(L + R) // 2]
    while L <= R:
        srav += 1
        while x[L] < X:
            srav += 1
            L += 1
        while x[R] > X:
            R -= 1
            srav += 1
        srav += 1
        if L <= R:
            c = x[L]
            x[L] = x[R]
            x[R] = c
            L += 1
            R -= 1
            swap1 += 1
    qsort(x, nstart, R)
    qsort(x, L, nend)

def QS(x, start, finish):
    qsort(x, start, finish)
    print("Быстрая сортировка")
    print("Количество сравнений", srav, "Количество перестановок", swap1)

a = []
print("ПОЛНОСТЬЮ СОРТИРОВАННЫЙ СПИСОК")
for i in range(0, 10):
    a.append(i)
arrpuz = a.copy()
arrvstavka = a.copy()
arrvibor = a.copy()
arrqs = a.copy()
arrshell = a.copy()
puz(arrpuz)
vstavka1(arrvstavka)
vibor(arrvibor)
QS(arrqs, 0, len(arrqs) - 1)
shell(arrshell)
print()
print("ЧАСТИЧНО СОРТИРОВАННЫЙ СПИСОК")
b = []
for i in range(0, 5):
    b.append(i)
for i in range(10, 5, -1):
    b.append(i)
arrpuz = b.copy()
arrvstavka = b.copy()
arrvibor = b.copy()
arrqs = b.copy()
arrshell = b.copy()
puz(arrpuz)
vstavka1(arrvstavka)
vibor(arrvibor)
QS(arrqs, 0, len(arrqs) - 1)
shell(arrshell)
print()
print("ПОЛНОСТЬЮ НЕСОРТИРОВАННЫЙ СПИСОК")
c = []
for i in range(10,0,-1):
    c.append(i)
arrpuz = c.copy()
arrvstavka = c.copy()
arrvibor = c.copy()
arrqs = c.copy()
arrshell = c.copy()
puz(arrpuz)
vstavka1(arrvstavka)
vibor(arrvibor)
QS(arrqs, 0, len(arrqs) - 1)
shell(arrshell)


import logging
from math import ceil
from fuzzywuzzy import fuzz
import numpy as np
from aiogram.dispatcher import FSMContext
from aiogram.dispatcher.filters import Text
from aiogram.types import Message, ReplyKeyboardRemove, InlineKeyboardMarkup, InlineKeyboardButton

from keyboards.default import main_menu, cancel_search
from loader import dp
from states.main_states import UserStates
from utils.db_api.models import DBCommands

db = DBCommands()

logging.basicConfig(format=u'%(filename)s [LINE:%(lineno)d] '
                           u'#%(levelname)-8s [%(asctime)s]  %(message)s',
                    level=logging.INFO)

stop_word_list = ['в', 'до', 'без', 'безо', 'во', 'за', 'из', 'из-за', 'к',
                  'ко', 'на', 'о', 'об', 'от', 'по', 'при', 'про', 'у', 'at',
                  'in', 'of', 'to', 'as', 'со', 'с', 'и']

# Таблица meme_datatable - Columns
# id = db.Column(db.Integer, db.Sequence('meme_id_seq'), primary_key=True)
# meme_name = db.Column(db.String, unique=True)
# describe = db.Column(db.String)
# pic_href = db.Column(db.String)
# meme_href = db.Column(db.String)

async def search(msg, all_memes):
    process_msg = msg.text.split()
    for word in process_msg:
        if word in stop_word_list:
            process_msg.remove(word)
    first_letters_msg = [word[:2].lower() for word in process_msg]
    process_msg_use = ''.join(process_msg)
    memes_name_list = [i.meme_name for i in all_memes]
    list_of_memes = []
    set_of_memes = set()
    for mem in memes_name_list:
        process_mem = mem.split()
        first_letters_mem = [word[:2].lower() for word in process_mem]
        result = list(set(first_letters_msg) & set(first_letters_mem))
        result_fuzz = fuzz.WRatio(process_msg_use, mem)
        if result_fuzz > 60 and result:
            set_of_memes.update({(mem, result_fuzz)})
            list_of_memes.append((mem, result_fuzz))
    for word in process_msg:
        set_of_memes.update({(mem, fuzz.WRatio(process_msg_use, mem)) for mem in filter(
            lambda memes: word.lower() in memes, memes_name_list
        )})
        set_of_memes.update({(mem, fuzz.WRatio(process_msg_use, mem)) for mem in filter(
            lambda memes: word.title() in memes, memes_name_list
        )})
    list_of_memes = [meme for meme in set_of_memes if meme[1] >= 60]
    sorted_list_of_memes = [res[0] for res in sorted(list_of_memes, key=lambda x: x[1], reverse=True)]
    return sorted_list_of_memes

@dp.message_handler(Text(equals=['Поиск мема 🗿']))
async def wait_for_mem_request(message: Message):
    # LOG you!!!!!!!
    logging.info(f'from: {message.chat.first_name}, text: {message.text}')
    # LOG you!!!!!!!
    await UserStates.search_input_key_words.set()
    await message.answer('Введите ключевые слова для поиска мема в базе',
                         reply_markup=ReplyKeyboardRemove())

@dp.message_handler(Text, state=UserStates.search_input_key_words)
async def search_and_show_results(message: Message, state: FSMContext):
    # LOG you!!!!!!!
    logging.info(f'from: {message.chat.first_name}, text: {message.text}')
    # LOG you!!!!!!!
    if message.text == 'Показать результаты поиска':
        data_from_state = await state.get_data()
        await message.answer(data_from_state.get('all_result_messages')[data_from_state.get('page')],
                             reply_markup=data_from_state.get('keyboards')[data_from_state.get('page')])
    elif message.text == 'Отмена':
        await state.finish()
        await message.answer('Отменено',
                             reply_markup=main_menu)
    else:
        await state.update_data(
            {'result_mem_search_by_page': {1: {}},
             'keyboards': {1: {}},
             'all_result_messages': {1: {}},
             'page': 1}
        )
        data_from_state = await state.get_data()
        memes_data_objects = await db.all_meme()
        result_search = await search(msg=message, all_memes=memes_data_objects)
        if not result_search:
            await message.answer('Ничего не найдено по запросу. '
                                 'Попробуй написать еще раз свой запрос, но другими словами.',
                                 reply_markup=cancel_search)
        elif len(result_search) <= 5:
            result_kb = InlineKeyboardMarkup(row_width=5)
            result_message = ''
            # result_mem_search_by_page.update({1: {}})
            for num, res in enumerate(result_search, 1):
                res_button = InlineKeyboardButton(str(num),
                                                  callback_data=f"res_{num}:{num}")
                data_from_state.get('result_mem_search_by_page')[1].update({str(num): res})
                result_kb.insert(res_button)
                result_message += f'{num}. {res}\n\n'
            data_from_state.get('keyboards').update({1: result_kb})
            data_from_state.get('all_result_messages').update({1: result_message})
            # all_result_messages.update({1: result_message})
            await state.update_data(data_from_state)
            await message.answer('Результат поиска:',
                                 reply_markup=cancel_search)
            await message.answer(data_from_state.get('all_result_messages')[data_from_state.get('page')],
                                 reply_markup=data_from_state.get('keyboards')[data_from_state.get('page')])
        elif len(result_search) > 5:
            data_from_state.get('result_mem_search_by_page').clear()
            number_of_pages = ceil(len(result_search) / 5)
            rule_np_list = []
            for i in range(number_of_pages):
                if i == 0:
                    rule_np_list.append(5)
                    continue
                rule_np_list.append(rule_np_list[i - 1] + 5)
            search_results_by_pages = np.array_split(result_search, rule_np_list)
            # Клавиатура для каждой страницы
            keyboards_inside = {}
            for page_num in range(number_of_pages):
                if page_num == 0:
                    keyboards_inside.update(
                        {page_num + 1: InlineKeyboardMarkup(row_width=5, inline_keyboard=[
                            [InlineKeyboardButton('➡', callback_data='next_page')]]
                                                                )})
                    continue
                if page_num == list(range(number_of_pages))[-1]:
                    keyboards_inside.update(
                        {page_num + 1: InlineKeyboardMarkup(row_width=5, inline_keyboard=[
                            [InlineKeyboardButton('⬅', callback_data='previous_page')]]
                                                                )})
                    continue
                keyboards_inside.update({page_num + 1: InlineKeyboardMarkup(row_width=5, inline_keyboard=[
                    [InlineKeyboardButton('⬅', callback_data='previous_page')],
                    [InlineKeyboardButton('➡', callback_data='next_page')]])})
            # Наверное стоит это как то в функцию захуярить, я валяюсь
            for page_num, page in enumerate(range(number_of_pages), 1):
                data_from_state.get('result_mem_search_by_page').update({page_num: {}})
                # Индивидуально для каждой страницы должно быть
                result_message = ''
                for num, res in enumerate(list(search_results_by_pages)[page], 1):
                    res_button = InlineKeyboardButton(str(num), callback_data=f"res_{num}:{num}")
                    data_from_state.get('result_mem_search_by_page')[page_num].update({str(num): res})
                    if num == 1:
                        keyboards_inside[page_num].add(res_button)
                        result_message += f'{num}. {res}\n\n'
                        continue
                    keyboards_inside[page_num].insert(res_button)
                    result_message += f'{num}. {res}\n\n'
                result_message += f'Страница {page_num} из {number_of_pages}'  # current_page
                data_from_state.get('all_result_messages').update({page_num: result_message})
            data_from_state.get('keyboards').update(keyboards_inside)
            await state.update_data(data_from_state)
            await message.answer('Результат поиска:',
                                 reply_markup=cancel_search)
            # Должно быть с первой страницы
            await message.answer(data_from_state.get('all_result_messages')[data_from_state.get('page')],
                                 reply_markup=data_from_state.get('keyboards')[data_from_state.get('page')])


# # Hight Resolution image shape is 96

# supressing tensorflow warning, info, or things.
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

import tensorflow as tf
from tensorflow.keras.models import load_model
import argparse
from mirnet import get_enhancement_model, get_denoising_model, get_super_resolution_model

parser = argparse.ArgumentParser()

parser.add_argument('--scale_factor', type=int, default=4)
parser.add_argument('--saved_model_path', type=str, default="checkpoint/super_resolution/best_model.h5")
parser.add_argument('--tflite_model_path', type=str, default='checkpoint/super_resolution/best_model.tflite')
parser.add_argument('--mode', type=str, default='super_resolution')
parser.add_argument('--num_rrg', type=int, default=3)
parser.add_argument('--num_mrb', type=int, default=1)
parser.add_argument('--num_channels', type=int, default=64)
parser.add_argument('--optimize', type=bool, default=False)

args = parser.parse_args()

def main():
    if args.mode == "super_resolution":
        model = get_super_resolution_model(
            num_rrg=args.num_rrg,
            num_mrb=args.num_mrb,
            num_channels=args.num_channels,
            scale_factor=args.scale_factor
        )

    if args.mode == "denoise":
        model = get_denoising_model(
            num_rrg=args.num_rrg,
            num_mrb=args.num_mrb,
            num_channels=args.num_channels
        )

    else:
        model = get_enhancement_model(
            num_rrg=args.num_rrg,
            num_mrb=args.num_mrb,
            num_channels=args.num_channels
        )

    model.load_weights(args.saved_model_path)

    # trained_model = load_model(args.saved_model_path, custom_objects={'tf': tf})
    # weights = trained_model.get_weights()
    # model.set_weights(weights)

    converter = tf.lite.TFLiteConverter.from_keras_model(model)

    if args.optimize:
        converter.optimizations = [tf.lite.Optimize.DEFAULT]

    tflite_model = converter.convert()
    with open(args.tflite_model_path, 'wb') as f:
        f.write(tflite_model)

    print("Tflite model is saved at: ", args.tflite_model_path)

if __name__ == '__main__':
    main()


# coding: utf-8

# In[11]:

import pandas as pd,numpy as np

# In[12]:

def event_dic(data,lables):#将数据按值根据离散类进行统计(为woe的计算做准备)
    val=np.unique(data)
    event_total = lables.sum()
    non_event_total = len(lables) - event_total
    eve_dic={}
    for i in val:
        event=lables[np.where(data==i)]
        event_count=event.sum()
        non_event_count=len(event)-event_count
        rate_event = 1.0 * event_count / event_total
        rate_non_event = 1.0 * non_event_count / non_event_total
        eve_dic[i]=[rate_event,rate_non_event]
    return eve_dic

# In[13]:

def woe_iv(data,lables,woe_min=-20,woe_max=20):#根据数据,计算woe与iv值
    woe_dic,iv_dic={},{}
    for col in data.columns:
        woe_dic1={}
        iv=0
        col_dic=event_dic(data[col],lables)
        print('='*100)
        for k,(rate_event, rate_non_event) in col_dic.items():
            if rate_event == 0:
                woek = woe_min
            elif rate_non_event == 0:
                woek = woe_max
            else:
                woek = np.log(rate_event / rate_non_event)
            woe_dic1[k]=woek
            iv += (rate_event - rate_non_event) * woek
        woe_dic[col]=woe_dic1
        iv_dic[col]=iv
        col_table=pd.DataFrame([woe_dic1],index=['woe'])
        print(col_table.T)
    return woe_dic,iv_dic

# In[14]:

#woe 编码
def data_woe_encode(data,woe_dic):
    data_woe=pd.DataFrame(np.zeros(data.shape),columns=data.columns)
    for col in data.columns:
        col_dic=woe_dic[col]
        data_woe[col]=data[col].map(col_dic)
    return data_woe


import numpy as np
import tensorflow as tf
import pandas as pd
import config
import os
import pickle
import pandas as pd
def preprocess(path = "train_data_raw",shuffle = False):
    np.random.seed(1234)
    dataset = []
    labelset = []
    close_price = []
    filtered = 0
    for root, dirs, files in os.walk(path):
        for file in files:
            df = np.array(pd.read_csv(os.path.join(root,file),sep = '\t'))
            data = df[:,3:6][::-1].astype(np.float32)
            for i in range(config.SEQ_LEN,data.shape[0]):
                if (data[i,-1] - data[i-1,-1])/(1e-9+data[i-1,-1]) >=  0.0055 or (data[i,-1] - data[i-1,-1])/(1e-9+data[i,-1]) <=  -0.005:
                    dataset.append(data[i-config.SEQ_LEN:i])
                    label = np.array([np.array([1,0]) if data[j+1,-1] > data[j,-1] else np.array([0,1])
                             for j in range(i-config.SEQ_LEN,i)])
                    labelset.append(np.array(label))
                else:
                    filtered += 1
    labelset = np.array(labelset)
    dataset = np.array(dataset)
    if shuffle:
        ind = np.arange(dataset.shape[0])
        np.random.shuffle(ind)
        dataset = dataset[ind]
        labelset = labelset[ind]
    return dataset, labelset
    #         f = open(os.path.join(root, file_))
    #         for line in f:
    #             count += 1
    #             split_line = line.split('\t')
    #             data.append(np.array([float(i) for i in split_line[3:6]]))
    #             close_price.append(float(split_line[5]))
    #             if data.__len__() >= config.SEQ_LEN:
    #                 dataset.append(np.array(data))
    #                 data.remove(data[0])
    #                 data = []
    # close_price = np.array(close_price)
    # for i in range(close_price.__len__()):
    #     if(i == close_price.__len__() - 1):
    #         label.append(np.array([0,1]))
    #     else:
    #         if(close_price[i+1] > close_price[i]):
    #             label.append(np.array([1,0]))
    #         else:
    #             label.append(np.array([0,1]))
    #     if label.__len__() >= config.SEQ_LEN:
    #         labelset.append(np.array(label))
    #         label.remove(label[0])
            # label = []
    # for i in range(close_price.__len__()):
    #     if(i == close_price.__len__() - 1):
    #         label.append(np.array([1]))
    #     else:
    #         label.append(close_price[i + 1])
    #     if label.__len__() >= config.SEQ_LEN:
    #         labelset.append(np.array(label))
    #         label.remove(label[0])
            # label = []

if __name__ == "__main__":
    # dataset, labelset = preprocess(path = "train_data_raw",shuffle=True)
    # f = open("dataset_train",'wb')
    # pickle.dump(dataset,f)
    # f.close()
    # f = open("labelset_train",'wb')
    # pickle.dump(labelset,f)
    # f.close()
    #
    # dataset, labelset = preprocess(path="test_data_raw", shuffle=False)
    # f = open("dataset_test", 'wb')
    # pickle.dump(dataset, f)
    # f.close()
    # f = open("labelset_test", 'wb')
    # pickle.dump(labelset, f)
    # f.close()

    dataset, labelset = preprocess(path="dev_data_raw", shuffle=False)
    f = open("dataset_dev", 'wb')
    pickle.dump(dataset, f)
    f.close()
    f = open("labelset_dev", 'wb')
    pickle.dump(labelset, f)
    f.close()


from bottle import request, Bottle, abort, template, static_file
from gevent.pywsgi import WSGIServer
from geventwebsocket import WebSocketError
from geventwebsocket.handler import WebSocketHandler

from pymongo import MongoClient

import json
import gevent

from socketio import socketio_manage
from socketio.server import SocketIOServer
from socketio.namespace import BaseNamespace

app = Bottle()

db = MongoClient('mongodb', 27017)

class SocketIOApp(object):
    """Stream sine values"""
    def __call__(self, environ, start_response):
        if environ['PATH_INFO'].startswith('/socket.io'):
            socketio_manage(environ, {'': ChatNamespace});

class ChatNamespace(BaseNamespace):
    _registry = {}

    def initialize(self):
        self._registry[id(self)] = self

    def disconnect(self, *args, **kwargs):
        del self._registry[id(self)]
        super(ChatNamespace, self).disconnect(*args, **kwargs)

    def on_login(self, message):
        pass

    def on_chat(self, message):
        db.bottle.messages.insert_one(json.loads(message));
        self._broadcast('chat', message)

    def _broadcast(self, event, message):
        for s in self._registry.values():
            s.emit(event, message)

@app.route('/static/<filepath:path>')
def server_static(filepath):
    return static_file(filepath, root='/app')

@app.route('/')
def index():
    return template('index')

@app.route('/history')
def getHistory():

    cursor = db.bottle.messages.find()

    messages = []

    for message in cursor:
        messages.append({'name': message['name'], 'message': message['message']})

    return {'messages' : messages}

# setup server to handle webserver requests
http_server = WSGIServer(('0.0.0.0', 8080), app, handler_class=WebSocketHandler)

# setup server to handle websocket requests
sio_server = SocketIOServer(
    ('0.0.0.0', 9999), SocketIOApp(),
    namespace="socket.io",
    policy_server=False
)

gevent.joinall([
    gevent.spawn(http_server.serve_forever),
    gevent.spawn(sio_server.serve_forever)
])


def load_data(file):
    data = open(file, 'r')
    sections = []
    for line in data:
        stripped_line = line.strip().split(",")
        temp_section = []
        for section in stripped_line:
            range = section.split("-")
            temp_section.append(int(range[0]))
            temp_section.append(int(range[1]))
        sections.append(temp_section)

    data.close()
    return sections

if __name__ == "__main__":
    result = load_data("data_4")

    full_overlapping_count = 0
    partial_overlapping_count = 0
    for entry in result:
        if all(e in range(entry[0], entry[1]+1) for e in range(entry[2], entry[3]+1)): # note ranges are not inclusive at max
            full_overlapping_count += 1
        elif all(e in range(entry[2], entry[3]+1) for e in range(entry[0], entry[1]+1)):
            full_overlapping_count += 1

        if entry[0] in range(entry[2], entry[3]+1) or entry[1] in range(entry[2], entry[3]+1):
            partial_overlapping_count += 1
        elif entry[2] in range(entry[0], entry[1]+1) or entry[3] in range(entry[0], entry[1]+1):
            partial_overlapping_count += 1

    print(f"full overlapping = {full_overlapping_count} partial overlapping = {partial_overlapping_count}")


from torch import nn
import torch.nn.functional as F

class ConvNet(nn.Module):
    def __init__(self):
        super(ConvNet, self).__init__()
        self.conv1 = nn.Conv2d(1, 16, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(16, 32, 3)
        self.fc1 = nn.Linear(32 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        if x.ndim != 4:
            raise ValueError('Expected input to a 4D tensor')
        if x.shape[1] != 1 or x.shape[2] != 28 or x.shape[3] != 28:
            raise ValueError('Expected each sample to have shape [1, 28, 28]')
        # -> n, 3, 32, 32
        x = self.pool(F.relu(self.conv1(x)))  # -> n, 16, 12, 12
        x = self.pool(F.relu(self.conv2(x)))  # -> n, 32, 5, 5
        x = x.view(-1, 32 * 5 * 5)            # -> n, 400
        x = F.relu(self.fc1(x))               # -> n, 120
        x = F.relu(self.fc2(x))               # -> n, 84
        x = self.fc3(x)                       # -> n, 10
        return x


#!/usr/bin/env python

def tsv2pdm(tsv_file_path):
    """Makes a Python data model from as tsv file. The first line is treated as a header. The rest of the table becomes an array of dicts, keyed on the header. (This data model follows the same pattern as dict_cursor making switching between a DB representation and simple TSVs easy as long as TSV headers follow the same pattern as DB column headers."""
    tsv_file_obj = open(tsv_file_path, "r")
    hstat = 0
    headers = []
    tab = []
    for line in tsv_file_obj:
        cline = line.rstrip("\n")
        row = {}
        if hstat == 0:
            headers = cline.split("\t")
            hstat = 1
        else:
            content = cline.split("\t")
            i = 0
            for head in headers:
                row[head]=content[i]
                i += 1
            tab.append(row)
    tsv_file_obj.close()
    return tab

def dictList2row_column_cell(tab_dictList, key_row):
    """Turns a list of table represented as a list of dicts into a dict of dicts keyed on the contents of a specified key row."""
    row_column_cell = {}

    for d in tab_dictList:
        row_key = d[key_row]
        for column_key in d:
            if row_key not in row_column_cell:
                row_column_cell[row_key] = {}
            row_column_cell[row_key][column_key] = d[column_key]
    return row_column_cell

def blank_tree_node_with_dom():
	return {u'extId': [], u'nodeId': u'', u'nodeState': {u'open': u'false', u'selected': u'false'}, u'domainData': {u'domainSelected': u'false', u'domainColour': [], u'domainCentre': [], u'domainId': u''}, u'name': u''}

def nodeIdList(treeContent):
    """Returns a list of nodeIds as integers from a treeContent JSON"""
    list = []
    for node in treeContent:
        list.append(int(node['nodeId']))
    return list

def populate_node(node, row_dict):
    """Takes a treeContent node as first arg and a dictionary for the correct row to populate it (making sure these match is the job of the code that calls this function."""
    ### Strip any existing domain data
#    if not node['extId']:
#        node['extId'].append(row_dict['oboId'])
    ### Populate domain data from table.
    node['name'] = row_dict['name']
    if row_dict['domainId']:
        node['domainData']['domainId']=row_dict['domainId']
    if row_dict['domainColour']:
        color = row_dict['domainColour'].split(",")
        color_float = []
        for c in color:
            color_float.append( float(c) )
        node['domainData']['domainColour'] = color_float
    if row_dict['domainCentre']:
        centre = row_dict['domainCentre'].split(",")
        centre_float = []
        for c in centre:
            centre_float.append( float(c) )
        node['domainData']['domainCentre'] = centre_float

def treeContent_remove_blanks(treeContent):
    to_clean = ['domainId', 'domainColour', 'domainCentre']
    for node in treeContent:
        for k in to_clean:
            if (k in node['domainData']):
                if (not node['domainData'][k]):
                    node['domainData'].pop(k)

def update_tree_jsons_from_tab(tab, treeContent, treeStructure):
    # Starting point is:
    ## 1. a table of domainIDs and mappings to FBbt + (potentially) domain centres and colours.
    ## 2. a treeContent file which needs new domain IDs for the domains it has (idenitfied by FBbt), + new domains added (Identified by there being no domain with this FBbt in the existing json file).
    # Make lookup list of FBbt in tree
    oboId_in_tc = []
    for node in treeContent:
         oboId_in_tc.append(node['extId'][0])
    # Roll lookup dict of tab row by FBbt.
    oboId_column_cell =  dictList2row_column_cell(tab, 'oboId')
    oboId_in_tab = oboId_column_cell.keys()
    toReplace = set(oboId_in_tab) & set(oboId_in_tc)
    toAdd = set(oboId_in_tab) - set(oboId_in_tc)

    # Update existing nodes
    for node in treeContent:
        # Strip potentially clashing domainData from all nodes
        if 'domainId' in node['domainData']:
            node['domainData'].pop('domainId')
        if 'domainCentre' in node['domainData']:
            node['domainData'].pop('domainCentre')
        #  Update nodes for which new domainData is available
        if node['extId'][0] in toReplace:
            populate_node(node, oboId_column_cell[node['extId'][0]])

    nodeIds = nodeIdList(treeContent)
    nodeId = 0 # Making nodeId integer to so it can be incremented.  Convert to string to use.
    # add new nodes
    for oboId in toAdd:
        node =  blank_tree_node_with_dom()
        populate_node(node, oboId_column_cell[oboId])
        # generate new nodeId
        while nodeId in nodeIds:
            nodeId +=1
        nodeIds.append(nodeId)
        node['nodeId'] = str(nodeId)
        treeStructure['node']['children'].append({ 'node': {'nodeId': str(nodeId)}})
        treeContent.append(node)


import transformers
from base import smart_tokenizer, SupervisedDataset, DataCollatorForSupervisedDataset
DEFAULT_PAD_TOKEN = "[PAD]"
DEFAULT_EOS_TOKEN = "</s>"
DEFAULT_BOS_TOKEN = "</s>"
DEFAULT_UNK_TOKEN = "</s>"

PROMPT_DICT = {
    "prompt_input": (
        "Below is an instruction that describes a task, paired with an input that provides further context. "
        "Write a response that appropriately completes the request.\n\n"
        "### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:"
    ),
    "prompt_no_input": (
        "Below is an instruction that describes a task. "
        "Write a response that appropriately completes the request.\n\n"
        "### Instruction:\n{instruction}\n\n### Response:"
    ),
}

def make_prompt(example, tokenizer):
    prompt_input, prompt_no_input = PROMPT_DICT["prompt_input"], PROMPT_DICT["prompt_no_input"]
    source = prompt_input.format_map(example) if example.get(
        "input", "") != "" else prompt_no_input.format_map(example)
    target = f"{example['output']}{tokenizer.eos_token}"
    return source, target

def get_dataset_and_collator(tokenizer):
    # tokenizer.model_max_length = max_seq_lengh
    smart_tokenizer(
        special_tokens_dict=dict(pad_token=DEFAULT_PAD_TOKEN),
        tokenizer=tokenizer
    )
    train_dataset = SupervisedDataset(
        tokenizer=tokenizer, data_path="./alpaca_data.json", make_prompt=make_prompt, num_proc=40)
    data_collator =  DataCollatorForSupervisedDataset(tokenizer=tokenizer)
    return dict(train_dataset=train_dataset, eval_dataset=None, data_collator=data_collator)

if __name__ == "__main__":
    tokenizer = transformers.LlamaTokenizer.from_pretrained(
        "decapoda-research/llama-7b-hf",
        padding_side="right",
        use_fast=False,
    )
    r = get_dataset_and_collator(tokenizer)


import sys
import optuna
import pandas as pd
import numpy as np
from model import *

exp_id = None
lat_bound = None
config_meta = pd.read_csv('../benchmark/meta/config_meta.csv')

best_reward = -100
best_config = None

progress = []
last_thr = 0

def objective(trial):
    global last_thr, best_reward, best_config, progress
    rec_config = np.array([])
    for index, row in config_meta.iterrows():
        param = row['name'].split('->')[1]
        rec_config = np.append(rec_config, trial.suggest_float(param, 0, 1))
    reward, thr, lat = get_reward(exp_id, rec_config, last_thr, lat_bound)
    last_thr = thr
    # log step
    if reward > best_reward:
        best_reward = reward
        best_config = rec_config

    record = {
        'best_config': best_config,
        'current_config': rec_config,
        'reward': reward,
        'throughput': thr,
        'latency': lat,
        'best_reward': best_reward,
        'latency_bound': lat_bound
    }
    progress.append(record)

    return -reward

def optimize():
    study = optuna.create_study()
    study.optimize(objective, n_trials=500, show_progress_bar=True)
    return study.best_params

if __name__ == "__main__":
    for alph in range(1, 2):
        exps = ['exp-%d' % i for i in range(9)]
        for exp in exps:
            progress = []
            best_reward = -100
            exp_id = exp
            def_config = (config_meta['default'] - config_meta['min']) / (config_meta['max'] - config_meta['min'])
            def_config = def_config.to_numpy()
            def_lat = get_lat(exp_id, def_config)
            lat_bound = def_lat / alph
            optimize()
            progress = pd.DataFrame(progress)
            progress.to_csv('results/optuna/progress_%s_%s.csv' % (exp_id, str(alph)), index=None)


n = int(input())
for _ in range(n):
    data = input()
    stack = []
    for i in data:
        if i == '(':
            stack.append('(')
        elif i == ')':
            if len(stack) != 0 and stack[-1] == '(':
                stack.pop()
            else:
                stack.append("exit")
                break

    if len(stack) == 0:
        print("YES")
    else:
        print("NO")


# All code related to emails is taken from collective.watcherlist
# https://github.com/collective/collective.watcherlist/blob/master/collective/watcherlist/utils.py

try:
    from email.utils import parseaddr, formataddr

    parseaddr, formataddr  # pyflakes
except ImportError:
    # BBB for python2.4 (Plone 3)
    from email.Utils import parseaddr, formataddr

from persistent.dict import PersistentDict
from persistent.list import PersistentList
from Products.CMFCore.utils import getToolByName
from Products.CMFPlone.utils import safe_unicode
from zope.component import getMultiAdapter

try:
    from zope.component.hooks import getSite

    getSite  # pyflakes
except ImportError:
    from zope.app.component.hooks import getSite

DEFAULT_CHARSET = "utf-8"

def get_charset():
    """Character set to use for encoding the email.

    If encoding fails we will try some other encodings.  We hope
    to get utf-8 here always actually.

    The getSiteEncoding call also works when portal is None, falling
    back to utf-8.  But that is only on Plone 4, not Plone 3.  So we
    handle that ourselves.
    """
    charset = None
    portal = getSite()
    if portal is None:
        return DEFAULT_CHARSET
    charset = portal.getProperty("email_charset", "")
    if not charset:
        charset = "utf-8"
    return charset

def su(value):
    """Return safe unicode version of value."""
    return safe_unicode(value, encoding=get_charset())

def get_mail_host():
    """Get the MailHost object.

    Return None in case of problems.
    """
    portal = getSite()
    if portal is None:
        return None
    request = portal.REQUEST
    ctrlOverview = getMultiAdapter((portal, request), name="overview-controlpanel")
    mail_settings_correct = not ctrlOverview.mailhost_warning()
    if mail_settings_correct:
        mail_host = getToolByName(portal, "MailHost", None)
        return mail_host

def get_mail_from_address():
    portal = getSite()
    if portal is None:
        return ""
    from_address = portal.getProperty("email_from_address", "")
    from_name = portal.getProperty("email_from_name", "")
    mfrom = formataddr((from_name, from_address))
    if parseaddr(mfrom)[1] != from_address:
        # formataddr probably got confused by special characters.
        mfrom = from_address
    return mfrom

def obj_to_pobj(o):
    """Return an object to a persistent object.
    Only works on list and dictionaries for now.
    """

    if isinstance(o, list):
        return list_to_plist(o)

    if isinstance(o, dict):
        return dict_to_pdict(o)

    return o

def list_to_plist(l):
    pl = PersistentList()
    for val in l:
        pl.append(obj_to_pobj(val))

    return pl

def dict_to_pdict(d):
    pd = PersistentDict()

    for k, v in d.items():
        pd[obj_to_pobj(k)] = obj_to_pobj(v)

    return pd


import inspect
import json
from os import getcwd, path
from typing import Dict, List

from .config_manager import BaseConfigManager
from .config_model import (ConfigData, ServerData, ServerGateData,
                           ServerRoleData)

class ServerGateDataFile(ServerGateData):
    @classmethod
    def from_json(cls, data: dict):
        # Data might be in the old format
        gate_enabled = data["gateEnabled"] if data.get(
            "gateEnabled") is not None else data.get("gate_enabled")
        allow_rejoin = data["allowRejoin"] if data.get(
            "allowRejoin") is not None else data.get("allow_rejoin")
        key_role_id = data["keyRoleId"] if data.get(
            "keyRoleId") is not None else data.get("key_role_id")
        if key_role_id == "" or key_role_id is None:
            key_role_id = "0"

        keyed_users: Dict[str, str] = dict()
        if data.get("keyedUser") is not None:
            keyed_users = data["keyedUsers"]
        elif data.get("keyed_users") is not None:
            keyed_users_raw = data.get("keyed_users")
            for key in keyed_users_raw:
                keyed_users[str(key)] = str(keyed_users_raw[key])

        return cls(bool(gate_enabled), bool(allow_rejoin), int(key_role_id), keyed_users)

    # def to_json(self):
    #     obj_json = dict()
    #     obj_json['allow_rejoin'] = self.allow_rejoin
    #     obj_json['gate_enabled'] = self.gate_enabled
    #     obj_json['key_role_id'] = self.key_role_id
    #     obj_json['keyed_users'] = self.keyed_users

class ServerRoleDataFile(ServerRoleData):
    @classmethod
    def from_json(cls, id: int, data: dict):
        # Data might be in the old format
        can_join = data["canJoin"] if data.get(
            "canJoin") is not None else data.get("can_join")

        commanders: List[int] = []
        if isinstance(data["commanders"], list):
            commanders = data["commanders"]
        elif isinstance(data["commanders"], dict):
            for key in data["commanders"].keys():
                if int(key) > 0:
                    commanders.append(int(key))

        return cls(id, data["name"], bool(can_join), commanders)

    @classmethod
    def from_json_list(cls, data: dict) -> Dict[int, ServerRoleData]:
        roles = {}
        for key in data.keys():
            if isinstance(key, int):
                id = key
            elif isinstance(key, str) and key.isdigit():
                id = int(str(key))
            else:
                continue

            value = data[key]
            roles.update({id: cls.from_json(id, value)})

        return roles

class ServerDataFile(ServerData):
    @ classmethod
    def from_json(cls, id, data: dict):
        # Data might be in the old format.
        greeting_message = data["greetingMessage"] if data.get(
            "greetingMessage") is not None else data.get("greeting_message")
        gate_data = data["gateData"] if data.get(
            "gateData") is not None else data.get("gate_data")
        role_data = data["roleData"] if data.get(
            "roleData") is not None else data.get("role_data")

        return cls(id, str(data["name"]), str(greeting_message), ServerGateDataFile.from_json(gate_data), ServerRoleDataFile.from_json_list(role_data))

    @classmethod
    def from_json_list(cls, data: dict) -> Dict[int, ServerData]:
        servers = {}
        for key in data.keys():
            if isinstance(key, int):
                id = key
            elif isinstance(key, str) and key.isdigit():
                id = int(str(key))
            else:
                continue
            value = data[key]
            servers.update({id: ServerDataFile.from_json(id, value)})
        return servers

class ObjectEncoder(json.JSONEncoder):
    def default(self, obj):
        if hasattr(obj, "to_json"):
            return self.default(obj.to_json())
        elif hasattr(obj, "__dict__"):
            d = dict(
                (key, value)
                for key, value in inspect.getmembers(obj)
                if not key.startswith("_")
                and not inspect.isabstract(value)
                and not inspect.isbuiltin(value)
                and not inspect.isfunction(value)
                and not inspect.isgenerator(value)
                and not inspect.isgeneratorfunction(value)
                and not inspect.ismethod(value)
                and not inspect.ismethoddescriptor(value)
                and not inspect.isroutine(value)
            )
            return self.default(d)
        return obj

class FileConfigManager(BaseConfigManager):
    def readConfig(self):
        try:
            print(getcwd())
            f = open('data/config.json', 'r')
            print(path.realpath(f.name))
            servers = ServerDataFile.from_json_list(json.load(f))
            self.config_data: ConfigData = {
                server_id: servers[server_id] for server_id in servers.keys()}
        except Exception as ex:
            self.config_data: ConfigData = {}

        return

    def writeConfig(self):
        f = open('data/config.json',  'w')

        json.dump(self.config_data, f, cls=ObjectEncoder, indent=2)


from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from sys import platform
import time
import shutil
import json
import os
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait

###########################################################################################
# Selects the correct chromedriver to run depending on the OS
###########################################################################################

if platform == "darwin":
    driver_path = "./chromedriver"
    # OS X
elif platform == "win32":
    driver_path = "./chromedriver.exe"
    # Windows...
else:
    driver_path = "ERROR"

print(platform)

###########################################
# GLOBAL DRIVER OPTIONS AND PATH
# Current options applied: Hides pop-up for browser
###########################################
chrome_options = Options()
chrome_options.headless = True
service = Service(executable_path=driver_path)

class diningHallWebScrapper:
    def __init__(self):
        self.isFirstInput = True  # Used to ensure that first JSON entry of a food entry does not contain a , at the start
        self.all_foods = {}  # Contains all the food items with their properties
        self.switch_for_dining_hall = {
            'College Nine/John R. Lewis Dining Hall': 'clgnine_johnlewis',
            'Cowell/Stevenson Dining Hall': 'cowell_steve',
            'Crown/Merrill Dining Hall': 'crown_merill',
            'Porter/Kresge Dining Hall': 'porter_kresge'
        }
        self.dining_halls = ['College Nine/John R. Lewis Dining Hall', 'Cowell/Stevenson Dining Hall',
                    'Crown/Merrill Dining Hall', 'Porter/Kresge Dining Hall']

        ###########################################################################################
        # Setup for the Webscrapper.
        # Sets start page to https://nutrition.sa.ucsc.edu/
        ###########################################################################################

        global service
        #self.driver = webdriver.Chrome(service=service, options=chrome_options) #Comment if (driver = ...) is uncommented
        self.driver = webdriver.Chrome(service=service) #Uncomment to see browser

        self.driver.get("https://nutrition.sa.ucsc.edu/")  # Starting website

    def getDiningHallUrl(self):
        return self.driver.current_url

    def startDiningHallScrapper(self):

        for curr_dining_hall in self.dining_halls:
            try:
                self.moveToNutritionPage(curr_dining_hall)
                self.webScrapAllMealsTimes(curr_dining_hall)
            except:
                print("Failed to extract " + curr_dining_hall + " dining hall")
                continue

        print("Total menu items scrapped: ", str(len(self.all_foods)))
        # driver.get("https://nutrition.sa.ucsc.edu/")
        self.endDiningHallScrapper()

    def endDiningHallScrapper(self):
        self.driver.quit()

    def createFoodResultsJson(self):
        try:
            self.removeCurrentJSON()
            self.convertToJSON(self.all_foods)
            self.moveToResultsFolder()
            return True
        except:
            return False

    def checkFoodResultsPath(self):
        if os.path.exists("./results/food_results.json"):
            return True
        return False

    def removeCurrentJSON(self):
        if self.checkFoodResultsPath():
            os.remove("./results/food_results.json")

    def moveToResultsFolder(self):
        shutil.move('./food_results.json', './results')

    # Converts the list of dictionaries for split_foods into a JSON file based on the dining hall
    def convertToJSON(self,all_foods):
        file_name = "./food_results.json"
        ofstrm = open(file_name, 'w+')
        size = len(all_foods)
        count = 0

        ofstrm.write('[\n')
        if self.isFirstInput:
            self.isFirstInput = False
        else:
            ofstrm.write(",")
        for key, value in all_foods.items():
            json.dump(value, ofstrm, indent=4)
            count += 1
            if count == size:
                ofstrm.write("\n")
            if count != size:
                ofstrm.write(",\n")
        ofstrm.write(']')
        ofstrm.close()

    def resetDiningHallPage(self):
        self.driver.get("https://nutrition.sa.ucsc.edu/")  # Resets the page incase url is different

    # Starting from the main dining hall page, the driver moves to the nutrition page for all the food of that dining hall
    def moveToNutritionPage(self,dining_hall):
        try:
            self.resetDiningHallPage()

            dining_hall_link = WebDriverWait(self.driver, timeout=3).until(
                lambda d: d.find_element(By.LINK_TEXT, dining_hall))
            dining_hall_link.click()
            return True
        except:
            return False

    def __getMealTimeLinks(self):
        # Setup: Obtaining len and name of meal times(Breakfast, Lunch, Dinner, Late_Night)
        nutrition_pages = WebDriverWait(self.driver, timeout=3).until(
            lambda d: d.find_elements(By.CLASS_NAME, "shortmenunutritive"))

        meal_time_elems = WebDriverWait(self.driver, timeout=3).until(
            lambda d: d.find_elements(By.CLASS_NAME, "shortmenumeals"))

        meal_times = []
        for meal_time in meal_time_elems:
            meal_times.append(meal_time.text)
        return meal_times

    # Calls each meal time for the specific dining hall and then calls getNutritionFacts() to start scrapping all the data
    def webScrapAllMealsTimes(self,dining_hall):
        try:
            # Setup: Obtaining len and name of meal times(Breakfast, Lunch, Dinner, Late_Night)
            meal_times = self.__getMealTimeLinks()
            total_meal_times = len(meal_times)

            # Webscrapper starts below this point using the data from setup:
            for curr_meal_type in range(total_meal_times):
                nutrition_pages = WebDriverWait(self.driver, timeout=3).until(
                    lambda d: d.find_elements(By.CLASS_NAME, "shortmenunutritive"))
                nutrition_pages[curr_meal_type].click()
                curr_meal_time = meal_times[curr_meal_type]
                if curr_meal_time == 'Late Night':  # Changed to Late_Night to avoid space conflicts
                    curr_meal_time = 'Late_Night'
                self.__getNutritionFacts(dining_hall, curr_meal_time)
            return True
        except:
            return False

    def __getFoodType(self, raw_food_type_text):
        REMOVE_FRONT_DASH = 3
        REMOVE_BACK_DASH = len(raw_food_type_text) - 3
        parsed_food_type = raw_food_type_text[REMOVE_FRONT_DASH:REMOVE_BACK_DASH]
        return parsed_food_type

    def __init_basic_food_obj(self,current_food_type,food_name):
        food_obj = {
            'food_type': current_food_type,
            'food_name': food_name,
            'dining_hall': {
                'clgnine_johnlewis': False,
                'cowell_steve': False,
                'crown_merill': False,
                'porter_kresge': False
            },
            'meal_time': {
                'Breakfast': False,
                'Lunch': False,
                'Dinner': False,
                'Late_Night': False
            }
        }
        self.all_foods[food_name] = food_obj
        return food_obj

    ###########################################################################################
    # Get functions for all nutrient/tag info about a specific food item
    # All get functions are called within singleNutrientFacts()
    ###########################################################################################

    # Used as an helper for the getter functions. Returns elem based on XPATH given
    def __searchNutritionFactBoxHTML(self,xpath_val):
        try:
            result_elem = WebDriverWait(self.driver, timeout=0.5).until(
                lambda d: d.find_element(By.XPATH, xpath_val))
            return result_elem.text.lstrip()
        except:
            return None

    def __getServing_size(self,curr_food_obj):
        curr_food_obj['serving_size'] = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[1]/td[1]/font[3]')

    def __getCalories(self,curr_food_obj):
        try:
            calories_text = WebDriverWait(self.driver, timeout=0.5).until(
                lambda d: d.find_element(By.XPATH, '/html/body/table[1]/tbody/tr/td/table/tbody/tr[1]/td[1]/font[4]'))

            calories_arr = calories_text.text.split(' ')

            NO_CALORIES_FOUND_BEYOND_2 = 2

            if len(calories_arr) < NO_CALORIES_FOUND_BEYOND_2:  # No Calories value
                curr_food_obj['calories'] = None
            else:
                curr_food_obj['calories'] = calories_arr[1]
        except:
            curr_food_obj['calories'] = None

    def __getTotalFat(self,curr_food_obj):
        curr_food_obj['total_fat'] = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[2]/td[1]/font[2]')

    def __getTotalFat_DV(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[2]/td[2]/font[1]/b')
        curr_food_obj['total_fat_DV'] = None if result == '' else result

    def __getSatFat(self,curr_food_obj):
        curr_food_obj['sat_fat'] = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[3]/td[1]/font[2]')

    def __getSatFat_DV(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[3]/td[2]/font[1]/b')
        curr_food_obj['sat_fat_DV'] = None if result == '' else result

    def __getTransFat(self,curr_food_obj):
        curr_food_obj['trans_fat'] = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[4]/td[1]/font[2]')

    def __getCholesterol(self,curr_food_obj):
        curr_food_obj['cholesterol'] = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[5]/td[1]/font[2]')

    def __getCholesterol_DV(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[5]/td[2]/font[1]/b')
        curr_food_obj['cholesterol_DV'] = None if result == '' else result

    def __getSodium(self,curr_food_obj):
        curr_food_obj['sodium'] = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[6]/td[1]/font[2]')

    def __getSodiumDV(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[6]/td[2]/font[1]/b')
        curr_food_obj['sodium_DV'] = None if result == '' else result

    def __getTotalCarbs(self,curr_food_obj):
        curr_food_obj['total_carbs'] = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[2]/td[3]/font[2]')

    def __getTotalCarbs_DV(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[2]/td[4]/font[1]/b')
        curr_food_obj['total_carbs_DV'] = None if result == '' else result

    def __getDietaryFiber(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[3]/td[3]/font[2]')
        curr_food_obj['dietary_fiber'] = '- - - g' if result is None else result

    def __getDietaryFiber_DV(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[3]/td[4]/font[1]/b')
        curr_food_obj['dietary_fiber_DV'] = None if result == '' else result

    def __getSugars(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[4]/td[3]/font[2]')
        curr_food_obj['sugars'] = '- - - g' if result is None else result

    def __getProtein(self,curr_food_obj):
        curr_food_obj['protein'] = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[5]/td[3]/font[2]')

    def __getIngredients(self,curr_food_obj):
        curr_food_obj['ingredients'] = self.__searchNutritionFactBoxHTML('/html/body/table[2]/tbody/tr/td/span[2]')

    def __getAllergens(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML('/html/body/table[3]/tbody/tr/td/span[2]')
        curr_food_obj['allergens'] = None if result == '' else result

    def __getVitamin_D(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[7]/td/table/tbody/tr/td[1]/table/tbody/tr/td/font[2]')
        curr_food_obj['vitamin_d'] = None if result == '' else result

    def __getCalcium(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[7]/td/table/tbody/tr/td[2]/table/tbody/tr/td/li/font[2]')
        curr_food_obj['calcium'] = None if result == '' else result

    def __getIron(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[7]/td/table/tbody/tr/td[3]/table/tbody/tr/td/li/font[2]')
        curr_food_obj['iron'] = None if result == '' else result

    def __getPotassium(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[7]/td/table/tbody/tr/td[4]/table/tbody/tr/td/li/font[2]')
        curr_food_obj['potassium'] = None if result == '' else result

    def __getFoodTags(self,curr_food_obj):
        tmp = {
            'Eggs': False,
            'Fish': False,
            'GlutenFree': False,
            'Dairy': False,
            'Peanuts': False,
            'Soy': False,
            'Vegan': False,
            'Vegetarian': False,
            'Pork': False,
            'Beef': False,
            'Halal': False,
            'Shellfish': False,
            'TreeNut': False

        }

        try:
            food_tag_elem = WebDriverWait(self.driver, timeout=0.5).until(
                lambda d: d.find_element(By.CLASS_NAME, 'labelwebcodesvalue'))

            all_tags = WebDriverWait(food_tag_elem, timeout=0.5).until(
                lambda d: d.find_elements(By.TAG_NAME, 'img'))

            curr_food_obj['food_tags'] = {

            }

            for curr_tag in all_tags:
                curr_tag_text = curr_tag.get_attribute('alt')
                if curr_tag_text == 'Egg/s':  # Egg/s changed to Eggs to avoid a possible URL path issue
                    tmp['Eggs'] = True
                elif curr_tag_text == 'Gluten Free' or curr_tag_text == 'Gluten-Free':
                    tmp['GlutenFree'] = True
                elif curr_tag_text == 'Tree Nut':
                    tmp['TreeNut'] = True
                else:
                    tmp[curr_tag_text] = True

            curr_food_obj['food_tags'] = tmp
        except:
            curr_food_obj['food_tags'] = tmp

    # Starting from the food item nutrition page, all nutrition facts are scrapped
    def singleNutritionFact(self,curr_food_obj):
        self.__getServing_size(curr_food_obj)
        self.__getCalories(curr_food_obj)
        self.__getTotalFat(curr_food_obj)
        self.__getSatFat(curr_food_obj)
        self.__getTransFat(curr_food_obj)
        self.__getCholesterol(curr_food_obj)
        self.__getSodium(curr_food_obj)
        self.__getTotalCarbs(curr_food_obj)
        self.__getDietaryFiber(curr_food_obj)
        self.__getSugars(curr_food_obj)
        self.__getProtein(curr_food_obj)
        self.__getIngredients(curr_food_obj)
        self.__getAllergens(curr_food_obj)
        self.__getVitamin_D(curr_food_obj)
        self.__getCalcium(curr_food_obj)
        self.__getIron(curr_food_obj)
        self.__getPotassium(curr_food_obj)
        self.__getFoodTags(curr_food_obj)
        self.__getTotalFat_DV(curr_food_obj)
        self.__getSatFat_DV(curr_food_obj)
        self.__getCholesterol_DV(curr_food_obj)
        self.__getSodiumDV(curr_food_obj)
        self.__getTotalCarbs_DV(curr_food_obj)
        self.__getDietaryFiber_DV(curr_food_obj)

    def __start_nutrient_fact_webscrapper(self, curr_foods):
        # Gets the nutrient facts for all food items that have just recently been added to all_foods{}.
        # Recent food names are in curr_foods[]
        for count in range(len(curr_foods)):
            link = WebDriverWait(self.driver, timeout=3).until(
                lambda d: d.find_element(By.LINK_TEXT, curr_foods[count]['food_name']))
            link.click()
            self.singleNutritionFact(curr_foods[count])
            self.driver.back()  # Returns to list of meal time page

    # Must call moveToNutritionPage() before calling this function.
    # Scraps all information(food genre, name, nutrition facts,etc) and writes it to a JSON file

    def __getNutritionFacts(self,dining_hall, meal_time):
        food_table = WebDriverWait(self.driver, timeout=3).until(lambda d: d.find_element(By.XPATH, '//table[@width="70%"]'))

        inner_table = WebDriverWait(food_table, timeout=3).until(lambda d: d.find_elements(By.TAG_NAME, 'table'))
        size = len(inner_table) - 1

        current_food_type = ''
        curr_foods = []
        for i in range(1, size, 2):
            try:
                try:  # Result is the food type. Parses the string and sets current_food_type
                    raw_food_type = inner_table[i].find_element(By.CLASS_NAME, "longmenucolmenucat")
                    current_food_type = self.__getFoodType(raw_food_type.text)

                except:  # Result is the food name. Checks if food_name is in all_foods{}.
                    # If not, create new obj with default parameters. Else update the object.
                    food_name = inner_table[i].find_element(By.CLASS_NAME, "longmenucoldispname")
                    food_obj = self.all_foods.get(food_name.text)
                    if food_obj is None:
                        food_obj = self.__init_basic_food_obj(current_food_type,food_name.text)
                        curr_foods.append(food_obj)
                    set_dining_hall = self.switch_for_dining_hall.get(dining_hall)
                    food_obj['dining_hall'][set_dining_hall] = True
                    food_obj['meal_time'][meal_time] = True
            except:
                print("FATAL ERROR: FOOD NAMES PARSED INCORRECTLY")

        self.__start_nutrient_fact_webscrapper(curr_foods)

        print("Webscrap " + dining_hall + ": " + meal_time + " Successful")
        self.driver.back()  # Returns back to dining hall's food list page


import re

from bs4 import BeautifulSoup, Tag
from undetected_chromedriver import Chrome
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.timeouts import Timeouts
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec

from models.text import Text
from models.browser import get_driver, scroll_down

class AliExpress:
    domain_url: str = "https://aliexpress.ru"
    pure_name: str = "aliexpress.ru"
    search_input: (str, str) = (By.ID, 'searchInput')
    content: (str, str) = (By.CSS_SELECTOR, 'div[class^="product-snippet_ProductSnippet__content__"')
    card_selen: (str, str) = (By.CSS_SELECTOR, 'div[class^="product-snippet_ProductSnippet__content__"')
    card_bs: dict = {"class_": re.compile('product-snippet_ProductSnippet__content__')}
    product_img: dict = {"class_": re.compile('gallery_Gallery__image__')}
    product_brand_name: dict = {"class_": re.compile('product-snippet_ProductSnippet__shop__')}
    product_goods_name: dict = {"class_": re.compile('product-snippet_ProductSnippet__name__')}
    current_price: dict = {"class_": re.compile('snow-price_SnowPrice__mainM__')}
    old_price: dict = {"class_": re.compile('snow-price_SnowPrice__secondPrice__')}
    reviews: dict = {"class_": re.compile('snippet_ProductSnippet__score__')}

def clean_url(raw_url: str) -> str:
    """Очистить ссылки от начальных '/' """
    if raw_url.startswith('/'):
        return clean_url(raw_url[1:])
    return raw_url

def parse_url(html_part: Tag) -> str:
    """Ссылка на товар"""
    item_url = html_part.find('a').get('href')
    item_url = clean_url(item_url)
    if item_url.startswith(AliExpress.domain_url):
        return item_url
    if item_url.startswith(AliExpress.pure_name):
        return 'https://' + item_url
    # Если ссылки относительные (/item/1005004065604805.html?sku_id=12000027923647408)
    # return aliexpress.domain_url + '/' + item_url[1:] if item_url[0] == '/' else item_url  # Оставить 1 слэш
    return AliExpress.domain_url + '/' + clean_url(item_url)

def parse_img_url(html_part: Tag) -> str:
    """Изображение товара"""
    # Берем первую картинку из карусели
    img_url = html_part.find(**AliExpress.product_img).get('src')
    if img_url and len(img_url) > 2 and img_url[:2] == '//':
        img_url = img_url[2:]
    return img_url

def parse_brand_name(html_part: Tag):
    """Название бренда"""
    brand_name = html_part.find(**AliExpress.product_brand_name)
    brand_name = brand_name.text if brand_name else None
    return brand_name

def parse_item_name(html_part: Tag):
    """Название товара"""
    item_name = html_part.find_next(**AliExpress.product_goods_name)
    item_name = item_name.text if item_name else None
    return item_name

def parse_price(text: str):
    no_spaces = re.sub(r'[\s.-]', '', text)
    split_symbol = re.search(r'[^,\d]+', no_spaces)[0]
    clean_price = re.sub(split_symbol, '', no_spaces)
    clean_price = re.sub(',', '.', clean_price)

    return float(clean_price)

def parse_current_price(html_part: Tag):
    """Текущая цена товара"""
    current_price_raw = html_part.find(**AliExpress.current_price)
    current_price = parse_price(current_price_raw.text) if current_price_raw else 0
    return current_price

def parse_old_price(html_part: Tag):
    """Старая цена товара"""
    old_price_raw = html_part.find(**AliExpress.old_price)
    old_price = parse_price(old_price_raw.text) if old_price_raw else 0
    return old_price

def parse_reviews(html_part: Tag):
    val = html_part.find(**AliExpress.reviews)
    val = float(val.text.replace(',', '.')) if val else 0
    return val

def parse_soup_html(input_name, card_elems):
    text_matcher = Text()
    res = []
    for card in card_elems:
        img = parse_img_url(card)
        current_price = parse_current_price(card)
        goods_name = parse_item_name(card)

        res.append({
            "url": parse_url(card),
            "img": img if img.startswith("https") else f"https://{img}",
            "current_price": current_price,
            "reverse_price": (1 / current_price) if current_price != 0 else 0,
            "old_price": parse_old_price(card),
            "brand_name": parse_brand_name(card),
            "goods_name": parse_item_name(card),
            "reviews": parse_reviews(card),
            "comments": 0,  # их нет в поисковой выдаче
            "market": "aliexpress",
            "text_accuracy": text_matcher.text_accuracy(input_name, goods_name)
        })
    return res

def main(input_name: str, driver: Chrome = None):
    if not driver:
        driver = get_driver()
    driver.timeouts = Timeouts(implicit_wait=3, page_load=3, script=60)

    try:
        driver.get(AliExpress.domain_url)
    except TimeoutException:
        WebDriverWait(driver, 10).until(ec.presence_of_element_located(AliExpress.search_input))

    elem = driver.find_element(*AliExpress.search_input)
    elem.clear()

    try:
        elem.send_keys(input_name + Keys.ENTER)
    except TimeoutException:
        WebDriverWait(driver, 10).until(ec.presence_of_element_located(AliExpress.card_selen))

    scroll_down(driver, 6)

    html_soup = BeautifulSoup(driver.page_source, features="lxml")
    card_elems = html_soup.find_all(**AliExpress.card_bs)

    res = parse_soup_html(input_name, card_elems)

    driver.close()
    return res


from uuid import uuid4

from pylm.parts.core import Inbound, Outbound,\
    zmq_context, BypassInbound
from pylm.parts.messages_pb2 import PalmMessage
from http.server import HTTPServer, BaseHTTPRequestHandler
import zmq
import sys

class RepService(Inbound):
    """
    RepService binds to a given socket and returns something.
    """
    def __init__(self,
                 name,
                 listen_address,
                 broker_address="inproc://broker",
                 logger=None,
                 cache=None,
                 messages=sys.maxsize):
        """
        :param name: Name of the service
        :param listen_address: ZMQ socket address to bind to
        :param broker_address: ZMQ socket address of the broker
        :param logger: Logger instance
        :param messages: Maximum number of messages. Defaults to infinity
        :return:
        """
        super(RepService, self).__init__(
            name,
            listen_address,
            zmq.REP,
            reply=True,
            broker_address=broker_address,
            bind=True,
            logger=logger,
            cache=cache,
            messages=messages
        )

class PullService(Inbound):
    """
    PullService binds to a socket waits for messages from a push-pull queue.
    """
    def __init__(self,
                 name,
                 listen_address,
                 broker_address="inproc://broker",
                 logger=None,
                 cache=None,
                 messages=sys.maxsize):
        """
        :param name: Name of the service
        :param listen_address: ZMQ socket address to bind to
        :param broker_address: ZMQ socket address of the broker
        :param logger: Logger instance
        :param messages: Maximum number of messages. Defaults to infinity.
        :return:
        """
        super(PullService, self).__init__(
            name,
            listen_address=listen_address,
            socket_type=zmq.PULL,
            reply=False,
            broker_address=broker_address,
            bind=True,
            logger=logger,
            cache=cache,
            messages=messages
        )

class PushService(Outbound):
    """
    PullService binds to a socket waits for messages from a push-pull queue.
    """
    def __init__(self,
                 name,
                 listen_address,
                 broker_address="inproc://broker",
                 logger=None,
                 cache=None,
                 messages=sys.maxsize):
        """
        :param name: Name of the service
        :param listen_address: ZMQ socket address to bind to
        :param broker_address: ZMQ socket address of the broker
        :param logger: Logger instance
        :param messages: Maximum number of messages. Defaults to infinity.
        :return:
        """
        super(PushService, self).__init__(
            name,
            listen_address=listen_address,
            socket_type=zmq.PUSH,
            reply=False,
            broker_address=broker_address,
            bind=True,
            logger=logger,
            cache=cache,
            messages=messages
        )

class PubService(Outbound):
    """
    PullService binds to a socket waits for messages from a push-pull queue.

    :param name: Name of the service
    :param listen_address: ZMQ socket address to bind to
    :param broker_address: ZMQ socket address of the broker
    :param logger: Logger instance
    :param messages: Maximum number of messages. Defaults to infinity.
    :param pipelined: Defaults to False. Pipelined if publishes to a
        server, False if publishes to a client.
    :param server: Name of the server, necessary to pipeline messages.
    """
    def __init__(self,
                 name,
                 listen_address,
                 broker_address="inproc://broker",
                 logger=None,
                 cache=None,
                 messages=sys.maxsize,
                 pipelined=False,
                 server=None):
        super(PubService, self).__init__(
            name,
            listen_address=listen_address,
            socket_type=zmq.PUB,
            reply=False,
            broker_address=broker_address,
            bind=True,
            logger=logger,
            cache=cache,
            messages=messages
        )
        self.name = server
        self.pipelined = pipelined

    def handle_stream(self, message):
        """
        Handle the stream of messages.

        :param message: The message about to be sent to the next step in the
            cluster
        :return: topic (str) and message (PalmMessage)

        The default behaviour is the following. If you leave this function
        unchanged and pipeline is set to False, the topic is the ID of the
        client, which makes the message return to the client. If the pipeline
        parameter is set to True, the topic is set as the name of the server and
        the step of the message is incremented by one.

        You can alter this default behaviour by overriding this function.
        Take into account that the message is also available in this function,
        and you can change other parameters like the stage or the function.
        """
        if self.pipelined:
            # If the master is pipelined,
            topic = self.name
            message.stage += 1
        else:
            topic = message.client

        return topic, message

    def start(self):
        """
        Call this function to start the component
        """
        message = PalmMessage()

        self.listen_to.bind(self.listen_address)
        self.logger.info('{} successfully started'.format(self.name))

        for i in range(self.messages):
            self.logger.debug('{} blocked waiting for broker'.format(self.name))
            message_data = self.broker.recv()
            self.logger.debug('{} Got message from broker'.format(self.name))
            message_data = self._translate_from_broker(message_data)
            message.ParseFromString(message_data)

            for scattered in self.scatter(message):
                topic, scattered = self.handle_stream(scattered)
                self.listen_to.send_multipart([topic.encode('utf-8'),
                                               message.SerializeToString()])
                self.logger.debug('Component {} Sent message. Topic {}'.format(
                    self.name, topic))

                if self.reply:
                    feedback = self.listen_to.recv()
                    feedback = self._translate_to_broker(feedback)
                    self.handle_feedback(feedback)

            self.broker.send(self.reply_feedback())

        return self.name

class WorkerPushService(PushService):
    """
    This is a particular push service that does not modify the messages that
    the broker sends.
    """
    def _translate_from_broker(self, message_data):
        """
        See help of parent
        :param message_data:
        :return:
        """
        self.logger.debug('{} translating from broker'.format(self.name))
        return message_data

    def _translate_to_broker(self, message_data):
        """
        See help of parent
        :param message_data:
        :return:
        """
        return message_data

class WorkerPullService(PullService):
    """
    This is a particular pull service that does not modify the messages that
    the broker sends.
    """
    def _translate_to_broker(self, message_data):
        """
        See help of parent
        :param message_data:
        :return:
        """
        return message_data

    def _translate_from_broker(self, message_data):
        """
        See help of parent
        :param message_data:
        :return:
        """
        return message_data

class PushPullService(object):
    """
    Push-Pull Service to connect to workers
    """
    def __init__(self,
                 name,
                 push_address,
                 pull_address,
                 broker_address="inproc://broker",
                 logger=None,
                 cache=None,
                 messages=sys.maxsize):
        """
        :param name: Name of the component
        :param listen_address: ZMQ socket address to listen to
        :param socket_type: ZMQ inbound socket type
        :param reply: True if the listening socket blocks waiting a reply
        :param broker_address: ZMQ socket address for the broker
        :param bind: True if socket has to bind, instead of connect.
        :param logger: Logger instance
        :param cache: Cache for shared data in the server
        :param messages: Maximum number of inbound messages. Defaults to infinity.
        :return:
        """
        self.name = name.encode('utf-8')

        self.push = zmq_context.socket(zmq.PUSH)
        self.pull = zmq_context.socket(zmq.PULL)
        self.push_address = push_address
        self.pull_address = pull_address
        self.push.bind(push_address)
        self.pull.bind(pull_address)

        self.broker = zmq_context.socket(zmq.REQ)
        self.broker.identity = self.name
        self.broker.connect(broker_address)

        self.logger = logger
        self.cache = cache
        self.messages = messages

        self.last_message = b''

    def scatter(self, message_data):
        """
        To be overriden. Picks a message and returns a generator that multiplies the messages
        to the broker.
        :param message_data:
        :return:
        """
        yield message_data

    def handle_feedback(self, message_data):
        """
        To be overriden. Handles the feedback from the broker
        :param message_data:
        :return:
        """
        self.last_message = message_data

    def reply_feedback(self):
        """
        To be overriden. Returns the feedback if the component has to reply.
        :return:
        """
        return self.last_message

    def start(self):
        message = PalmMessage
        self.logger.info('{} Successfully started'.format(self.name))
        initial_message = PalmMessage()
        initial_message.pipeline = '0'
        initial_message.client = '0'
        initial_message.stage = 0
        initial_message.function = ''
        initial_message.payload = b'0'
        self.broker.send(initial_message.SerializeToString())

        for i in range(self.messages):
            self.logger.debug('{} blocked waiting for broker'.format(self.name))
            message_data = self.broker.recv()
            self.logger.debug('Got message {} from broker'.format(i))
            message.ParseFromString(message_data)

            for scattered in self.scatter(message):
                self.push.send(scattered)
                self.handle_feedback(self.pull.recv())

            self.broker.send(self.reply_feedback())

        return self.name

    def cleanup(self):
        self.push.close()
        self.pull.close()
        self.broker.close()

class RepBypassService(BypassInbound):
    """
    Generic connection that opens a Rep socket and bypasses the broker.
    """
    def __init__(self, name, listen_address, logger=None,
                 cache=None, messages=sys.maxsize):
        """
        :param name: Name of the connection
        :param listen_address: ZMQ socket address to listen to
        :param logger: Logger instance
        :param messages:
        :return:
        """
        super(RepBypassService, self).__init__(name, listen_address, zmq.REP,
                                               reply=True, bind=True,
                                               logger=logger, cache=cache,
                                               messages=messages)

class HttpService(Inbound):
    """
    Similar to PullService, but the connection offered is an HTTP server
    that deals with inbound messages.

    ACHTUNG: this thing is deliberately single threaded
    """
    def __init__(self,
                 name,
                 hostname,
                 port,
                 broker_address="inproc://broker",
                 logger=None,
                 cache=None):
        self.name = name.encode('utf-8')
        self.hostname = hostname
        self.port = port
        self.logger = logger
        self.broker = zmq_context.socket(zmq.REQ)
        self.broker.identity = self.name
        self.broker.connect(broker_address)
        self.cache = cache

    def _make_handler(self):
        """
        This is serious meta programming. Note that the handler reuses
        the socket that connects to the router. This is intentional and
        makes the handler strictly single threaded.

        :return: Returns a PalmHandler class, that is a subclass of
          BaseHttpRequestHandler
        """
        # Clarify the scope since self is masked by the returned class
        scatter = self.scatter
        _translate_to_broker = self._translate_to_broker
        broker = self.broker
        handle_feedback = self.handle_feedback
        reply_feedback = self.reply_feedback

        class PalmHandler(BaseHTTPRequestHandler):
            def do_POST(self):
                """
                Note that this http server always replies
                """
                message = PalmMessage()
                self.send_response(200)
                self.end_headers()
                message_data = self.rfile.read(
                    int(
                        self.headers.get('Content-Length')
                    )
                )

                message.ParseFromString(message_data)
                scattered_messages = scatter(message)

                if not scattered_messages:
                    self.wfile.write(b'0')

                else:
                    for scattered in scattered_messages:
                        scattered = _translate_to_broker(scattered)

                        if scattered:
                            broker.send(scattered.SerializeToString())
                            handle_feedback(broker.recv())

                    self.wfile.write(reply_feedback())

        return PalmHandler

    def debug(self):
        """
        Starts the component and serves the http server forever.
        """
        server = HTTPServer((self.hostname, self.port), self._make_handler())
        server.handle_request()

    def start(self):
        """
        Starts the component and serves the http server forever.
        """
        server = HTTPServer((self.hostname, self.port), self._make_handler())
        server.serve_forever()

class CacheService(RepBypassService):
    """
    Cache service for clients and workers
    """
    def recv(self):
        message_data = self.listen_to.recv()
        message = PalmMessage()
        message.ParseFromString(message_data)
        instruction = message.function.split('.')[1]

        if instruction == 'set':
            if message.cache:
                key = message.cache
            else:
                key = str(uuid4())

            self.logger.debug('Cache Service: Set key {}'.format(key))
            value = message.payload
            self.cache.set(key, value)
            return_value = key.encode('utf-8')

        elif instruction == 'get':
            key = message.payload.decode('utf-8')
            self.logger.debug('Cache Service: Get key {}'.format(key))
            value = self.cache.get(key)
            if not value:
                self.logger.error('key {} not present'.format(key))
                return_value = b''
            else:
                return_value = value

        elif instruction == 'delete':
            key = message.payload.decode('utf-8')
            self.logger.debug('Cache Service: Delete key {}'.format(key))
            self.cache.delete(key)
            return_value = key.encode('utf-8')

        else:
            self.logger.error(
                'Cache {}:Key not found in the database'.format(self.name)
            )
            return_value = b''

        if isinstance(return_value, str):
            self.listen_to.send_string(return_value)
        else:
            self.listen_to.send(return_value)


import os, sys, time
rd,wd = os.pipe()
# son file descriptors, no file objects
r, w  = os.fdopen(rd,'rb',0), os.fdopen(wd,'wb',0) # file objects
pid = os.fork()
if pid: # padre
    w.close()
    while True:
        data = r.readline()
        if not data:
            break
        print("el padre lee: " + data.decode('utf8').strip())
else:  # hijo
    r.close()
    for i in range(10):
        mensaje = "linea %s\n" % i
        w.write(mensaje.encode('utf8'))
        w.flush()
        time.sleep(1)


import numpy as np
import pandas as pd
import sklearn
import argparse
import sys

import kmapper as km
import sklearn.manifold as skm
import sklearn.decomposition as skd
import sklearn.neighbors as skn

import scipy.spatial.distance as ssd

from a_patent_data import PatentData
from a_analyzer import Analyzer
from a_utilities import color_averager, is_empty_data

import mappertools.outputs.text_dump as tdump

from analysis_merged_pca import get_labels_and_data

def get_common_parser():
    common_parser = argparse.ArgumentParser(add_help = False)
    common_parser.add_argument("--verbose", "-v", action="store_true", help="verbose.")

    # Data choice
    group_data_choice = common_parser.add_argument_group("Data choice")
    group_data_choice.add_argument("--data", help="data choice: 0 (folder 180901_csv) or 1 (folder 200110_csv) (default: 1).", type=int, default=1, choices=[0,1])
    group_data_choice.add_argument("--mode", "-m", help="mode choice: 0 or 1 or 2 (default: 0; both modes: 2).", type=int, default=0, choices=[0,1,2])

    group_data_choice.add_argument("--from_year", "-f", help="starting year to do analysis.", type=int,default=1976)
    group_data_choice.add_argument("--to_year", "-g", help="ending year (inclusive) to do analysis.", type=int,default=2005)

    # Processing
    group_processing = common_parser.add_argument_group("Processing")
    group_processing.add_argument("--keep_zeros", "-z", action="store_true", help="preserve zeros columns in data. Do not use. Otherwise, drop zero columns.")
    group_processing.add_argument("--cos_trans", "-c", action="store_true", help="use cosine distances to transform data.")

    # output choices
    group_output = common_parser.add_argument_group("Output options")
    group_output.add_argument("--clustermap", action="store_true", help="Do clustermap.")
    group_output.add_argument("--no_dump_raw", action="store_true", help="Skip dumping raw data.")
    # group_output.add_argument("--interactive", action="store_true", help="interactive plot of lens.")
    group_output.add_argument("--kclusters", help="number(s) of k-Medoids and k-Means to compute and append to cyjs output. Note that k-Means ignores --metric and always computes with Euclidean distances.", type=int, nargs="+", default=None)

    group_output.add_argument("--char_limit", help="limit chars for firms and patent class names", type=int, default=None)
    # group_output.add_argument("--no_mapper", action="store_true", help="Skip Mapper computation entirely.")
    group_output.add_argument("--output", "-o", help="output base folder.", type=str)

    return common_parser

def get_parser():
    parser = argparse.ArgumentParser(description="Program for performing Mapper analysis on patent data.")
    parser.set_defaults(verbosity=0, procedure=None)
    subparsers = parser.add_subparsers(help="choose mode of operation:")
    common_parser = get_common_parser()

    # ACCUMULATE
    accum_parser = subparsers.add_parser("accumulate",
                                         help="accumulate all chosen years. Operation order: cos-dist-transform (if enabled), then accumulate, then log (if enabled)",
                                         parents=[common_parser])
    accum_parser.set_defaults(procedure="accumulate")

    # MERGE
    merge_parser = subparsers.add_parser("merge",
                                         help="merge all chosen years",
                                         parents=[common_parser])
    merge_parser.set_defaults(procedure="merge")

    # MERGE-ACCUMULATE
    common_parser_copy = get_common_parser()
    matching_groups = (g for g in common_parser_copy._action_groups if g.title == 'Data choice')
    group = next(matching_groups, None) or common_parser_copy
    group.add_argument("--window", "-w", type=int, help="window size (default=5)",default=5)
    group.add_argument("--shift", "-s", type=int, help="window shift (default=5)",default=5)

    merge_accum_parser = subparsers.add_parser("ma",
                                               help="accumulate over window, then merge across shifted windows.",
                                               parents=[common_parser_copy])
    merge_accum_parser.set_defaults(procedure="merge_accumulate")

    return parser

def do_final_year_analysis(proc, data):
    ans_columns = ["yfinal--yfinal_neighbor", "yfinal--yfinal_neighborname", "yfinal--yfinal_popmean"]
    ans = pd.DataFrame(index=proc.labels.unique_members, columns=ans_columns)

    # extract final year
    final_year = max(proc.labels.years_data)
    final_year_bool = (proc.labels.years_data == final_year)
    final_year_data = data[final_year_bool]
    final_year_firms = proc.labels.intemporal_index[final_year_bool]

    # final year distances
    final_year_distances = ssd.squareform(ssd.pdist(final_year_data,metric="cosine"))

    # output distances
    main_folder = proc.get_main_folder()
    name = "{:s}_{:s}_finalyeardistances.csv".format(proc.labels.transforms_name, proc.labels.data_name)
    output_fname = main_folder.joinpath(name)
    pd.DataFrame(final_year_distances, index=final_year_firms, columns=final_year_firms).to_csv(str(output_fname))

    # final year pop mean
    final_year_popmean = np.mean(final_year_data, axis=0)

    # final year nn
    nbrs = skn.NearestNeighbors(n_neighbors=2, algorithm='auto', metric="precomputed").fit(final_year_distances)
    final_year_nn_distances, final_year_nn_indices = nbrs.kneighbors(n_neighbors=1)

    for i, firm in enumerate(final_year_firms):
        ans.loc[firm, "yfinal--yfinal_neighbor"] = final_year_nn_distances[i][0]
        ans.loc[firm, "yfinal--yfinal_neighborname"] = final_year_firms[final_year_nn_indices[i][0]]

        ans.loc[firm, "yfinal--yfinal_popmean"] = ssd.cosine(final_year_data.iloc[i,:], final_year_popmean)

    return ans

def do_year_mean_analysis(proc, data):
    ans_columns = ["ymean--ymean_neighbor", "ymean--ymean_neighborname", "ymean--ymean_popmean"]
    ans = pd.DataFrame(index=proc.labels.unique_members, columns=ans_columns)

    # create means for each firm
    firm_mean_data = pd.DataFrame(columns = data.columns)
    for firm in proc.labels.unique_members:
        firm_bool = proc.labels.intemporal_index == firm
        firm_data = np.mean(data[firm_bool])
        firm_mean_data.loc[firm] = firm_data
    # remove all-zero rows (firms):
    firm_mean_data.dropna(axis=0, how="all", inplace=True)

    # output year means
    main_folder = proc.get_main_folder()
    name = "{:s}_{:s}_firmmeans.csv".format(proc.labels.transforms_name, proc.labels.data_name)
    output_fname = main_folder.joinpath(name)
    firm_mean_data.to_csv(str(output_fname))

    # yearmean distances
    firm_mean_distances = ssd.squareform(ssd.pdist(firm_mean_data,metric="cosine"))

    # output distances
    name = "{:s}_{:s}_firmmeansdistances.csv".format(proc.labels.transforms_name, proc.labels.data_name)
    output_fname = main_folder.joinpath(name)
    pd.DataFrame(firm_mean_distances, index=firm_mean_data.index, columns=firm_mean_data.index).to_csv(str(output_fname))

    # firm mean pop mean
    firm_mean_popmean = np.mean(firm_mean_data, axis=0)

    # firm mean nn
    nbrs = skn.NearestNeighbors(n_neighbors=2, algorithm='auto', metric="precomputed").fit(firm_mean_distances)
    firm_mean_nn_distances, firm_mean_nn_indices = nbrs.kneighbors(n_neighbors=1)

    for i, firm in enumerate(firm_mean_data.index):
        ans.loc[firm, "ymean--ymean_neighbor"] = firm_mean_nn_distances[i][0]
        ans.loc[firm, "ymean--ymean_neighborname"] = firm_mean_data.index[firm_mean_nn_indices[i][0]]

        ans.loc[firm, "ymean--ymean_popmean"] = ssd.cosine(firm_mean_data.loc[firm,:], firm_mean_popmean)

    return ans

def do_jaffe_measures(args, labels, data, verbosity):
    if is_empty_data(data, args.from_year, args.to_year): return

    if args.sum_to_one:
        summed = data.sum(axis=1)
        assert np.allclose(summed.loc[summed!=0],1)

    # prepare mapper data and lens
    if args.mds == True:
        lens_name = "mds{}d".format(args.dimension)
    else:
        lens_name = "pca{}d".format(args.dimension)

    proc = Analyzer(data, labels=labels,
                          lens=None, lens_name=lens_name, metric=args.metric,
                          verbose=verbosity)

    if args.mds:
        if proc.metric == "precomputed":
            dists = proc.distance_matrix
        else:
            X = ssd.pdist(data, metric=args.metric)
            dists = ssd.squareform(X)
        proc.lens = skm.MDS(n_components=args.dimension, dissimilarity="precomputed").fit_transform(dists)
    else:
        proc.lens = skd.PCA(n_components=args.dimension).fit_transform(data)

    # initialize analyzer
    proc.initialize(base_path=args.output)

    # Other outputs
    if args.clustermap: proc.do_clustermap()
    if not args.no_dump_raw: proc.dump_data()
    if True: proc.plot_lens(np.array(labels.rgb_colors)/255., show=False)
    if args.kclusters:
        kClusters = proc.dataframe_kClusters(args.kclusters, dump_summary=True, dump_aggregates=True)

    ans = do_final_year_analysis(proc, data)
    ans = ans.join(do_year_mean_analysis(proc,data), how="outer")
    print(ans)

    main_folder = proc.get_main_folder()
    name = "{:s}_{:s}_jaffemeasures.csv".format(proc.labels.transforms_name, proc.labels.data_name)
    output_fname = main_folder.joinpath(name)
    ans.to_csv(str(output_fname))

def main(raw_args):
    parser = get_parser()
    args = parser.parse_args(raw_args)

    if args.procedure == None:
        parser.print_help()
        exit()

    # fix choices for Jaffe measure
    args.transpose = False
    args.log = False
    args.sum_to_one = True
    args.metric = "cosine"

    labels, data = get_labels_and_data(args)

    args.mds = False
    args.dimension = 2

    do_jaffe_measures(args, labels, data, verbosity=(2 if args.verbose else 0))

if __name__ == "__main__":
    main(sys.argv[1:])