"""
Santander Web Pay Plus
"""
import re
import asyncio
import os
import urllib
import xml.etree.ElementTree as ET

from dotenv import load_dotenv
import requests

from lib.AESEncryption import AES128Encryption
from lib.exceptions import (
    CitasConnectionError,
    CitasMissingConfigurationError,
    CitasNotValidAnswerError,
    CitasTimeoutError,
    CitasRequestError,
    CitasUnknownError,
    CitasEncryptError,
    CitasGetURLFromXMLEncryptedError,
    CitasDesencryptError,
    CitasBankResponseInvalidError,
    CitasXMLReadError,
)

XML_ENCRYPT_REGEXP = r"^[a-zA-Z0-9=+\/]{32,}$"

RESPUESTA_EXITO = "approved"
RESPUESTA_DENEGADA = "denied"
RESPUESTA_ERROR = "error"

load_dotenv()
WPP_COMMERCE_ID = os.getenv("WPP_COMMERCE_ID", None)
WPP_COMPANY_ID = os.getenv("WPP_COMPANY_ID", None)
WPP_BRANCH_ID = os.getenv("WPP_BRANCH_ID", None)
WPP_KEY = os.getenv("WPP_KEY", None)
WPP_PASS = os.getenv("WPP_PASS", None)
WPP_TIMEOUT = int(os.getenv("WPP_TIMEOUT", "12"))
WPP_URL = os.getenv("WPP_URL", None)
WPP_USER = os.getenv("WPP_USER", None)

def create_chain_xml(
    pago_id: int,
    amount: float,
    email: str,
    description: str,
    cit_client_id: int,
) -> str:
    """Crear cadena XML"""

    # Validar WPP_COMPANY_ID
    if WPP_COMPANY_ID is None:
        raise CitasMissingConfigurationError("Falta declarar la variable de entorno WPP_COMPANY_ID.")

    # Validar WPP_BRANCH_ID
    if WPP_BRANCH_ID is None:
        raise CitasMissingConfigurationError("Falta declarar la variable de entorno WPP_BRANCH_ID.")

    # Validar WPP_USER
    if WPP_USER is None:
        raise CitasMissingConfigurationError("Falta declarar la variable de entorno WPP_USER.")

    # Validar WPP_PASS
    if WPP_PASS is None:
        raise CitasMissingConfigurationError("Falta declarar la variable de entorno WPP_PASS.")

    # Inicializar XML
    root = ET.Element("P")

    # Inicializar datos de la organización
    business = ET.SubElement(root, "business")
    ET.SubElement(business, "id_company").text = WPP_COMPANY_ID
    ET.SubElement(business, "id_branch").text = WPP_BRANCH_ID
    ET.SubElement(business, "user").text = WPP_USER
    ET.SubElement(business, "pwd").text = WPP_PASS

    # Inicializar datos de la version
    ET.SubElement(root, "version").text = "IntegraWPP"

    # Inicializar datos del cliente y del pago
    url = ET.SubElement(root, "url")
    ET.SubElement(url, "reference").text = str(pago_id)
    ET.SubElement(url, "amount").text = str(amount)
    ET.SubElement(url, "moneda").text = "MXN"
    ET.SubElement(url, "canal").text = "W"
    ET.SubElement(url, "omitir_notif_default").text = "0"
    ET.SubElement(url, "st_correo").text = "1"
    ET.SubElement(url, "mail_cliente").text = email

    # Inicializar datos adicionales
    data = ET.SubElement(url, "datos_adicionales")

    # Inicializar descripción del servicio
    data1 = ET.SubElement(data, "data")
    data1.attrib["id"] = "1"
    data1.attrib["display"] = "true"
    label1 = ET.SubElement(data1, "label")
    label1.text = "Servicio"
    value1 = ET.SubElement(data1, "value")
    value1.text = description

    # Inicializar descripción del cliente
    data2 = ET.SubElement(data, "data")
    data2.attrib["id"] = "2"
    data2.attrib["display"] = "false"
    label2 = ET.SubElement(data2, "label")
    label2.text = "Cliente ID"
    value2 = ET.SubElement(data2, "value")
    value2.text = str(cit_client_id)

    # Entregar cadena XML
    return ET.tostring(root, encoding="unicode")

def encrypt_chain(chain: str) -> str:
    """Cifrar cadena XML"""

    # Validar WPP_KEY
    if WPP_KEY is None:
        raise CitasMissingConfigurationError("Falta declarar la variable de entorno WPP_KEY.")

    # Cifrar la cadena
    aes_encryptor = AES128Encryption()
    ciphertext = aes_encryptor.encrypt(chain, WPP_KEY)

    # Entregar cadena cifrada
    return ciphertext

def decrypt_chain(chain_encrypted: str) -> str:
    """Descifrar cadena XML"""

    # Validar WPP_KEY
    if WPP_KEY is None:
        raise CitasMissingConfigurationError("Falta declarar la variable de entorno WPP_KEY.")

    # Eliminar avances de línea y espacios en blanco
    chain_encrypted = chain_encrypted.replace("\n", "").replace(" ", "")

    # Validar la cadena cifrada con la expresión regular
    if re.fullmatch(XML_ENCRYPT_REGEXP, chain_encrypted) is None:
        raise CitasBankResponseInvalidError(f"Error porque la respuesta del banco no pasa la validación por expresión regular: [{chain_encrypted}]")

    # Descifrar la cadena
    aes_encryptor = AES128Encryption()
    try:
        plaintext = aes_encryptor.decrypt(WPP_KEY, chain_encrypted)
    except Exception as error:
        raise CitasDesencryptError("Error porque no se pudo desencritar la respuesta del banco.") from error

    # Entregar cadena descifrada
    return plaintext

def create_chain_xml_sender(chain: str) -> str:
    """Crear cadena para XML de envío WPP"""

    # Validar WPP_COMMERCE_ID
    if WPP_COMMERCE_ID is None:
        raise CitasMissingConfigurationError("Falta declarar la variable de entorno WPP_COMMERCE_ID.")

    # Validar WPP_URL
    if WPP_URL is None:
        raise CitasMissingConfigurationError("Falta declarar la variable de entorno WPP_URL.")

    # Empacar la cadena
    root = ET.Element("pgs")
    ET.SubElement(root, "data0").text = WPP_COMMERCE_ID
    ET.SubElement(root, "data").text = chain
    chain_bytes = ET.tostring(root, encoding="unicode")

    # Entregar cadena
    return urllib.parse.quote(chain_bytes, "utf-8")

async def send_chain(chain: str) -> str:
    """Enviar cadena a WPP"""

    # Validar WPP_URL
    if WPP_URL is None:
        raise CitasMissingConfigurationError("Falta declarar la variable de entorno WPP_URL.")

    # Validar WPP_TIMEOUT
    if WPP_TIMEOUT is None:
        raise CitasMissingConfigurationError("Falta declarar la variable de entorno WPP_TIMEOUT.")

    # Preparar la carga para la petición
    payload = "xml=" + create_chain_xml_sender(chain)
    headers = {"Content-Type": "application/x-www-form-urlencoded"}

    # Enviar la petición al banco
    try:
        response = requests.request(
            "POST",
            WPP_URL,
            headers=headers,
            data=payload,
            timeout=WPP_TIMEOUT,
        )
    except requests.exceptions.ConnectionError as error:
        raise CitasConnectionError("Error porque no se pudo conectar al banco.") from error
    except requests.exceptions.Timeout as error:
        raise CitasTimeoutError("Error porque se agoto el tiempo de espera con el banco.") from error
    except requests.exceptions.RequestException as error:
        raise CitasRequestError("Error al enviar la cadena al banco.") from error
    except Exception as error:
        raise CitasUnknownError("Error desconocido al enviar la cadena al banco.") from error

    # Entregar
    return response.text

def get_url_from_xml_encrypt(xml_encrypt: str):
    """Extrae la url del xml de respuesta"""

    # Desencriptar el XML
    try:
        xml_str = decrypt_chain(xml_encrypt)
    except Exception as error:
        raise CitasDesencryptError(f"No se puede desencriptar el XML del banco. {str(error)}") from error

    # Leer el contenido XML
    try:
        root = ET.fromstring(xml_str)
    except ET.ParseError as error:
        raise CitasXMLReadError("Error porque el XML del banco no es válido.") from error

    # Obtener el URL
    url = root.find("nb_url").text
    if url is None or url == "":
        raise CitasXMLReadError("Error porque el XML del banco no tiene la URL.")

    # Entregar
    return url

def create_pay_link(
    pago_id: int,
    email: str,
    service_detail: str,
    cit_client_id: int,
    amount: float,
) -> str:
    """Regresa el link para mostrar el formulario de pago"""

    # Crear cadena XML
    chain = create_chain_xml(
        pago_id=pago_id,
        amount=amount,
        email=email,
        description=service_detail,
        cit_client_id=cit_client_id,
    )

    # Encriptación del XML
    try:
        chain_encrypt = encrypt_chain(chain).decode()  # bytes
    except Exception as error:
        raise CitasEncryptError("Error al encriptar el XML") from error

    # Enviar cadena XML a WPP
    respuesta = None
    try:
        respuesta = asyncio.run(send_chain(chain_encrypt))
    except asyncio.TimeoutError as error:
        raise CitasTimeoutError("Error porque se agoto el tiempo de espera con el banco.") from error
    except Exception as error:
        raise CitasUnknownError("Error al tratar de enviar la cadena XML al banco.") from error

    # Si no hay respuesta, causar error
    if respuesta is None or respuesta == "" or respuesta == "\n":
        raise CitasNotValidAnswerError("Error en la respuesta del banco (respuesta vacía).")

    # Descifrar la respuesta y extraer la url
    try:
        url = get_url_from_xml_encrypt(respuesta)
    except Exception as error:
        raise CitasGetURLFromXMLEncryptedError(f"Error al obtener la URL del banco desde su XML encriptado. {str(error)}") from error

    # Entregar
    return url

def convert_xml_to_dict(xml_str: str) -> dict:
    """Convertir el xml descifrado a un diccionario"""

    # Leer el contenido XML
    try:
        root = ET.fromstring(xml_str)
    except ET.ParseError as error:
        raise CitasXMLReadError("Error porque el XML que dio el banco no es válido.") from error

    # Obtener pago_id
    pago_id = root.find("reference").text
    if pago_id is None or pago_id == "":
        raise CitasXMLReadError("Error porque el XML no tiene el pago_id.")

    # Obtener respuesta
    respuesta = root.find("response").text
    if respuesta is None or respuesta == "":
        raise CitasXMLReadError("Error porque el XML no tiene la respuesta.")

    # Obtener folio
    folio = root.find("foliocpagos").text
    if folio is None or folio == "":
        raise CitasXMLReadError("Error porque el XML no tiene el folio.")

    # Obtener auth
    auth = root.find("auth").text
    if auth is None or auth == "":
        raise CitasXMLReadError("Error porque el XML no tiene el auth.")

    # Obtener email
    email = root.find("email").text
    if email is None or email == "":
        raise CitasXMLReadError("Error porque el XML no tiene el email.")

    # Entregar diccionario
    return {
        "pago_id": pago_id,
        "respuesta": respuesta,
        "folio": folio,
        "auth": auth,
        "email": email,
    }


from __future__ import annotations

from typing import Iterable
from unittest import mock

from airflow.utils.entry_points import entry_points_with_dist, metadata

class MockDistribution:
    def __init__(self, name: str, entry_points: Iterable[metadata.EntryPoint]) -> None:
        self.metadata = {"Name": name}
        self.entry_points = entry_points

class MockMetadata:
    def distributions(self):
        return [
            MockDistribution(
                "dist1",
                [metadata.EntryPoint("a", "b", "group_x"), metadata.EntryPoint("c", "d", "group_y")],
            ),
            MockDistribution("Dist2", [metadata.EntryPoint("e", "f", "group_x")]),
            MockDistribution("dist2", [metadata.EntryPoint("g", "h", "group_x")]),  # Duplicated name.
        ]

@mock.patch("airflow.utils.entry_points.metadata", MockMetadata())
def test_entry_points_with_dist():
    entries = list(entry_points_with_dist("group_x"))

    # Only "group_x" entries are loaded. Distributions are not deduplicated.
    assert [dist.metadata["Name"] for _, dist in entries] == ["dist1", "Dist2", "dist2"]
    assert [ep.name for ep, _ in entries] == ["a", "e", "g"]


from django.conf.urls import patterns, url
from surfice import views, ajax

# Delete when deploying!
from django.contrib.staticfiles.urls import staticfiles_urlpatterns

urlpatterns = patterns('',
	# Home
	url(r'^$', views.index, name='index'),

	#url(r'^ajax', ajax.ajax, name='ajax'), # No params
	#url(r'^about/$', views.about, name='about'),
	# surf_url is passed to the surf method in views.py
	# regex to look for any sequence of alphanumeric characters
	# and underscores before the trailing slash
	#url(r'^surf/(?P<surf_url>\w+)/$', views.surf, name='surf'),

	# AJAX
	url(r'^.*?ajax/(?P<action>[^/]*)', ajax.dispatch, name='ajax'),

	# Admin
	url(r'^admin/$', views.admin, name='admin'),
	url(r'^admin/surfs/$', views.surfs, name='surfs'),
	url(r'^admin/surfices/$', views.surfices, name='surfices'),
	url(r'^admin/settings/$', views.settings, name='settings'),
	url(r'^admin/statuses/$', views.statuses, name='statuses'),

	url(r'^admin/events/(?P<page>\w*)(?:\/)(?P<order_by>[-\w]*)', views.events, name='events'),
	url(r'^admin/events/(?P<page>\w*)', views.events, name='events'),
	url(r'^admin/events/$', views.events, name='events'),

	url(r'^admin/dings/(?P<page>\w*)(?:\/)(?P<order_by>[-\w]*)', views.dings, name='dings'),
	url(r'^admin/dings/(?P<page>\w*)', views.dings, name='dings'),
	url(r'^admin/dings/$', views.dings, name='dings'),

	# Need to comment out because I'm not going to use any except ding
	#url(r'^admin/surfice/(?P<surfice>.*)/$', views.surfice, name='surfice'),
	#url(r'^admin/status/(?P<status>.*)/$', views.status, name='status'),
	url(r'^admin/ding/(?P<ding>.*)', views.ding, name='ding'),
)


from pprint import pprint
from datetime import timedelta

from django.conf import settings
from django.utils import timezone

from django.core.paginator import Paginator
from django.core.paginator import InvalidPage

from rest_framework import pagination
from rest_framework.response import Response
from rest_framework.authentication import SessionAuthentication, BasicAuthentication

from rest_framework_simplejwt.authentication import JWTTokenUserAuthentication

from sendgrid.sendgrid import SendGridAPIClient
from sendgrid.helpers.mail import Mail

from sentry_sdk import capture_exception

class CurrentPagePagination(pagination.PageNumberPagination):
    page_size = 8
    invalid_page_message = "server_invalid_page_number"

    def get_paginated_response(self, data):
        return Response(
            {
                "items_per_page": self.page_size,
                "current_page": self.page.number,
                "total_pages": self.page.paginator.num_pages,
                "total_items": self.page.paginator.count,
                "results": data,
            }
        )

class CommonUtil:
    @staticmethod
    def get_authentication_classes():
        authentication_classes = [
            JWTTokenUserAuthentication,
        ]

        if settings.ENVIRONMENT != "production":
            authentication_classes.append(SessionAuthentication)
            authentication_classes.append(BasicAuthentication)

        return authentication_classes

class EmailUtil:
    @staticmethod
    def send_signup_email(instance, verification_token):

        verify_url = settings.DOMAIN_NAME + "/#/signup/verify?"

        verify_url += (
            "email="
            + str(instance.email)
            + "&"
            + "token="
            + str(verification_token.token)
        )

        message = Mail(
            from_email=settings.DEFAULT_FROM_EMAIL,
            to_emails=instance.email,
        )

        message.dynamic_template_data = {"verify_url": verify_url}

        # NOTE: DYNAMIC TEMPLATE ID OF THE TEMPLATE IN THE SENDGRID ACCOUNT for Sign Up email
        message.template_id = settings.SENDGRID_TEMPLATE_ID_LABORATORY_SIGNUP_EMAIL

        try:
            sendgrid_client = SendGridAPIClient(settings.SENDGRID_API_KEY)
            response = sendgrid_client.send(message)

        except Exception as error:
            # log to sentry
            capture_exception(error)

    @staticmethod
    def send_request_password_reset_email(instance, verification_token):

        verify_url = settings.DOMAIN_NAME + "/#/password/reset?"

        verify_url += (
            "email="
            + str(instance.email)
            + "&"
            + "token="
            + str(verification_token.token)
        )

        message = Mail(
            from_email=settings.DEFAULT_FROM_EMAIL,
            to_emails=instance.email,
        )

        message.dynamic_template_data = {"verify_url": verify_url}

        # NOTE: DYNAMIC TEMPLATE ID OF THE TEMPLATE IN THE RESET PASSWORD email
        message.template_id = settings.SENDGRID_TEMPLATE_ID_REQUEST_PASSWORD_RESET_EMAIL

        try:
            sendgrid_client = SendGridAPIClient(settings.SENDGRID_API_KEY)
            response = sendgrid_client.send(message)

        except Exception as error:
            # log to sentry
            capture_exception(error)

class TimeUtil:
    @staticmethod
    def get_minutes_from_now(MINUTES):
        return timezone.now() + timedelta(minutes=MINUTES)


import torch

from vision.datasets.voc_dataset import VOCDataset

from vision.utils.misc import str2bool, Timer
import argparse
import pathlib
import numpy as np

import random

from vision.ssd.imJnet_ssd_lite import create_imJnet_ssd_lite
from vision.ssd.imJnet_ssd_lite import create_imJnet_ssd_lite_predictor
import cv2
import string
import imutils
parser = argparse.ArgumentParser(description="SSD Evaluation on VOC Dataset.")
# model_log/jnet-ssd-lite-Epoch-210-Loss-0.6506194928113151.pth
parser.add_argument("--trained_model", default= 'model_log/jnet-ssd-lite-Epoch-1080-Loss-0.6063801158558239.pth', type=str)

parser.add_argument("--dataset", default='/media/handsome/backupdata/hanson/ocr_table_dataset_v2/Cropped_v1', type=str, help="The root directory of the VOC dataset or Open Images dataset.")
parser.add_argument("--label_file", default='model_log/voc-model-labels.txt' ,type=str, help="The label file path.")
parser.add_argument("--use_cuda", type=str2bool, default=True)
parser.add_argument("--use_2007_metric", type=str2bool, default=True)
parser.add_argument("--nms_method", type=str, default="hard")
parser.add_argument("--iou_threshold", type=float, default=0.5, help="The threshold of Intersection over Union.")
parser.add_argument("--eval_dir", default="eval_results", type=str, help="The directory to store evaluation results.")
parser.add_argument('--mb2_width_mult', default=1.0, type=float,
                    help='Width Multiplifier for MobilenetV2')
args = parser.parse_args()
DEVICE = torch.device("cuda:0" if torch.cuda.is_available() and args.use_cuda else "cpu")
# DEVICE = torch.device("cpu")

def group_annotation_by_class(dataset):
    true_case_stat = {}
    all_gt_boxes = {}
    all_difficult_cases = {}
    for i in range(len(dataset)):
        image_id, annotation = dataset.get_annotation(i)
        gt_boxes, classes, is_difficult = annotation
        gt_boxes = torch.from_numpy(gt_boxes)
        for i, difficult in enumerate(is_difficult):
            class_index = int(classes[i])
            gt_box = gt_boxes[i]
            if not difficult:
                true_case_stat[class_index] = true_case_stat.get(class_index, 0) + 1

            if class_index not in all_gt_boxes:
                all_gt_boxes[class_index] = {}
            if image_id not in all_gt_boxes[class_index]:
                all_gt_boxes[class_index][image_id] = []
            all_gt_boxes[class_index][image_id].append(gt_box)
            if class_index not in all_difficult_cases:
                all_difficult_cases[class_index]={}
            if image_id not in all_difficult_cases[class_index]:
                all_difficult_cases[class_index][image_id] = []
            all_difficult_cases[class_index][image_id].append(difficult)

    for class_index in all_gt_boxes:
        for image_id in all_gt_boxes[class_index]:
            all_gt_boxes[class_index][image_id] = torch.stack(all_gt_boxes[class_index][image_id])
    for class_index in all_difficult_cases:
        for image_id in all_difficult_cases[class_index]:
            all_gt_boxes[class_index][image_id] = torch.tensor(all_gt_boxes[class_index][image_id])
    return true_case_stat, all_gt_boxes, all_difficult_cases

if __name__ == '__main__':
    eval_path = pathlib.Path(args.eval_dir)
    eval_path.mkdir(exist_ok=True)
    timer = Timer()
    class_names = [name.strip() for name in open(args.label_file).readlines()]
    dataset = VOCDataset(args.dataset, is_test=True)

    true_case_stat, all_gb_boxes, all_difficult_cases = group_annotation_by_class(dataset)

    net = create_imJnet_ssd_lite(len(class_names), width_mult=args.mb2_width_mult, is_test=True)

    import collections
    timer.start("Load Model")
    # net.load(args.trained_model)
    pretrained_dict = torch.load(args.trained_model)
    # new_state_dict = collections.OrderedDict()
    # for k, v in pretrained_dict.items():
    #     print(k)
    #     name = k[7:]
    #     new_state_dict[name] = v
    net.load_state_dict(pretrained_dict)
    net = net.to(DEVICE)
    print(f'It took {timer.end("Load Model")} seconds to load the model.')

    predictor = create_imJnet_ssd_lite_predictor(net, nms_method=args.nms_method, device=DEVICE)

    for i in range(len(dataset)):
        print("process image", i)
        timer.start("Load Image")
        src_image, gt_mask = dataset.get_image(i)
        # image = imutils.resize(image,image.shape[1], image.shape[0] * 2)
        image = cv2.resize(src_image,(src_image.shape[1], src_image.shape[0] * 2))
        max_edge = max(image.shape[0], image.shape[1])
        image = cv2.copyMakeBorder(image, 0, max_edge - image.shape[0], 0, max_edge - image.shape[1],cv2.BORDER_CONSTANT)

        print("Load Image: {:4f} seconds.".format(timer.end("Load Image")))
        timer.start("Predict")
        boxes, labels, probs, masks, angle, Matrix, factor= predictor.predict(image, gt_mask)
        print("Prediction: {:4f} seconds.".format(timer.end("Predict")))

        orig_image = image.copy()
        image_id, annotation = dataset.get_annotation(i)
        gt_boxes, classes, is_difficult = annotation

        seg_mask = masks[0]
        seg_mask = torch.squeeze(seg_mask)
        seg_mask = seg_mask.cpu().detach().numpy().astype(np.float32)

        ran_str = ''.join(random.sample(string.ascii_letters + string.digits, 20))
        ocr_cropped_bbox = 'eval_results/bbox/'+ ran_str + ".png"
        ocr_cropped_heatmap = 'eval_results/heatmap/' + ran_str + ".png"
        print(factor)
        final_pts = []
        for i in range(boxes.size(0)):
            if probs[i] > 0.4:
                box = boxes[i, :]

                x1 = min(box[0] + 5, 768-1) / factor[0]
                y1 = min(box[1] + 5, 768-1) / factor[1]
                x2 = max(box[2] - 5, box[0]) / factor[0]
                y2 = max(box[3] - 5, box[1]) / factor[1]
                pts = np.array([[x1, y1, 1], [x2, y1, 1], [x2, y2, 1], [x1, y2, 1]])
                # print(pts.T.shape, M.shape)
                dst_ps = np.dot(Matrix, pts.T).T
                for p in range(dst_ps.shape[0]):
                    dst_ps[p, 0] = int(dst_ps[p, 0] * (image.shape[1] / 768.0))
                    dst_ps[p, 1] = int(dst_ps[p, 1] * (image.shape[1] / 768.0) / 2.0)

                final_pts.append(dst_ps.astype(np.int32))

        for dps in final_pts:
            b = random.randint(0, 255)
            g = random.randint(0, 255)
            r = random.randint(0, 255)
            cv2.line(src_image, (dps[0,0],dps[0,1]), (dps[2,0],dps[2,1]), (b, g, r), 2)
            # for p in range(dps.shape[0]):
                # cv2.circle(src_image, (dps[p,0], dps[p,1]), 2, (b, g, r), 2)
        cv2.imshow('src_image',src_image)
        cv2.imshow('seg_mask', seg_mask)
        cv2.waitKey(0)


r"""
The Trust model.
Documentation: https://api.infynity.com.au/v1/doc#!reference/trust.md
"""
from .base import InfynipyBase
from .entity.entity import ClientAccountEntity
from .entity.address import Address
from ..exceptions import ClientException

class Trust(ClientAccountEntity):

    def __init__(self, infynity, broker_id, data=None):
        """Initialize a trust instance.
        This class is intended to be interfaced with through ``infynity.broker.trust``.
        """
        super().__init__(infynity, 'trust', broker_id, data=data)

    @property
    def addresses(self):
        if not hasattr(self, 'trust_id'):
            raise ClientException('Trust does not exist.')

        endpoint = InfynipyBase.ENDPOINT + f'address/trust/{self.trust_id}'
        return [Address(d) for d in self._infynity.get(endpoint)]

    def new_address(self, address_type, data):
        if not hasattr(self, 'trust_id'):
            raise ClientException('Trust does not exist.')

        endpoint = InfynipyBase.ENDPOINT + f'address/TrustMailing/{self.trust_id}'
        self._infynity.post(endpoint, data)


import urllib as ul

base_url = 'http://www.ux.uis.no/~tranden/brodatz/D'
dataset_size = 112

for i in range(1,dataset_size+1):
    ul.urlretrieve(base_url+str(i)+'.gif', './../data/brodatz/D'+str(i)+'.gif')

#>Data preparation script: ----------------------------------------------

#It is necessary to convert the images to .bmp before running this script
# OpenCV does not support .gif images

# # Resample Brodatz input images
# data_path = './../data/brodatz/'

# WS = 64
# for i in range(1,112+1):
#     if (i == 14):
#         continue

#     img = cv2.imread(data_path + 'D' + str(i) +'.bmp', cv2.IMREAD_GRAYSCALE)

#     for rb in range(0, 10):
#         for cb in range(0, 10):
#             img_block = img[rb*WS:(rb+1)*WS, cb*WS:(cb+1)*WS]
#             # cv2.imshow('Test Image', img_block)
#             # cv2.waitKey(0)

#             cv2.imwrite(data_path + 'resampled/D' +
#               str(i) + '_' + str(rb) + '_' + str(cb) + '.bmp' , img_block)


from django.http.response import Http404
from rest_framework import serializers

from rest_framework import status
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework.permissions import IsAuthenticated, AllowAny
from rest_framework.decorators import api_view, permission_classes
from .models import HousingData
from .serializers import HousingDataSerializer
# from .models import DataCart
# from .serializers import DataCartSerializer
from django.contrib.auth.models import User

# Create your views here.
class HousingDataList(APIView):

    permission_classes = [AllowAny]
    # change to [IsAuthenticated for saved data shopping cart model where reqs userId]

    def get(self, request):
        qs = HousingData.objects.all()
        country = request.GET.get('country')
        state = request.GET.get('state')
        city = request.GET.get('city')
        lower_price = request.GET.get('lower')

        if city:
            qs = qs.filter(city=city)

        if state:
            qs = qs.filter(state=state)

        if country:
            qs = qs.filter(country=country)

        if lower_price:
            qs = qs.filter(rent1YearAvg__gte=lower_price)
            # modify and/or and more?

        serializer = HousingDataSerializer(qs, many=True)
        return Response(serializer.data)

    def post(self, request):
        serializer = HousingDataSerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class HousingDataDetail(APIView):

    def get_object(self,pk):
        try:
            return HousingData.objects.get(pk=pk)
        except HousingData.DoesNotExist:
            raise Http404

    def get(self,request,pk):
        housingdata = self.get_object(pk)
        serializer = HousingDataSerializer(housingdata)
        return Response(serializer.data)

    def put(self, request, pk):
        housingdata = self.get_object(pk)
        serializer = HousingDataSerializer(housingdata, data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    def delete(self, request, pk):
        housingdata = self.get_object(pk)
        housingdata.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

# class DataCartList(APIView):

#     permission_classes = [AllowAny]
#     # change to [IsAuthenticated for saved data shopping cart model where reqs userId]

#     def get(self, request):
#         stats = DataCart.objects.all()
#         serializer = DataCartSerializer(DataCart, many=True)
#         return Response(serializer.data)


## https://leetcode.com/problems/letter-combinations-of-a-phone-number/

from typing import List
from collections import deque

class Solution:
    def letterCombinations(self, digits: str) -> List[str]:
        alphas = {"2" : ["a", "b", "c"], "3" : ["d", "e", "f"], "4" : ["g", "h", "i"], "5" : ["j", "k", "l"],
                    "6" : ["m", "n", "o"], "7" : ["p", "q", "r", "s"], "8" : ["t", "u", "v"], "9" : ["w", "x", "y", "z"]}

        if not digits:
            return []

        q = deque()
        q.extend([(alpha, 0) for alpha in alphas[digits[0]]])

        while len(q[0][0]) < len(digits):
            string, length = q.popleft()
            for alpha in alphas[digits[length+1]]:
                q.append((string + alpha, length + 1))

        return [string for string, length in q]

print(Solution().letterCombinations("23"))


import jax
import jax.numpy as jnp
from functools import partial

def get_eval_fn(pure_logits_fn, task):

    @jax.jit
    def pure_eval_fn(params, key, token_ids):
        logits = pure_logits_fn.apply(params, key,
                                      token_ids, training=False,
                                      task=task)
        return logits

    return pure_eval_fn

@partial(jax.jit, static_argnums = (1,2,))
def greedy(key, pure_logits_eval_fn, max_length, params, initial_token_ids):

    token_ids = initial_token_ids
    for i in range(max_length):

        key, subkey = jax.random.split(key)
        logits = pure_logits_eval_fn(params, subkey, token_ids)

        predictions = jnp.argmax(logits[:,i,:], axis=-1)

        token_ids = jax.ops.index_update(token_ids, jax.ops.index[:,i+1],
                                         predictions)
    return token_ids

def first_padded_idx(x, pad_id):
    return x.shape[-1]-jnp.sum(x==pad_id, axis=-1)[0]

def seeded_greedy(key, pure_logits_eval_fn, config, params, initial_token_ids):

    token_ids = initial_token_ids
    j = first_padded_idx(initial_token_ids, config['pad_id'])
    for i in range(j, config['max_length']):

        key, subkey = jax.random.split(key)
        logits = pure_logits_eval_fn(params, subkey, token_ids)

        predictions = jnp.argmax(logits[:,i,:], axis=-1)

        token_ids = jax.ops.index_update(token_ids, jax.ops.index[:,i+1],
                                         predictions)
    return token_ids


# -*- coding: utf-8 -*-
"""
Created on Sun May 17 18:39:27 2020

Implementing a simple VAE.
Following the tutorial at: https://graviraja.github.io/vanillavae/#

@author: Jack
"""

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import DataLoader
from torchvision import datasets, transforms

import umap
import seaborn as sns

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

import sklearn
from sklearn.preprocessing import MinMaxScaler
from sklearn.cluster import KMeans
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import confusion_matrix

import itertools

#%%

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

transforms = transforms.Compose([transforms.ToTensor()])
train_ds = datasets.MNIST('./data', train=True, download=True,
                          transform=transforms)

test_ds = datasets.MNIST('./data', train=False, download=True,
        transform=transforms
    )

''' TODO: remove - the DataLoader does scaling
scaler = MinMaxScaler()
scaler.fit(train_ds.data.view(-1,28*28))
scaled = scaler.transform(train_ds.data.view(-1,28*28))
train_ds_sc = torch.Tensor(scaled).view(-1,28,28)

test_ds_sc = torch.Tensor(scaler.transform(test_ds.data.view(-1,28*28)))
test_ds_sc = test_ds_sc.view(-1,28,28)
'''

class Encoder(nn.Module):

    def __init__(self, input_dim, hidden_dim1, hidden_dim2, hidden_dim3, latent_dim):

        super().__init__()
        self.linear1 = nn.Linear(input_dim, hidden_dim1)
        self.linear2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.linear3 = nn.Linear(hidden_dim2, hidden_dim3)
        self.mu = nn.Linear(hidden_dim3, latent_dim)
        self.var = nn.Linear(hidden_dim3, latent_dim)

    def forward(self, data):

        hidden1 = F.relu(self.linear1(data))  #Not sure about using ReLu
        hidden2 = F.relu(self.linear2(hidden1))
        hidden3 = F.relu(self.linear3(hidden2))
        self.z_mu = self.mu(hidden3)
        self.z_var = self.var(hidden3)

        return self.z_mu, self.z_var

class Decoder(nn.Module):

    def __init__(self, latent_dim, hidden_dim3, hidden_dim2, hidden_dim1, out_dim):

        super().__init__()
        self.linear1 = nn.Linear(latent_dim, hidden_dim3)
        self.linear2 = nn.Linear(hidden_dim3, hidden_dim2)
        self.linear3 = nn.Linear(hidden_dim2, hidden_dim1)
        self.out = nn.Linear(hidden_dim1, out_dim)

    def forward(self, data):

        hidden1 = F.relu(self.linear1(data))
        hidden2 = F.relu(self.linear2(hidden1))
        hidden3 = F.relu(self.linear3(hidden2))
        pred = F.relu(self.out(hidden3))  #Not sure about sigmoid output

        return pred

class VAE(nn.Module):

    def __init__(self, encoder, decoder):

        super().__init__()
        self.encoder = encoder
        self.decoder = decoder

    def forward(self, data):

        self.z_mu, self.z_var = self.encoder(data)

        #Reparameterize:
        std = torch.exp(self.z_var/2)

        e = torch.randn_like(std)   #Samples a N(0,1) in the shape of std
        latent_sample = e.mul(std).add_(self.z_mu)   #Does elementwise multiplication and addition

        prediction = self.decoder(latent_sample)

        return prediction, self.z_mu, self.z_var

    def sample(self, num=1):
        z = torch.randn(num, z_dim)
        output = self.decoder(z)
        return output

in_dim = 28*28
hidden_dim1 = 256
hidden_dim2 = 128
z_dim = 12
lr = 0.001
batches = 50

train_it = DataLoader(train_ds, batch_size=batches, shuffle=True, drop_last=True)
test_it = DataLoader(test_ds, batch_size=batches, drop_last=True)

#KL divergence between N(mu,var) and N(0,1)
def KL_div(mu, var):
    mu_sq = pow(mu,2)
    sig_sq = torch.exp(var) #doesnt need /2
    loss = 0.5 * torch.sum(mu_sq - var - 1 + sig_sq )
    return loss

def get_loss(input_x, output_x, mu, logvar):
    recon_loss = F.mse_loss(output_x, input_x, reduction='sum')
    kl_loss = KL_div(mu, logvar)
    return recon_loss + kl_loss

def train(mod, e):

    mod.train()
    train_loss = 0
    loss_list = []
    average_loss = 999
    #while average_loss > 40:
    percent = 0
    for i, (data, target) in enumerate(train_it):
        data = data.view(-1,28*28)
        optimizer.zero_grad()

        output, mu, logvar = mod(data)
        loss = get_loss(data, output, mu, logvar)

        loss.backward()
        optimizer.step()
        train_loss += loss.item()
        loss_list.append(loss.item())
        percent = i*batches*100/len(train_it.dataset)
        #print(loss.item())
        if percent % 20 == 0:
            print(f"{percent}% , {loss.item()}")
        #if i*batches % len(train_it.dataset) == 0:
            #print(f"{100*i/len(train_it.dataset)}% - Loss = {loss}")
    average_loss = train_loss/len(train_it.dataset)
    print(f"====> Epoch {e} - Average train loss = {average_loss}")

    return train_loss, loss_list

def test(mod):

    mod.eval()

    test_loss = 0
    mu_list = torch.Tensor()
    logvar_list = torch.Tensor()
    output_list = torch.Tensor()

    with torch.no_grad():
        for i, (data, target) in enumerate(test_it):

            data = data.view(-1, 28*28)
            data = data.to(device)

            sample, z_mu, z_var = mod(data)
            mu_list = torch.cat([mu_list, z_mu], dim=0)
            logvar_list = torch.cat([logvar_list, z_var], dim=0)
            output_list = torch.cat([output_list, sample], dim=0)
            loss = get_loss(data, sample, z_mu, z_var)
            test_loss += loss.item()

    return test_loss, output_list, mu_list, logvar_list

num_tests = 5
epochs = 2

test_output_scores = []
test_latent_scores = []
loss_lists = []
fit_sizes = [15,20,30,40,50,60,70,80,90,100,150,200,250,300,350,400,450,500,600,700,
             800,900,1000,2000,4000,6000,8000,8250,8500,8600,8700,8800,8900,9000]

for t in range(num_tests):

    best_loss = 1e10
    wins = 0

    all_losses = []

    dim_list = [28*28, 256, 128, 64, 12]
    enc = Encoder(*dim_list)
    dim_list.reverse()
    dec = Decoder(*dim_list)
    model = VAE(enc, dec)
    optimizer = optim.Adam(model.parameters(), lr=lr)

    for e in range(epochs):

        train_loss, loss_list = train(model, e)
        all_losses.append(loss_list)
        test_loss, outputs, z_mus, z_vars = test(model)

        train_loss /= len(train_ds)
        test_loss /= len(test_ds)
        print(f"Epoch {e} - Train loss = {round(train_loss,2)}, Test loss = {round(test_loss,2)}")

        if test_loss < best_loss:
            best_loss = test_loss
        else:
            wins += 1

        if wins > 3:
            print(f"{e} epochs completed.")
            break

    loss_lists.append(all_losses)

    num = 15
    #size = 9000
    test_size = 1000
    test_loss, outputs, z_mus, z_vars = test(model)

    score_list = []
    latent_score_list = []

    for size in fit_sizes:

        knn_output = KNeighborsClassifier(n_neighbors=num)
        knn_output.fit(outputs[0:size].detach().view(-1,28*28), test_ds.targets[0:size])
        score = knn_output.score(outputs[size:size+test_size].detach().view(-1,28*28),
                                 test_ds.targets[size:size+test_size])

        knn_latent = KNeighborsClassifier(n_neighbors=num)
        knn_latent.fit(z_mus[0:size].detach(), test_ds.targets[0:size])
        latent_score = knn_latent.score(z_mus[size:size+test_size].detach(),
                                        test_ds.targets[size:size+test_size])

        score_list.append(score)
        latent_score_list.append(latent_score)

    test_output_scores.append(score_list)
    test_latent_scores.append(latent_score_list)
    print(f"Test {t} done.")

test_output_scores = pd.DataFrame(test_output_scores)
test_latent_scores = pd.DataFrame(test_latent_scores)
loss_lists = pd.DataFrame(loss_lists)

mean_outputs = test_output_scores.mean(axis=0)
mean_latent = test_latent_scores.mean(axis=0)
std_output = test_output_scores.std(axis=0)
std_latent = test_latent_scores.std(axis=0)

x = loss_lists.loc[0][0]
x += loss_lists.loc[0][1]

y = loss_lists.loc[1][0]
y += loss_lists.loc[1][1]

plt.scatter([i for i in range(len(x))], x)
plt.scatter([i for i in range(len(y))], y)

fig = plt.figure()
ax1 = fig.add_subplot(111)
ax1.errorbar(fit_sizes, mean_outputs, yerr=std_output,
             c='b', label='Output space')
ax1.errorbar(fit_sizes, mean_latent, yerr=std_latent,
             c='r', label='Latent space')
plt.xlabel("Number of supervised examples")
plt.ylabel("Mean accuracy")
plt.title("Effect of supervision on Architecture 1")
plt.legend(loc='middle right');
#plt.savefig("saved_plots/VAE_MNIST_arch1_AccuracyVSnum_08-06.png")

test_loss, outputs, z_mus, z_vars = test(model)

knn_latent = KNeighborsClassifier(n_neighbors=num)
knn_latent.fit(z_mus[0:size].detach(), test_ds.targets[0:size])
predictions = knn_latent.predict(z_mus[size:size+test_size].detach())

conf_mat = confusion_matrix(test_ds.targets[size:size+test_size], predictions)
plt.imshow(conf_mat, cmap=plt.cm.Blues)

def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix',
                          cmap=plt.cm.Blues, save_name=None):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    plt.figure(figsize=(9,7))
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    if not normalize:
        plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        cm = cm.round(2)

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")
    #plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    if save_name != None:
        plt.savefig(f"{save_name}")
        print("Saving...")

plot_confusion_matrix(conf_mat, classes=range(10), normalize=True,
                      save_name="saved_plots/VAE_MNIST_confusion-matrix-normalised_9kSupervisedKNN_08-06.png")

'''
sample = model.sample(1)
img = sample.view(28,28).data
plt.imshow(img, cmap='gray')

losses = []
for i in range(len(all_losses)):
    for j in range(len(all_losses[i])):
        losses.append(all_losses[i][j])

plt.scatter([i for i in range(len(losses))], losses)

losses, outputs, mus, logvars = test(model)
plt.imshow(outputs[0].view(28,28), cmap='gray')
plt.imshow(test_ds.data[0].view(28,28), cmap='gray')
'''

#%%

#Attempts at UMAP, with trained model:

'''
#Plot the separations in the data
fit = umap.UMAP(random_state=42)
u = fit.fit_transform(train_ds.data.view(-1,28*28))
fig,ax = plt.subplots(figsize=(10,8))
cmap = plt.get_cmap('gist_rainbow', 10)
cax = ax.scatter(u[:,0], u[:,1], s=5, c=train_ds.targets, cmap=cmap,
                 vmin=0, vmax=train_ds.targets.max())
fig.colorbar(cax)#, extend='min')
#plt.savefig(fname="saved_plots/MNIST_UMAP.png", quality=100)
'''

'''
#test_data = model.z_mu.detach().numpy() Needed since z_mu requires grad
fit2 = umap.UMAP(n_neighbors=50)
emb = fit2.fit_transform(z_mu_tensor)

fig,ax = plt.subplots(figsize=(10,8))
cmap = plt.get_cmap('Spectral', 10)
cax = ax.scatter(emb[:,0], emb[:,1], s=1, c=targets, cmap=cmap,
                 vmin=0, vmax=train_ds.targets.max())
fig.colorbar(cax)
plt.title('Latent mean-vector space of the MNIST dataset', fontsize=14)
plt.savefig(fname=f'saved_plots/latent_dim/MNIST_scaled_{z_dim}.png', quality=100)
'''


def calculate_size(age, weight):
    if age > 32 and age != 32:
        raise ValueError("Your cat is older than 32 weeks")
    else:
        size = int(round((weight/age)*32, 2))
        return size

def calculate_age(age):
    try:
        if age <= 6:
                return round((age*19) / 3  + 1)
        if age > 6:
                return round((age-6)*4 + 40)
    except ValueError:
        raise ValueError("Please enter an interger")

def calculate_bmi(rib, leg):
    bmi = (rib / 0.7062 - leg) /0.9156 - leg
    if bmi < 15:
        message = "underweight"
    if 29.9 >= bmi >= 15:
        message = "normal weight"
    if 29.9 < bmi < 42:
        message = "overweight"
    if bmi >= 42:
        message = "obese"
    return [round(bmi), message]

def header():
    print("=====================================================")
    print("============== WELCOME TO CATCULATOR ================")
    print("=====================================================")

def display_menu():
    print(f"\n")
    print("=============== Select a Catculator! ===============")
    print("= 1 - How big will my kitten get?                  =")
    print("= 2 - How old is my cat in human years?            =")
    print("= 3 - Is my cat overweight?                        =")
    print("= Q - Quit                                         =")
    print("================ ᨐᵉᵒʷ ᨐᵉᵒʷ ᨐᵉᵒʷ ᨐᵉᵒʷ ===============")
    print(f"\n")

def thanks():
    print(f"\n")
    print("==== Thanks for using Catculator! =====")
    print(f"\n")

def main():
    header()
    while True:
        quit = False
        display_menu()
        choice = input("Choose a calculator: ").strip()
        if choice == "1":
            while True:
                age = input("How old is your kitten in weeks? ").strip()
                if age.upper() =="Q":
                    quit = True
                    break
                elif age.isnumeric() == False:
                        print("Please enter a whole positive number!!")
                elif int(age) > 32:
                        print("Your cat is older than 32 weeks, which is when they become an adult! Press Q to exit")
                else:
                    age = int(age)
                    break
            while True:
                if quit == True:
                    break
                weight = input("How much does your kitten weigh in grams? ").upper().replace("G", "").strip()
                if weight.upper()== "Q":
                    quit = True
                    break
                elif weight.isnumeric()== False:
                    print("Please enter a whole positive number!")
                else:
                    weight = int(weight)
                    break
            if quit == False:
                print(f"Your kitten will grow to {calculate_size(age, weight)} grams.")
        if choice == "2":
            while True:
                age = input("How old is your cat in years? ").strip()
                if age.upper()=="Q":
                    break
                elif age.isnumeric == False:
                    print("Please enter a whole positive number!")
                    continue
                else:
                    age= int(age)
                    print(f"Your cat's age in human years is {calculate_age(age)} years old.")
                    break
        if choice == "3":
            while True:
                rib = input("What is the circumference of your cat's rib cage in centimeters? ").upper().replace("CM", "").strip()
                if rib.upper()=="Q":
                    quit = True
                    break
                if rib.isnumeric()==False:
                    print("Please enter a whole positive number!")
                else:
                    rib= int(rib)
                    break
            while True:
                if quit == True:
                    break
                leg = input("What is the distance from your cat's back knee to their ankle in centimeters? ").upper().replace("CM", "").strip()
                if leg.upper()=="Q":
                    quit=True
                    break
                elif leg.isnumeric() == False:
                    print("Please enter a whole positive number!")
                else:
                    leg= int(leg)
                    break
            if quit == False:
                print(f"Your cat's BMI is {calculate_bmi(rib, leg)[0]}. This means that they are {calculate_bmi(rib, leg)[1]}.")
        if choice.upper().strip() == "Q":
            thanks()
            break
        if input("Press enter to return to the main menu or Q to quit: ").upper().strip() == "Q":
            thanks()
            break

if __name__ == "__main__":
    main()


import subprocess

def solve(grid,solver_path='./Solver/./a.out',num_threads=4):
    """
        passes the digitized sudoku puzzle to the solver and return the final answer
            grid: the sudoku grid(9x9) in numeric format (int array)
            solver_path: the path to the solver being used
            num_threads: number of threads to be created by the solver to solve the puzzle
            (by default a parallel solver is used)
    """
    # converting the grid into a string
    grid_str = '\n'.join([str(i) for j in grid for i in j])

    # sending the string to the solver and reading the output
    proc = subprocess.run(solver_path,stdout=subprocess.PIPE,text=True,input=grid_str)
    data = proc.stdout

    # converting the output back into an array of numbers
    grid = [[0 for x in range(9)] for y in range(9)]
    nums = [float(i) for i in data.split() if i != '\n']
    for i in range(9):
        for j in range(9):
            grid[i][j] = int(nums[i*9+j])
    time_taken = nums[81]*1000
    print("solved in " + str(time_taken)+'ms')

    # returning the solved puzzle
    return grid


import re
import bs4
import string
import nltk

def strip_json(file_path):
    """
    Strips lines of html and css. Converts file contents from dictionary to string format. Removes broken unicode
    :param file_path: Path to json file
    :return: List of strings decoded from html and css format
    """

    # List of laws
    contents = []

    # Open json file
    with open(file_path) as f:

        # Loop through each line in file
        for line in f:

            # Create Beautiful soup html data_parser object
            soup = bs4.BeautifulSoup(line, "html.parser")

            # Remove tags (soup.get_text) and json format (10:-4)
            stripped = str(soup.get_text()[10:-4])

            # Remove unicode that was decoded incorrectly (\\u201a, \\n, ...)
            stripped = re.sub(r'\\[\w][\d|\w][\d|\w][\d|\w][\d|\w]||\\n', '', stripped)

            # Remove things like
            #   section 3151(e)(2) of this title
            #   September 30, 2009,
            #   paragraph (1)
            #   subsection (a)
            #   subparagraph (A)(iv)
            #   clause (ii)
            #   (D)
            #   (1)
            stripped = re.sub(r"((\w+)\.?\s(\d+)\,\s(\d{4}))|"
                              r"(\w*(?<=section).*of this title)|"
                              r"(\w*(?<=sections).*of this title)|"
                              r"(\w*(?<=section).*of title \d+)|"
                              r"(\w*(?<=sections).*of title \d+)|"
                              r"(paragraph \(\d+\))|"
                              r"(subsection \(\w+\))|"
                              r"(subparagraph \(\w+\)\(\w+\))|"
                              r"(clause \(\w+\))|"
                              r"(\(\w+\))|"
                              r"(\(\d+\))"
                              , "", stripped)

            # Add lines that have text on them
            if len(stripped) > 0:
                contents.append(stripped)

    return contents

def list_to_file(data, file_path):
    """
    Writes a list of strings to a file
    :param data: List of strings
    :param file_path: Path to file
    :return: None
    """

    # Open target file
    with open(file_path, "w") as f:

        # Write each element of list to a new line
        for line in data:
            f.write(line + "\n")

def file_to_list(file_path):
    """
    Convert file contents into a list of strings
    :param file_path: Path to file
    :return: List of strings
    """

    # List of lines in file
    contents = []

    # Open file
    with open(file_path) as f:

        # Loop through each line in file
        for line in f:

            # Add line to list
            contents.append(line)

    return contents

def create_n_grams(data, n_grams=None, max_n_gram=7):
    """
    Splits data into n-grams, ranging from n=1 to n=max_n_gram
    :param data: List of strings
    :param n_grams: Dictionary of old n_grams, if avaliable
    :return: All n-grams found for each string element in data
    """

    # If n-gram dictionary doesnt exist yet
    if n_grams is None:

        # Initialize n-gram dictionary
        n_grams = {}
        for i in range(1, max_n_gram+1):
            n_grams[i] = []

    # Fill n-gram dictionary
    for line in data:

        # Change text to lowercase
        line = line.lower()

        # Remove punctuation
        line = line.translate(str.maketrans("", "", string.punctuation))

        # Split the line by space
        split = line.split()

        # Create n-grams ranging from n=1 to n=max_n_gram
        for n in range(1, max_n_gram+1):

            # Check if current sentence is long enough to support the n-gram
            if len(split) >= n:
                current_n_grams = list(nltk.ngrams(split, n))

                # Append each n-gram
                for n_gram in current_n_grams:
                    n_grams[n].append(n_gram)

            # else, sentence is too short for current n-gram and subsequent larger n-grams
            else:
                break

    return n_grams

def get_frequency(n_grams):
    """
    Finds the number of unique n-grams
    :param n_grams: Dictionary of lists of n-grams
    :return: Dictionary of lists of unique n-grams
    """

    # Unique n-grams
    n_gram_frequency = {}

    # Loop through all n-grams starting from 1-grams
    for i in range(1, len(n_grams)+1):

        # Create a dictionary for each layer (i) of n-grams
        n_gram_frequency[i] = {}

        # Loop through all the tokens in a layer and
        for token in n_grams[i]:

            # Add tokens to dictionary
            if token not in n_gram_frequency[i]:
                n_gram_frequency[i][token] = 1
            else:

                # Count number of times the specific token is repeated
                n_gram_frequency[i][token] = n_gram_frequency[i][token] + 1

    return n_gram_frequency


# -*- coding:utf-8 -*-
import sys
import os
import time
import json
import re
import random

def get_millis_time():
    return int(round(time.time() * 1000))

def dict_to_query_str(param_dict):
    s = ''
    for key in param_dict:
        s += "{}={}&".format(key, param_dict[key])
    return s[0:-1]

def jsonp_to_dict(jsonp):
    if isinstance(jsonp, str):
        match = re.findall(r'.+\((.+)\)', jsonp)
        if len(match) == 0:
            return False
        try:
            d = json.loads(match[0])
        except:
            return False
        finally:
            if isinstance(d, dict):
                return d
    return False

def random_str(length, no_repeat=False):
    if length > 18:
        length = 18
    s = '1234567890qwertyuiopasdfghjklzxcvbnm'
    ls = len(s)
    t = []
    while True:
        j = s[random.randint(0, ls-1)]
        if no_repeat:
            if j not in t:
                t.append(j)
        else:
            t.append(j)
        if len(t) >= length:
            break

    return "".join(t)

def main_is_frozen():
    """Return ''True'' if we're running from a frozen program."""
    import imp
    return (
        # new py2exe
        hasattr(sys, "frozen") or
        # tools/freeze
        imp.is_frozen("__main__"))

def get_main_dir():
    """Return the script directory - whether we're frozen or not."""
    if main_is_frozen():
        return os.path.abspath(os.path.dirname(sys.executable))
    return os.path.abspath(os.path.dirname(sys.argv[0]))


# Тонкая настройка моделей методом сеточного поиска (Grid Search)

# В машинном обучении существует 2 типа параметров: параметры, извлекаемые из тренировочных данных, например весовые коэффициенты в
# логистической регрессии, и параметры алгоритма обучения, которые оптимизируются отдельно.

# Вторые параметры - это настроечные (гиперпараметры) параметры, которые оптимизируются отдельно, например, параметр регуляризации в
# логистической регрессии или глубина дерева решений.

# Принцип работы данного алгоритма весьма прост, мы создаем список значений для различных гиперпараметров, после этого компьютер оценивает
# качество модели для каждой их комбинации.

# Вместо раздельного выполнения шагов по подгонке и преобразованию тренировочного и тестового наборов данных мы можем расположить бъекты
# StandardScaler, PCA и LogisticRegression друг за другом в конвейере.

# -=-=-=-=-=-=-=- Этап чтения данных
import pandas as pd
import numpy as np
# url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/breast-cancer-wisconsin/wdbc.data'
# df = pd.read_csv(url, header=None)

url = '/Users/grigorev-ee/Work/AnacondaProjects/My_projects/ml_w_python/data/cancer.base'
df = pd.read_csv(url)

names = ['id_number', 'diagnosis', 'radius_mean',
         'texture_mean', 'perimeter_mean', 'area_mean',
         'smoothness_mean', 'compactness_mean', 'concavity_mean',
         'concave_points_mean', 'symmetry_mean',
         'fractal_dimension_mean', 'radius_se', 'texture_se',
         'perimeter_se', 'area_se', 'smoothness_se',
         'compactness_se', 'concavity_se', 'concave_points_se',
         'symmetry_se', 'fractal_dimension_se',
         'radius_worst', 'texture_worst', 'perimeter_worst',
         'area_worst', 'smoothness_worst',
         'compactness_worst', 'concavity_worst',
         'concave_points_worst', 'symmetry_worst',
         'fractal_dimension_worst']

df = df[names]

# Присвоим 30 признакоы массиву X библиотеки NumPy.
# При помощи LabelEncoder преобразуем метки классов из их исходного строкового представления (M и B) в целочисленное.
from sklearn.preprocessing import LabelEncoder
X = df.iloc[:,2:].values
y = df.iloc[:,1].values

print(y[:36])

le = LabelEncoder()
y=le.fit_transform(y)

# M-злокачественная опухоль (malignant), B-доброкачественная (benign). Выведем первые 36 элементов из набора данных.
print(y[:36])

# Выведем пример того, как работает кодировщик на тестовом наборе данных
le.transform(['M','M','B'])

# Прежде, чем в следующем подразделе мы построим наш первый модельный конвейер, рзделим выборку на обучающую и тренировочную.
# Оценка модели на ранее не встречавшихся данных
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)

# -=-=-=-=-=-=-= Основной алгоритм -=-=-=-=-=-=-=-=-=-=
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn. model_selection import GridSearchCV
from sklearn.svm import SVC
pipe_svc = Pipeline([('scl', StandardScaler()), ('clf', SVC(random_state=1))])
param_range = [0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0, 1000.0]

param_grid = [{'clf__C': param_range,
               'clf__kernel': ['linear']},
              {'clf__C': param_range,
               'clf__gamma': param_range,
               'clf__kernel': ['rbf']}]

# Инициализируем объект GridSearchCV, чтобы натренировать и настроить конвейер методо опорных векторов (SVM).
# Мы назначаем параметру param_grid объекта GridSearchCV список словарей с определением параметров, которые мы хотели бы настроить.
# Для линейного классификатора SVM мы настраиваем только один параметр, параметр обратной регуляризации C, для ядерного SVM c РБФ в
# качестве ядра мы выполнили настройку двух параметров: C и gamma. Отметим, что параметр gamma имеет непосредственное отношение к
# ядерным методам SVM. По результатам применения оптимизационного поиска по сетке параметров в трибуте best_score мы получим оценку
# наиболее качественной модели и обратились к его параметрам через атрибут best_params.
gs=GridSearchCV(estimator=pipe_svc, param_grid=param_grid, scoring='accuracy', cv=10, n_jobs=-1)
gs=gs.fit(X_train, y_train)
print('Перекрестно-проверочная верность: ', gs.best_score_)
print('Лучшие параметры: ', gs.best_params_)

# Воспользуемся независимым тестовым набором данных для оценки качества наилучшей отобранной модели, которая доступна через атрибут
# best_estimator_ объекта GridSearchCV
clf = gs.best_estimator_
clf.fit(X_train, y_train)
print('Верность на тестовом наборе: %.3f' % clf.score(X_test, y_test))

# Поиск по сетке параметров является мощным методом нахождения оптимальных параметров, но при этом очень затратным в плане ресурсов.
# Альтернативным подходом является метод рандомизированного поиска (RandomizedSearchCV).
# Дополнительные примеры можно найти по ссылке:
# http://scikit-learn.org/stable/modules/grid_search.html#randomized-parameter-optimization


import concurrent
import logging
import os
import re
import sys
import tarfile
import time
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from logging import Logger
from typing import Any, Callable, Dict, List

from spark_rapids_tools import CspEnv
from spark_rapids_pytools.cloud_api.sp_types import get_platform, \
    ClusterBase, DeployMode, NodeHWInfo
from spark_rapids_pytools.common.prop_manager import YAMLPropertiesContainer
from spark_rapids_pytools.common.sys_storage import FSUtil, FileVerifier
from spark_rapids_pytools.common.utilities import ToolLogging, Utils, ToolsSpinner
from spark_rapids_pytools.rapids.rapids_job import RapidsJobPropContainer
from spark_rapids_pytools.rapids.tool_ctxt import ToolContext

@dataclass
class RapidsTool(object):
    """
    A generic class that represents a RAPIDS plugin tool.
    :param platform_type: the type of platform associated with the current execution.
    :param cluster: name of the cluster on which the application will be running
    :param output_folder: location to store the output of the execution
    :param config_path: location of the configuration file of the current tool
    :param wrapper_options: dictionary containing options specific to the wrapper tool execution.
    :param rapids_options: dictionary containing the options to be passed as CLI arguments to the RAPIDS Accelerator.
    :param name: the name of the tool
    :param ctxt: context manager for the current tool execution.
    :param logger: the logger instant associated to the current tool.
    """
    platform_type: CspEnv
    cluster: str = None
    output_folder: str = None
    config_path: str = None
    wrapper_options: dict = field(default_factory=dict)
    rapids_options: dict = field(default_factory=dict)
    name: str = field(default=None, init=False)
    ctxt: ToolContext = field(default=None, init=False)
    logger: Logger = field(default=None, init=False)

    def pretty_name(self):
        return self.name.capitalize()

    def get_exec_cluster(self) -> ClusterBase:
        return self.ctxt.get_ctxt('execCluster')

    def is_remote_cluster_execution(self) -> bool:
        """
        used to verify whether a rapids tool runs on a remote cluster submission.
        This does not include the serverlessMode
        :return: True when the tool needs to have a remote cluster established
        """
        return self.ctxt.get_deploy_mode() == DeployMode.REMOTE_CLUSTER

    def requires_remote_folder(self) -> bool:
        """
        used to verify whether a rapids tool running remotely has a defined remote path to generate the
        rapids tool output.
        :return: True when the tool needs to have a remote cluster folder
        """
        deply_mode: DeployMode = self.ctxt.get_deploy_mode()
        return deply_mode.requires_remote_storage()

    def requires_cluster_connection(self) -> bool:
        return self.is_remote_cluster_execution()

    def timeit(timed_item: str):  # pylint: disable=no-self-argument
        def decorator(func_cb: Callable):
            def wrapper(self, *args, **kwargs):
                start_time = time.monotonic()
                func_cb(self, *args, **kwargs)  # pylint: disable=not-callable
                end_time = time.monotonic()
                self.logger.info('Total Execution Time: %s => %s seconds', timed_item,
                                 f'{(end_time-start_time):,.3f}')
            return wrapper
        return decorator

    def phase_banner(phase_name: str,  # pylint: disable=no-self-argument
                     enable_prologue: bool = True,
                     enable_epilogue: bool = True):
        def decorator(func_cb: Callable):
            def wrapper(self, *args, **kwargs):
                try:
                    if enable_prologue:
                        self.logger.info('******* [%s]: Starting *******', phase_name)
                    func_cb(self, *args, **kwargs)  # pylint: disable=not-callable
                    if enable_epilogue:
                        self.logger.info('======= [%s]: Finished =======', phase_name)
                except Exception:    # pylint: disable=broad-except
                    logging.exception('%s. Raised an error in phase [%s]\n',
                                      self.pretty_name(),
                                      phase_name)
                    sys.exit(1)
            return wrapper
        return decorator

    def __post_init__(self):
        # when debug is set to true set it in the environment.
        self.logger = ToolLogging.get_and_setup_logger(f'rapids.tools.{self.name}')

    def _check_environment(self) -> None:
        self.ctxt.platform.setup_and_validate_env()

    def _process_output_args(self):
        self.logger.debug('Processing Output Arguments')
        # make sure that output_folder is being absolute
        if self.output_folder is None:
            self.output_folder = Utils.get_rapids_tools_env('OUTPUT_DIRECTORY', os.getcwd())
        self.output_folder = FSUtil.get_abs_path(self.output_folder)
        self.logger.debug('Root directory of local storage is set as: %s', self.output_folder)
        self.ctxt.set_local_workdir(self.output_folder)
        self.ctxt.load_prepackaged_resources()

    def _process_rapids_args(self):
        pass

    def _process_custom_args(self):
        pass

    def _process_job_submission_args(self):
        pass

    @phase_banner('Process-Arguments')
    def _process_arguments(self):
        # 0- process the output location
        self._process_output_args()
        # 1- process any arguments to be passed to the RAPIDS tool
        self._process_rapids_args()
        # 2- we need to process the arguments of the CLI
        self._process_custom_args()
        # 3- process submission arguments
        self._process_job_submission_args()

    @phase_banner('Initialization')
    def _init_tool(self):
        self._init_ctxt()
        self._check_environment()

    def _init_ctxt(self):
        if self.config_path is None:
            self.config_path = Utils.resource_path(f'{self.name}-conf.yaml')
        self.ctxt = ToolContext(platform_cls=get_platform(self.platform_type),
                                platform_opts=self.wrapper_options.get('platformOpts'),
                                prop_arg=self.config_path,
                                name=self.name)

    def _run_rapids_tool(self):
        # 1- copy dependencies to remote server
        # 2- prepare the arguments
        # 3- create a submission job
        # 4- execute
        pass

    @phase_banner('Execution')
    def _execute(self):
        """
        Phase representing actual execution of the wrapper command.
        """
        self._run_rapids_tool()

    def _process_output(self):
        pass

    def _delete_local_dep_folder(self):
        # clean_up the local dependency folder
        local_dep_folder = self.ctxt.get_local_work_dir()
        if self.ctxt.platform.storage.resource_exists(local_dep_folder):
            self.ctxt.platform.storage.remove_resource(local_dep_folder)

    def _delete_remote_dep_folder(self):
        # clean up the remote dep folder first
        remote_dep_folder = self.ctxt.get_remote('depFolder')
        if self.ctxt.platform.storage.resource_exists(remote_dep_folder):
            # delete the folder. Note that for dataproc this will also delete the parent directory when it is empty
            self.ctxt.platform.storage.remove_resource(remote_dep_folder)

    def _download_remote_output_folder(self):
        # download the output folder in to the local one with overriding
        remote_output_folder = self.ctxt.get_remote('workDir')
        # for dataproc it is possible that the entire directory has been deleted when it is empty
        if self.ctxt.platform.storage.resource_exists(remote_output_folder):
            local_folder = self.ctxt.get_local('outputFolder')
            self.ctxt.platform.storage.download_resource(remote_output_folder, local_folder)

    def _download_output(self):
        self._delete_local_dep_folder()
        # clean up the remote dep folder first
        self._delete_remote_dep_folder()
        # download the output folder in to the local one with overriding
        self._download_remote_output_folder()

    @phase_banner('Generating Report Summary',
                  enable_epilogue=False)
    def _finalize(self):
        print(Utils.gen_str_header(f'{self.pretty_name().upper()} Report',
                                   ruler='_',
                                   line_width=100))
        self._write_summary()

    def _write_summary(self):
        pass

    @phase_banner('Archiving Tool Output')
    def _archive_phase(self):
        self._archive_results()

    def _archive_results(self):
        pass

    @phase_banner('Collecting-Results')
    def _collect_result(self):
        """
        Following a successful run, collect and process data as needed
        :return:
        """
        self._download_output()
        self._process_output()

    @phase_banner('Connecting to Execution Cluster')
    def _connect_to_execution_cluster(self):
        """
        Connecting to execution cluster
        :return:
        """
        if self.requires_cluster_connection():
            self.logger.info('%s requires the execution cluster %s to be running. '
                             'Establishing connection to cluster',
                             self.pretty_name(),
                             self.cluster)
            exec_cluster = self.ctxt.platform.connect_cluster_by_name(self.cluster)
            self.ctxt.set_ctxt('execCluster', exec_cluster)
            self._verify_exec_cluster()
        else:
            self.logger.info('%s requires no execution cluster. Skipping phase', self.pretty_name())

    def _handle_non_running_exec_cluster(self, err_msg: str) -> None:
        self.logger.warning(err_msg)

    def _verify_exec_cluster(self):
        # For remote job we should fail once we find that the cluster is not actually running
        exec_cluster = self.get_exec_cluster()
        if exec_cluster and exec_cluster.is_cluster_running():
            return
        # For remote cluster mode, the execution cluster must be running
        if not exec_cluster:
            msg = 'An error initializing the execution cluster'
        else:
            msg = f'Remote execution Cluster [{exec_cluster.get_name()}] is not active. ' \
                  f'The execution cluster should be in RUNNING state'
        self._handle_non_running_exec_cluster(msg)

    def launch(self):
        with ToolsSpinner(in_debug_mode=ToolLogging.is_debug_mode_enabled()):
            self._init_tool()
            self._connect_to_execution_cluster()
            self._process_arguments()
            self._execute()
            self._collect_result()
            self._archive_phase()
            self._finalize()

    def _report_tool_full_location(self) -> str:
        pass

    def _report_results_are_empty(self):
        return [f'The {self.pretty_name()} tool did not generate any output. Nothing to display.']

    def _generate_section_lines(self, sec_conf: dict) -> List[str]:
        all_lines = sec_conf['content'].get('lines')
        if all_lines:
            return all_lines
        return None

    def _generate_section_content(self, sec_conf: dict) -> List[str]:
        sec_title = sec_conf.get('sectionName')
        rep_lines = []
        if sec_title:
            rep_lines.append(Utils.gen_report_sec_header(sec_title, title_width=20))
        if sec_conf.get('content'):
            headers = sec_conf['content'].get('header')
            if headers:
                rep_lines.extend(headers)
            all_lines = self._generate_section_lines(sec_conf)
            if all_lines:
                rep_lines.extend(all_lines)
        return rep_lines

    def _generate_platform_report_sections(self) -> List[str]:
        section_arr = self.ctxt.platform.configs.get_value_silent('wrapperReporting',
                                                                  self.name,
                                                                  'sections')
        if section_arr:
            rep_lines = []
            for curr_sec in section_arr:
                required_flag = curr_sec.get('requiresBoolFlag')
                # if section requires a condition that was not enabled the section is skipped
                if not required_flag or self.ctxt.get_ctxt(required_flag):
                    rep_lines.extend(self._generate_section_content(curr_sec))
            return rep_lines
        return None

    def _calculate_spark_settings(self, worker_info: NodeHWInfo) -> dict:
        """
        Calculate the cluster properties that we need to append to the /etc/defaults of the spark
        if necessary.
        :param worker_info: the hardware info as extracted from the worker. Note that we assume
                            that all the workers have the same configurations.
        :return: dictionary containing 7 spark properties to be set by default on the cluster.
        """
        num_gpus = worker_info.gpu_info.num_gpus
        gpu_mem = worker_info.gpu_info.gpu_mem
        num_cpus = worker_info.sys_info.num_cpus
        cpu_mem = worker_info.sys_info.cpu_mem

        config_path = Utils.resource_path('cluster-configs.yaml')
        constants = YAMLPropertiesContainer(prop_arg=config_path).get_value('clusterConfigs', 'constants')
        executors_per_node = num_gpus
        num_executor_cores = max(1, num_cpus // executors_per_node)
        gpu_concurrent_tasks = min(constants.get('maxGpuConcurrent'), gpu_mem // constants.get('gpuMemPerTaskMB'))
        # account for system overhead
        usable_worker_mem = max(0, cpu_mem - constants.get('systemReserveMB'))
        executor_container_mem = usable_worker_mem // executors_per_node
        # reserve 10% of heap as memory overhead
        max_executor_heap = max(0, int(executor_container_mem * (1 - constants.get('heapOverheadFraction'))))
        # give up to 2GB of heap to each executor core
        executor_heap = min(max_executor_heap, constants.get('heapPerCoreMB') * num_executor_cores)
        executor_mem_overhead = int(executor_heap * constants.get('heapOverheadFraction'))
        # use default for pageable_pool to add to memory overhead
        pageable_pool = constants.get('defaultPageablePoolMB')
        # pinned memory uses any unused space up to 4GB
        pinned_mem = min(constants.get('maxPinnedMemoryMB'),
                         executor_container_mem - executor_heap - executor_mem_overhead - pageable_pool)
        executor_mem_overhead += pinned_mem + pageable_pool
        res = {
            'spark.executor.cores': num_executor_cores,
            'spark.executor.memory': f'{executor_heap}m',
            'spark.executor.memoryOverhead': f'{executor_mem_overhead}m',
            'spark.rapids.sql.concurrentGpuTasks': gpu_concurrent_tasks,
            'spark.rapids.memory.pinnedPool.size': f'{pinned_mem}m',
            'spark.sql.files.maxPartitionBytes': f'{constants.get("maxSqlFilesPartitionsMB")}m',
            'spark.task.resource.gpu.amount': 1 / num_executor_cores,
            'spark.rapids.shuffle.multiThreaded.reader.threads': num_executor_cores,
            'spark.rapids.shuffle.multiThreaded.writer.threads': num_executor_cores,
            'spark.rapids.sql.multiThreadedRead.numThreads': max(20, num_executor_cores)
        }
        return res

@dataclass
class RapidsJarTool(RapidsTool):
    """
    A wrapper class to represent wrapper commands that require RAPIDS jar file.
    """

    def _process_jar_arg(self):
        tools_jar_url = self.wrapper_options.get('toolsJar')
        if tools_jar_url is None:
            tools_jar_url = self.ctxt.get_rapids_jar_url()
        # download the jar
        jar_path = self.ctxt.platform.storage.download_resource(tools_jar_url,
                                                                self.ctxt.get_local_work_dir(),
                                                                fail_ok=False,
                                                                create_dir=True)
        self.logger.info('RAPIDS accelerator jar is downloaded to work_dir %s', jar_path)
        # get the jar file name and add it to the tool args
        jar_file_name = FSUtil.get_resource_name(jar_path)
        self.ctxt.add_rapids_args('jarFileName', jar_file_name)
        self.ctxt.add_rapids_args('jarFilePath', jar_path)

    def __accept_tool_option(self, option_key: str) -> bool:
        defined_tool_options = self.ctxt.get_value_silent('sparkRapids', 'cli', 'toolOptions')
        if defined_tool_options is not None:
            if option_key not in defined_tool_options:
                self.logger.warning('Ignoring tool option [%s]. Invalid option.', option_key)
                return False
        return True

    def _process_tool_args_from_input(self) -> list:
        """
        Process the arguments passed from the CLI if any and return a list of strings representing
        the arguments to be passed to the final command running the job. This needs processing
        because we need to verify the arguments and handle hiphens
        :return: list of the rapids arguments added by the user
        """
        arguments_list = []
        self.logger.debug('Processing Rapids plugin Arguments %s', self.rapids_options)
        raw_tool_opts: Dict[str, Any] = {}
        for key, value in self.rapids_options.items():
            if not isinstance(value, bool):
                # a boolean flag, does not need to have its value added to the list
                if isinstance(value, str):
                    # if the argument is multiple word, then protect it with single quotes.
                    if re.search(r'\s|\(|\)|,', value):
                        value = f"'{value}'"
                raw_tool_opts.setdefault(key, []).append(value)
            else:
                if value:
                    raw_tool_opts.setdefault(key, [])
                else:
                    # argument parser removes the "no-" prefix and set the value to false.
                    # we need to restore the original key
                    raw_tool_opts.setdefault(f'no{key}', [])
        for key, value in raw_tool_opts.items():
            self.logger.debug('Processing tool CLI argument.. %s:%s', key, value)
            if len(key) > 1:
                # python forces "_" to "-". we need to reverse that back.
                fixed_key = key.replace('_', '-')
                prefix = '--'
            else:
                # shortcut argument
                fixed_key = key
                prefix = '-'
            if self.__accept_tool_option(fixed_key):
                k_arg = f'{prefix}{fixed_key}'
                if len(value) >= 1:
                    # handle list options
                    for value_entry in value[0:]:
                        arguments_list.append(f'{k_arg}')
                        arguments_list.append(f'{value_entry}')
                else:
                    # this could be a boolean type flag that has no arguments
                    arguments_list.append(f'{k_arg}')
        return arguments_list

    def _process_tool_args(self):
        """
        Process the arguments passed from the CLI if any and return a string representing the
        arguments to be passed to the final command running the job.
        :return:
        """
        self.ctxt.add_rapids_args('rapidsOpts', self._process_tool_args_from_input())

    def _process_dependencies(self):
        """
        For local deployment mode, we need to process the extra dependencies specific to the platform
        :return:
        """
        if 'deployMode' in self.ctxt.platform_opts:
            # process the deployment
            # we need to download the dependencies locally if necessary
            self._download_dependencies()

    def timeit(timed_item: str):  # pylint: disable=no-self-argument
        def decorator(func_cb: Callable):
            def wrapper(self, *args, **kwargs):
                start_time = time.monotonic()
                func_cb(self, *args, **kwargs)  # pylint: disable=not-callable
                end_time = time.monotonic()
                self.logger.info('Total Execution Time: %s => %s seconds', timed_item,
                                 f'{(end_time-start_time):,.3f}')
            return wrapper
        return decorator

    @timeit('Downloading dependencies for local Mode')  # pylint: disable=too-many-function-args
    def _download_dependencies(self):

        def exception_handler(future):
            # Handle any exceptions raised by the task
            exception = future.exception()
            if exception:
                self.logger.error('Error while downloading dependency: %s', exception)

        def cache_single_dependency(dep: dict) -> str:
            """
            Downloads the specified URL and saves it to disk
            """
            start_time = time.monotonic()
            self.logger.info('Checking dependency %s', dep['name'])
            dest_folder = self.ctxt.get_cache_folder()
            resource_file_name = FSUtil.get_resource_name(dep['uri'])
            resource_file = FSUtil.build_path(dest_folder, resource_file_name)
            file_check_dict = {'size': dep['size']}
            signature_file = FileVerifier.get_signature_file(dep['uri'], dest_folder)
            if signature_file is not None:
                file_check_dict['signatureFile'] = signature_file
            algorithm = FileVerifier.get_integrity_algorithm(dep)
            if algorithm is not None:
                file_check_dict['hashlib'] = {
                    'algorithm': algorithm,
                    'hash': dep[algorithm]
                }
            is_created = FSUtil.cache_from_url(dep['uri'], resource_file, file_checks=file_check_dict)
            if is_created:
                self.logger.info('The dependency %s has been downloaded into %s', dep['uri'],
                                 resource_file)
                # check if we need to decompress files
            if dep['type'] == 'archive':
                destination_path = self.ctxt.get_local_work_dir()
                with tarfile.open(resource_file, mode='r:*') as tar:
                    tar.extractall(destination_path)
                    tar.close()
                dep_item = FSUtil.remove_ext(resource_file_name)
                if dep.get('relativePath') is not None:
                    dep_item = FSUtil.build_path(dep_item, dep.get('relativePath'))
                dep_item = FSUtil.build_path(destination_path, dep_item)
            else:
                # copy the jar into dependency folder
                dep_item = self.ctxt.platform.storage.download_resource(resource_file,
                                                                        self.ctxt.get_local_work_dir())
            end_time = time.monotonic()
            self.logger.info('Completed downloading of dependency [%s] => %s seconds',
                             dep['name'],
                             f'{(end_time-start_time):,.3f}')
            return dep_item

        def cache_all_dependencies(dep_arr: List[dict]):
            """
            Create a thread pool and download specified urls
            """
            futures_list = []
            results = []
            with ThreadPoolExecutor(max_workers=4) as executor:
                for dep in dep_arr:
                    futures = executor.submit(cache_single_dependency, dep)
                    futures.add_done_callback(exception_handler)
                    futures_list.append(futures)
                try:
                    # set the timeout to 30 minutes.
                    for future in concurrent.futures.as_completed(futures_list, timeout=1800):
                        result = future.result()
                        results.append(result)
                except Exception as ex:    # pylint: disable=broad-except
                    self.logger.error('Failed to download dependencies %s', ex)
                    raise ex
            return results

        # TODO: Verify the downloaded file by checking their MD5
        deploy_mode = DeployMode.tostring(self.ctxt.get_deploy_mode())
        depend_arr = self.ctxt.platform.configs.get_value_silent('dependencies',
                                                                 'deployMode',
                                                                 deploy_mode)
        if depend_arr:
            dep_list = cache_all_dependencies(depend_arr)
            if any(dep_item is None for dep_item in dep_list):
                raise RuntimeError('Could not download all dependencies. Aborting Executions.')
            self.logger.info('Dependencies are processed as: %s',
                             Utils.gen_joined_str(join_elem='; ',
                                                  items=dep_list))
            self.ctxt.add_rapids_args('javaDependencies', dep_list)

    def _process_rapids_args(self):
        # add a dictionary to hold the rapids arguments
        self._process_jar_arg()
        self._process_dependencies()
        self._process_tool_args()

    def _process_offline_cluster_args(self):
        pass

    def _process_gpu_cluster_args(self, offline_cluster_opts: dict = None) -> bool:
        pass

    def _copy_dependencies_to_remote(self):
        self.logger.info('Skipping preparing remote dependency folder')

    def _prepare_job_arguments(self):
        self._prepare_local_job_arguments()

    def _run_rapids_tool(self):
        # 1- copy dependencies to remote server
        self._copy_dependencies_to_remote()
        # 2- prepare the arguments
        #  2.a -check if the app_id is not none
        self._prepare_job_arguments()
        #
        # 3- create a submission job
        # 4- execute

    def _get_main_cluster_obj(self):
        return self.ctxt.get_ctxt('cpuClusterProxy')

    def _process_eventlogs_args(self):
        eventlog_arg = self.wrapper_options.get('eventlogs')
        if eventlog_arg is None:
            # get the eventlogs from spark properties
            cpu_cluster_obj = self._get_main_cluster_obj()
            if cpu_cluster_obj:
                spark_event_logs = cpu_cluster_obj.get_eventlogs_from_config()
            else:
                self.logger.warning('Eventlogs is not set properly. The property cannot be pulled '
                                    'from cluster because it is not defined')
                spark_event_logs = []
        else:
            if isinstance(eventlog_arg, tuple):
                spark_event_logs = List[eventlog_arg]
            elif isinstance(eventlog_arg, str):
                spark_event_logs = eventlog_arg.split(',')
            else:
                spark_event_logs = eventlog_arg
        if len(spark_event_logs) < 1:
            self.logger.error('Eventlogs list is empty. '
                              'The cluster Spark properties may be missing "spark.eventLog.dir". '
                              'Re-run the command passing "--eventlogs" flag to the wrapper.')
            raise RuntimeError('Invalid arguments. The list of Apache Spark event logs is empty.')
        self.ctxt.set_ctxt('eventLogs', spark_event_logs)

    def _create_migration_cluster(self, cluster_type: str, cluster_arg: str) -> ClusterBase:
        if cluster_arg is None:
            raise RuntimeError(f'The {cluster_type} cluster argument is not set.')
        arg_is_file = self.ctxt.platform.storage.is_file_path(cluster_arg)
        if not arg_is_file:
            self.logger.info('Loading %s cluster properties by name %s. Note that this will fail '
                             'if the cluster was permanently deleted.',
                             cluster_type,
                             cluster_arg)
            # create a cluster by name
            cluster_obj = self.ctxt.platform.connect_cluster_by_name(cluster_arg)
        else:
            self.logger.info('Loading %s cluster properties from file %s',
                             cluster_type,
                             cluster_arg)
            # create cluster by loading properties files
            # download the file to the working directory
            cluster_conf_path = self.ctxt.platform.storage.download_resource(cluster_arg,
                                                                             self.ctxt.get_local_work_dir())
            cluster_obj = self.ctxt.platform.load_cluster_by_prop_file(cluster_conf_path)
        return cluster_obj

    def _gen_output_tree(self) -> List[str]:
        tree_conf = self.ctxt.get_value('local', 'output', 'treeDirectory')
        if tree_conf and tree_conf.get('enabled'):
            level = tree_conf.get('depthLevel')
            indentation = tree_conf.get('indentation', '\t')
            ex_patterns = tree_conf.get('excludedPatterns', {})
            exc_dirs = ex_patterns.get('directories')
            exc_files = ex_patterns.get('files')
            out_folder_path = self.ctxt.get_local('outputFolder')
            out_tree_list = FSUtil.gen_dir_tree(out_folder_path,
                                                depth_limit=level,
                                                indent=indentation,
                                                exec_dirs=exc_dirs,
                                                exec_files=exc_files)
            doc_url = self.ctxt.get_value('sparkRapids', 'outputDocURL')
            out_tree_list.append(f'{indentation}- To learn more about the output details, visit {doc_url}')
            return out_tree_list
        return None

    def _report_tool_full_location(self) -> str:
        if not self._rapids_jar_tool_has_output():
            return None
        out_folder_path = self.ctxt.get_rapids_output_folder()
        res_arr = [Utils.gen_report_sec_header('Output'),
                   f'{self.pretty_name()} tool output: {out_folder_path}']
        out_tree_list = self._gen_output_tree()
        return Utils.gen_multiline_str(res_arr, out_tree_list)

    def _evaluate_rapids_jar_tool_output_exist(self) -> bool:
        """
        Used as a subtask of self._process_output(). this method has the responsibility of
        checking if the tools produced no output and take the necessary action
        :return: True if the tool has generated an output
        """
        rapids_output_dir = self.ctxt.get_rapids_output_folder()
        res = True
        if not self.ctxt.platform.storage.resource_exists(rapids_output_dir):
            res = False
            self.ctxt.set_ctxt('wrapperOutputContent',
                               self._report_results_are_empty())
            self.logger.info('The Rapids jar tool did not generate an output directory')
        self.ctxt.set_ctxt('rapidsOutputIsGenerated', res)
        return res

    def _rapids_jar_tool_has_output(self) -> bool:
        return self.ctxt.get_ctxt('rapidsOutputIsGenerated')

    @timeit('Processing job submission arguments')  # pylint: disable=too-many-function-args
    def _process_job_submission_args(self):
        self._process_local_job_submission_args()

    def _set_remote_folder_for_submission(self, requires_remote_storage: bool) -> dict:
        res = {}
        submission_args = self.wrapper_options.get('jobSubmissionProps')
        # get the root remote folder and make sure it exists
        remote_folder = submission_args.get('remoteFolder')
        # If remote_folder is not specified, then ignore it
        if requires_remote_storage:
            # if the remote storage required and no remote folder specified. then try to assign the
            # tmp storage of the exec_cluster to be used for storage
            archive_enabled = True
            if not remote_folder:
                # get the execCluster
                exec_cluster = self.get_exec_cluster()
                if exec_cluster:
                    remote_folder = exec_cluster.get_tmp_storage()
                    if remote_folder:
                        archive_enabled = False
            self.ctxt.set_ctxt('archiveToRemote', archive_enabled)
        if remote_folder is None:
            # the output is only for local machine
            self.logger.info('No remote output folder specified.')
            if requires_remote_storage:
                raise RuntimeError(f'Remote folder [{remote_folder}] is invalid.')
        else:
            if not self.ctxt.platform.storage.resource_exists(remote_folder):
                raise RuntimeError(f'Remote folder invalid path. [{remote_folder}] does not exist.')
            # now we should make the subdirectory to indicate the output folder,
            # by appending the name of the execution folder
            exec_full_name = self.ctxt.get_ctxt('execFullName')
            remote_workdir = FSUtil.build_url_from_parts(remote_folder, exec_full_name)
            self.ctxt.set_remote('rootFolder', remote_folder)
            self.ctxt.set_remote('workDir', remote_workdir)
            self.logger.info('Remote workdir is set as %s', remote_workdir)
            remote_dep_folder = FSUtil.build_url_from_parts(remote_workdir,
                                                            self.ctxt.get_ctxt('depFolderName'))
            self.ctxt.set_remote('depFolder', remote_dep_folder)
            self.logger.info('Remote dependency folder is set as %s', remote_dep_folder)
        if requires_remote_storage:
            res.update({'outputDirectory': self.ctxt.get_remote('workDir')})
        else:
            # the output folder has to be set any way
            res.update({'outputDirectory': self.ctxt.get_output_folder()})
        return res

    def _process_local_job_submission_args(self):
        job_args = {}
        submission_args = self.wrapper_options.get('jobSubmissionProps')
        job_args.update(self._set_remote_folder_for_submission(self.requires_remote_folder()))
        platform_args = submission_args.get('platformArgs')
        if platform_args is not None:
            processed_platform_args = self.ctxt.platform.cli.build_local_job_arguments(platform_args)
            ctxt_rapids_args = self.ctxt.get_ctxt('rapidsArgs')
            dependencies = ctxt_rapids_args.get('javaDependencies')
            processed_platform_args.update({'dependencies': dependencies})
            job_args['platformArgs'] = processed_platform_args
        self.ctxt.update_job_args(job_args)

    def _init_rapids_arg_list(self) -> List[str]:
        return []

    @timeit('Building Job Arguments and Executing Job CMD')  # pylint: disable=too-many-function-args
    def _prepare_local_job_arguments(self):
        job_args = self.ctxt.get_ctxt('jobArgs')
        # now we can create the job object
        # Todo: For dataproc, this can be autogenerated from cluster name
        rapids_arg_list = self._init_rapids_arg_list()
        ctxt_rapids_args = self.ctxt.get_ctxt('rapidsArgs')
        jar_file_path = ctxt_rapids_args.get('jarFilePath')
        rapids_opts = ctxt_rapids_args.get('rapidsOpts')
        if rapids_opts:
            rapids_arg_list.extend(rapids_opts)
        # add the eventlogs at the end of all the tool options
        rapids_arg_list.extend(self.ctxt.get_ctxt('eventLogs'))
        class_name = self.ctxt.get_value('sparkRapids', 'mainClass')
        rapids_arg_obj = {
            'jarFile': jar_file_path,
            'jarArgs': rapids_arg_list,
            'className': class_name
        }
        platform_args = job_args.get('platformArgs')
        spark_conf_args = {}
        job_properties_json = {
            'outputDirectory': job_args.get('outputDirectory'),
            'rapidsArgs': rapids_arg_obj,
            'sparkConfArgs': spark_conf_args,
            'platformArgs': platform_args
        }
        job_properties = RapidsJobPropContainer(prop_arg=job_properties_json,
                                                file_load=False)
        job_obj = self.ctxt.platform.create_local_submission_job(job_prop=job_properties,
                                                                 ctxt=self.ctxt)
        job_obj.run_job()

    def _archive_results(self):
        self._archive_local_results()

    def _archive_local_results(self):
        remote_work_dir = self.ctxt.get_remote('workDir')
        if remote_work_dir and self._rapids_jar_tool_has_output():
            local_folder = self.ctxt.get_output_folder()
            # TODO make sure it worth issuing the command
            self.ctxt.platform.storage.upload_resource(local_folder, remote_work_dir)


n = int(input())

def ok(word):
    word = [ord(i) for i in word]
    return max(word) - min(word) <= 10

ans = []
while n:
    a = input()
    if ok(a):
        ans.append(a)
    n-=1
print(*ans, sep='\n')


import re

started=[]
can_start=[]
workers = {}
for i in range(0,2):
    workers[i] = {'curr':'.', 'steps_remaining':0}
ticks = {}
for i,c in zip(range(1,27),list('ABCDEFGHIJKLMNOPQRSTUVWXYZ')):
    ticks[c]=i
times=0
tt = {}
def part1():
    global can_start
    global started
    events = {}
    line_re = re.compile("^Step (.*) must be finished before step (.*) can begin.")
    for line in open("test.txt","r"):
        parsed_line = re.match(line_re,line)
        step = parsed_line.group(1)
        before = parsed_line.group(2)
        if step not in events:
            events[step] = [before]
        else:
            events[step].append(before)
        events[step] = sorted(events[step])
        print(f"{step} -> {before}")
    print(events)
    print(ticks)
    print()
    traverse(events)

def traverse(events):
    global can_start
    global started
    global times

    for k in events:
        if k in started: continue
        if can_it_start(k,events,started):
            if k not in can_start:
                can_start.append(k)
    can_start = sorted(can_start)
    print("can_start", can_start)
    if len(can_start) < 1:
        return
    ss = can_start[0]
    started.append(ss)
    times += ticks[ss]
    for xx in can_start:
        tt[xx] = ticks[xx] - times

    can_start.remove(ss)
    print(''.join(started), "time:", times, len(started), tt)
    input()
    traverse(events)

def can_it_start(k,dic,started):
    if k not in dic:
        return False
    for i in dic:
        if i in started:
            continue
        if k in dic[i]:
            return False
    return True

if __name__ == "__main__":
    part1()


import requests
import json

URL = "http://127.0.0.1:8000/deserializer/"

data = {
    'name': 'Tushar',
    'roll': 101,
    'city': 'Dahaka'
}

json_data = json.dumps(data)

req = requests.post(url=URL, data=json_data)

data = req.json()
print(data)


class Solution:
    def longestCommonPrefix(self, strs) -> str:
        prefix = ""
        shortest = min(strs, key=len)
        for i in range(len(shortest)):
            letter = set()
            for word in strs:
                letter.add(word[i])
            if len(letter)==1:
                prefix += word[i]
            else:
                return prefix
        return prefix


class Solution:
    def pancakeSort(self, arr: List[int]) -> List[int]:
        def flip(sub_array, k):
            sub_array[:k] = sub_array[:k][::-1]

        res = []
        n = len(arr)
        for i in range(n, 0, -1):
            max_idx = arr.index(max(arr[:i]))
            if max_idx != i-1:
                if max_idx != 0:
                    flip(arr, max_idx+1)
                    res.append(max_idx+1)
                flip(arr, i)
                res.append(i)
        return res